diff --git a/Makefile b/Makefile index 89ea80accd19c..692526a77cf2f 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,12 @@ # % cd matrixone # % MO_CL_CUDA=1 make +# Go toolchain (override with `make GO=/path/to/go ...`); defaults to `go`. +# Requires Go 1.26+ for the arch-specific SIMD kernels (built by default on x86_64). +ifeq ($(GO),) + GO=go +endif + # where am I ROOT_DIR = $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) BIN_NAME := mo-service @@ -185,6 +191,28 @@ DEBUG_OPT := CGO_DEBUG_OPT := TAGS := +# Env-var prefix for the build command. On x86_64 the arch-specific SIMD kernels in +# pkg/vectorindex/metric are compiled by default (ARCHSIMD=1): GOAMD64 defaults to v3 +# (Haswell baseline -- AVX2/FMA/BMI, required by the Go simd experiment) and +# GOEXPERIMENT defaults to simd (enables the goexperiment.simd build tag on Go 1.26+). +# Disable the SIMD kernels with: +# make ARCHSIMD=0 build # plain x86 build, no SIMD kernels +# Either default can still be overridden individually, e.g. `make GOAMD64=v4 build`. +GOEXPERIMENT_OPT ?= +ifeq ("$(UNAME_M)", "x86_64") + ARCHSIMD ?= 1 + ifeq ($(ARCHSIMD),1) + GOAMD64 ?= v3 + GOEXPERIMENT_SIMD ?= simd + endif + ifneq ($(GOAMD64),) + GOEXPERIMENT_OPT += GOAMD64=$(GOAMD64) + endif + ifneq ($(GOEXPERIMENT_SIMD),) + GOEXPERIMENT_OPT += GOEXPERIMENT=$(GOEXPERIMENT_SIMD) + endif +endif + ifeq ($(MO_CL_CUDA),1) ifeq ($(CONDA_PREFIX),) $(error CONDA_PREFIX env variable not found.) @@ -235,7 +263,7 @@ jieba-dict: .PHONY: build build: config cgo thirdparties jieba-dict $(info [Build binary]) - $(CGO_OPTS) go build $(TAGS) $(RACE_OPT) $(GOLDFLAGS) $(DEBUG_OPT) $(GOBUILD_OPT) -o $(BIN_NAME) ./cmd/mo-service + $(GOEXPERIMENT_OPT) $(CGO_OPTS) $(GO) build $(TAGS) $(RACE_OPT) $(GOLDFLAGS) $(DEBUG_OPT) $(GOBUILD_OPT) -o $(BIN_NAME) ./cmd/mo-service # https://wiki.musl-libc.org/getting-started.html # https://musl.cc/ @@ -265,13 +293,13 @@ musl: override TAGS := -tags musl musl: musl-install musl-cgo config musl-thirdparties jieba-dict musl: $(info [Build binary(musl)]) - $(CGO_OPTS) go build $(TAGS) $(RACE_OPT) $(GOLDFLAGS) $(DEBUG_OPT) $(GOBUILD_OPT) -o $(BIN_NAME) ./cmd/mo-service + $(GOEXPERIMENT_OPT) $(CGO_OPTS) $(GO) build $(TAGS) $(RACE_OPT) $(GOLDFLAGS) $(DEBUG_OPT) $(GOBUILD_OPT) -o $(BIN_NAME) ./cmd/mo-service # build mo-tool .PHONY: mo-tool mo-tool: config cgo thirdparties $(info [Build mo-tool tool]) - $(CGO_OPTS) go build $(GOLDFLAGS) -o mo-tool ./cmd/mo-tool + $(GOEXPERIMENT_OPT) $(CGO_OPTS) $(GO) build $(GOLDFLAGS) -o mo-tool ./cmd/mo-tool # build mo-service binary for debugging with go's race detector enabled # produced executable is 10x slower and consumes much more memory diff --git a/cgo/cuvs/Makefile b/cgo/cuvs/Makefile index 5386d81ea2b25..cee1b339eb15f 100644 --- a/cgo/cuvs/Makefile +++ b/cgo/cuvs/Makefile @@ -128,6 +128,19 @@ test_kmeans: obj/test/test_kmeans.o $(OBJS) @echo "Linking $@" $(NVCC) $(LDFLAGS) $^ $(LIBS) -o $@ +# Standalone reproducer for the uint8 quantization recall collapse +# (f32->uint8 and f16->uint8). Has its own main(); runs in isolation +# rather than as part of the full test_cuvs_worker suite. +uint8_quant_bug: obj/test/uint8_quant_bug.o $(OBJS) + @echo "Linking $@" + $(NVCC) $(LDFLAGS) $^ $(LIBS) -o $@ + +# Real-dataset (wiki_all_1M) version of the uint8 collapse reproducer. Loads the +# .fbin base/queries + .ibin ground truth and grades recall@k at 1M scale. +wiki1m_uint8_bug: obj/test/wiki1m_uint8_bug.o $(OBJS) + @echo "Linking $@" + $(NVCC) $(LDFLAGS) $^ $(LIBS) -o $@ + # Standalone reproducer for the cuvs::neighbors::dynamic_batching deadlock # (conservative_dispatch=true). Intentionally depends on nothing in this # project — links only against the cuVS / RAFT / RMM libraries we already diff --git a/cgo/cuvs/brute_force.hpp b/cgo/cuvs/brute_force.hpp index f0046c3304a1e..7afc03ce3309f 100644 --- a/cgo/cuvs/brute_force.hpp +++ b/cgo/cuvs/brute_force.hpp @@ -102,7 +102,7 @@ namespace matrixone { // is_loaded_=true, then clears flattened_host_dataset to free memory. // The built index holds a device pointer to the dataset via // dataset_device_ptr_ (shared_ptr kept alive by index_). -// 4. search() / search_float() — dispatched via submit() (round-robin, but with +// 4. search() / search_quantize() — dispatched via submit() (round-robin, but with // SINGLE_GPU there is only one device). // 5. Destructor calls destroy() which stops the worker and resets index_. // @@ -125,7 +125,7 @@ namespace matrixone { // search_internal() holds a shared_lock during the GPU search call (read-only // access to index_). This is fine because brute_force is SINGLE_GPU and has no // concurrent extend path. -// search_float_internal() converts float queries to T on the device before +// search_quantize_internal() converts base (B) queries to T on the device before // searching (quantize for 1-byte T, half-cast for T=half, direct for T=float). // // SOFT-DELETE BITSET @@ -160,15 +160,21 @@ struct brute_force_search_result_t { /** * @brief gpu_brute_force_t implements a Brute Force index that can run on a single GPU. */ -template -class gpu_brute_force_t : public gpu_index_base_t { +// [B,Q] design (B = base/query element type, T = storage element type), mirroring +// gpu_cagra_t / gpu_ivf_pq_t. For the unquantized cases B==T; the overflow of a +// quantized index stores T (e.g. half) while the base/query is B (e.g. float), +// so search_quantize() converts B -> T (cast for f32->f16, learned SQ for 1-byte). +template +class gpu_brute_force_t : public gpu_index_base_t { public: + using base_type = B; + using storage_type = T; // We force DistT=float for all our indices to avoid template bloat and satisfy cuVS using brute_force_index = cuvs::neighbors::brute_force::index; using search_result_t = brute_force_search_result_t; // Inherited dependent type — bring into scope so search_internal can take a // const host_mask_bundle_t* parameter without `typename Base::...` everywhere. - using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; + using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; // Internal index storage std::unique_ptr index_; @@ -536,71 +542,73 @@ class gpu_brute_force_t : public gpu_index_base_tsearch_float_async(queries_data, num_queries, query_dimension, limit, sp); + // Sync quantize entry — wraps search_quantize_async + search_wait. The query + // is the BASE type B; search_quantize_internal converts it to storage T. + search_result_t search_quantize(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp) { + uint64_t job_id = this->search_quantize_async(queries_data, num_queries, query_dimension, limit, sp); return this->search_wait(job_id); } - uint64_t search_float_async(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp) { - if constexpr (std::is_same_v) return search_async(queries_data, num_queries, query_dimension, limit, sp); + uint64_t search_quantize_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp) { + if constexpr (std::is_same_v) return search_async(queries_data, num_queries, query_dimension, limit, sp); if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); // Reject a mismatched caller dim instead of silently coercing to - // this->dimension. search_float_internal sizes its H2D extent by + // this->dimension. search_quantize_internal sizes its H2D extent by // this->dimension; if caller's query_dimension differed we'd either // OOB-read or under-copy host queries. Fail loudly so the caller bug // surfaces here rather than as wrong search results. if (query_dimension != this->dimension) { throw std::invalid_argument( - "search_float_async: query_dimension (" + std::to_string(query_dimension) + + "search_quantize_async: query_dimension (" + std::to_string(query_dimension) + ") does not match index dimension (" + std::to_string(this->dimension) + ")"); } if (!this->is_loaded_ || !index_) throw std::runtime_error("search_async: index not loaded"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp); }; return this->worker->submit(task); } - // Sync float filtered entry — wraps search_float_with_filter_async + search_wait. - search_result_t search_float_with_filter(const float* queries_data, uint64_t num_queries, + // Sync quantize filtered entry — wraps search_quantize_with_filter_async + search_wait. + search_result_t search_quantize_with_filter(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp, const std::string& preds_json) { - uint64_t job_id = this->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); + uint64_t job_id = this->search_quantize_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); return this->search_wait(job_id); } - // Async variant of search_float_with_filter. Brute force is single-GPU + // Async variant of search_quantize_with_filter. Brute force is single-GPU // only, so the bitmap eval stays on the calling thread and the GPU // search goes through worker->submit so concurrent calls can be // auto-batched in the device queue. Used by the multi-index brute-force // fallback so it dispatches in parallel with the primary IVF/CAGRA shards. - uint64_t search_float_with_filter_async(const float* queries_data, uint64_t num_queries, + // The query is the BASE type B; search_quantize_internal converts it to T. + uint64_t search_quantize_with_filter_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const brute_force_search_params_t& sp, const std::string& preds_json) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); - // See search_float_async() above for the rationale. + // See search_quantize_async() above for the rationale. if (query_dimension != this->dimension) { throw std::invalid_argument( - "search_float_with_filter_async: query_dimension (" + std::to_string(query_dimension) + + "search_quantize_with_filter_async: query_dimension (" + std::to_string(query_dimension) + ") does not match index dimension (" + std::to_string(this->dimension) + ")"); } if (!this->is_loaded_ || !index_) throw std::runtime_error("search_async: index not loaded"); if (!this->worker) throw std::runtime_error("Worker not initialized"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); auto mask = this->build_filter_single_mask(preds_json); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy, mask](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); }; return this->worker->submit(task); } @@ -608,7 +616,11 @@ class gpu_brute_force_t : public gpu_index_base_t int8/uint8; the remaining + // (B=float, T=half) instantiation casts f32 -> f16. + search_result_t search_quantize_internal(raft_handle_wrapper_t& handle, const B* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const brute_force_search_params_t& /*sp*/, const std::string& /*preds_json*/ = "", const host_mask_bundle_t* prebuilt = nullptr) { // Same snapshot pattern as search_internal — see comment there. const brute_force_index* local_index = nullptr; uint64_t local_count = 0; @@ -616,7 +628,7 @@ class gpu_brute_force_t : public gpu_index_base_t lock(this->mutex_); if (!this->is_loaded_ || !this->index_) { - throw std::runtime_error("search_float_internal: index not loaded"); + throw std::runtime_error("search_quantize_internal: index not loaded"); } local_index = this->index_.get(); local_count = this->count; @@ -627,19 +639,20 @@ class gpu_brute_force_t : public gpu_index_base_t(*res, num_queries, this->dimension); - if constexpr (std::is_same_v) { - raft::copy(*res, q_dev_t.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + if constexpr (std::is_same_v) { + // B == T: no conversion. + raft::copy(*res, q_dev_t.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + } else if constexpr (sizeof(T) == 1) { + // sizeof(T) == 1: quantize the base-typed query B -> int8/uint8. + if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); + auto q_dev_b = raft::make_device_matrix(*res, num_queries, this->dimension); + raft::copy(*res, q_dev_b.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + this->quantizer_.template transform(*res, q_dev_b.view(), q_dev_t.data_handle(), true); } else { - auto q_dev_f = raft::make_device_matrix(*res, num_queries, this->dimension); - raft::copy(*res, q_dev_f.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - - if constexpr (sizeof(T) == 1) { - if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); - this->quantizer_.template transform(*res, q_dev_f.view(), q_dev_t.data_handle(), true); - } else { - // T is half - raft::copy(*res, q_dev_t.view(), q_dev_f.view()); - } + // B != T and sizeof(T) != 1: (B=float, T=half) — cast f32 -> f16 on-device. + auto q_dev_b = raft::make_device_matrix(*res, num_queries, this->dimension); + raft::copy(*res, q_dev_b.view(), raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + raft::copy(*res, q_dev_t.view(), q_dev_b.view()); } // Legacy path syncs so the deletes-only sync_device_bitset below can // drain on the same stream. Prebuilt path skips: bitset H2D queues @@ -729,7 +742,7 @@ class gpu_brute_force_t : public gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"Brute-Force\", \"brute_force\": {"; if (index_) json += "\"built\": true"; else json += "\"built\": false"; diff --git a/cgo/cuvs/brute_force_c.cpp b/cgo/cuvs/brute_force_c.cpp index 2ae451dd44eb1..1d79da8fd9fb7 100644 --- a/cgo/cuvs/brute_force_c.cpp +++ b/cgo/cuvs/brute_force_c.cpp @@ -1,4 +1,4 @@ -/* +/* * Copyright 2021 Matrix Origin * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,20 @@ /* * Brute-Force C Wrapper Implementation - * Supported data types (via quantization_t): Quantization_F32, Quantization_F16 + * + * Two type axes via quantization_t: + * btype = base / query / quantizer-SOURCE element type (Quantization_F32 or F16) + * qtype = storage element type (Quantization_F32 or F16) + * + * Wired (btype, qtype) combinations: + * F32 base: F32, F16 storage + * F16 base: F16 storage + * Any other combination throws "unsupported (base,storage) type combination". + * + * NOTE: unlike CAGRA, cuVS brute_force only provides build/search for + * index and index — there is NO int8_t/uint8_t + * storage path. So the INT8/UINT8 qtype combos are intentionally omitted + * here (instantiating them would fail to bind cuvs::brute_force::build/search). */ #include "brute_force_c.h" @@ -28,39 +41,71 @@ #include #include #include +#include +#include -struct gpu_brute_force_any_t { +using namespace matrixone; - quantization_t qtype; +struct gpu_brute_force_any_t { + quantization_t btype; // base / query / quantizer-source element type + quantization_t qtype; // storage element type void* ptr; - gpu_brute_force_any_t(quantization_t q, void* p) : qtype(q), ptr(p) {} - ~gpu_brute_force_any_t() { - switch (qtype) { - case Quantization_F32: delete static_cast*>(ptr); break; - case Quantization_F16: delete static_cast*>(ptr); break; - default: break; + gpu_brute_force_any_t(quantization_t b, quantization_t q, void* p) + : btype(b), qtype(q), ptr(p) {} + ~gpu_brute_force_any_t(); +}; + +// Static dispatch: resolves the concrete gpu_brute_force_t for (btype,qtype) +// and invokes fn with a typed pointer. fn is a generic lambda; recover B/Q inside +// it via decltype(idx)::base_type / ::storage_type. Throws on unsupported combos. +template +static auto brute_force_dispatch(const gpu_brute_force_any_t* a, Fn&& fn) { + switch (a->btype) { + case Quantization_F32: + switch (a->qtype) { + case Quantization_F32: return fn(static_cast*>(a->ptr)); + case Quantization_F16: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + case Quantization_F16: + switch (a->qtype) { + case Quantization_F16: return fn(static_cast*>(a->ptr)); + default: break; } + break; + default: break; } -}; + throw std::runtime_error("gpu_brute_force: unsupported (base,storage) type combination"); +} + +gpu_brute_force_any_t::~gpu_brute_force_any_t() { + if (!ptr) return; + try { + brute_force_dispatch(this, [](auto* idx) { + delete idx; + }); + } catch (...) { + // unsupported combo never gets a live ptr — nothing to free + } +} extern "C" { -gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, uint32_t nthread, int device_id, quantization_t qtype, const int64_t* ids, void* errmsg) { +gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, uint32_t nthread, int device_id, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { void* index_ptr = nullptr; if (errmsg) *(static_cast(errmsg)) = nullptr; try { - switch (qtype) { - case Quantization_F32: - index_ptr = new matrixone::gpu_brute_force_t(static_cast(dataset_data), count_vectors, dimension, metric_c, nthread, device_id, ids); - break; - case Quantization_F16: - index_ptr = new matrixone::gpu_brute_force_t(static_cast(dataset_data), count_vectors, dimension, metric_c, nthread, device_id, ids); - break; - default: - throw std::runtime_error("Unsupported quantization type for brute force (only f32 and f16 supported)"); - } - return static_cast(new gpu_brute_force_any_t(qtype, index_ptr)); + // Construct the right gpu_brute_force_t; the native build + // constructor takes storage-typed (T) data. + gpu_brute_force_any_t key(btype, qtype, nullptr); + index_ptr = brute_force_dispatch(&key, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using T = typename std::remove_pointer_t::storage_type; + return new gpu_brute_force_t(static_cast(dataset_data), count_vectors, dimension, metric_c, nthread, device_id, ids); + }); + return static_cast(new gpu_brute_force_any_t(btype, qtype, index_ptr)); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_new", e.what()); @@ -72,21 +117,17 @@ gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_v } } -gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, uint32_t nthread, int device_id, quantization_t qtype, const int64_t* ids, void* errmsg) { +gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, uint32_t nthread, int device_id, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { void* index_ptr = nullptr; if (errmsg) *(static_cast(errmsg)) = nullptr; try { - switch (qtype) { - case Quantization_F32: - index_ptr = new matrixone::gpu_brute_force_t(total_count, dimension, metric_c, nthread, device_id, ids); - break; - case Quantization_F16: - index_ptr = new matrixone::gpu_brute_force_t(total_count, dimension, metric_c, nthread, device_id, ids); - break; - default: - throw std::runtime_error("Unsupported quantization type for brute force (only f32 and f16 supported)"); - } - return static_cast(new gpu_brute_force_any_t(qtype, index_ptr)); + gpu_brute_force_any_t key(btype, qtype, nullptr); + index_ptr = brute_force_dispatch(&key, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using T = typename std::remove_pointer_t::storage_type; + return new gpu_brute_force_t(total_count, dimension, metric_c, nthread, device_id, ids); + }); + return static_cast(new gpu_brute_force_any_t(btype, qtype, index_ptr)); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_new_empty", e.what()); @@ -101,12 +142,7 @@ gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimen void gpu_brute_force_start(gpu_brute_force_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->start(); break; - case Quantization_F16: static_cast*>(any->ptr)->start(); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [](auto* idx) { idx->start(); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_start", e.what()); @@ -119,12 +155,7 @@ void gpu_brute_force_start(gpu_brute_force_c index_c, void* errmsg) { void gpu_brute_force_build(gpu_brute_force_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->build(); break; - case Quantization_F16: static_cast*>(any->ptr)->build(); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [](auto* idx) { idx->build(); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_build", e.what()); @@ -137,12 +168,10 @@ void gpu_brute_force_build(gpu_brute_force_c index_c, void* errmsg) { void gpu_brute_force_add_chunk(gpu_brute_force_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_add_chunk", e.what()); @@ -152,45 +181,35 @@ void gpu_brute_force_add_chunk(gpu_brute_force_c index_c, const void* chunk_data } } -void gpu_brute_force_add_chunk_float(gpu_brute_force_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { +// Base-typed (B) add: the add counterpart of search_quantize. queries_data is in +// the base element type B; add_chunk_quantize converts B -> storage T (native +// store when B==T, f32->f16 cast for (float,half), learned SQ for 1-byte). Used +// by the CDC overflow build to store base vectors at the index's storage type. +void gpu_brute_force_add_chunk_quantize(gpu_brute_force_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->add_chunk_quantize(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, - "Error in gpu_brute_force_add_chunk_float", e.what()); + "Error in gpu_brute_force_add_chunk_quantize", e.what()); } catch (...) { matrixone::set_errmsg(errmsg, - "Error in gpu_brute_force_add_chunk_float", "unknown C++ exception"); + "Error in gpu_brute_force_add_chunk_quantize", "unknown C++ exception"); } } gpu_brute_force_search_result_c gpu_brute_force_search(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto res = std::make_unique::search_result_t>(); - *res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, brute_force_search_params_default()); - result_ptr = res.release(); - break; - } - case Quantization_F16: { - auto res = std::make_unique::search_result_t>(); - *res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, brute_force_search_params_default()); - result_ptr = res.release(); - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search(static_cast(queries_data), num_queries, query_dimension, limit, brute_force_search_params_default()); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search", e.what()); @@ -202,49 +221,35 @@ gpu_brute_force_search_result_c gpu_brute_force_search(gpu_brute_force_c index_c } } -gpu_brute_force_search_result_c gpu_brute_force_search_float(gpu_brute_force_c index_c, const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { +gpu_brute_force_search_result_c gpu_brute_force_search_quantize(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto res = std::make_unique::search_result_t>(); - *res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, brute_force_search_params_default()); - result_ptr = res.release(); - break; - } - case Quantization_F16: { - auto res = std::make_unique::search_result_t>(); - *res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, brute_force_search_params_default()); - result_ptr = res.release(); - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize(static_cast(queries_data), num_queries, query_dimension, limit, brute_force_search_params_default()); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, - "Error in gpu_brute_force_search_float", e.what()); + "Error in gpu_brute_force_search_quantize", e.what()); return nullptr; } catch (...) { matrixone::set_errmsg(errmsg, - "Error in gpu_brute_force_search_float", "unknown C++ exception"); + "Error in gpu_brute_force_search_quantize", "unknown C++ exception"); return nullptr; } } -uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, +uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t search_params; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_async", e.what()); return 0; @@ -254,22 +259,20 @@ uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* que } } -uint64_t gpu_brute_force_search_float_async(gpu_brute_force_c index_c, const float* queries_data, uint64_t num_queries, +uint64_t gpu_brute_force_search_quantize_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t search_params; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_async", "unknown C++ exception"); return 0; } } @@ -277,24 +280,11 @@ uint64_t gpu_brute_force_search_float_async(gpu_brute_force_c index_c, const flo gpu_brute_force_search_result_c gpu_brute_force_search_wait(gpu_brute_force_c index_c, uint64_t job_id, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); - result_ptr = cpp_res; - break; - } - case Quantization_F16: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); - result_ptr = cpp_res; - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + *cpp_res = idx->search_wait(job_id); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_wait", e.what()); return nullptr; @@ -307,7 +297,7 @@ gpu_brute_force_search_result_c gpu_brute_force_search_wait(gpu_brute_force_c in void gpu_brute_force_get_results(gpu_brute_force_search_result_c result_c, uint64_t num_queries, uint32_t limit, int64_t* neighbors, float* distances) { try { if (!result_c) return; - auto* search_result = static_cast::search_result_t*>(result_c); + auto* search_result = static_cast(result_c); size_t total = num_queries * limit; if (search_result->neighbors.size() >= total) { @@ -329,7 +319,7 @@ void gpu_brute_force_get_results(gpu_brute_force_search_result_c result_c, uint6 void gpu_brute_force_free_search_result(gpu_brute_force_search_result_c result_c) { try { if (!result_c) return; - delete static_cast::search_result_t*>(result_c); + delete static_cast(result_c); } catch (...) { matrixone::log_err("gpu_brute_force_free_search_result: unknown C++ exception (swallowed)"); } @@ -338,12 +328,7 @@ void gpu_brute_force_free_search_result(gpu_brute_force_search_result_c result_c uint64_t gpu_brute_force_cap(gpu_brute_force_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->cap(); - case Quantization_F16: return static_cast*>(any->ptr)->cap(); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->cap(); }); } catch (...) { return 0; } @@ -352,12 +337,7 @@ uint64_t gpu_brute_force_cap(gpu_brute_force_c index_c) { uint64_t gpu_brute_force_len(gpu_brute_force_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->len(); - case Quantization_F16: return static_cast*>(any->ptr)->len(); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->len(); }); } catch (...) { return 0; } @@ -367,13 +347,7 @@ char* gpu_brute_force_info(gpu_brute_force_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return nullptr; try { - auto* any = static_cast(index_c); - std::string info; - switch (any->qtype) { - case Quantization_F32: info = static_cast*>(any->ptr)->info(); break; - case Quantization_F16: info = static_cast*>(any->ptr)->info(); break; - default: return nullptr; - } + std::string info = brute_force_dispatch(static_cast(index_c), [](auto* idx) -> std::string { return idx->info(); }); return strdup(info.c_str()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, @@ -389,8 +363,7 @@ char* gpu_brute_force_info(gpu_brute_force_c index_c, void* errmsg) { void gpu_brute_force_destroy(gpu_brute_force_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - delete any; + delete static_cast(index_c); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_destroy", e.what()); @@ -406,13 +379,10 @@ void gpu_brute_force_set_filter_columns(gpu_brute_force_c index_c, const char* c uint64_t total_count, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string meta = col_meta_json ? col_meta_json : ""; - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_filter_columns(meta, total_count); break; - case Quantization_F16: static_cast*>(any->ptr)->set_filter_columns(meta, total_count); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + idx->set_filter_columns(meta, total_count); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_set_filter_columns", e.what()); } catch (...) { @@ -425,12 +395,9 @@ void gpu_brute_force_add_filter_chunk(gpu_brute_force_c index_c, uint32_t col_id uint64_t nrows, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_F16: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - default: break; - } + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_filter_chunk(col_idx, data, null_bitmap, nrows); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_add_filter_chunk", e.what()); } catch (...) { @@ -445,26 +412,14 @@ gpu_brute_force_search_result_c gpu_brute_force_search_with_filter(gpu_brute_for void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t sp; std::string preds = preds_json ? preds_json : ""; - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); - result_ptr = cpp_res; - break; - } - case Quantization_F16: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); - result_ptr = cpp_res; - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_with_filter", e.what()); return nullptr; @@ -474,62 +429,73 @@ gpu_brute_force_search_result_c gpu_brute_force_search_with_filter(gpu_brute_for } } -gpu_brute_force_search_result_c gpu_brute_force_search_float_with_filter(gpu_brute_force_c index_c, - const float* queries_data, +gpu_brute_force_search_result_c gpu_brute_force_search_quantize_with_filter(gpu_brute_force_c index_c, + const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t sp; std::string preds = preds_json ? preds_json : ""; - void* result_ptr = nullptr; - switch (any->qtype) { - case Quantization_F32: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); - result_ptr = cpp_res; - break; - } - case Quantization_F16: { - auto* cpp_res = new matrixone::gpu_brute_force_t::search_result_t(); - *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); - result_ptr = cpp_res; - break; - } - default: break; - } - return static_cast(result_ptr); + auto cpp_res = std::make_unique(); + brute_force_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); + return static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_with_filter", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_with_filter", e.what()); return nullptr; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_with_filter", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_with_filter", "unknown C++ exception"); return nullptr; } } -uint64_t gpu_brute_force_search_float_with_filter_async(gpu_brute_force_c index_c, - const float* queries_data, +uint64_t gpu_brute_force_search_quantize_with_filter_async(gpu_brute_force_c index_c, + const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); brute_force_search_params_t sp; std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - default: return 0; - } + return brute_force_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_with_filter_async", e.what()); + return 0; + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_quantize_with_filter_async", "unknown C++ exception"); + return 0; + } +} + +// Native-typed (T) async filtered search. queries_data is in the index storage +// type T; no quantization/widening. Returns a job_id collected with +// gpu_brute_force_search_wait. Lets the filtered overflow stay native. +uint64_t gpu_brute_force_search_with_filter_async(gpu_brute_force_c index_c, + const void* queries_data, + uint64_t num_queries, uint32_t query_dimension, + uint32_t limit, const char* preds_json, + void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + brute_force_search_params_t sp; + std::string preds = preds_json ? preds_json : ""; + return brute_force_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_with_filter_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_with_filter_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_float_with_filter_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_brute_force_search_with_filter_async", "unknown C++ exception"); return 0; } } @@ -537,6 +503,7 @@ uint64_t gpu_brute_force_search_float_with_filter_async(gpu_brute_force_c index_ } // extern "C" namespace matrixone { -template class gpu_brute_force_t; -template class gpu_brute_force_t; +template class gpu_brute_force_t; +template class gpu_brute_force_t; +template class gpu_brute_force_t; } diff --git a/cgo/cuvs/brute_force_c.h b/cgo/cuvs/brute_force_c.h index ea989839f4c68..4d7d56e866fb5 100644 --- a/cgo/cuvs/brute_force_c.h +++ b/cgo/cuvs/brute_force_c.h @@ -31,10 +31,10 @@ typedef void* gpu_brute_force_c; typedef void* gpu_brute_force_search_result_c; // Constructor for gpu_brute_force_t -gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, uint32_t nthread, int device_id, quantization_t qtype, const int64_t* ids, void* errmsg); +gpu_brute_force_c gpu_brute_force_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, uint32_t nthread, int device_id, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Constructor for an empty index (pre-allocates) -gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, uint32_t nthread, int device_id, quantization_t qtype, const int64_t* ids, void* errmsg); +gpu_brute_force_c gpu_brute_force_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, uint32_t nthread, int device_id, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Starts the worker and initializes resources void gpu_brute_force_start(gpu_brute_force_c index_c, void* errmsg); @@ -45,20 +45,21 @@ void gpu_brute_force_build(gpu_brute_force_c index_c, void* errmsg); // Add chunk of data (same type as index quantization) void gpu_brute_force_add_chunk(gpu_brute_force_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); -// Add chunk of data (from float, with on-the-fly conversion if needed) -void gpu_brute_force_add_chunk_float(gpu_brute_force_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); +// Add chunk of base-typed (B) data; converts B -> storage T (the add counterpart +// of search_quantize: native store when B==T, f32->f16 cast, or learned SQ for 1-byte). +void gpu_brute_force_add_chunk_quantize(gpu_brute_force_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); // Performs a search operation gpu_brute_force_search_result_c gpu_brute_force_search(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); -// Performs a search operation with float32 queries -gpu_brute_force_search_result_c gpu_brute_force_search_float(gpu_brute_force_c index_c, const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); +// Performs a search operation with base-typed (B) queries; quantizes B -> storage T internally. +gpu_brute_force_search_result_c gpu_brute_force_search_quantize(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); // Asynchronous search functions -uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, +uint64_t gpu_brute_force_search_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); -uint64_t gpu_brute_force_search_float_async(gpu_brute_force_c index_c, const float* queries_data, uint64_t num_queries, +uint64_t gpu_brute_force_search_quantize_async(gpu_brute_force_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, void* errmsg); gpu_brute_force_search_result_c gpu_brute_force_search_wait(gpu_brute_force_c index_c, uint64_t job_id, void* errmsg); @@ -88,20 +89,30 @@ gpu_brute_force_search_result_c gpu_brute_force_search_with_filter(gpu_brute_for uint32_t limit, const char* preds_json, void* errmsg); -gpu_brute_force_search_result_c gpu_brute_force_search_float_with_filter(gpu_brute_force_c index_c, - const float* queries_data, +gpu_brute_force_search_result_c gpu_brute_force_search_quantize_with_filter(gpu_brute_force_c index_c, + const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const char* preds_json, void* errmsg); -// Async variant of gpu_brute_force_search_float_with_filter. Returns a job_id +// Async variant of gpu_brute_force_search_quantize_with_filter. Returns a job_id // that is collected with the existing gpu_brute_force_search_wait. -uint64_t gpu_brute_force_search_float_with_filter_async(gpu_brute_force_c index_c, - const float* queries_data, +uint64_t gpu_brute_force_search_quantize_with_filter_async(gpu_brute_force_c index_c, + const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const char* preds_json, void* errmsg); +// Native-typed (T) async variant of gpu_brute_force_search_with_filter: the +// query stays in the index element type T (f32 or f16), no widening. Returns a +// job_id collected with gpu_brute_force_search_wait. Lets the filtered overflow +// stay native half. +uint64_t gpu_brute_force_search_with_filter_async(gpu_brute_force_c index_c, + const void* queries_data, + uint64_t num_queries, uint32_t query_dimension, + uint32_t limit, const char* preds_json, + void* errmsg); + // Returns the capacity of the index buffer uint64_t gpu_brute_force_cap(gpu_brute_force_c index_c); diff --git a/cgo/cuvs/cagra.hpp b/cgo/cuvs/cagra.hpp index 078239ca96fb9..d9fbe68c3ffc3 100644 --- a/cgo/cuvs/cagra.hpp +++ b/cgo/cuvs/cagra.hpp @@ -97,7 +97,7 @@ namespace matrixone { // 2. Call start() — initializes the worker thread pool and CUDA context. // 3. Call build() — triggers CAGRA graph construction (or file load). // If is_loaded_ is already true (private constructor path), build() is a no-op. -// 4. Call search() / search_float() to query. +// 4. Call search() / search_quantize() to query. // 5. Call extend() / extend_float() to add new vectors (SINGLE_GPU or REPLICATED). // 6. Destructor calls destroy() which calls stop() on the worker. // @@ -183,7 +183,7 @@ namespace matrixone { // and for snapshotting count/dataset in build() // - NO lock during GPU calls themselves (build, extend, search kernels) // - shared_lock IS held during post-GPU CPU-side ID translation in search_internal / -// search_float_internal (protects host_ids and shard_sizes_ against concurrent extend) +// search_quantize_internal (protects host_ids and shard_sizes_ against concurrent extend) // - extend_mutex_ (std::mutex in base) serializes concurrent extend() callers // - Per-device bitset cache uses its own std::mutex (not the shared_mutex) // @@ -201,14 +201,16 @@ struct cagra_search_result_t { /** * @brief gpu_cagra_t implements a CAGRA index that can run on a single GPU or sharded across multiple GPUs. */ -template -class gpu_cagra_t : public gpu_index_base_t { +template +class gpu_cagra_t : public gpu_index_base_t { public: + using base_type = B; + using storage_type = T; using cagra_index = cuvs::neighbors::cagra::index; using search_result_t = cagra_search_result_t; // Inherited dependent type — bring into scope so search_internal can take a // const host_mask_bundle_t* parameter without `typename Base::...` everywhere. - using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; + using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; // Internal index storage std::unique_ptr index_; @@ -336,7 +338,7 @@ class gpu_cagra_t : public gpu_index_base_t { * @brief Merges multiple CAGRA indices into a single index. * Only works for SINGLE_GPU indices. */ - static std::unique_ptr> merge(const std::vector*>& base_indices, uint32_t nthread, const std::vector& devs) { + static std::unique_ptr> merge(const std::vector*>& base_indices, uint32_t nthread, const std::vector& devs) { if (base_indices.empty()) throw std::invalid_argument("base_indices empty"); uint32_t dim = base_indices[0]->dimension; @@ -351,7 +353,7 @@ class gpu_cagra_t : public gpu_index_base_t { std::vector cagra_indices; for (auto* bi : base_indices) { - auto* idx = static_cast*>(bi); + auto* idx = static_cast*>(bi); if (!idx->is_loaded_ || !idx->index_) { throw std::runtime_error("One of the indices to merge is not loaded or is a multi-GPU index."); } @@ -375,7 +377,7 @@ class gpu_cagra_t : public gpu_index_base_t { std::unique_ptr merged_idx(merged_idx_ptr); transient_worker.stop(); - auto new_idx = std::make_unique>( + auto new_idx = std::make_unique>( std::move(merged_idx), dim, m, nthread, devs ); @@ -428,6 +430,9 @@ class gpu_cagra_t : public gpu_index_base_t { // std::cout << "[DEBUG] CAGRA build: Starting build count=" << this->count << " dim=" << this->dimension << " metric=" << (int)this->metric << std::endl; + // 1-byte storage T: train the B-source quantizer on the buffered B + // sample, transform B->T, and store as T. For float/half storage this + // is a no-op. this->train_quantizer_if_needed(); if (!this->worker) throw std::runtime_error("Worker not initialized"); @@ -741,7 +746,32 @@ class gpu_cagra_t : public gpu_index_base_t { return this->search_wait(job_id); } - // Async T-typed filtered search. Mirrors search_float_with_filter_async + // Quantize a B-source query to the 1-byte storage type T via the B-source + // quantizer, writing num_queries*dimension T values into `out`. The caller + // then runs the normal native search(const T*) path. No f32 detour. + void quantize_query(const B* queries_data, uint64_t num_queries, T* out) { + if constexpr (sizeof(T) != 1) { + throw std::runtime_error("quantize_query requires a 1-byte storage type (int8/uint8)"); + } else { + uint64_t job = this->worker->submit_main( + [this, queries_data, num_queries, out](raft_handle_wrapper_t& handle) -> std::any { + auto res = handle.get_raft_resources(); + auto q_b_host = raft::make_host_matrix_view(queries_data, num_queries, this->dimension); + auto q_b_dev = raft::make_device_matrix(*res, num_queries, this->dimension); + raft::copy(*res, q_b_dev.view(), q_b_host); + if (!this->quantizer_.is_trained()) throw std::runtime_error("quantizer not trained"); + auto q_t_dev = raft::make_device_matrix(*res, num_queries, this->dimension); + this->quantizer_.template transform(*res, q_b_dev.view(), q_t_dev.data_handle(), true); + raft::copy(*res, raft::make_host_matrix_view(out, num_queries, this->dimension), q_t_dev.view()); + handle.sync(); + return std::any(); + }); + auto r = this->worker->wait(job).get(); + if (r.error) std::rethrow_exception(r.error); + } + } + + // Async T-typed filtered search. Mirrors search_quantize_with_filter_async // but for the T-typed query path (T may be float / half / int8 / uint8). // Build masks on the caller's thread, copy queries into a shared_ptr so // they outlive the Go caller, capture both in the worker lambda. @@ -1029,31 +1059,32 @@ class gpu_cagra_t : public gpu_index_base_t { } } - transform_distance(this->metric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } - // Sync float entry — wraps search_float_async + search_wait. - search_result_t search_float(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp) { - uint64_t job_id = this->search_float_async(queries_data, num_queries, query_dimension, limit, sp); + // Sync quantize entry — wraps search_quantize_async + search_wait. + search_result_t search_quantize(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp) { + uint64_t job_id = this->search_quantize_async(queries_data, num_queries, query_dimension, limit, sp); return this->search_wait(job_id); } - // Sync float filtered entry — wraps search_float_with_filter_async + search_wait. - search_result_t search_float_with_filter(const float* queries_data, uint64_t num_queries, + // Sync quantize filtered entry — wraps search_quantize_with_filter_async + search_wait. + search_result_t search_quantize_with_filter(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp, const std::string& preds_json) { - uint64_t job_id = this->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); + uint64_t job_id = this->search_quantize_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); return this->search_wait(job_id); } - // Async variant of search_float_with_filter. Builds the host mask bundle on + // Async variant of search_quantize_with_filter. Builds the host mask bundle on // the calling thread (off-worker), copies queries into a shared_ptr so they // outlive the Go caller, captures both in the worker lambda, and returns a // job_id that search_wait() can collect. Used by the multi-index filter - // path so per-shard searches run in parallel. - uint64_t search_float_with_filter_async(const float* queries_data, uint64_t num_queries, + // path so per-shard searches run in parallel. The query is the BASE type B + // (float or half); search_quantize_internal converts it to storage T. + uint64_t search_quantize_with_filter_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp, const std::string& preds_json) { @@ -1066,7 +1097,7 @@ class gpu_cagra_t : public gpu_index_base_t { } if (!this->worker) throw std::runtime_error("Worker not initialized"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Bitmap eval runs on the caller's (Go) thread — off-worker — so @@ -1076,7 +1107,7 @@ class gpu_cagra_t : public gpu_index_base_t { auto shard_masks = this->build_filter_shard_masks(preds_json); auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy, shard_masks](raft_handle_wrapper_t& gpu_handle) -> std::any { int rank = gpu_handle.get_rank(); - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -1088,12 +1119,12 @@ class gpu_cagra_t : public gpu_index_base_t { // would force serialization through main_thread_ and lose batching. auto mask = this->build_filter_single_mask(preds_json); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy, mask](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); }; return this->worker->submit(task); } - uint64_t search_float_async(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp) { + uint64_t search_quantize_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const cagra_search_params_t& sp) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); @@ -1102,13 +1133,13 @@ class gpu_cagra_t : public gpu_index_base_t { if (!this->is_loaded_ || (!index_ && this->replicated_indices_.empty())) throw std::runtime_error("search_async: index not loaded"); } - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Same shape as search_async — fan out, hand back a composite id, // let search_wait() do the merge on the caller's thread. auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy](raft_handle_wrapper_t& gpu_handle) -> std::any { - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -1116,16 +1147,16 @@ class gpu_cagra_t : public gpu_index_base_t { // Single-GPU / replicated: the helper decides standalone vs fused; the // shared_ptr keeps the copied queries alive until the search runs. - return this->search_batchable_float(queries_copy, queries_copy->data(), num_queries, limit, sp); + return this->search_batchable_quantize(queries_copy, queries_copy->data(), num_queries, limit, sp); } - // float32-input search. Mirrors search_batchable_typed but calls - // search_float_internal; request-level batching (if enabled) happens inside it. - uint64_t search_batchable_float(std::shared_ptr> owner, const float* queries_data, + // Base-typed (B) quantize search. Mirrors search_batchable_typed but calls + // search_quantize_internal; request-level batching (if enabled) happens inside it. + uint64_t search_batchable_quantize(std::shared_ptr> owner, const B* queries_data, uint64_t num_queries, uint32_t limit, const cagra_search_params_t& sp) { if (!this->worker) throw std::runtime_error("Worker not initialized"); auto task = [this, owner, queries_data, num_queries, limit, sp](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_data, num_queries, this->dimension, limit, sp); + return this->search_quantize_internal(handle, queries_data, num_queries, this->dimension, limit, sp); }; return this->worker->submit(task); } @@ -1134,7 +1165,14 @@ class gpu_cagra_t : public gpu_index_base_t { // semantics here (off-worker CPU mask eval, skip queries-H2D sync_stream // when prebuilt is non-null, kernel queues naturally behind the H2Ds on // the same stream). - search_result_t search_float_internal(raft_handle_wrapper_t& handle, const float* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, + // + // Takes the query in the BASE element type B (float or half) and converts + // it to the storage type T on-device: B==T is a plain copy, sizeof(T)==1 + // quantizes B -> int8/uint8 via the learned scalar quantizer, and the + // remaining (B=float, T=half) instantiation casts f32 -> f16. This is the + // "quantize" entry — see search_internal() for the already-storage-typed T + // path that performs no conversion. + search_result_t search_quantize_internal(raft_handle_wrapper_t& handle, const B* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const cagra_search_params_t& sp, const std::string& preds_json = "", const host_mask_bundle_t* prebuilt = nullptr) { // No top-level lock: see search_internal() above — pointer fetched // via per-handle cache / narrow inner shared_lock, GPU work runs @@ -1147,26 +1185,30 @@ class gpu_cagra_t : public gpu_index_base_t { auto q_dev_t = raft::make_device_matrix_view( q_buf_t.data(), static_cast(num_queries), static_cast(this->dimension)); - if constexpr (std::is_same_v) { - raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - } else if constexpr (std::is_same_v) { - // Host-side fp32 → fp16 cast (F16C / AVX, IEEE round-to-nearest-even - // — bit-identical to mdspan_copy_kernel<__half>) into a pinned - // staging buffer, then a single half-sized H2D copy. Skips the - // q_dev_f device allocation and the mdspan_copy_kernel dispatch. + if constexpr (std::is_same_v) { + // B == T (float->float or half->half): no conversion, copy straight + // into the storage-typed workspace. + raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + } else if constexpr (sizeof(T) == 1) { + // sizeof(T) == 1: quantize the base-typed query B -> int8/uint8. + // Stage the B query on its own per-thread device workspace (distinct + // from q_buf_t — see q_dev_buf), then transform B -> T on-device. + if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); + auto& q_buf_b = handle.template q_dev_buf(n_q_elems); + auto q_dev_b = raft::make_device_matrix_view( + q_buf_b.data(), static_cast(num_queries), static_cast(this->dimension)); + raft::copy(*res, q_dev_b, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + this->quantizer_.template transform(*res, q_dev_b, q_buf_t.data(), true); + } else { + // B != T and sizeof(T) != 1: the only such instantiation is + // B=float, T=half (f32 base -> fp16 storage). Host-side fp32 -> fp16 + // cast (F16C / AVX, IEEE round-to-nearest-even — bit-identical to + // mdspan_copy_kernel<__half>) into a pinned staging buffer, then a + // single half-sized H2D copy. __half* host_h = handle.ensure_host_half_buf(n_q_elems); matrixone::cast_float_to_half_host(queries_data, host_h, n_q_elems); raft::copy(*res, q_dev_t, raft::make_host_matrix_view(host_h, num_queries, this->dimension)); - } else { - // sizeof(T) == 1: int8 quantizer needs the fp32 device matrix. - auto& q_buf_f = handle.q_dev_buf_float(n_q_elems); - auto q_dev_f = raft::make_device_matrix_view( - q_buf_f.data(), static_cast(num_queries), static_cast(this->dimension)); - raft::copy(*res, q_dev_f, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - - if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); - this->quantizer_.template transform(*res, q_dev_f, q_buf_t.data(), true); } // Legacy path syncs so build_search_bitset's stack-local host bitmap // can drain on the same stream. Prebuilt path skips: bitset H2D queues @@ -1311,12 +1353,12 @@ class gpu_cagra_t : public gpu_index_base_t { } } - transform_distance(this->metric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } std::string info() const override { - std::string json = gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"CAGRA\", \"cagra\": {"; std::shared_lock lock(this->mutex_); if (index_) json += "\"mode\": \"Single-GPU\", \"size\": " + std::to_string(index_->size()); diff --git a/cgo/cuvs/cagra_c.cpp b/cgo/cuvs/cagra_c.cpp index 907d52ff3ab95..5314367c76117 100644 --- a/cgo/cuvs/cagra_c.cpp +++ b/cgo/cuvs/cagra_c.cpp @@ -1,4 +1,4 @@ -/* +/* * Copyright 2021 Matrix Origin * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,15 @@ /* * CAGRA C Wrapper Implementation - * Supported data types (via quantization_t): Quantization_F32, Quantization_F16, Quantization_INT8, Quantization_UINT8 + * + * Two type axes via quantization_t: + * btype = base / query / quantizer-SOURCE element type (Quantization_F32 or F16) + * qtype = storage element type (Quantization_F32, F16, INT8, UINT8) + * + * Wired (btype, qtype) combinations: + * F32 base: F32, F16, INT8, UINT8 storage + * F16 base: F16, INT8, UINT8 storage + * Any other combination throws "unsupported (base,storage) type combination". */ #include "cagra_c.h" @@ -27,78 +35,111 @@ #include #include #include +#include using namespace matrixone; struct gpu_cagra_any_t { - quantization_t qtype; + quantization_t btype; // base / query / quantizer-source element type + quantization_t qtype; // storage element type void* ptr; - gpu_cagra_any_t(quantization_t q, void* p) : qtype(q), ptr(p) {} - ~gpu_cagra_any_t() { + gpu_cagra_any_t(quantization_t b, quantization_t q, void* p) + : btype(b), qtype(q), ptr(p) {} + ~gpu_cagra_any_t(); +}; + +// Static dispatch: resolves the concrete gpu_cagra_t for (btype,qtype) and +// invokes fn with a typed pointer. fn is a generic lambda; recover B/Q inside it +// via decltype(idx)::base_type / ::storage_type. Throws on unsupported combos. +template +static auto cagra_dispatch(const gpu_cagra_any_t* a, Fn&& fn) { + switch (a->btype) { + case Quantization_F32: + switch (a->qtype) { + case Quantization_F32: return fn(static_cast*>(a->ptr)); + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + case Quantization_F16: + switch (a->qtype) { + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + default: break; + } + throw std::runtime_error("gpu_cagra: unsupported (base,storage) type combination"); +} + +gpu_cagra_any_t::~gpu_cagra_any_t() { + if (!ptr) return; + try { + cagra_dispatch(this, [](auto* idx) { + idx->destroy(); + delete idx; + }); + } catch (...) { + // unsupported combo never gets a live ptr — nothing to free + } +} + +// Construct a new gpu_cagra_t for the wired (btype,qtype) combos. +// Maker is a generic lambda invoked as maker(static type tag) -> void*; it +// receives a null typed pointer purely to recover B and Q. +template +static void* cagra_construct(quantization_t btype, quantization_t qtype, Maker&& maker) { + switch (btype) { + case Quantization_F32: switch (qtype) { - case Quantization_F32: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_F16: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_INT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_UINT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - default: break; + case Quantization_F32: return maker(static_cast*>(nullptr)); + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; + } + break; + case Quantization_F16: + switch (qtype) { + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; } + break; + default: break; } -}; + throw std::runtime_error("gpu_cagra: unsupported (base,storage) type combination"); +} extern "C" { gpu_cagra_c gpu_cagra_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, cagra_build_params_t build_params, const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_cagra_any_t(qtype, ptr)); + void* ptr = cagra_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + // The dataset-providing constructor takes storage-typed (Q) data and + // copies it directly into flattened_host_dataset (no quantization here; + // quantization happens via add_chunk_quantize / add_chunk_float). + return new gpu_cagra_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(new gpu_cagra_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_new", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_new", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_new", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_new", "unknown C++ exception"); } return nullptr; } @@ -106,68 +147,42 @@ gpu_cagra_c gpu_cagra_new(const void* dataset_data, uint64_t count_vectors, uint gpu_cagra_c gpu_cagra_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, cagra_build_params_t build_params, const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_cagra_any_t(qtype, ptr)); + void* ptr = cagra_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_cagra_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(new gpu_cagra_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_new_empty", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_new_empty", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_new_empty", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_new_empty", "unknown C++ exception"); } return nullptr; } gpu_cagra_c gpu_cagra_load_file(const char* filename, uint32_t dimension, distance_type_t metric_c, cagra_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg) { + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_F16: - ptr = new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_INT8: - ptr = new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_UINT8: - ptr = new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - default: return nullptr; - } - return static_cast(new gpu_cagra_any_t(qtype, ptr)); + void* ptr = cagra_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_cagra_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); + }); + return static_cast(new gpu_cagra_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_load_file", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_load_file", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_load_file", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_load_file", "unknown C++ exception"); } return nullptr; } @@ -177,205 +192,152 @@ void gpu_cagra_destroy(gpu_cagra_c index_c, void* errmsg) { try { delete static_cast(index_c); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_destroy", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_destroy", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_destroy", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_destroy", "unknown C++ exception"); } } void gpu_cagra_start(gpu_cagra_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->start(); break; - case Quantization_F16: static_cast*>(any->ptr)->start(); break; - case Quantization_INT8: static_cast*>(any->ptr)->start(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->start(); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [](auto* idx) { idx->start(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_start", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_start", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_start", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_start", "unknown C++ exception"); } } void gpu_cagra_build(gpu_cagra_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->build(); break; - case Quantization_F16: static_cast*>(any->ptr)->build(); break; - case Quantization_INT8: static_cast*>(any->ptr)->build(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->build(); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [](auto* idx) { idx->build(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_build", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_build", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_build", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_build", "unknown C++ exception"); } } void gpu_cagra_add_chunk(gpu_cagra_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_add_chunk", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_add_chunk", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk", "unknown C++ exception"); } } void gpu_cagra_add_chunk_float(gpu_cagra_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_chunk_float(chunk_data, chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_add_chunk_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk_float", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_add_chunk_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk_float", "unknown C++ exception"); } } -void gpu_cagra_train_quantizer(gpu_cagra_c index_c, const float* train_data, uint64_t n_samples, void* errmsg) { +void gpu_cagra_add_chunk_quantize(gpu_cagra_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_F16: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_INT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_UINT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->add_chunk_quantize(static_cast(base_data), chunk_count, -1, ids); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk_quantize", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_chunk_quantize", "unknown C++ exception"); + } +} + +void gpu_cagra_quantize_query(gpu_cagra_c index_c, const void* base_data, uint64_t num_queries, void* out, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + idx->quantize_query(static_cast(base_data), num_queries, static_cast(out)); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_train_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_quantize_query", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_train_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_quantize_query", "unknown C++ exception"); + } +} + +void gpu_cagra_train_quantizer(gpu_cagra_c index_c, const void* train_data, uint64_t n_samples, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->train_quantizer(static_cast(train_data), n_samples); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_train_quantizer", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_train_quantizer", "unknown C++ exception"); } } void gpu_cagra_set_batch_window(gpu_cagra_c index_c, int64_t window_us, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_F16: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_batch_window(window_us); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_batch_window", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_batch_window", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_batch_window", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_batch_window", "unknown C++ exception"); } } void gpu_cagra_set_dynb_conservative_dispatch(gpu_cagra_c index_c, bool enable, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_F16: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_dynb_conservative_dispatch(enable); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_dynb_conservative_dispatch", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_dynb_conservative_dispatch", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_dynb_conservative_dispatch", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_dynb_conservative_dispatch", "unknown C++ exception"); } } void gpu_cagra_set_quantizer(gpu_cagra_c index_c, float min, float max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_set_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_quantizer", "unknown C++ exception"); } } void gpu_cagra_get_quantizer(gpu_cagra_c index_c, float* min, float* max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->get_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_get_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_get_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_get_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_get_quantizer", "unknown C++ exception"); } } void gpu_cagra_save(gpu_cagra_c index_c, const char* filename, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save(filename); break; - case Quantization_F16: static_cast*>(any->ptr)->save(filename); break; - case Quantization_INT8: static_cast*>(any->ptr)->save(filename); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save(filename); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->save(filename); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_save", e.what()); } catch (...) { @@ -386,14 +348,7 @@ void gpu_cagra_save(gpu_cagra_c index_c, const char* filename, void* errmsg) { void gpu_cagra_save_dir(gpu_cagra_c index_c, const char* dir, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_F16: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_INT8: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save_dir(dir); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->save_dir(dir); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_save_dir", e.what()); } catch (...) { @@ -404,14 +359,7 @@ void gpu_cagra_save_dir(gpu_cagra_c index_c, const char* dir, void* errmsg) { void gpu_cagra_delete_id(gpu_cagra_c index_c, int64_t id, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_F16: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_INT8: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_UINT8: static_cast*>(any->ptr)->delete_id(id); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->delete_id(id); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_delete_id", e.what()); } catch (...) { @@ -423,14 +371,7 @@ void gpu_cagra_load_dir(gpu_cagra_c index_c, const char* dir, distribution_mode_t target_mode, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_F16: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_INT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_UINT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { idx->load_dir(dir, target_mode); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_load_dir", e.what()); } catch (...) { @@ -444,63 +385,49 @@ gpu_cagra_search_res_t gpu_cagra_search(gpu_cagra_c index_c, const void* queries if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_search", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_search", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search", "unknown C++ exception"); } return result; } -gpu_cagra_search_res_t gpu_cagra_search_float(gpu_cagra_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_cagra_search_res_t gpu_cagra_search_quantize(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize", "unknown C++ exception"); } return result; } -uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_async", e.what()); return 0; @@ -510,24 +437,20 @@ uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, u } } -uint64_t gpu_cagra_search_float_async(gpu_cagra_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_cagra_search_quantize_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_async", "unknown C++ exception"); return 0; } } @@ -536,15 +459,10 @@ gpu_cagra_search_res_t gpu_cagra_search_wait(gpu_cagra_c index_c, uint64_t job_i if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + *cpp_res = idx->search_wait(job_id); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_wait", e.what()); @@ -601,14 +519,7 @@ void gpu_cagra_free_result(gpu_cagra_result_c result_c) { uint64_t gpu_cagra_cap(gpu_cagra_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->cap(); - case Quantization_F16: return static_cast*>(any->ptr)->cap(); - case Quantization_INT8: return static_cast*>(any->ptr)->cap(); - case Quantization_UINT8: return static_cast*>(any->ptr)->cap(); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->cap(); }); } catch (...) { return 0; } @@ -617,14 +528,7 @@ uint64_t gpu_cagra_cap(gpu_cagra_c index_c) { uint64_t gpu_cagra_len(gpu_cagra_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->len(); - case Quantization_F16: return static_cast*>(any->ptr)->len(); - case Quantization_INT8: return static_cast*>(any->ptr)->len(); - case Quantization_UINT8: return static_cast*>(any->ptr)->len(); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->len(); }); } catch (...) { return 0; } @@ -640,15 +544,9 @@ char* gpu_cagra_get_filter_col_meta_json(gpu_cagra_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return strdup(""); try { - auto* any = static_cast(index_c); - std::string json; - switch (any->qtype) { - case Quantization_F32: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_F16: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_INT8: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_UINT8: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - default: return strdup(""); - } + std::string json = cagra_dispatch(static_cast(index_c), [](auto* idx) -> std::string { + return matrixone::format_filter_col_meta(idx->filter_host_.columns); + }); return strdup(json.c_str()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_get_filter_col_meta_json", e.what()); @@ -663,23 +561,13 @@ char* gpu_cagra_info(gpu_cagra_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return nullptr; try { - auto* any = static_cast(index_c); - std::string info; - switch (any->qtype) { - case Quantization_F32: info = static_cast*>(any->ptr)->info(); break; - case Quantization_F16: info = static_cast*>(any->ptr)->info(); break; - case Quantization_INT8: info = static_cast*>(any->ptr)->info(); break; - case Quantization_UINT8: info = static_cast*>(any->ptr)->info(); break; - default: return nullptr; - } + std::string info = cagra_dispatch(static_cast(index_c), [](auto* idx) -> std::string { return idx->info(); }); return strdup(info.c_str()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_info", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_info", e.what()); return nullptr; } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_info", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_info", "unknown C++ exception"); return nullptr; } } @@ -688,14 +576,10 @@ void gpu_cagra_extend(gpu_cagra_c index_c, const void* additional_data, uint64_t const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend(static_cast(additional_data), num_vectors, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend(static_cast(additional_data), num_vectors, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend(static_cast(additional_data), num_vectors, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend(static_cast(additional_data), num_vectors, new_ids); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->extend(static_cast(additional_data), num_vectors, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_extend", e.what()); } catch (...) { @@ -708,44 +592,21 @@ gpu_cagra_c gpu_cagra_merge(gpu_cagra_c* indices_c, int num_indices, uint32_t nt try { if (num_indices <= 0) return nullptr; auto* first = static_cast(indices_c[0]); - quantization_t qtype = first->qtype; std::vector devs(devices, devices + device_count); - void* merged_ptr = nullptr; - - switch (qtype) { - case Quantization_F32: { - std::vector*> base_indices; - for (int i = 0; i < num_indices; ++i) base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); - merged_ptr = gpu_cagra_t::merge(base_indices, nthread, devs).release(); - break; - } - case Quantization_F16: { - std::vector*> base_indices; - for (int i = 0; i < num_indices; ++i) base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); - merged_ptr = gpu_cagra_t::merge(base_indices, nthread, devs).release(); - break; + void* merged_ptr = cagra_construct(first->btype, first->qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + std::vector*> base_indices; + for (int i = 0; i < num_indices; ++i) { + base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); } - case Quantization_INT8: { - std::vector*> base_indices; - for (int i = 0; i < num_indices; ++i) base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); - merged_ptr = gpu_cagra_t::merge(base_indices, nthread, devs).release(); - break; - } - case Quantization_UINT8: { - std::vector*> base_indices; - for (int i = 0; i < num_indices; ++i) base_indices.push_back(static_cast*>(static_cast(indices_c[i])->ptr)); - merged_ptr = gpu_cagra_t::merge(base_indices, nthread, devs).release(); - break; - } - default: return nullptr; - } - return static_cast(new gpu_cagra_any_t(qtype, merged_ptr)); + return gpu_cagra_t::merge(base_indices, nthread, devs).release(); + }); + return static_cast(new gpu_cagra_any_t(first->btype, first->qtype, merged_ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_merge", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_merge", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_cagra_merge", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_merge", "unknown C++ exception"); } return nullptr; } @@ -756,15 +617,10 @@ void gpu_cagra_set_filter_columns(gpu_cagra_c index_c, const char* col_meta_json uint64_t total_count, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string s = col_meta_json ? col_meta_json : ""; - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_F16: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + idx->set_filter_columns(s, total_count); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_set_filter_columns", e.what()); } catch (...) { @@ -777,14 +633,9 @@ void gpu_cagra_add_filter_chunk(gpu_cagra_c index_c, uint32_t col_idx, uint64_t nrows, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_F16: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_filter_chunk(col_idx, data, null_bitmap, nrows); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_add_filter_chunk", e.what()); } catch (...) { @@ -799,16 +650,12 @@ gpu_cagra_search_res_t gpu_cagra_search_with_filter(gpu_cagra_c index_c, const v if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_with_filter", e.what()); @@ -818,52 +665,44 @@ gpu_cagra_search_res_t gpu_cagra_search_with_filter(gpu_cagra_c index_c, const v return result; } -gpu_cagra_search_res_t gpu_cagra_search_float_with_filter(gpu_cagra_c index_c, const float* queries_data, +gpu_cagra_search_res_t gpu_cagra_search_quantize_with_filter(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_cagra_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + cagra_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_with_filter", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_with_filter", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_with_filter", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_with_filter", "unknown C++ exception"); } return result; } -uint64_t gpu_cagra_search_float_with_filter_async(gpu_cagra_c index_c, const float* queries_data, +uint64_t gpu_cagra_search_quantize_with_filter_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - default: return 0; - } + return cagra_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_with_filter_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_with_filter_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_float_with_filter_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_cagra_search_quantize_with_filter_async", "unknown C++ exception"); return 0; } } @@ -871,8 +710,11 @@ uint64_t gpu_cagra_search_float_with_filter_async(gpu_cagra_c index_c, const flo } // extern "C" namespace matrixone { -template class gpu_cagra_t; -template class gpu_cagra_t; -template class gpu_cagra_t; -template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; +template class gpu_cagra_t; } // namespace matrixone diff --git a/cgo/cuvs/cagra_c.h b/cgo/cuvs/cagra_c.h index 0e6a4fb463215..0d0a7229c7521 100644 --- a/cgo/cuvs/cagra_c.h +++ b/cgo/cuvs/cagra_c.h @@ -31,18 +31,22 @@ typedef void* gpu_cagra_c; // Opaque pointer to the C++ CAGRA search result object typedef void* gpu_cagra_result_c; +// btype = base/query/quantizer-source element type (Quantization_F32 or F16). +// qtype = storage element type. Wired combos: F32 base {F32,F16,INT8,UINT8}; +// F16 base {F16,INT8,UINT8}. Other combinations set errmsg and return NULL. + // Constructor for building from dataset gpu_cagra_c gpu_cagra_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, cagra_build_params_t build_params, const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Constructor for loading from file gpu_cagra_c gpu_cagra_load_file(const char* filename, uint32_t dimension, distance_type_t metric, cagra_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg); + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg); // Destructor void gpu_cagra_destroy(gpu_cagra_c index_c, void* errmsg); @@ -57,7 +61,7 @@ void gpu_cagra_build(gpu_cagra_c index_c, void* errmsg); gpu_cagra_c gpu_cagra_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, cagra_build_params_t build_params, const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Add chunk of data (same type as index quantization) @@ -66,8 +70,16 @@ void gpu_cagra_add_chunk(gpu_cagra_c index_c, const void* chunk_data, uint64_t c // Add chunk of data (from float, with on-the-fly quantization if needed) void gpu_cagra_add_chunk_float(gpu_cagra_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); +// Add chunk of base-typed (B) data, quantizing natively to a 1-byte storage type +// (int8/uint8) via the B-source quantizer. Requires int8/uint8 storage. +void gpu_cagra_add_chunk_quantize(gpu_cagra_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); + +// Quantize a base-typed (B) query to the 1-byte storage type via the B-source +// quantizer, writing num_queries*dimension bytes into out. Requires int8/uint8 storage. +void gpu_cagra_quantize_query(gpu_cagra_c index_c, const void* base_data, uint64_t num_queries, void* out, void* errmsg); + // Trains the scalar quantizer (if T is 1-byte) -void gpu_cagra_train_quantizer(gpu_cagra_c index_c, const float* train_data, uint64_t n_samples, void* errmsg); +void gpu_cagra_train_quantizer(gpu_cagra_c index_c, const void* train_data, uint64_t n_samples, void* errmsg); void gpu_cagra_set_batch_window(gpu_cagra_c index_c, int64_t window_us, void* errmsg); void gpu_cagra_set_dynb_conservative_dispatch(gpu_cagra_c index_c, bool enable, void* errmsg); @@ -100,17 +112,19 @@ gpu_cagra_search_res_t gpu_cagra_search(gpu_cagra_c index_c, const void* queries uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg); -gpu_cagra_search_res_t gpu_cagra_search_float(gpu_cagra_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +// Quantize search: query in the BASE element type B (float or half); the index +// converts it to storage type T (copy / quantize / f32->f16 cast) internally. +gpu_cagra_search_res_t gpu_cagra_search_quantize(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg); // Asynchronous search functions -uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_cagra_search_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg); -uint64_t gpu_cagra_search_float_async(gpu_cagra_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_cagra_search_quantize_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, void* errmsg); gpu_cagra_search_res_t gpu_cagra_search_wait(gpu_cagra_c index_c, uint64_t job_id, void* errmsg); @@ -162,22 +176,23 @@ void gpu_cagra_add_filter_chunk(gpu_cagra_c index_c, uint32_t col_idx, const void* data, const uint32_t* null_bitmap, uint64_t nrows, void* errmsg); -// Filtered variants of gpu_cagra_search / gpu_cagra_search_float. preds_json is a JSON +// Filtered variants of gpu_cagra_search / gpu_cagra_search_quantize. preds_json is a JSON // predicate array; passing NULL or "" yields unfiltered behavior. gpu_cagra_search_res_t gpu_cagra_search_with_filter(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, const char* preds_json, void* errmsg); -gpu_cagra_search_res_t gpu_cagra_search_float_with_filter(gpu_cagra_c index_c, const float* queries_data, +// Query in the BASE element type B (float or half); converted to storage T internally. +gpu_cagra_search_res_t gpu_cagra_search_quantize_with_filter(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, const char* preds_json, void* errmsg); -// Async variant of gpu_cagra_search_float_with_filter. Returns a job_id that +// Async variant of gpu_cagra_search_quantize_with_filter. Returns a job_id that // is collected with the existing gpu_cagra_search_wait. Lets multi-index // callers fan out filtered searches across shards in parallel. -uint64_t gpu_cagra_search_float_with_filter_async(gpu_cagra_c index_c, const float* queries_data, +uint64_t gpu_cagra_search_quantize_with_filter_async(gpu_cagra_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, cagra_search_params_t search_params, const char* preds_json, void* errmsg); diff --git a/cgo/cuvs/cuvs_worker.hpp b/cgo/cuvs/cuvs_worker.hpp index 0e7eb624edc8c..d6581dec2b964 100644 --- a/cgo/cuvs/cuvs_worker.hpp +++ b/cgo/cuvs/cuvs_worker.hpp @@ -112,7 +112,7 @@ inline rmm::mr::device_memory_resource* worker_pool_mr(int device_id) { try { cudaSetDevice(device_id); auto base = std::make_shared(); - // Initial pool = 10% of free GPU memory; max = unbounded — pool + // Initial pool = 2% of free GPU memory; max = unbounded — pool // grows by allocating more from the upstream as needed. // Kept small so a subsequent huge-index load (e.g. an IVF-PQ or // CAGRA index needing ≥¾ VRAM in a single allocation) can still @@ -123,7 +123,7 @@ inline rmm::mr::device_memory_resource* worker_pool_mr(int device_id) { auto pool = std::make_shared< rmm::mr::pool_memory_resource>( base.get(), - rmm::percent_of_free_device_memory(10)); + rmm::percent_of_free_device_memory(2)); std::lock_guard lk(keepalive_mu); keepalives.push_back(pool); // pool outlives every device_uvector keepalives.push_back(base); // base outlives the pool diff --git a/cgo/cuvs/index_base.hpp b/cgo/cuvs/index_base.hpp index 6b4c6c810bb35..7cbf0bb192054 100644 --- a/cgo/cuvs/index_base.hpp +++ b/cgo/cuvs/index_base.hpp @@ -62,12 +62,12 @@ using ::distribution_mode_t; // // OVERVIEW // -------- -// gpu_index_base_t is the CRTP-style base class shared by +// gpu_index_base_t is the CRTP-style base class shared by // all three GPU index types: // -// gpu_ivf_flat_t (IdT = int64_t) -// gpu_ivf_pq_t (IdT = int64_t) -// gpu_cagra_t (IdT = uint32_t) +// gpu_ivf_flat_t (IdT = int64_t) // base hardcoded to float +// gpu_ivf_pq_t (IdT = int64_t) // B = base/source element type, T = storage type +// gpu_cagra_t (IdT = uint32_t) // B = base/source element type, T = storage type // // It provides: // - Pre-build vector buffering (flattened_host_dataset) @@ -202,17 +202,26 @@ using ::distribution_mode_t; // // QUANTIZER (1-byte types only: int8_t, uint8_t) // ------------------------------------------------ -// scalar_quantizer_t quantizer_ maps float32 values to [min, max] range -// and packs them into int8/uint8. It must be trained before add_chunk_float() -// or extend_float() is called for 1-byte types. +// scalar_quantizer_t quantizer_ maps source-type B values into the storage +// range [min, max] and packs them into int8/uint8. // -// Training: quantizer_.train(res, train_matrix) or train_quantizer(data, n). -// - Auto-training occurs in add_chunk_float if not yet trained (uses up to 500 -// samples from the first chunk). -// - For extend_float, the quantizer MUST already be trained (throws otherwise). +// Training (on the ORIGINAL float/half source data only): +// - add_chunk_float() / add_chunk_quantize() buffer their raw B chunks in +// pending_float_chunks_; flush_pending_float_chunks_internal() — invoked at +// build time via train_quantizer_if_needed() — trains the quantizer on ALL +// buffered rows at once, then quantizes them into storage. (No "first chunk" +// or 500-sample heuristic; the full buffered set is used.) +// - The quantizer is NEVER trained from flattened_host_dataset: for a 1-byte T +// that buffer holds only storage bytes, so training on it would learn the +// COMPRESSED range, not the original float range. +// - A pre-quantized index (rows added via add_chunk(T*), with no original +// floats and no set_quantizer()) therefore leaves the quantizer UNTRAINED. +// Base-typed (B) search/extend on it requires an explicit range via +// set_quantizer() first — quantize_query() (search) and +// upload_float_matrix_as_T() (extend) throw "quantizer not trained" otherwise. // -// Extended vectors must lie within the trained [min, max] range; vectors outside -// this range will be clamped and produce degraded search quality. +// Extended/searched vectors must lie within the trained [min, max] range; values +// outside it are clamped and produce degraded search quality. // // // SERIALIZATION (save_dir / load_dir) @@ -330,27 +339,37 @@ inline void fill_all_sentinel(NeighborT* neighbors, float* distances, std::fill_n(distances, count, std::numeric_limits::max()); } -// InnerProduct sign flip on the search result's distances. cuvs returns -// inner-product distances negated (so smaller is "closer") — we flip back so -// downstream callers see the true inner product. ±FLT_MAX sentinels are -// preserved (they mark padded / filtered-out slots from scatter_with_padding -// or fill_all_sentinel above). No-op for any other metric. +// Post-process a search result's distances in place: +// - InnerProduct: flip the sign. cuvs returns inner-product distances negated +// (so smaller is "closer"); we flip back so callers see the true IP. +// - quantized L2 (dequant_factor != 1): rescale the quantized-domain distance +// back to the base (f32) scale, so a 1-byte (int8/uint8) main index merges +// on the same scale as the base-typed CDC overflow brute force. The factor +// comes from quantized_l2_dequant_factor() (1/scalar^2 for squared L2). IP +// and L2-dequant are mutually exclusive — IP/cosine + int8/uint8 is rejected +// at plan time (an affine quantizer is not a pure rescale for IP/cosine). +// ±FLT_MAX sentinels (padded / filtered-out slots from scatter_with_padding or +// fill_all_sentinel above) are preserved. No-op for plain f32/f16 L2. inline void transform_distance(distance_type_t metric, - float* distances, size_t count) { - if (metric != DistanceType_InnerProduct) return; + float* distances, size_t count, + double dequant_factor = 1.0) { + const bool flip = (metric == DistanceType_InnerProduct); + const bool rescale = (dequant_factor != 1.0); + if (!flip && !rescale) return; const float kSentinel = std::numeric_limits::max(); for (size_t i = 0; i < count; ++i) { - if (distances[i] != kSentinel && distances[i] != -kSentinel) { - distances[i] *= -1.0f; - } + if (distances[i] == kSentinel || distances[i] == -kSentinel) continue; + if (flip) distances[i] *= -1.0f; + else distances[i] = static_cast(static_cast(distances[i]) * dequant_factor); } } // Convenience overload for the persistent-index path where distances live in // a std::vector. Same semantics as the (float*, size_t) form. inline void transform_distance(distance_type_t metric, - std::vector& distances) { - transform_distance(metric, distances.data(), distances.size()); + std::vector& distances, + double dequant_factor = 1.0) { + transform_distance(metric, distances.data(), distances.size(), dequant_factor); } /** @@ -359,13 +378,16 @@ inline void transform_distance(distance_type_t metric, * See the Developer Guide block above for full details on lifecycle, locking, * distribution modes, ID mapping, and the soft-delete bitset system. * - * @tparam T Element type: float, half (__half), int8_t, uint8_t + * @tparam B Base/query/quantizer-SOURCE element type: float or half + * @tparam T Storage element type: float, half (__half), int8_t, uint8_t * @tparam BuildParams Index-specific build parameter struct * @tparam IdT Neighbor ID type: int64_t (IVF) or uint32_t (CAGRA) */ -template +template class gpu_index_base_t { public: + using base_type = B; + using storage_type = T; // ---- Index configuration (immutable after build) ---- uint32_t dimension = 0; ///< Vector dimensionality distance_type_t metric; ///< Distance metric (L2, IP, cosine, ...) @@ -495,6 +517,42 @@ class gpu_index_base_t { return it->second; } + // Factor that rescales a quantized-domain L2 distance back to the base (f32) + // scale, for transform_distance(). For 1-byte storage (int8/uint8) the index + // computes L2 over the quantized vectors, where each element is + // q(x)=scalar*x+offset with scalar=255/(max-min); the per-element offset is a + // constant translation that cancels in a difference, so + // ||q(a)-q(b)||^2 = scalar^2*||a-b||^2 (and scalar*||a-b|| for the sqrt + // metrics). Returning 1/scalar^2 (resp. 1/scalar) undoes that, so a quantized + // main-index distance lands on the SAME scale as the base-typed CDC overflow + // brute force — otherwise mergeMultiResults compares scalar^2-scaled main + // distances against base-scale overflow distances and the overflow rows + // wrongly dominate the top-k. Also makes the reported l2_distance correct. + // + // Returns 1.0 (no-op) for plain f32/f16 storage, an untrained quantizer, a + // degenerate range, or a non-L2 metric (IP/cosine are not a pure rescale + // under an affine quantizer and are rejected at plan time). + double quantized_l2_dequant_factor() const { + if constexpr (sizeof(T) == 1) { + if (!this->quantizer_.is_trained()) return 1.0; + const double range = static_cast(this->quantizer_.max()) - + static_cast(this->quantizer_.min()); + if (!(range > 0.0)) return 1.0; + const double s = 255.0 / range; // scalar + switch (this->metric) { + case DistanceType_L2Expanded: + case DistanceType_L2Unexpanded: + return 1.0 / (s * s); // distances are squared L2 + case DistanceType_L2SqrtExpanded: + case DistanceType_L2SqrtUnexpanded: + return 1.0 / s; + default: + return 1.0; // IP / cosine: scale alone can't reconcile them + } + } + return 1.0; + } + // Sync a shard-local slice of the deleted bitset to device (SHARDED mode). // shard_offset must be a multiple of 32 (enforced at build time). // Bit j of the resulting device bitset = global bit (shard_offset + j). @@ -1060,15 +1118,15 @@ class gpu_index_base_t { auto res = handle.get_raft_resources(); - // --- GPU work: train quantizer on ALL pending float data — NO LOCK --- - std::vector all_floats; + // --- GPU work: train quantizer on ALL pending B-source data — NO LOCK --- + std::vector all_floats; all_floats.reserve(total * dimension); for (auto& c : chunks) { all_floats.insert(all_floats.end(), c.data.begin(), c.data.end()); } - auto train_host_view = raft::make_host_matrix_view( + auto train_host_view = raft::make_host_matrix_view( all_floats.data(), static_cast(total), static_cast(dimension)); - auto train_device = raft::make_device_matrix(*res, total, dimension); + auto train_device = raft::make_device_matrix(*res, total, dimension); raft::copy(*res, train_device.view(), train_host_view); // Train without holding the lock: GPU kernels run while lock is not held, // so concurrent readers are not blocked for the duration of training. @@ -1079,28 +1137,11 @@ class gpu_index_base_t { // acquisition by a reader is guaranteed to see is_trained() == true. { std::unique_lock _pub_lock(mutex_); } - // --- GPU work + locked store: process each buffered chunk --- + // --- Quantize each buffered chunk on the CPU and store. The quantizer + // is trained (above), so the B->T transform is a pure host affine map — + // no per-chunk GPU round-trip (this is what made a 1M-row f16-base / + // add_chunk_quantize build crawl). See transform_host(). --- for (auto& c : chunks) { - // Upload and quantize — NO LOCK - auto chunk_host_view = raft::make_host_matrix_view( - c.data.data(), static_cast(c.count), static_cast(dimension)); - auto chunk_device = raft::make_device_matrix(*res, c.count, dimension); - raft::copy(*res, chunk_device.view(), chunk_host_view); - - auto chunk_device_target = raft::make_device_matrix(*res, c.count, dimension); - - { - std::shared_lock lock(mutex_); - quantizer_.template transform(*res, chunk_device.view(), chunk_device_target.data_handle(), true); - } - - std::vector chunk_host_target(c.count * dimension); - raft::copy(*res, - raft::make_host_matrix_view(chunk_host_target.data(), static_cast(c.count), static_cast(dimension)), - chunk_device_target.view()); - handle.sync(); - - // Store into shared state — unique_lock std::unique_lock lock(mutex_); uint64_t target_offset; if (c.offset == -1) { @@ -1118,8 +1159,10 @@ class gpu_index_base_t { if (flattened_host_dataset.size() < required_elements) { flattened_host_dataset.resize(required_elements); } - std::copy(chunk_host_target.begin(), chunk_host_target.end(), - flattened_host_dataset.begin() + target_offset * dimension); + quantizer_.template transform_host( + c.data.data(), + flattened_host_dataset.data() + target_offset * dimension, + static_cast(c.count) * dimension); if (this->dist_mode == DistributionMode_SHARDED) { int num_shards = static_cast(this->devices_.size()); @@ -1152,10 +1195,16 @@ class gpu_index_base_t { if (is_loaded_) throw std::runtime_error("Cannot add chunk to built index"); } - auto res = handle.get_raft_resources(); - // If quantization is needed (T is 1-byte) if constexpr (sizeof(T) == 1) { + // The deferred-quantize buffer and the quantizer both work on + // the SOURCE type B. Convert the incoming f32 chunk to B once + // (identical bytes when B==float; per-element float->half cast + // when B==half). + std::vector chunk_b(chunk_count * dimension); + for (size_t i = 0; i < chunk_count * dimension; ++i) { + chunk_b[i] = static_cast(chunk_data[i]); + } bool trained; { std::shared_lock lock(mutex_); @@ -1165,7 +1214,7 @@ class gpu_index_base_t { if (!trained) { // Buffer this chunk for deferred training. pending_float_chunk_t c; - c.data.assign(chunk_data, chunk_data + chunk_count * dimension); + c.data = chunk_b; c.count = chunk_count; c.offset = offset; if (ids) c.ids.assign(ids, ids + chunk_count); @@ -1195,22 +1244,14 @@ class gpu_index_base_t { // c was NOT pushed to pending, so fall through to process chunk_data directly. } - // Quantizer already trained: quantize this chunk immediately. - auto queries_host_view = raft::make_host_matrix_view(chunk_data, chunk_count, dimension); - auto queries_device = raft::make_device_matrix(*res, chunk_count, dimension); - raft::copy(*res, queries_device.view(), queries_host_view); - - auto chunk_device_target = raft::make_device_matrix(*res, chunk_count, dimension); - - { - std::shared_lock lock(mutex_); - quantizer_.template transform(*res, queries_device.view(), chunk_device_target.data_handle(), true); - } - - std::vector chunk_host_target(chunk_count * dimension); - raft::copy(*res, raft::make_host_matrix_view(chunk_host_target.data(), chunk_count, dimension), chunk_device_target.view()); - handle.sync(); - + // Quantizer already trained: quantize on the CPU and write + // directly into flattened_host_dataset. Scalar quantization + // is a pure affine map from the trained [min,max], so no GPU + // round-trip (malloc + H2D copy + kernel + D2H copy + sync) + // is needed per chunk — this is the same host-only fast path + // as float/half storage. transform_host() produces bytes + // identical to the device transform(), so a CPU-quantized + // base stays consistent with a GPU-quantized query at search. std::unique_lock lock(mutex_); uint64_t target_offset; if (offset == -1) { @@ -1228,7 +1269,10 @@ class gpu_index_base_t { if (flattened_host_dataset.size() < required_elements) { flattened_host_dataset.resize(required_elements); } - std::copy(chunk_host_target.begin(), chunk_host_target.end(), flattened_host_dataset.begin() + (target_offset * dimension)); + quantizer_.template transform_host( + chunk_b.data(), + flattened_host_dataset.data() + (target_offset * dimension), + static_cast(chunk_count) * dimension); if (this->dist_mode == DistributionMode_SHARDED) { int num_shards = static_cast(this->devices_.size()); @@ -1288,12 +1332,12 @@ class gpu_index_base_t { if (res.error) std::rethrow_exception(res.error); } - void train_quantizer(const float* train_data, uint64_t n_samples) { + void train_quantizer(const B* train_data, uint64_t n_samples) { uint64_t job_id = worker->submit_main( [this, train_data, n_samples](raft_handle_wrapper_t& handle) -> std::any { auto res = handle.get_raft_resources(); - auto train_host_view = raft::make_host_matrix_view(train_data, n_samples, dimension); - auto train_device = raft::make_device_matrix(*res, n_samples, dimension); + auto train_host_view = raft::make_host_matrix_view(train_data, n_samples, dimension); + auto train_device = raft::make_device_matrix(*res, n_samples, dimension); raft::copy(*res, train_device.view(), train_host_view); quantizer_.train(*res, train_device.view()); handle.sync(); @@ -1311,44 +1355,23 @@ class gpu_index_base_t { // 1. Flush any buffered chunks first flush_pending_float_chunks_internal(handle); - // 2. Check if still not trained (might have used add_chunk instead of float). - // WARNING: if data was added via add_chunk(T*) rather than add_chunk_float(), - // flattened_host_dataset already holds T values (e.g. int8 in [-128,127]). - // Casting them to float trains the quantizer on the compressed range, not the - // original float range. extend_float() will then clamp to the wrong range. - // If extend_float() is needed after add_chunk(T*), call train_quantizer() - // explicitly with representative original float data before calling build(). - bool needs_training; - uint64_t n_train = 0; - { - std::shared_lock lock(mutex_); - needs_training = !quantizer_.is_trained() && !flattened_host_dataset.empty(); - if (needs_training) { - n_train = std::min(static_cast(500), count); - if (n_train == 0) needs_training = false; - } - } - - if (needs_training) { - std::vector train_data(n_train * dimension); - { - std::shared_lock lock(mutex_); - for (size_t i = 0; i < n_train * dimension; ++i) { - train_data[i] = static_cast(flattened_host_dataset[i]); - } - } - - auto res = handle.get_raft_resources(); - auto train_host_view = raft::make_host_matrix_view(train_data.data(), n_train, dimension); - auto train_device = raft::make_device_matrix(*res, n_train, dimension); - raft::copy(*res, train_device.view(), train_host_view); - - { - std::unique_lock lock(mutex_); - quantizer_.train(*res, train_device.view()); - } - handle.sync(); - } + // 2. Do NOT auto-train the quantizer from flattened_host_dataset. + // For a 1-byte storage type that buffer only ever holds STORAGE + // bytes — raw T from add_chunk(T*) (a pre-quantized index) or the + // post-flush quantized output — never original floats. Training on + // it would learn the COMPRESSED range (e.g. int8 [-128,127]) instead + // of the true float range, so later base-typed search/extend would + // silently quantize against the wrong min/max. + // + // Correct training happens above in flush_pending_float_chunks_internal() + // on the ORIGINAL floats buffered by add_chunk_float()/add_chunk_quantize(). + // A pre-quantized index (built solely via add_chunk(T*)) therefore + // leaves the quantizer untrained; base-typed (B) search/extend on it + // requires an explicit range via set_quantizer() first. Both base-typed + // entry points already throw "quantizer not trained" while it is + // untrained — search via quantize_query() and extend via + // upload_float_matrix_as_T() — so the op fails loudly instead of + // mis-quantizing against a wrong range. return std::any(); } ); @@ -1364,8 +1387,41 @@ class gpu_index_base_t { void get_quantizer(float* min, float* max) { std::shared_lock lock(mutex_); - *min = quantizer_.min(); - *max = quantizer_.max(); + *min = static_cast(quantizer_.min()); + *max = static_cast(quantizer_.max()); + } + + // ---- Native B-source quantization (base element B -> 1-byte T) ---- + // Base-typed (B) add: converts the SOURCE-typed chunk to storage T, the add + // counterpart of search_quantize (symmetric: same B->T conversion). Routes by + // (B,T): B==T is a native store; sizeof(T)==1 buffers the B chunk for deferred + // quantizer training (the B->T transform happens at build via + // flush_pending_float_chunks_internal — B==float and B==half both supported, + // no f32 detour for half); the remaining (B=float, T=half) case casts f32->f16 + // via add_chunk_float (std::copy into vector = __half assignment). + void add_chunk_quantize(const B* chunk_data, uint64_t chunk_count, int64_t offset = -1, const IdT* ids = nullptr) { + if constexpr (std::is_same_v) { + // B == T: no conversion — native storage add. + this->add_chunk(chunk_data, chunk_count, offset, ids); + } else if constexpr (sizeof(T) == 1) { + { + std::shared_lock lock(mutex_); + if (is_loaded_) throw std::runtime_error("Cannot add chunk to built index"); + } + pending_float_chunk_t c; + c.data.assign(chunk_data, chunk_data + chunk_count * dimension); + c.count = chunk_count; + c.offset = offset; + if (ids) c.ids.assign(ids, ids + chunk_count); + std::unique_lock lock(mutex_); + pending_total_count_ += chunk_count; + pending_float_chunks_.push_back(std::move(c)); + } else if constexpr (std::is_same_v) { + // B=float, T=half (sizeof(T)!=1): f32 -> T cast via add_chunk_float. + this->add_chunk_float(chunk_data, chunk_count, offset, ids); + } else { + throw std::runtime_error("add_chunk_quantize: unsupported (base,storage) type combination"); + } } // Returns a snapshot of host_ids by value. The previous signature @@ -1498,10 +1554,10 @@ class gpu_index_base_t { struct manifest_data_t { std::string raw; // full manifest.json content std::string comp_json; // "components" sub-object - bool has_ids = false; - bool has_quantizer = false; - bool has_bitset = false; - bool has_filter = false; + bool has_ids = false; + bool has_quantizer = false; + bool has_bitset = false; + bool has_filter = false; }; // Saves ids, quantizer, bitset, and filter data (when present) to dir. @@ -1513,23 +1569,23 @@ class gpu_index_base_t { FilterStore filter_snapshot; { std::shared_lock lock(mutex_); - has_ids = !this->host_ids.empty(); - has_quantizer = this->quantizer_.is_trained(); - has_bitset = !this->deleted_bitset_.empty(); - has_filter = !this->filter_host_.empty(); + has_ids = !this->host_ids.empty(); + has_quantizer = this->quantizer_.is_trained(); + has_bitset = !this->deleted_bitset_.empty(); + has_filter = !this->filter_host_.empty(); if (has_filter) filter_snapshot = this->filter_host_; // copy } - if (has_ids) this->save_ids(dir + "/ids.bin"); - if (has_quantizer) this->quantizer_.save_to_file(dir + "/quantizer.bin"); - if (has_bitset) this->save_bitset(dir); - if (has_filter) filter_snapshot.save(dir + "/filter_data.bin"); + if (has_ids) this->save_ids(dir + "/ids.bin"); + if (has_quantizer) this->quantizer_.save_to_file(dir + "/quantizer.bin"); + if (has_bitset) this->save_bitset(dir); + if (has_filter) filter_snapshot.save(dir + "/filter_data.bin"); std::vector entries; - if (has_ids) entries.push_back(" \"ids\": \"ids.bin\""); - if (has_quantizer) entries.push_back(" \"quantizer\": \"quantizer.bin\""); - if (has_bitset) entries.push_back(" \"bitset\": \"bitset.bin\""); - if (has_filter) entries.push_back(" \"filter_data\": \"filter_data.bin\""); + if (has_ids) entries.push_back(" \"ids\": \"ids.bin\""); + if (has_quantizer) entries.push_back(" \"quantizer\": \"quantizer.bin\""); + if (has_bitset) entries.push_back(" \"bitset\": \"bitset.bin\""); + if (has_filter) entries.push_back(" \"filter_data\": \"filter_data.bin\""); return entries; } @@ -1554,10 +1610,10 @@ class gpu_index_base_t { uint64_t cap_val, len_val, del_count, bs_ver; { std::shared_lock lock(mutex_); - has_ids = !this->host_ids.empty(); - has_quantizer = this->quantizer_.is_trained(); - has_bitset = !this->deleted_bitset_.empty(); - has_filter = !this->filter_host_.empty(); + has_ids = !this->host_ids.empty(); + has_quantizer = this->quantizer_.is_trained(); + has_bitset = !this->deleted_bitset_.empty(); + has_filter = !this->filter_host_.empty(); cap_val = this->count; len_val = this->current_offset_; del_count = this->deleted_count_; @@ -1624,11 +1680,11 @@ class gpu_index_base_t { manifest_data_t m; m.raw = raw; - m.comp_json = json_object(raw, "components"); - m.has_ids = json_bool(raw, "has_ids"); - m.has_quantizer = json_bool(raw, "has_quantizer"); - m.has_bitset = json_bool(raw, "has_bitset"); - m.has_filter = json_bool(raw, "has_filter"); + m.comp_json = json_object(raw, "components"); + m.has_ids = json_bool(raw, "has_ids"); + m.has_quantizer = json_bool(raw, "has_quantizer"); + m.has_bitset = json_bool(raw, "has_bitset"); + m.has_filter = json_bool(raw, "has_filter"); return m; } @@ -1670,7 +1726,10 @@ class gpu_index_base_t { } protected: - scalar_quantizer_t quantizer_; + // Scalar quantizer over the SOURCE element type B (float or half). Used only + // when the STORAGE type T is 1-byte (int8/uint8); for float/half storage the + // add path casts B->T directly with no quantizer. + scalar_quantizer_t quantizer_; uint64_t current_offset_ = 0; // Serializes concurrent extend() calls. Held across GPU work and count update so that // set_ids() offsets always match the GPU execution order. Does NOT block searches. @@ -1820,8 +1879,20 @@ class gpu_index_base_t { throw std::runtime_error( "upload_float_matrix_as_T: quantizer not trained"); } - this->quantizer_.template transform( - *res, float_view, storage.data(), true); + if constexpr (std::is_same_v) { + this->quantizer_.template transform( + *res, float_view, storage.data(), true); + } else { + // B == half: quantizer is half-source. Cast the f32 input to + // half on-device, then transform half -> T. + rmm::device_uvector b_storage( + static_cast(n_rows) * this->dimension, stream, matrixone::raw_device_mr()); + auto b_view = raft::make_device_matrix_view( + b_storage.data(), (int64_t)n_rows, (int64_t)this->dimension); + raft::copy(*res, b_view, float_view); + this->quantizer_.template transform( + *res, b_view, storage.data(), true); + } } else { // T is half — cast float → half raft::copy(*res, device_view, float_view); @@ -1830,13 +1901,15 @@ class gpu_index_base_t { return storage; } - // Deferred float chunk buffer for quantizer training (1-byte types only). - // See class-level comment block above for full description. + // Deferred B-source chunk buffer for quantizer training (1-byte storage T). + // See class-level comment block above for full description. Holds the raw + // SOURCE element type B (float or half); the quantizer trains on B and + // transforms B->T at flush time. struct pending_float_chunk_t { - std::vector data; ///< count * dimension floats - uint64_t count; - int64_t offset; ///< -1 = append; >= 0 = explicit position - std::vector ids; ///< empty if caller supplied no IDs + std::vector data; ///< count * dimension B elements + uint64_t count; + int64_t offset; ///< -1 = append; >= 0 = explicit position + std::vector ids; ///< empty if caller supplied no IDs }; static constexpr uint64_t kQuantizerTrainThreshold = 1000; std::vector pending_float_chunks_; diff --git a/cgo/cuvs/ivf_flat.hpp b/cgo/cuvs/ivf_flat.hpp index 414fd169dd831..e23d78893f134 100644 --- a/cgo/cuvs/ivf_flat.hpp +++ b/cgo/cuvs/ivf_flat.hpp @@ -106,10 +106,10 @@ namespace matrixone { // - Non-SHARDED: submit() (round-robin GPU assignment) // - SHARDED: submit_all_devices_no_wait() → matrixone::cpu_topk_merge_sharded() // -// search_float() is the same but accepts float32 queries and converts on the fly +// search_quantize() is the same but accepts base-typed (B) queries and converts on the fly // (via quantizer for 1-byte T, via half conversion for T=half, direct for T=float). // -// search_batchable_typed() / search_batchable_float() just submit the search to +// search_batchable_typed() / search_batchable_quantize() just submit the search to // the worker; request-level batching, when enabled (batch_window() > 0), // happens inside search_internal via cuVS dynamic_batching (see dynamic_batching.hpp). // @@ -139,15 +139,17 @@ struct ivf_flat_search_result_t { /** * @brief gpu_ivf_flat_t implements an IVF-Flat index that can run on a single GPU or sharded across multiple GPUs. */ -template -class gpu_ivf_flat_t : public gpu_index_base_t { +template +class gpu_ivf_flat_t : public gpu_index_base_t { public: + using base_type = B; + using storage_type = T; using ivf_flat_index = cuvs::neighbors::ivf_flat::index; using mg_index = cuvs::neighbors::mg_index; using search_result_t = ivf_flat_search_result_t; // Inherited dependent type — bring into scope so search_internal can take a // const host_mask_bundle_t* parameter without `typename Base::...` everywhere. - using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; + using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; std::unique_ptr index_; std::string data_filename_; @@ -611,8 +613,8 @@ class gpu_ivf_flat_t : public gpu_index_base_tsearch_wait(job_id); } - // Async T-typed filtered search. Mirrors search_float_with_filter_async - // but uses search_internal (T) instead of search_float_internal (float). + // Async T-typed filtered search. Mirrors search_quantize_with_filter_async + // but uses search_internal (T) instead of search_quantize_internal (B). uint64_t search_with_filter_async(const T* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp, @@ -715,27 +717,28 @@ class gpu_ivf_flat_t : public gpu_index_base_tworker->submit(task); } - // Sync float entry — wraps search_float_async + search_wait. - search_result_t search_float(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp) { - uint64_t job_id = this->search_float_async(queries_data, num_queries, query_dimension, limit, sp); + // Sync quantize entry — wraps search_quantize_async + search_wait. + search_result_t search_quantize(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp) { + uint64_t job_id = this->search_quantize_async(queries_data, num_queries, query_dimension, limit, sp); return this->search_wait(job_id); } - // Sync float filtered entry — wraps search_float_with_filter_async + search_wait. - search_result_t search_float_with_filter(const float* queries_data, uint64_t num_queries, + // Sync quantize filtered entry — wraps search_quantize_with_filter_async + search_wait. + search_result_t search_quantize_with_filter(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp, const std::string& preds_json) { - uint64_t job_id = this->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); + uint64_t job_id = this->search_quantize_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); return this->search_wait(job_id); } - // Async variant of search_float_with_filter. Builds the host mask bundle on + // Async variant of search_quantize_with_filter. Builds the host mask bundle on // the calling thread (off-worker), copies queries into a shared_ptr so they // outlive the Go caller, captures both in the worker lambda, and returns a // job_id that search_wait() can collect. Used by the multi-index filter - // path so per-shard searches run in parallel. - uint64_t search_float_with_filter_async(const float* queries_data, uint64_t num_queries, + // path so per-shard searches run in parallel. The query is the BASE type B + // (float or half); search_quantize_internal converts it to storage T. + uint64_t search_quantize_with_filter_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp, const std::string& preds_json) { @@ -748,7 +751,7 @@ class gpu_ivf_flat_t : public gpu_index_base_tworker) throw std::runtime_error("Worker not initialized"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Bitmap eval runs on the caller's (Go) thread; per-shard searches @@ -757,7 +760,7 @@ class gpu_ivf_flat_t : public gpu_index_base_tbuild_filter_shard_masks(preds_json); auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy, shard_masks](raft_handle_wrapper_t& gpu_handle) -> std::any { int rank = gpu_handle.get_rank(); - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -769,12 +772,12 @@ class gpu_ivf_flat_t : public gpu_index_base_tbuild_filter_single_mask(preds_json); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy, mask](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); }; return this->worker->submit(task); } - uint64_t search_float_async(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp) { + uint64_t search_quantize_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_flat_search_params_t& sp) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); @@ -783,13 +786,13 @@ class gpu_ivf_flat_t : public gpu_index_base_tis_loaded_ || (!index_ && this->replicated_indices_.empty())) throw std::runtime_error("search_async: index not loaded"); } - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Same shape as search_async — fan out, hand back a composite id, // let search_wait() do the merge on the caller's thread. auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy](raft_handle_wrapper_t& gpu_handle) -> std::any { - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -797,16 +800,16 @@ class gpu_ivf_flat_t : public gpu_index_base_tsearch_batchable_float(queries_copy, queries_copy->data(), num_queries, limit, sp); + return this->search_batchable_quantize(queries_copy, queries_copy->data(), num_queries, limit, sp); } - // float32-input search. Mirrors search_batchable_typed but calls - // search_float_internal; request-level batching (if enabled) happens inside it. - uint64_t search_batchable_float(std::shared_ptr> owner, const float* queries_data, + // Base-typed (B) quantize search. Mirrors search_batchable_typed but calls + // search_quantize_internal; request-level batching (if enabled) happens inside it. + uint64_t search_batchable_quantize(std::shared_ptr> owner, const B* queries_data, uint64_t num_queries, uint32_t limit, const ivf_flat_search_params_t& sp) { if (!this->worker) throw std::runtime_error("Worker not initialized"); auto task = [this, owner, queries_data, num_queries, limit, sp](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_data, num_queries, this->dimension, limit, sp); + return this->search_quantize_internal(handle, queries_data, num_queries, this->dimension, limit, sp); }; return this->worker->submit(task); } @@ -983,14 +986,18 @@ class gpu_ivf_flat_t : public gpu_index_base_tmetric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } // See search_internal() above for the prebuilt-bundle contract; identical // semantics here (off-worker CPU mask eval, skip queries-H2D sync_stream // when prebuilt is non-null). - search_result_t search_float_internal(raft_handle_wrapper_t& handle, const float* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const ivf_flat_search_params_t& sp, const std::string& preds_json = "", const host_mask_bundle_t* prebuilt = nullptr) { + // Takes the query in the BASE element type B (float or half) and converts + // it to the storage type T on-device — see the cagra search_quantize_internal + // comment. B==T copies straight, sizeof(T)==1 quantizes B -> int8/uint8, and + // the (B=float, T=half) instantiation casts f32 -> f16 on the host. + search_result_t search_quantize_internal(raft_handle_wrapper_t& handle, const B* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const ivf_flat_search_params_t& sp, const std::string& preds_json = "", const host_mask_bundle_t* prebuilt = nullptr) { // No top-level lock: see search_internal() above — pointer fetched // via per-handle cache / narrow inner shared_lock, GPU work runs // unlocked. @@ -1002,26 +1009,28 @@ class gpu_ivf_flat_t : public gpu_index_base_t( q_buf_t.data(), static_cast(num_queries), static_cast(this->dimension)); - if constexpr (std::is_same_v) { - raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - } else if constexpr (std::is_same_v) { - // Host-side fp32 → fp16 cast (F16C / AVX, IEEE round-to-nearest-even - // — bit-identical to mdspan_copy_kernel<__half>) into a pinned - // staging buffer, then a single half-sized H2D copy. Skips the - // q_dev_f device allocation and the mdspan_copy_kernel dispatch. + if constexpr (std::is_same_v) { + // B == T (float->float or half->half): no conversion. + raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + } else if constexpr (sizeof(T) == 1) { + // sizeof(T) == 1: quantize the base-typed query B -> int8/uint8. + // Stage the B query on its own per-thread device workspace (distinct + // from q_buf_t — see q_dev_buf), then transform B -> T on-device. + if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); + auto& q_buf_b = handle.template q_dev_buf(n_q_elems); + auto q_dev_b = raft::make_device_matrix_view( + q_buf_b.data(), static_cast(num_queries), static_cast(this->dimension)); + raft::copy(*res, q_dev_b, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + this->quantizer_.template transform(*res, q_dev_b, q_buf_t.data(), true); + } else { + // B != T and sizeof(T) != 1: only (B=float, T=half). Host fp32 -> fp16 + // cast (F16C / AVX, IEEE round-to-nearest-even — bit-identical to + // mdspan_copy_kernel<__half>) into a pinned staging buffer, then a + // single half-sized H2D copy. __half* host_h = handle.ensure_host_half_buf(n_q_elems); matrixone::cast_float_to_half_host(queries_data, host_h, n_q_elems); raft::copy(*res, q_dev_t, raft::make_host_matrix_view(host_h, num_queries, this->dimension)); - } else { - // sizeof(T) == 1: int8 quantizer needs the fp32 device matrix. - auto& q_buf_f = handle.q_dev_buf_float(n_q_elems); - auto q_dev_f = raft::make_device_matrix_view( - q_buf_f.data(), static_cast(num_queries), static_cast(this->dimension)); - raft::copy(*res, q_dev_f, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - - if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); - this->quantizer_.template transform(*res, q_dev_f, q_buf_t.data(), true); } // Legacy path syncs so build_search_bitset's stack-local host bitmap can // drain on the same stream. Prebuilt path skips: bitset H2D queues @@ -1175,7 +1184,7 @@ class gpu_ivf_flat_t : public gpu_index_base_tmetric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } @@ -1451,7 +1460,14 @@ class gpu_ivf_flat_t : public gpu_index_base_t(*res, n_centers, dim); if constexpr (sizeof(T) == 1) { auto centers_float_view = raft::make_device_matrix_view(centers_view.data_handle(), n_centers, dim); - this->quantizer_.template transform(*res, centers_float_view, centers_device_target.data_handle(), true); + if constexpr (std::is_same_v) { + this->quantizer_.template transform(*res, centers_float_view, centers_device_target.data_handle(), true); + } else { + // B == half: cast the float centers to B on-device, then quantize B -> T. + auto centers_b = raft::make_device_matrix(*res, n_centers, dim); + raft::copy(*res, centers_b.view(), centers_float_view); + this->quantizer_.template transform(*res, centers_b.view(), centers_device_target.data_handle(), true); + } } else { raft::copy(*res, centers_device_target.view(), centers_view); } @@ -1470,7 +1486,7 @@ class gpu_ivf_flat_t : public gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"IVF-Flat\", \"ivf_flat\": {"; if (index_) json += "\"mode\": \"Single-GPU\", \"size\": " + std::to_string(index_->size()); else if (!this->replicated_indices_.empty()) json += "\"mode\": \"Local-Indices\", \"ranks\": " + std::to_string(this->replicated_indices_.size()); diff --git a/cgo/cuvs/ivf_flat_c.cpp b/cgo/cuvs/ivf_flat_c.cpp index 6f59272749330..7b0f20c27c30f 100644 --- a/cgo/cuvs/ivf_flat_c.cpp +++ b/cgo/cuvs/ivf_flat_c.cpp @@ -1,4 +1,4 @@ -/* +/* * Copyright 2021 Matrix Origin * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,15 @@ /* * IVF-Flat C Wrapper Implementation - * Supported data types (via quantization_t): Quantization_F32, Quantization_F16, Quantization_INT8, Quantization_UINT8 + * + * Two type axes via quantization_t: + * btype = base / query / quantizer-SOURCE element type (Quantization_F32 or F16) + * qtype = storage element type (Quantization_F32, F16, INT8, UINT8) + * + * Wired (btype, qtype) combinations: + * F32 base: F32, F16, INT8, UINT8 storage + * F16 base: F16, INT8, UINT8 storage + * Any other combination throws "unsupported (base,storage) type combination". */ #include "ivf_flat_c.h" @@ -27,146 +35,152 @@ #include #include #include +#include using namespace matrixone; struct gpu_ivf_flat_any_t { - quantization_t qtype; + quantization_t btype; // base / query / quantizer-source element type + quantization_t qtype; // storage element type void* ptr; - gpu_ivf_flat_any_t(quantization_t q, void* p) : qtype(q), ptr(p) {} - ~gpu_ivf_flat_any_t() { + gpu_ivf_flat_any_t(quantization_t b, quantization_t q, void* p) + : btype(b), qtype(q), ptr(p) {} + ~gpu_ivf_flat_any_t(); +}; + +// Static dispatch: resolves the concrete gpu_ivf_flat_t for (btype,qtype) and +// invokes fn with a typed pointer. fn is a generic lambda; recover B/Q inside it +// via decltype(idx)::base_type / ::storage_type. Throws on unsupported combos. +template +static auto ivf_flat_dispatch(const gpu_ivf_flat_any_t* a, Fn&& fn) { + switch (a->btype) { + case Quantization_F32: + switch (a->qtype) { + case Quantization_F32: return fn(static_cast*>(a->ptr)); + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + case Quantization_F16: + switch (a->qtype) { + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + default: break; + } + throw std::runtime_error("gpu_ivf_flat: unsupported (base,storage) type combination"); +} + +gpu_ivf_flat_any_t::~gpu_ivf_flat_any_t() { + if (!ptr) return; + try { + ivf_flat_dispatch(this, [](auto* idx) { + idx->destroy(); + delete idx; + }); + } catch (...) { + // unsupported combo never gets a live ptr — nothing to free + } +} + +// Construct a new gpu_ivf_flat_t for the wired (btype,qtype) combos. +// Maker is a generic lambda invoked as maker(static type tag) -> void*; it +// receives a null typed pointer purely to recover B and Q. +template +static void* ivf_flat_construct(quantization_t btype, quantization_t qtype, Maker&& maker) { + switch (btype) { + case Quantization_F32: switch (qtype) { - case Quantization_F32: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_F16: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_INT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_UINT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - default: break; + case Quantization_F32: return maker(static_cast*>(nullptr)); + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; + } + break; + case Quantization_F16: + switch (qtype) { + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; } + break; + default: break; } -}; + throw std::runtime_error("gpu_ivf_flat: unsupported (base,storage) type combination"); +} extern "C" { -gpu_ivf_flat_c gpu_ivf_flat_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, +gpu_ivf_flat_c gpu_ivf_flat_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_flat_any_t(qtype, ptr)); + void* ptr = ivf_flat_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + // The dataset-providing constructor takes storage-typed (Q) data. + return new gpu_ivf_flat_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(new gpu_ivf_flat_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_new", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_new", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_new", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_new", "unknown C++ exception"); } return nullptr; } -gpu_ivf_flat_c gpu_ivf_flat_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, +gpu_ivf_flat_c gpu_ivf_flat_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } return static_cast(new gpu_ivf_flat_any_t(qtype, ptr)); - } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_new_empty", e.what()); - } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_new_empty", "unknown C++ exception"); + void* ptr = ivf_flat_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_flat_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(new gpu_ivf_flat_any_t(btype, qtype, ptr)); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_new_empty", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_new_empty", "unknown C++ exception"); } return nullptr; } gpu_ivf_flat_c gpu_ivf_flat_load_file(const char* filename, uint32_t dimension, distance_type_t metric_c, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg) { + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_F16: - ptr = new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_INT8: - ptr = new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_flat_any_t(qtype, ptr)); + void* ptr = ivf_flat_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_flat_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); + }); + return static_cast(new gpu_ivf_flat_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_load_file", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_load_file", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_load_file", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_load_file", "unknown C++ exception"); } return nullptr; } @@ -176,51 +190,31 @@ void gpu_ivf_flat_destroy(gpu_ivf_flat_c index_c, void* errmsg) { try { delete static_cast(index_c); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_destroy", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_destroy", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_destroy", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_destroy", "unknown C++ exception"); } } void gpu_ivf_flat_start(gpu_ivf_flat_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->start(); break; - case Quantization_F16: static_cast*>(any->ptr)->start(); break; - case Quantization_INT8: static_cast*>(any->ptr)->start(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->start(); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [](auto* idx) { idx->start(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_start", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_start", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_start", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_start", "unknown C++ exception"); } } void gpu_ivf_flat_build(gpu_ivf_flat_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->build(); break; - case Quantization_F16: static_cast*>(any->ptr)->build(); break; - case Quantization_INT8: static_cast*>(any->ptr)->build(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->build(); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [](auto* idx) { idx->build(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_build", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_build", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_build", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_build", "unknown C++ exception"); } } @@ -228,14 +222,10 @@ void gpu_ivf_flat_extend(gpu_ivf_flat_c index_c, const void* new_data, uint64_t const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->extend(static_cast(new_data), n_rows, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_extend", e.what()); } catch (...) { @@ -247,14 +237,9 @@ void gpu_ivf_flat_extend_float(gpu_ivf_flat_c index_c, const float* new_data, ui const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + idx->extend_float(new_data, n_rows, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_extend_float", e.what()); } catch (...) { @@ -265,174 +250,127 @@ void gpu_ivf_flat_extend_float(gpu_ivf_flat_c index_c, const float* new_data, ui void gpu_ivf_flat_add_chunk(gpu_ivf_flat_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_add_chunk", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_add_chunk", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk", "unknown C++ exception"); } } void gpu_ivf_flat_add_chunk_float(gpu_ivf_flat_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_chunk_float(chunk_data, chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_add_chunk_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk_float", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_add_chunk_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk_float", "unknown C++ exception"); + } +} + +void gpu_ivf_flat_add_chunk_quantize(gpu_ivf_flat_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->add_chunk_quantize(static_cast(base_data), chunk_count, -1, ids); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk_quantize", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_chunk_quantize", "unknown C++ exception"); } } void gpu_ivf_flat_train_quantizer(gpu_ivf_flat_c index_c, const float* train_data, uint64_t n_samples, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_F16: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_INT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_UINT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + // train_quantizer takes base-typed (B) data. The C ABI hands us + // float32 (every existing caller has an F32 base); for an F16 base + // convert the host buffer to half first so all instantiations are + // both compilable and correct. + if constexpr (std::is_same_v) { + idx->train_quantizer(train_data, n_samples); + } else { + std::vector conv(static_cast(n_samples) * idx->dimension); + matrixone::cast_float_to_half_host(train_data, conv.data(), conv.size()); + idx->train_quantizer(conv.data(), n_samples); + } + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_train_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_train_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_train_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_train_quantizer", "unknown C++ exception"); } } void gpu_ivf_flat_set_batch_window(gpu_ivf_flat_c index_c, int64_t window_us, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_F16: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_batch_window(window_us); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_batch_window", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_batch_window", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_batch_window", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_batch_window", "unknown C++ exception"); } } void gpu_ivf_flat_set_dynb_conservative_dispatch(gpu_ivf_flat_c index_c, bool enable, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_F16: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_dynb_conservative_dispatch(enable); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_dynb_conservative_dispatch", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_dynb_conservative_dispatch", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_dynb_conservative_dispatch", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_dynb_conservative_dispatch", "unknown C++ exception"); } } void gpu_ivf_flat_set_quantizer(gpu_ivf_flat_c index_c, float min, float max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_set_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_quantizer", "unknown C++ exception"); } } void gpu_ivf_flat_get_quantizer(gpu_ivf_flat_c index_c, float* min, float* max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->get_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_get_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_get_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_get_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_get_quantizer", "unknown C++ exception"); } } void gpu_ivf_flat_save(gpu_ivf_flat_c index_c, const char* filename, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save(filename); break; - case Quantization_F16: static_cast*>(any->ptr)->save(filename); break; - case Quantization_INT8: static_cast*>(any->ptr)->save(filename); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save(filename); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->save(filename); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_save", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_save", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_save", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_save", "unknown C++ exception"); } } void gpu_ivf_flat_save_dir(gpu_ivf_flat_c index_c, const char* dir, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_F16: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_INT8: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save_dir(dir); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->save_dir(dir); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_save_dir", e.what()); } catch (...) { @@ -443,14 +381,7 @@ void gpu_ivf_flat_save_dir(gpu_ivf_flat_c index_c, const char* dir, void* errmsg void gpu_ivf_flat_delete_id(gpu_ivf_flat_c index_c, int64_t id, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_F16: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_INT8: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_UINT8: static_cast*>(any->ptr)->delete_id(id); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->delete_id(id); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_delete_id", e.what()); } catch (...) { @@ -462,14 +393,7 @@ void gpu_ivf_flat_load_dir(gpu_ivf_flat_c index_c, const char* dir, distribution_mode_t target_mode, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_F16: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_INT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_UINT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { idx->load_dir(dir, target_mode); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_load_dir", e.what()); } catch (...) { @@ -477,69 +401,55 @@ void gpu_ivf_flat_load_dir(gpu_ivf_flat_c index_c, const char* dir, } } -gpu_ivf_flat_search_res_t gpu_ivf_flat_search(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_ivf_flat_search_res_t gpu_ivf_flat_search(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_search", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_search", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search", "unknown C++ exception"); } return result; } -gpu_ivf_flat_search_res_t gpu_ivf_flat_search_float(gpu_ivf_flat_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_ivf_flat_search_res_t gpu_ivf_flat_search_quantize(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize", "unknown C++ exception"); } return result; } -uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_async", e.what()); return 0; @@ -549,24 +459,20 @@ uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_d } } -uint64_t gpu_ivf_flat_search_float_async(gpu_ivf_flat_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_flat_search_quantize_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_async", "unknown C++ exception"); return 0; } } @@ -575,15 +481,10 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search_wait(gpu_ivf_flat_c index_c, uint6 if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + *cpp_res = idx->search_wait(job_id); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_wait", e.what()); @@ -642,14 +543,7 @@ void gpu_ivf_flat_free_result(gpu_ivf_flat_result_c result_c) { uint64_t gpu_ivf_flat_cap(gpu_ivf_flat_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->cap(); - case Quantization_F16: return static_cast*>(any->ptr)->cap(); - case Quantization_INT8: return static_cast*>(any->ptr)->cap(); - case Quantization_UINT8: return static_cast*>(any->ptr)->cap(); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->cap(); }); } catch (...) { return 0; } @@ -658,14 +552,7 @@ uint64_t gpu_ivf_flat_cap(gpu_ivf_flat_c index_c) { uint64_t gpu_ivf_flat_len(gpu_ivf_flat_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->len(); - case Quantization_F16: return static_cast*>(any->ptr)->len(); - case Quantization_INT8: return static_cast*>(any->ptr)->len(); - case Quantization_UINT8: return static_cast*>(any->ptr)->len(); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->len(); }); } catch (...) { return 0; } @@ -675,23 +562,13 @@ char* gpu_ivf_flat_info(gpu_ivf_flat_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return nullptr; try { - auto* any = static_cast(index_c); - std::string info; - switch (any->qtype) { - case Quantization_F32: info = static_cast*>(any->ptr)->info(); break; - case Quantization_F16: info = static_cast*>(any->ptr)->info(); break; - case Quantization_INT8: info = static_cast*>(any->ptr)->info(); break; - case Quantization_UINT8: info = static_cast*>(any->ptr)->info(); break; - default: return nullptr; - } + std::string info = ivf_flat_dispatch(static_cast(index_c), [](auto* idx) -> std::string { return idx->info(); }); return strdup(info.c_str()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_info", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_info", e.what()); return nullptr; } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_info", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_info", "unknown C++ exception"); return nullptr; } } @@ -699,50 +576,22 @@ char* gpu_ivf_flat_info(gpu_ivf_flat_c index_c, void* errmsg) { void gpu_ivf_flat_get_centers(gpu_ivf_flat_c index_c, void* centers, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: { - auto host_centers = static_cast*>(any->ptr)->get_centers(); - if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); - break; - } - case Quantization_F16: { - auto host_centers = static_cast*>(any->ptr)->get_centers(); - if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); - break; - } - case Quantization_INT8: { - auto host_centers = static_cast*>(any->ptr)->get_centers(); - if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); - break; - } - case Quantization_UINT8: { - auto host_centers = static_cast*>(any->ptr)->get_centers(); - if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); - break; - } - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + auto host_centers = idx->get_centers(); + if (!host_centers.empty()) std::copy(host_centers.begin(), host_centers.end(), static_cast(centers)); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_get_centers", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_get_centers", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_flat_get_centers", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_get_centers", "unknown C++ exception"); } } uint32_t gpu_ivf_flat_get_n_list(gpu_ivf_flat_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_F16: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_n_list(); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_n_list(); }); } catch (...) { return 0; } @@ -754,15 +603,10 @@ void gpu_ivf_flat_set_filter_columns(gpu_ivf_flat_c index_c, const char* col_met uint64_t total_count, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string s = col_meta_json ? col_meta_json : ""; - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_F16: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + idx->set_filter_columns(s, total_count); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_set_filter_columns", e.what()); } catch (...) { @@ -775,14 +619,9 @@ void gpu_ivf_flat_add_filter_chunk(gpu_ivf_flat_c index_c, uint32_t col_idx, uint64_t nrows, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_F16: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_filter_chunk(col_idx, data, null_bitmap, nrows); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_add_filter_chunk", e.what()); } catch (...) { @@ -797,16 +636,12 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search_with_filter(gpu_ivf_flat_c index_c if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_with_filter", e.what()); @@ -816,52 +651,44 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search_with_filter(gpu_ivf_flat_c index_c return result; } -gpu_ivf_flat_search_res_t gpu_ivf_flat_search_float_with_filter(gpu_ivf_flat_c index_c, const float* queries_data, +gpu_ivf_flat_search_res_t gpu_ivf_flat_search_quantize_with_filter(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_flat_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_with_filter", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_with_filter", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_with_filter", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_with_filter", "unknown C++ exception"); } return result; } -uint64_t gpu_ivf_flat_search_float_with_filter_async(gpu_ivf_flat_c index_c, const float* queries_data, +uint64_t gpu_ivf_flat_search_quantize_with_filter_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - default: return 0; - } + return ivf_flat_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_with_filter_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_with_filter_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_float_with_filter_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_flat_search_quantize_with_filter_async", "unknown C++ exception"); return 0; } } @@ -869,8 +696,11 @@ uint64_t gpu_ivf_flat_search_float_with_filter_async(gpu_ivf_flat_c index_c, con } // extern "C" namespace matrixone { -template class gpu_ivf_flat_t; -template class gpu_ivf_flat_t; -template class gpu_ivf_flat_t; -template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; +template class gpu_ivf_flat_t; } // namespace matrixone diff --git a/cgo/cuvs/ivf_flat_c.h b/cgo/cuvs/ivf_flat_c.h index 14c6693bdd33b..8822119641379 100644 --- a/cgo/cuvs/ivf_flat_c.h +++ b/cgo/cuvs/ivf_flat_c.h @@ -31,18 +31,22 @@ typedef void* gpu_ivf_flat_c; // Opaque pointer to the C++ IVF-Flat search result object typedef void* gpu_ivf_flat_result_c; +// btype = base/query/quantizer-source element type (Quantization_F32 or F16). +// qtype = storage element type. Wired combos: F32 base {F32,F16,INT8,UINT8}; +// F16 base {F16,INT8,UINT8}. Other combinations set errmsg and return NULL. + // Constructor for building from dataset -gpu_ivf_flat_c gpu_ivf_flat_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, +gpu_ivf_flat_c gpu_ivf_flat_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Constructor for loading from file gpu_ivf_flat_c gpu_ivf_flat_load_file(const char* filename, uint32_t dimension, distance_type_t metric, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg); + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg); // Destructor void gpu_ivf_flat_destroy(gpu_ivf_flat_c index_c, void* errmsg); @@ -54,10 +58,10 @@ void gpu_ivf_flat_start(gpu_ivf_flat_c index_c, void* errmsg); void gpu_ivf_flat_build(gpu_ivf_flat_c index_c, void* errmsg); // Constructor for an empty index (pre-allocates) -gpu_ivf_flat_c gpu_ivf_flat_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, +gpu_ivf_flat_c gpu_ivf_flat_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, ivf_flat_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Add chunk of data (same type as index quantization) void gpu_ivf_flat_add_chunk(gpu_ivf_flat_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); @@ -74,6 +78,9 @@ void gpu_ivf_flat_extend_float(gpu_ivf_flat_c index_c, const float* new_data, ui // Add chunk of data (from float, with on-the-fly quantization if needed) void gpu_ivf_flat_add_chunk_float(gpu_ivf_flat_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); +// Add chunk of base-typed (B) data; the index converts B -> storage on device. +void gpu_ivf_flat_add_chunk_quantize(gpu_ivf_flat_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); + // Trains the scalar quantizer (if T is 1-byte) void gpu_ivf_flat_train_quantizer(gpu_ivf_flat_c index_c, const float* train_data, uint64_t n_samples, void* errmsg); @@ -107,17 +114,19 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search(gpu_ivf_flat_c index_c, const void uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg); -gpu_ivf_flat_search_res_t gpu_ivf_flat_search_float(gpu_ivf_flat_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +// Quantize search: query in the BASE element type B (float or half); the index +// converts it to storage type T (copy / quantize / f32->f16 cast) internally. +gpu_ivf_flat_search_res_t gpu_ivf_flat_search_quantize(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg); // Asynchronous search functions -uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_flat_search_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg); -uint64_t gpu_ivf_flat_search_float_async(gpu_ivf_flat_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_flat_search_quantize_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, void* errmsg); gpu_ivf_flat_search_res_t gpu_ivf_flat_search_wait(gpu_ivf_flat_c index_c, uint64_t job_id, void* errmsg); @@ -160,15 +169,16 @@ gpu_ivf_flat_search_res_t gpu_ivf_flat_search_with_filter(gpu_ivf_flat_c index_c uint32_t limit, ivf_flat_search_params_t search_params, const char* preds_json, void* errmsg); -gpu_ivf_flat_search_res_t gpu_ivf_flat_search_float_with_filter(gpu_ivf_flat_c index_c, const float* queries_data, +// Query in the BASE element type B (float or half); converted to storage T internally. +gpu_ivf_flat_search_res_t gpu_ivf_flat_search_quantize_with_filter(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, const char* preds_json, void* errmsg); -// Async variant of gpu_ivf_flat_search_float_with_filter. Returns a job_id +// Async variant of gpu_ivf_flat_search_quantize_with_filter. Returns a job_id // that is collected with the existing gpu_ivf_flat_search_wait. Lets // multi-index callers fan out filtered searches across shards in parallel. -uint64_t gpu_ivf_flat_search_float_with_filter_async(gpu_ivf_flat_c index_c, const float* queries_data, +uint64_t gpu_ivf_flat_search_quantize_with_filter_async(gpu_ivf_flat_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_flat_search_params_t search_params, const char* preds_json, void* errmsg); diff --git a/cgo/cuvs/ivf_pq.hpp b/cgo/cuvs/ivf_pq.hpp index 2cfba5311ac99..cee7908a4251e 100644 --- a/cgo/cuvs/ivf_pq.hpp +++ b/cgo/cuvs/ivf_pq.hpp @@ -71,8 +71,9 @@ namespace matrixone { // // OVERVIEW // -------- -// gpu_ivf_pq_t implements an IVF-PQ (Inverted File with Product Quantization) -// approximate nearest-neighbor index backed by cuVS. +// gpu_ivf_pq_t implements an IVF-PQ (Inverted File with Product Quantization) +// approximate nearest-neighbor index backed by cuVS (B = base/source element +// type, T = storage element type; T is 1-byte for scalar-quantized indexes). // // cuVS type: cuvs::neighbors::ivf_pq::index // Note: the cuVS IVF-PQ index type is NOT templated on T — it always stores @@ -114,7 +115,7 @@ namespace matrixone { // replicated_indices_[rank] holds a full copy per rank (cast to ivf_pq_index*). // The replicated dataset pointers (replicated_datasets_) are used during build // and erased after the first extend on each device. -// search_internal / search_float_internal use per-thread cached index ptr +// search_internal / search_quantize_internal use per-thread cached index ptr // (handle.get_index_ptr()) to avoid repeated map lookups. // // SHARDED: @@ -143,7 +144,7 @@ namespace matrixone { // For REPLICATED: uses per-thread cached index ptr to avoid mutex on hot path. // For SHARDED: called once per shard with the shard's local index. // -// search_float_internal(handle, float* queries, ...) +// search_quantize_internal(handle, B* queries, ...) // Converts float → T on device (quantize for 1-byte T, half-cast for T=half, // direct copy for T=float), then searches the same way as search_internal. // @@ -152,7 +153,7 @@ namespace matrixone { // - SHARDED: sync_shard_bitset() → bitset_filter over shard-local bit slice // Bit j of the shard bitset = global bit (rank * rows_per_shard + j) // -// search_batchable_typed() / search_batchable_float() just submit the search to +// search_batchable_typed() / search_batchable_quantize() just submit the search to // the worker; request-level batching, when enabled (batch_window() > 0), // happens inside search_internal via cuVS dynamic_batching (see dynamic_batching.hpp). // @@ -177,14 +178,16 @@ struct ivf_pq_search_result_t { /** * @brief gpu_ivf_pq_t implements an IVF-PQ index that can run on a single GPU or sharded/replicated across multiple GPUs. */ -template -class gpu_ivf_pq_t : public gpu_index_base_t { +template +class gpu_ivf_pq_t : public gpu_index_base_t { public: + using base_type = B; + using storage_type = T; using ivf_pq_index = cuvs::neighbors::ivf_pq::index; using search_result_t = ivf_pq_search_result_t; // Inherited dependent type — bring into scope so search_internal can take a // const host_mask_bundle_t* parameter without `typename Base::...` everywhere. - using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; + using host_mask_bundle_t = typename gpu_index_base_t::host_mask_bundle_t; // Internal index storage std::unique_ptr index_; @@ -368,6 +371,9 @@ class gpu_ivf_pq_t : public gpu_index_base_t return; } } + // 1-byte storage T: train the B-source quantizer on the buffered B + // sample, transform B->T, and store as T. For float/half storage this + // is a no-op. this->train_quantizer_if_needed(); if (!this->worker) throw std::runtime_error("Worker not initialized"); @@ -805,8 +811,34 @@ class gpu_ivf_pq_t : public gpu_index_base_t return this->search_wait(job_id); } - // Async T-typed filtered search. Mirrors search_float_with_filter_async - // but uses search_internal (T) instead of search_float_internal (float). + // Quantize a B-source query to the 1-byte storage type T via the B-source + // quantizer, writing num_queries*dimension T values into `out`. The caller + // then runs the normal native search(const T*) path — so sharding, overflow + // and result merge are reused unchanged. No f32 detour. + void quantize_query(const B* queries_data, uint64_t num_queries, T* out) { + if constexpr (sizeof(T) != 1) { + throw std::runtime_error("quantize_query requires a 1-byte storage type (int8/uint8)"); + } else { + uint64_t job = this->worker->submit_main( + [this, queries_data, num_queries, out](raft_handle_wrapper_t& handle) -> std::any { + auto res = handle.get_raft_resources(); + auto q_b_host = raft::make_host_matrix_view(queries_data, num_queries, this->dimension); + auto q_b_dev = raft::make_device_matrix(*res, num_queries, this->dimension); + raft::copy(*res, q_b_dev.view(), q_b_host); + if (!this->quantizer_.is_trained()) throw std::runtime_error("quantizer not trained"); + auto q_t_dev = raft::make_device_matrix(*res, num_queries, this->dimension); + this->quantizer_.template transform(*res, q_b_dev.view(), q_t_dev.data_handle(), true); + raft::copy(*res, raft::make_host_matrix_view(out, num_queries, this->dimension), q_t_dev.view()); + handle.sync(); + return std::any(); + }); + auto r = this->worker->wait(job).get(); + if (r.error) std::rethrow_exception(r.error); + } + } + + // Async T-typed filtered search. Mirrors search_quantize_with_filter_async + // but uses search_internal (T) instead of search_quantize_internal (B). uint64_t search_with_filter_async(const T* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp, @@ -1221,44 +1253,46 @@ class gpu_ivf_pq_t : public gpu_index_base_t } } - transform_distance(this->metric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } - // Sync float entry — wraps search_float_async + search_wait. - search_result_t search_float(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp) { - uint64_t job_id = this->search_float_async(queries_data, num_queries, query_dimension, limit, sp); + // Sync quantize entry — wraps search_quantize_async + search_wait. + search_result_t search_quantize(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp) { + uint64_t job_id = this->search_quantize_async(queries_data, num_queries, query_dimension, limit, sp); return this->search_wait(job_id); } - // Sync float filtered entry — wraps search_float_with_filter_async + search_wait. - search_result_t search_float_with_filter(const float* queries_data, uint64_t num_queries, + // Sync quantize filtered entry — wraps search_quantize_with_filter_async + search_wait. + search_result_t search_quantize_with_filter(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp, const std::string& preds_json) { - uint64_t job_id = this->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); + uint64_t job_id = this->search_quantize_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds_json); return this->search_wait(job_id); } - // Async variant of search_float_with_filter. Builds the host mask bundle on + // Async variant of search_quantize_with_filter. Builds the host mask bundle on // the calling thread (same off-worker pattern as the sync filter), copies // queries into a shared_ptr so they outlive the Go caller, captures both in // the worker lambda, and returns a job_id that search_wait() can collect. // Used by the multi-index filter path so per-shard searches run in parallel. - uint64_t search_float_with_filter_async(const float* queries_data, uint64_t num_queries, + // The query is the BASE type B (float or half); search_quantize_internal + // converts it to storage T. + uint64_t search_quantize_with_filter_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp, const std::string& preds_json) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); - // Reject mismatched caller dim. search_float_internal sizes its H2D + // Reject mismatched caller dim. search_quantize_internal sizes its H2D // extent by this->dimension (query_dimension param is unused inside), // so passing a different value here would either OOB-read or // under-copy host queries. See the T-typed sibling at line ~762. if (query_dimension != this->dimension) { throw std::invalid_argument( - "search_float_with_filter_async: query_dimension (" + std::to_string(query_dimension) + + "search_quantize_with_filter_async: query_dimension (" + std::to_string(query_dimension) + ") does not match index dimension (" + std::to_string(this->dimension) + ")"); } { @@ -1267,7 +1301,7 @@ class gpu_ivf_pq_t : public gpu_index_base_t } if (!this->worker) throw std::runtime_error("Worker not initialized"); - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Bitmap eval runs on the caller's (Go) thread; per-shard searches @@ -1276,7 +1310,7 @@ class gpu_ivf_pq_t : public gpu_index_base_t auto shard_masks = this->build_filter_shard_masks(preds_json); auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy, shard_masks](raft_handle_wrapper_t& gpu_handle) -> std::any { int rank = gpu_handle.get_rank(); - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", shard_masks[rank].get()); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -1288,19 +1322,19 @@ class gpu_ivf_pq_t : public gpu_index_base_t // would force serialization through main_thread_ and lose batching. auto mask = this->build_filter_single_mask(preds_json); auto task = [this, num_queries, query_dimension, limit, sp, queries_copy, mask](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); + return this->search_quantize_internal(handle, queries_copy->data(), num_queries, query_dimension, limit, sp, /*preds_json=*/"", mask.get()); }; return this->worker->submit(task); } - uint64_t search_float_async(const float* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp) { + uint64_t search_quantize_async(const B* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, const ivf_pq_search_params_t& sp) { if (!queries_data) throw std::invalid_argument("search_async: queries_data is null"); if (num_queries == 0) throw std::invalid_argument("search_async: num_queries is 0"); if (this->dimension == 0) throw std::runtime_error("search_async: index dimension is 0"); - // Reject mismatched caller dim — see search_float_with_filter_async. + // Reject mismatched caller dim — see search_quantize_with_filter_async. if (query_dimension != this->dimension) { throw std::invalid_argument( - "search_float_async: query_dimension (" + std::to_string(query_dimension) + + "search_quantize_async: query_dimension (" + std::to_string(query_dimension) + ") does not match index dimension (" + std::to_string(this->dimension) + ")"); } { @@ -1308,13 +1342,13 @@ class gpu_ivf_pq_t : public gpu_index_base_t if (!this->is_loaded_ || (!index_ && this->replicated_indices_.empty())) throw std::runtime_error("search_async: index not loaded"); } - auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); + auto queries_copy = std::make_shared>(queries_data, queries_data + num_queries * query_dimension); if (this->dist_mode == DistributionMode_SHARDED) { // Same shape as search_async — fan out, hand back a composite id, // let search_wait() do the merge on the caller's thread. auto shard_search_task = [this, num_queries, query_dimension, limit, sp, queries_copy](raft_handle_wrapper_t& gpu_handle) -> std::any { - return this->search_float_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); + return this->search_quantize_internal(gpu_handle, queries_copy->data(), num_queries, query_dimension, limit, sp); }; auto job_ids = this->worker->submit_all_devices_no_wait(shard_search_task); return this->worker->submit_composite_pending(std::move(job_ids), num_queries, limit); @@ -1322,22 +1356,26 @@ class gpu_ivf_pq_t : public gpu_index_base_t // Single-GPU / replicated: the helper decides standalone vs fused; the // shared_ptr keeps the copied queries alive until the search runs. - return this->search_batchable_float(queries_copy, queries_copy->data(), num_queries, limit, sp); + return this->search_batchable_quantize(queries_copy, queries_copy->data(), num_queries, limit, sp); } - // float32-input search. Mirrors search_batchable_typed but calls - // search_float_internal; request-level batching (if enabled) happens inside it. - uint64_t search_batchable_float(std::shared_ptr> owner, const float* queries_data, + // Base-typed (B) quantize search. Mirrors search_batchable_typed but calls + // search_quantize_internal; request-level batching (if enabled) happens inside it. + uint64_t search_batchable_quantize(std::shared_ptr> owner, const B* queries_data, uint64_t num_queries, uint32_t limit, const ivf_pq_search_params_t& sp) { if (!this->worker) throw std::runtime_error("Worker not initialized"); auto task = [this, owner, queries_data, num_queries, limit, sp](raft_handle_wrapper_t& handle) -> std::any { - return this->search_float_internal(handle, queries_data, num_queries, this->dimension, limit, sp); + return this->search_quantize_internal(handle, queries_data, num_queries, this->dimension, limit, sp); }; return this->worker->submit(task); } // See `search_internal` for the contract on `prebuilt`. - search_result_t search_float_internal(raft_handle_wrapper_t& handle, const float* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, + // Takes the query in the BASE element type B (float or half) and converts + // it to the storage type T on-device — see the cagra search_quantize_internal + // comment. B==T copies straight, sizeof(T)==1 quantizes B -> int8/uint8, and + // the (B=float, T=half) instantiation casts f32 -> f16 on the host. + search_result_t search_quantize_internal(raft_handle_wrapper_t& handle, const B* queries_data, uint64_t num_queries, uint32_t /*query_dimension*/, uint32_t limit, const ivf_pq_search_params_t& sp, const std::string& preds_json = "", const host_mask_bundle_t* prebuilt = nullptr) { auto res = handle.get_raft_resources(); // Step C: reuse the per-thread T-typed query workspace buffer. @@ -1346,29 +1384,28 @@ class gpu_ivf_pq_t : public gpu_index_base_t auto q_dev_t = raft::make_device_matrix_view( q_buf_t.data(), static_cast(num_queries), static_cast(this->dimension)); - if constexpr (std::is_same_v) { - raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - } else if constexpr (std::is_same_v) { - // Cast fp32 → fp16 on the host (F16C / AVX, IEEE round-to-nearest-even - // — bit-identical to mdspan_copy_kernel<__half>) into a pinned - // staging buffer, then a single H2D copy moves half the bytes. - // This eliminates one device alloc (q_dev_f), one full H2D fp32 - // upload, and the per-search mdspan_copy_kernel<__half> dispatch. + if constexpr (std::is_same_v) { + // B == T (float->float or half->half): no conversion. + raft::copy(*res, q_dev_t, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + } else if constexpr (sizeof(T) == 1) { + // sizeof(T) == 1: quantize the base-typed query B -> int8/uint8. + // Stage the B query on its own per-thread device workspace (distinct + // from q_buf_t — see q_dev_buf), then transform B -> T on-device. + if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); + auto& q_buf_b = handle.template q_dev_buf(n_q_elems); + auto q_dev_b = raft::make_device_matrix_view( + q_buf_b.data(), static_cast(num_queries), static_cast(this->dimension)); + raft::copy(*res, q_dev_b, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); + this->quantizer_.template transform(*res, q_dev_b, q_buf_t.data(), true); + } else { + // B != T and sizeof(T) != 1: only (B=float, T=half). Cast fp32 → fp16 + // on the host (F16C / AVX, IEEE round-to-nearest-even — bit-identical + // to mdspan_copy_kernel<__half>) into a pinned staging buffer, then a + // single H2D copy moves half the bytes. __half* host_h = handle.ensure_host_half_buf(n_q_elems); matrixone::cast_float_to_half_host(queries_data, host_h, n_q_elems); raft::copy(*res, q_dev_t, raft::make_host_matrix_view(host_h, num_queries, this->dimension)); - } else { - // sizeof(T) == 1: int8 quantizer path keeps an fp32 device copy - // because quantizer_.transform reads it on-device. Reuse the - // per-thread float workspace too. - auto& q_buf_f = handle.q_dev_buf_float(n_q_elems); - auto q_dev_f = raft::make_device_matrix_view( - q_buf_f.data(), static_cast(num_queries), static_cast(this->dimension)); - raft::copy(*res, q_dev_f, raft::make_host_matrix_view(queries_data, num_queries, this->dimension)); - - if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); - this->quantizer_.template transform(*res, q_dev_f, q_buf_t.data(), true); } // Legacy path syncs to drain queries DMA before the stack-local host // bitmap inside build_search_bitset goes through its own sync. Prebuilt @@ -1554,7 +1591,7 @@ class gpu_ivf_pq_t : public gpu_index_base_t } } - transform_distance(this->metric, search_res.distances); + transform_distance(this->metric, search_res.distances, this->quantized_l2_dequant_factor()); return search_res; } @@ -1591,7 +1628,14 @@ class gpu_ivf_pq_t : public gpu_index_base_t if constexpr (sizeof(T) == 1) { if (!this->quantizer_.is_trained()) throw std::runtime_error("Quantizer not trained"); auto centers_float_view = raft::make_device_matrix_view(centers_view.data_handle(), n_centers, dim_ext); - this->quantizer_.template transform(*res, centers_float_view, centers_device_target.data_handle(), true); + if constexpr (std::is_same_v) { + this->quantizer_.template transform(*res, centers_float_view, centers_device_target.data_handle(), true); + } else { + // B == half: cast cuVS's float centers to half, then transform. + auto centers_b = raft::make_device_matrix(*res, n_centers, dim_ext); + raft::copy(*res, centers_b.view(), centers_float_view); + this->quantizer_.template transform(*res, centers_b.view(), centers_device_target.data_handle(), true); + } } else { raft::copy(*res, centers_device_target.view(), centers_view); } @@ -1615,7 +1659,7 @@ class gpu_ivf_pq_t : public gpu_index_base_t } std::string info() const override { - std::string json = gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"IVF-PQ\", \"ivf_pq\": {"; if (index_) json += "\"mode\": \"Single-GPU\", \"size\": " + std::to_string(index_->size()); else if (!this->replicated_indices_.empty()) json += "\"mode\": \"Local-Indices\", \"ranks\": " + std::to_string(this->replicated_indices_.size()); diff --git a/cgo/cuvs/ivf_pq_c.cpp b/cgo/cuvs/ivf_pq_c.cpp index 642f2acecc87d..a61b77c002886 100644 --- a/cgo/cuvs/ivf_pq_c.cpp +++ b/cgo/cuvs/ivf_pq_c.cpp @@ -1,4 +1,4 @@ -/* +/* * Copyright 2021 Matrix Origin * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,15 @@ /* * IVF-PQ C Wrapper Implementation - * Supported data types (via quantization_t): Quantization_F32, Quantization_F16, Quantization_INT8, Quantization_UINT8 + * + * Two type axes via quantization_t: + * btype = base / query / quantizer-SOURCE element type (Quantization_F32 or F16) + * qtype = storage element type (Quantization_F32, F16, INT8, UINT8) + * + * Wired (btype, qtype) combinations: + * F32 base: F32, F16, INT8, UINT8 storage + * F16 base: F16, INT8, UINT8 storage + * Any other combination throws "unsupported (base,storage) type combination". */ #include "ivf_pq_c.h" @@ -27,181 +35,175 @@ #include #include #include +#include using namespace matrixone; struct gpu_ivf_pq_any_t { - quantization_t qtype; + quantization_t btype; // base / query / quantizer-source element type + quantization_t qtype; // storage element type void* ptr; - gpu_ivf_pq_any_t(quantization_t q, void* p) : qtype(q), ptr(p) {} - ~gpu_ivf_pq_any_t() { + gpu_ivf_pq_any_t(quantization_t b, quantization_t q, void* p) + : btype(b), qtype(q), ptr(p) {} + ~gpu_ivf_pq_any_t(); +}; + +// Static dispatch: resolves the concrete gpu_ivf_pq_t for (btype,qtype) and +// invokes fn with a typed pointer. fn is a generic lambda; recover B/Q inside it +// via decltype(idx)::base_type / ::storage_type. Throws on unsupported combos. +template +static auto ivf_pq_dispatch(const gpu_ivf_pq_any_t* a, Fn&& fn) { + switch (a->btype) { + case Quantization_F32: + switch (a->qtype) { + case Quantization_F32: return fn(static_cast*>(a->ptr)); + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + case Quantization_F16: + switch (a->qtype) { + case Quantization_F16: return fn(static_cast*>(a->ptr)); + case Quantization_INT8: return fn(static_cast*>(a->ptr)); + case Quantization_UINT8: return fn(static_cast*>(a->ptr)); + default: break; + } + break; + default: break; + } + throw std::runtime_error("gpu_ivf_pq: unsupported (base,storage) type combination"); +} + +gpu_ivf_pq_any_t::~gpu_ivf_pq_any_t() { + if (!ptr) return; + try { + ivf_pq_dispatch(this, [](auto* idx) { + idx->destroy(); + delete idx; + }); + } catch (...) { + // unsupported combo never gets a live ptr — nothing to free + } +} + +// Construct a new gpu_ivf_pq_t for the wired (btype,qtype) combos. +// Maker is a generic lambda invoked as maker(static type tag) -> void*; it +// receives a null typed pointer purely to recover B and Q. +template +static void* ivf_pq_construct(quantization_t btype, quantization_t qtype, Maker&& maker) { + switch (btype) { + case Quantization_F32: switch (qtype) { - case Quantization_F32: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_F16: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_INT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - case Quantization_UINT8: { - auto* p = static_cast*>(ptr); - p->destroy(); - delete p; - break; - } - default: break; + case Quantization_F32: return maker(static_cast*>(nullptr)); + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; + } + break; + case Quantization_F16: + switch (qtype) { + case Quantization_F16: return maker(static_cast*>(nullptr)); + case Quantization_INT8: return maker(static_cast*>(nullptr)); + case Quantization_UINT8: return maker(static_cast*>(nullptr)); + default: break; } + break; + default: break; } -}; + throw std::runtime_error("gpu_ivf_pq: unsupported (base,storage) type combination"); +} extern "C" { -gpu_ivf_pq_c gpu_ivf_pq_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, +gpu_ivf_pq_c gpu_ivf_pq_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric_c, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_pq_any_t(qtype, ptr)); + void* ptr = ivf_pq_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + // The dataset-providing constructor takes storage-typed (Q) data and + // copies it directly into flattened_host_dataset (no quantization here; + // quantization happens via add_chunk_quantize / add_chunk_float). + return new gpu_ivf_pq_t(static_cast(dataset_data), count_vectors, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(new gpu_ivf_pq_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new", "unknown C++ exception"); } return nullptr; } -gpu_ivf_pq_c gpu_ivf_pq_new_from_data_file(const char* data_filename, distance_type_t metric_c, +gpu_ivf_pq_c gpu_ivf_pq_new_from_data_file(const char* data_filename, distance_type_t metric_c, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg) { + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_F16: - ptr = new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_INT8: - ptr = new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_pq_any_t(qtype, ptr)); + void* ptr = ivf_pq_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_pq_t(std::string(data_filename), metric_c, build_params, devs, nthread, dist_mode); + }); + return static_cast(new gpu_ivf_pq_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new_from_data_file", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new_from_data_file", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new_from_data_file", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new_from_data_file", "unknown C++ exception"); } return nullptr; } -gpu_ivf_pq_c gpu_ivf_pq_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, +gpu_ivf_pq_c gpu_ivf_pq_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric_c, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_F16: - ptr = new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_INT8: - ptr = new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_pq_any_t(qtype, ptr)); + void* ptr = ivf_pq_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_pq_t(total_count, dimension, metric_c, build_params, devs, nthread, dist_mode, ids); + }); + return static_cast(new gpu_ivf_pq_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new_empty", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new_empty", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_new_empty", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_new_empty", "unknown C++ exception"); } return nullptr; } gpu_ivf_pq_c gpu_ivf_pq_load_file(const char* filename, uint32_t dimension, distance_type_t metric_c, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg) { + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { std::vector devs(devices, devices + device_count); - void* ptr = nullptr; - switch (qtype) { - case Quantization_F32: - ptr = new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_F16: - ptr = new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_INT8: - ptr = new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - case Quantization_UINT8: - ptr = new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); - break; - default: return nullptr; - } - return static_cast(new gpu_ivf_pq_any_t(qtype, ptr)); + void* ptr = ivf_pq_construct(btype, qtype, [&](auto* tag) -> void* { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + return new gpu_ivf_pq_t(std::string(filename), dimension, metric_c, build_params, devs, nthread, dist_mode); + }); + return static_cast(new gpu_ivf_pq_any_t(btype, qtype, ptr)); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_load_file", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_load_file", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_load_file", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_load_file", "unknown C++ exception"); } return nullptr; } @@ -211,51 +213,31 @@ void gpu_ivf_pq_destroy(gpu_ivf_pq_c index_c, void* errmsg) { try { delete static_cast(index_c); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_destroy", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_destroy", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_destroy", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_destroy", "unknown C++ exception"); } } void gpu_ivf_pq_start(gpu_ivf_pq_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->start(); break; - case Quantization_F16: static_cast*>(any->ptr)->start(); break; - case Quantization_INT8: static_cast*>(any->ptr)->start(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->start(); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [](auto* idx) { idx->start(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_start", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_start", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_start", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_start", "unknown C++ exception"); } } void gpu_ivf_pq_build(gpu_ivf_pq_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->build(); break; - case Quantization_F16: static_cast*>(any->ptr)->build(); break; - case Quantization_INT8: static_cast*>(any->ptr)->build(); break; - case Quantization_UINT8: static_cast*>(any->ptr)->build(); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [](auto* idx) { idx->build(); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_build", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_build", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_build", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_build", "unknown C++ exception"); } } @@ -263,14 +245,10 @@ void gpu_ivf_pq_extend(gpu_ivf_pq_c index_c, const void* new_data, uint64_t n_ro const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend(static_cast(new_data), n_rows, new_ids); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->extend(static_cast(new_data), n_rows, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_extend", e.what()); } catch (...) { @@ -282,14 +260,9 @@ void gpu_ivf_pq_extend_float(gpu_ivf_pq_c index_c, const float* new_data, uint64 const int64_t* new_ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_F16: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->extend_float(new_data, n_rows, new_ids); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + idx->extend_float(new_data, n_rows, new_ids); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_extend_float", e.what()); } catch (...) { @@ -300,174 +273,132 @@ void gpu_ivf_pq_extend_float(gpu_ivf_pq_c index_c, const float* new_data, uint64 void gpu_ivf_pq_add_chunk(gpu_ivf_pq_c index_c, const void* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + idx->add_chunk(static_cast(chunk_data), chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_add_chunk", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_add_chunk", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk", "unknown C++ exception"); } } void gpu_ivf_pq_add_chunk_float(gpu_ivf_pq_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_F16: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_chunk_float(chunk_data, chunk_count, -1, ids); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_chunk_float(chunk_data, chunk_count, -1, ids); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_add_chunk_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk_float", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_add_chunk_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk_float", "unknown C++ exception"); } } -void gpu_ivf_pq_train_quantizer(gpu_ivf_pq_c index_c, const float* train_data, uint64_t n_samples, void* errmsg) { +void gpu_ivf_pq_add_chunk_quantize(gpu_ivf_pq_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_F16: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_INT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - case Quantization_UINT8: static_cast*>(any->ptr)->train_quantizer(train_data, n_samples); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->add_chunk_quantize(static_cast(base_data), chunk_count, -1, ids); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk_quantize", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_chunk_quantize", "unknown C++ exception"); + } +} + +void gpu_ivf_pq_quantize_query(gpu_ivf_pq_c index_c, const void* base_data, uint64_t num_queries, void* out, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + using Q = typename std::remove_pointer_t::storage_type; + idx->quantize_query(static_cast(base_data), num_queries, static_cast(out)); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_train_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_quantize_query", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_train_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_quantize_query", "unknown C++ exception"); + } +} + +void gpu_ivf_pq_train_quantizer(gpu_ivf_pq_c index_c, const void* train_data, uint64_t n_samples, void* errmsg) { + if (errmsg) *(static_cast(errmsg)) = nullptr; + try { + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + idx->train_quantizer(static_cast(train_data), n_samples); + }); + } catch (const std::exception& e) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_train_quantizer", e.what()); + } catch (...) { + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_train_quantizer", "unknown C++ exception"); } } void gpu_ivf_pq_set_batch_window(gpu_ivf_pq_c index_c, int64_t window_us, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_F16: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_batch_window(window_us); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_batch_window(window_us); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_batch_window", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_batch_window", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_batch_window", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_batch_window", "unknown C++ exception"); } } void gpu_ivf_pq_set_dynb_conservative_dispatch(gpu_ivf_pq_c index_c, bool enable, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_F16: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_dynb_conservative_dispatch(enable); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_dynb_conservative_dispatch(enable); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_dynb_conservative_dispatch", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_dynb_conservative_dispatch", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_dynb_conservative_dispatch", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_dynb_conservative_dispatch", "unknown C++ exception"); } } void gpu_ivf_pq_set_quantizer(gpu_ivf_pq_c index_c, float min, float max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_quantizer(min, max); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->set_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_set_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_quantizer", "unknown C++ exception"); } } void gpu_ivf_pq_get_quantizer(gpu_ivf_pq_c index_c, float* min, float* max, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_F16: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_INT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - case Quantization_UINT8: static_cast*>(any->ptr)->get_quantizer(min, max); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->get_quantizer(min, max); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_get_quantizer", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_quantizer", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_get_quantizer", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_quantizer", "unknown C++ exception"); } } void gpu_ivf_pq_save(gpu_ivf_pq_c index_c, const char* filename, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save(filename); break; - case Quantization_F16: static_cast*>(any->ptr)->save(filename); break; - case Quantization_INT8: static_cast*>(any->ptr)->save(filename); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save(filename); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->save(filename); }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_save", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_save", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_save", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_save", "unknown C++ exception"); } } void gpu_ivf_pq_save_dir(gpu_ivf_pq_c index_c, const char* dir, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_F16: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_INT8: static_cast*>(any->ptr)->save_dir(dir); break; - case Quantization_UINT8: static_cast*>(any->ptr)->save_dir(dir); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->save_dir(dir); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_save_dir", e.what()); } catch (...) { @@ -478,14 +409,7 @@ void gpu_ivf_pq_save_dir(gpu_ivf_pq_c index_c, const char* dir, void* errmsg) { void gpu_ivf_pq_delete_id(gpu_ivf_pq_c index_c, int64_t id, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_F16: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_INT8: static_cast*>(any->ptr)->delete_id(id); break; - case Quantization_UINT8: static_cast*>(any->ptr)->delete_id(id); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->delete_id(id); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_delete_id", e.what()); } catch (...) { @@ -497,14 +421,7 @@ void gpu_ivf_pq_load_dir(gpu_ivf_pq_c index_c, const char* dir, distribution_mode_t target_mode, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_F16: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_INT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - case Quantization_UINT8: static_cast*>(any->ptr)->load_dir(dir, target_mode); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { idx->load_dir(dir, target_mode); }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_load_dir", e.what()); } catch (...) { @@ -512,69 +429,55 @@ void gpu_ivf_pq_load_dir(gpu_ivf_pq_c index_c, const char* dir, } } -gpu_ivf_pq_search_res_t gpu_ivf_pq_search(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_ivf_pq_search_res_t gpu_ivf_pq_search(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_search", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_search", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search", "unknown C++ exception"); } return result; } -gpu_ivf_pq_search_res_t gpu_ivf_pq_search_float(gpu_ivf_pq_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +gpu_ivf_pq_search_res_t gpu_ivf_pq_search_quantize(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float(queries_data, num_queries, query_dimension, limit, search_params); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize", "unknown C++ exception"); } return result; } -uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using Q = typename std::remove_pointer_t::storage_type; + return idx->search_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_async", e.what()); return 0; @@ -584,24 +487,20 @@ uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, } } -uint64_t gpu_ivf_pq_search_float_async(gpu_ivf_pq_c index_c, const float* queries_data, uint64_t num_queries, +uint64_t gpu_ivf_pq_search_quantize_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_async(queries_data, num_queries, query_dimension, limit, search_params); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_async(static_cast(queries_data), num_queries, query_dimension, limit, search_params); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_async", "unknown C++ exception"); return 0; } } @@ -610,15 +509,10 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search_wait(gpu_ivf_pq_c index_c, uint64_t jo if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_wait(job_id); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + *cpp_res = idx->search_wait(job_id); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_wait", e.what()); @@ -677,14 +571,7 @@ void gpu_ivf_pq_free_result(gpu_ivf_pq_result_c result_c) { uint64_t gpu_ivf_pq_cap(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->cap(); - case Quantization_F16: return static_cast*>(any->ptr)->cap(); - case Quantization_INT8: return static_cast*>(any->ptr)->cap(); - case Quantization_UINT8: return static_cast*>(any->ptr)->cap(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->cap(); }); } catch (...) { return 0; } @@ -693,14 +580,7 @@ uint64_t gpu_ivf_pq_cap(gpu_ivf_pq_c index_c) { uint64_t gpu_ivf_pq_len(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->len(); - case Quantization_F16: return static_cast*>(any->ptr)->len(); - case Quantization_INT8: return static_cast*>(any->ptr)->len(); - case Quantization_UINT8: return static_cast*>(any->ptr)->len(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint64_t { return idx->len(); }); } catch (...) { return 0; } @@ -713,15 +593,9 @@ char* gpu_ivf_pq_get_filter_col_meta_json(gpu_ivf_pq_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return strdup(""); try { - auto* any = static_cast(index_c); - std::string json; - switch (any->qtype) { - case Quantization_F32: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_F16: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_INT8: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - case Quantization_UINT8: json = matrixone::format_filter_col_meta(static_cast*>(any->ptr)->filter_host_.columns); break; - default: return strdup(""); - } + std::string json = ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> std::string { + return matrixone::format_filter_col_meta(idx->filter_host_.columns); + }); return strdup(json.c_str()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_filter_col_meta_json", e.what()); @@ -736,23 +610,13 @@ char* gpu_ivf_pq_info(gpu_ivf_pq_c index_c, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; if (!index_c) return nullptr; try { - auto* any = static_cast(index_c); - std::string info; - switch (any->qtype) { - case Quantization_F32: info = static_cast*>(any->ptr)->info(); break; - case Quantization_F16: info = static_cast*>(any->ptr)->info(); break; - case Quantization_INT8: info = static_cast*>(any->ptr)->info(); break; - case Quantization_UINT8: info = static_cast*>(any->ptr)->info(); break; - default: return nullptr; - } + std::string info = ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> std::string { return idx->info(); }); return strdup(info.c_str()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_info", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_info", e.what()); return nullptr; } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_info", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_info", "unknown C++ exception"); return nullptr; } } @@ -768,34 +632,20 @@ void gpu_ivf_pq_get_centers(gpu_ivf_pq_c index_c, void* centers, uint64_t count, std::copy(src.begin(), src.begin() + n, static_cast(dst)); }; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: copy_clamped(static_cast*>(any->ptr)->get_centers(), centers); break; - case Quantization_F16: copy_clamped(static_cast*>(any->ptr)->get_centers(), centers); break; - case Quantization_INT8: copy_clamped(static_cast*>(any->ptr)->get_centers(), centers); break; - case Quantization_UINT8: copy_clamped(static_cast*>(any->ptr)->get_centers(), centers); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + copy_clamped(idx->get_centers(), centers); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_get_centers", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_centers", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, - "Error in gpu_ivf_pq_get_centers", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_get_centers", "unknown C++ exception"); } } uint32_t gpu_ivf_pq_get_n_list(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_F16: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_n_list(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_n_list(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_n_list(); }); } catch (...) { return 0; } @@ -804,14 +654,7 @@ uint32_t gpu_ivf_pq_get_n_list(gpu_ivf_pq_c index_c) { uint32_t gpu_ivf_pq_get_dim(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_dim(); - case Quantization_F16: return static_cast*>(any->ptr)->get_dim(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_dim(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_dim(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_dim(); }); } catch (...) { return 0; } @@ -820,14 +663,7 @@ uint32_t gpu_ivf_pq_get_dim(gpu_ivf_pq_c index_c) { uint32_t gpu_ivf_pq_get_rot_dim(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_rot_dim(); - case Quantization_F16: return static_cast*>(any->ptr)->get_rot_dim(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_rot_dim(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_rot_dim(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_rot_dim(); }); } catch (...) { return 0; } @@ -836,14 +672,7 @@ uint32_t gpu_ivf_pq_get_rot_dim(gpu_ivf_pq_c index_c) { uint32_t gpu_ivf_pq_get_dim_ext(gpu_ivf_pq_c index_c) { try { if (!index_c) return 0; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->get_dim_ext(); - case Quantization_F16: return static_cast*>(any->ptr)->get_dim_ext(); - case Quantization_INT8: return static_cast*>(any->ptr)->get_dim_ext(); - case Quantization_UINT8: return static_cast*>(any->ptr)->get_dim_ext(); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [](auto* idx) -> uint32_t { return idx->get_dim_ext(); }); } catch (...) { return 0; } @@ -853,30 +682,11 @@ void gpu_ivf_pq_get_dataset(gpu_ivf_pq_c index_c, void* out_data) { // This is for debugging, we just copy the host dataset if it exists try { if (!index_c) return; - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: { - auto& ds = static_cast*>(any->ptr)->flattened_host_dataset; - if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); - break; - } - case Quantization_F16: { - auto& ds = static_cast*>(any->ptr)->flattened_host_dataset; - if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); - break; - } - case Quantization_INT8: { - auto& ds = static_cast*>(any->ptr)->flattened_host_dataset; - if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); - break; - } - case Quantization_UINT8: { - auto& ds = static_cast*>(any->ptr)->flattened_host_dataset; - if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); - break; - } - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + auto& ds = idx->flattened_host_dataset; + if (!ds.empty()) std::copy(ds.begin(), ds.end(), static_cast(out_data)); + }); } catch (...) { matrixone::log_err("gpu_ivf_pq_get_dataset: unknown C++ exception (swallowed)"); } @@ -888,15 +698,10 @@ void gpu_ivf_pq_set_filter_columns(gpu_ivf_pq_c index_c, const char* col_meta_js uint64_t total_count, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string s = col_meta_json ? col_meta_json : ""; - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_F16: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_INT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - case Quantization_UINT8: static_cast*>(any->ptr)->set_filter_columns(s, total_count); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + idx->set_filter_columns(s, total_count); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_set_filter_columns", e.what()); } catch (...) { @@ -909,14 +714,9 @@ void gpu_ivf_pq_add_filter_chunk(gpu_ivf_pq_c index_c, uint32_t col_idx, uint64_t nrows, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); - switch (any->qtype) { - case Quantization_F32: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_F16: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_INT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - case Quantization_UINT8: static_cast*>(any->ptr)->add_filter_chunk(col_idx, data, null_bitmap, nrows); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + idx->add_filter_chunk(col_idx, data, null_bitmap, nrows); + }); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_add_filter_chunk", e.what()); } catch (...) { @@ -931,16 +731,12 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search_with_filter(gpu_ivf_pq_c index_c, cons if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using Q = typename std::remove_pointer_t::storage_type; + *cpp_res = idx->search_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_with_filter", e.what()); @@ -950,52 +746,44 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search_with_filter(gpu_ivf_pq_c index_c, cons return result; } -gpu_ivf_pq_search_res_t gpu_ivf_pq_search_float_with_filter(gpu_ivf_pq_c index_c, const float* queries_data, +gpu_ivf_pq_search_res_t gpu_ivf_pq_search_quantize_with_filter(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; gpu_ivf_pq_search_res_t result = {nullptr}; try { - auto* any = static_cast(index_c); auto cpp_res = std::make_unique(); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_F16: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_INT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - case Quantization_UINT8: *cpp_res = static_cast*>(any->ptr)->search_float_with_filter(queries_data, num_queries, query_dimension, limit, sp, preds); break; - default: break; - } + ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) { + using B = typename std::remove_pointer_t::base_type; + *cpp_res = idx->search_quantize_with_filter(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); result.result_ptr = static_cast(cpp_res.release()); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_with_filter", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_with_filter", e.what()); } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_with_filter", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_with_filter", "unknown C++ exception"); } return result; } -uint64_t gpu_ivf_pq_search_float_with_filter_async(gpu_ivf_pq_c index_c, const float* queries_data, +uint64_t gpu_ivf_pq_search_quantize_with_filter_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t sp, const char* preds_json, void* errmsg) { if (errmsg) *(static_cast(errmsg)) = nullptr; try { - auto* any = static_cast(index_c); std::string preds = preds_json ? preds_json : ""; - switch (any->qtype) { - case Quantization_F32: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_F16: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_INT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - case Quantization_UINT8: return static_cast*>(any->ptr)->search_float_with_filter_async(queries_data, num_queries, query_dimension, limit, sp, preds); - default: return 0; - } + return ivf_pq_dispatch(static_cast(index_c), [&](auto* idx) -> uint64_t { + using B = typename std::remove_pointer_t::base_type; + return idx->search_quantize_with_filter_async(static_cast(queries_data), num_queries, query_dimension, limit, sp, preds); + }); } catch (const std::exception& e) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_with_filter_async", e.what()); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_with_filter_async", e.what()); return 0; } catch (...) { - matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_float_with_filter_async", "unknown C++ exception"); + matrixone::set_errmsg(errmsg, "Error in gpu_ivf_pq_search_quantize_with_filter_async", "unknown C++ exception"); return 0; } } @@ -1003,8 +791,11 @@ uint64_t gpu_ivf_pq_search_float_with_filter_async(gpu_ivf_pq_c index_c, const f } // extern "C" namespace matrixone { -template class gpu_ivf_pq_t; -template class gpu_ivf_pq_t; -template class gpu_ivf_pq_t; -template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; +template class gpu_ivf_pq_t; } // namespace matrixone diff --git a/cgo/cuvs/ivf_pq_c.h b/cgo/cuvs/ivf_pq_c.h index 4d68b6e23e27d..eeb30a6da0e86 100644 --- a/cgo/cuvs/ivf_pq_c.h +++ b/cgo/cuvs/ivf_pq_c.h @@ -31,30 +31,34 @@ typedef void* gpu_ivf_pq_c; // Opaque pointer to the C++ IVF-PQ search result object typedef void* gpu_ivf_pq_result_c; +// btype = base/query/quantizer-source element type (Quantization_F32 or F16). +// qtype = storage element type. Wired combos: F32 base {F32,F16,INT8,UINT8}; +// F16 base {F16,INT8,UINT8}. Other combinations set errmsg and return NULL. + // Constructor for building from dataset -gpu_ivf_pq_c gpu_ivf_pq_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, +gpu_ivf_pq_c gpu_ivf_pq_new(const void* dataset_data, uint64_t count_vectors, uint32_t dimension, distance_type_t metric, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Constructor for building from MODF datafile -gpu_ivf_pq_c gpu_ivf_pq_new_from_data_file(const char* data_filename, distance_type_t metric, +gpu_ivf_pq_c gpu_ivf_pq_new_from_data_file(const char* data_filename, distance_type_t metric, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg); + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg); // Constructor for loading from file gpu_ivf_pq_c gpu_ivf_pq_load_file(const char* filename, uint32_t dimension, distance_type_t metric, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, void* errmsg); + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, void* errmsg); // Constructor for an empty index (pre-allocates) -gpu_ivf_pq_c gpu_ivf_pq_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, +gpu_ivf_pq_c gpu_ivf_pq_new_empty(uint64_t total_count, uint32_t dimension, distance_type_t metric, ivf_pq_build_params_t build_params, - const int* devices, int device_count, uint32_t nthread, - distribution_mode_t dist_mode, quantization_t qtype, + const int* devices, int device_count, uint32_t nthread, + distribution_mode_t dist_mode, quantization_t btype, quantization_t qtype, const int64_t* ids, void* errmsg); // Add chunk of data (same type as index quantization) @@ -72,8 +76,19 @@ void gpu_ivf_pq_extend_float(gpu_ivf_pq_c index_c, const float* new_data, uint64 // Add chunk of data (from float, with on-the-fly quantization if needed) void gpu_ivf_pq_add_chunk_float(gpu_ivf_pq_c index_c, const float* chunk_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); -// Trains the scalar quantizer (if T is 1-byte) -void gpu_ivf_pq_train_quantizer(gpu_ivf_pq_c index_c, const float* train_data, uint64_t n_samples, void* errmsg); +// Add chunk of base-typed (B) data, quantizing natively to a 1-byte storage type +// (int8/uint8) via the B-source quantizer. base_data is a host buffer of +// chunk_count*dimension B elements (passed as raw bytes; B = btype). Requires int8/uint8 storage. +void gpu_ivf_pq_add_chunk_quantize(gpu_ivf_pq_c index_c, const void* base_data, uint64_t chunk_count, const int64_t* ids, void* errmsg); + +// Quantize a base-typed (B) query to the 1-byte storage type via the B-source +// quantizer, writing num_queries*dimension bytes into out. The caller then runs +// the normal native search with the quantized query. Requires int8/uint8 storage. +void gpu_ivf_pq_quantize_query(gpu_ivf_pq_c index_c, const void* base_data, uint64_t num_queries, void* out, void* errmsg); + +// Trains the scalar quantizer (if storage is 1-byte). train_data is a host buffer +// of n_samples*dimension B elements (B = btype), passed as raw bytes. +void gpu_ivf_pq_train_quantizer(gpu_ivf_pq_c index_c, const void* train_data, uint64_t n_samples, void* errmsg); void gpu_ivf_pq_set_batch_window(gpu_ivf_pq_c index_c, int64_t window_us, void* errmsg); void gpu_ivf_pq_set_dynb_conservative_dispatch(gpu_ivf_pq_c index_c, bool enable, void* errmsg); @@ -115,17 +130,19 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search(gpu_ivf_pq_c index_c, const void* quer uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg); -gpu_ivf_pq_search_res_t gpu_ivf_pq_search_float(gpu_ivf_pq_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +// Quantize search: query in the BASE element type B (float or half); the index +// converts it to storage type T (copy / quantize / f32->f16 cast) internally. +gpu_ivf_pq_search_res_t gpu_ivf_pq_search_quantize(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg); // Asynchronous search functions -uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_pq_search_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg); -uint64_t gpu_ivf_pq_search_float_async(gpu_ivf_pq_c index_c, const float* queries_data, uint64_t num_queries, - uint32_t query_dimension, uint32_t limit, +uint64_t gpu_ivf_pq_search_quantize_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, + uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, void* errmsg); gpu_ivf_pq_search_res_t gpu_ivf_pq_search_wait(gpu_ivf_pq_c index_c, uint64_t job_id, void* errmsg); @@ -189,15 +206,16 @@ gpu_ivf_pq_search_res_t gpu_ivf_pq_search_with_filter(gpu_ivf_pq_c index_c, cons uint32_t limit, ivf_pq_search_params_t search_params, const char* preds_json, void* errmsg); -gpu_ivf_pq_search_res_t gpu_ivf_pq_search_float_with_filter(gpu_ivf_pq_c index_c, const float* queries_data, +// Query in the BASE element type B (float or half); converted to storage T internally. +gpu_ivf_pq_search_res_t gpu_ivf_pq_search_quantize_with_filter(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, const char* preds_json, void* errmsg); -// Async variant of gpu_ivf_pq_search_float_with_filter. Returns a job_id that +// Async variant of gpu_ivf_pq_search_quantize_with_filter. Returns a job_id that // is collected with the existing gpu_ivf_pq_search_wait. Lets multi-index // callers fan out filtered searches across shards in parallel. -uint64_t gpu_ivf_pq_search_float_with_filter_async(gpu_ivf_pq_c index_c, const float* queries_data, +uint64_t gpu_ivf_pq_search_quantize_with_filter_async(gpu_ivf_pq_c index_c, const void* queries_data, uint64_t num_queries, uint32_t query_dimension, uint32_t limit, ivf_pq_search_params_t search_params, const char* preds_json, void* errmsg); diff --git a/cgo/cuvs/kmeans.hpp b/cgo/cuvs/kmeans.hpp index 2c12e0053ec59..82b26327b1fb5 100644 --- a/cgo/cuvs/kmeans.hpp +++ b/cgo/cuvs/kmeans.hpp @@ -74,8 +74,10 @@ struct kmeans_result_t { * Note: cuVS KMeans fits and predicts always use float centroids internally. */ template -class gpu_kmeans_t : public gpu_index_base_t { +class gpu_kmeans_t : public gpu_index_base_t { public: + using base_type = float; + using storage_type = T; // Internal centroids storage - ALWAYS float for cuVS KMeans std::unique_ptr> centroids_; @@ -184,6 +186,12 @@ class gpu_kmeans_t : public gpu_index_base_t std::unique_lock lock(this->mutex_); auto res = handle.get_raft_resources(); + // cuVS kmeans is not safe to run twice at once on one GPU (the same + // reason ivf_flat/cagra/ivf_pq/brute_force take this lock for build). + // Without it, concurrent async ivfflat reindexes each run their own + // GpuKMeans::fit on device 0 and race on GPU/RMM state -> SIGABRT. + std::lock_guard build_lk(matrixone::device_build_mutex(handle.get_device_id())); + cuvs::cluster::kmeans::balanced_params kmeans_params; kmeans_params.metric = static_cast(this->metric); kmeans_params.n_iters = static_cast(this->build_params.max_iter); @@ -220,6 +228,13 @@ class gpu_kmeans_t : public gpu_index_base_t std::unique_lock lock(this->mutex_); auto res = handle.get_raft_resources(); + // cuVS kmeans is not safe to run twice at once on one GPU (the + // same reason ivf_flat/cagra/ivf_pq/brute_force take this lock + // for build). Without it, concurrent async ivfflat reindexes + // each run their own GpuKMeans::fit on device 0 and race on + // GPU/RMM state -> SIGABRT. + std::lock_guard build_lk(matrixone::device_build_mutex(handle.get_device_id())); + cuvs::cluster::kmeans::balanced_params kmeans_params; kmeans_params.metric = static_cast(this->metric); kmeans_params.n_iters = static_cast(this->build_params.max_iter); @@ -345,6 +360,13 @@ class gpu_kmeans_t : public gpu_index_base_t std::unique_lock lock(this->mutex_); auto res = handle.get_raft_resources(); + // cuVS kmeans is not safe to run twice at once on one GPU (the + // same reason ivf_flat/cagra/ivf_pq/brute_force take this lock + // for build). Without it, concurrent async ivfflat reindexes + // each run their own GpuKMeans::fit on device 0 and race on + // GPU/RMM state -> SIGABRT. + std::lock_guard build_lk(matrixone::device_build_mutex(handle.get_device_id())); + cuvs::cluster::kmeans::balanced_params kmeans_params; kmeans_params.metric = static_cast(this->metric); kmeans_params.n_iters = static_cast(this->build_params.max_iter); @@ -389,12 +411,16 @@ class gpu_kmeans_t : public gpu_index_base_t kmeans_result_t fit_predict_float(const float* dataset_data, uint64_t count_vectors) { this->count = count_vectors; this->train_quantizer_if_needed(); - + uint64_t job_id = this->worker->submit_main( [&](raft_handle_wrapper_t& handle) -> std::any { std::unique_lock lock(this->mutex_); auto res = handle.get_raft_resources(); + // cuVS kmeans is not safe to run twice at once on one GPU; see + // the device_build_mutex note in fit()/build_internal(). + std::lock_guard build_lk(matrixone::device_build_mutex(handle.get_device_id())); + auto dataset_device_f = raft::make_device_matrix(*res, (int64_t)this->count, (int64_t)this->dimension); raft::copy(*res, dataset_device_f.view(), raft::make_host_matrix_view(dataset_data, this->count, this->dimension)); raft::resource::sync_stream(*res); @@ -458,7 +484,7 @@ class gpu_kmeans_t : public gpu_index_base_t } std::string info() const override { - std::string json = gpu_index_base_t::info(); + std::string json = gpu_index_base_t::info(); json += ", \"type\": \"KMeans\", \"kmeans\": {"; if (centroids_) json += "\"clusters\": " + std::to_string(centroids_->extent(0)); else json += "\"built\": false"; diff --git a/cgo/cuvs/python/cuvs.py b/cgo/cuvs/python/cuvs.py index 6a578c7beb0ec..14f2d7a40c49a 100644 --- a/cgo/cuvs/python/cuvs.py +++ b/cgo/cuvs/python/cuvs.py @@ -64,6 +64,20 @@ class Quantization(IntEnum): INT8 = 2 UINT8 = 3 +# numpy dtype for a base/storage element type. Base-typed buffers (dataset, +# query, quantizer training data) must carry these exact bytes so the C side +# reads them per btype — e.g. a vecf16 base must be passed as float16, NOT +# coerced to float32 (which would double the width and misread the halfs). +_NP_DTYPE = { + Quantization.F32: np.float32, + Quantization.F16: np.float16, + Quantization.INT8: np.int8, + Quantization.UINT8: np.uint8, +} + +def _np_dtype_for(quant): + return _NP_DTYPE[Quantization(int(quant))] + class DistributionMode(IntEnum): SINGLE_GPU = 0 SHARDED = 1 @@ -139,11 +153,11 @@ def _check_error(errmsg_ptr): _lib.gpu_adhoc_brute_force_search_float.argtypes = [ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.POINTER(ctypes.c_float), ctypes.c_void_p] # CAGRA - _lib.gpu_cagra_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_cagra_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_cagra_new.restype = ctypes.c_void_p - _lib.gpu_cagra_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_cagra_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_cagra_new_empty.restype = ctypes.c_void_p - _lib.gpu_cagra_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + _lib.gpu_cagra_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, CagraBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] _lib.gpu_cagra_load_file.restype = ctypes.c_void_p _lib.gpu_cagra_destroy.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_cagra_start.argtypes = [ctypes.c_void_p, ctypes.c_void_p] @@ -161,12 +175,12 @@ def _check_error(errmsg_ptr): _lib.gpu_cagra_delete_id.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p] _lib.gpu_cagra_search.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] _lib.gpu_cagra_search.restype = CagraSearchRes - _lib.gpu_cagra_search_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] - _lib.gpu_cagra_search_float.restype = CagraSearchRes + _lib.gpu_cagra_search_quantize.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] + _lib.gpu_cagra_search_quantize.restype = CagraSearchRes _lib.gpu_cagra_search_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] _lib.gpu_cagra_search_async.restype = ctypes.c_uint64 - _lib.gpu_cagra_search_float_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] - _lib.gpu_cagra_search_float_async.restype = ctypes.c_uint64 + _lib.gpu_cagra_search_quantize_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_void_p] + _lib.gpu_cagra_search_quantize_async.restype = ctypes.c_uint64 _lib.gpu_cagra_search_wait.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_cagra_search_wait.restype = CagraSearchRes _lib.gpu_cagra_get_neighbors.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64)] @@ -184,15 +198,15 @@ def _check_error(errmsg_ptr): _lib.gpu_cagra_add_filter_chunk.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_cagra_search_with_filter.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_char_p, ctypes.c_void_p] _lib.gpu_cagra_search_with_filter.restype = CagraSearchRes - _lib.gpu_cagra_search_float_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_char_p, ctypes.c_void_p] - _lib.gpu_cagra_search_float_with_filter.restype = CagraSearchRes + _lib.gpu_cagra_search_quantize_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, CagraSearchParams, ctypes.c_char_p, ctypes.c_void_p] + _lib.gpu_cagra_search_quantize_with_filter.restype = CagraSearchRes # IVF-Flat - _lib.gpu_ivf_flat_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_ivf_flat_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_ivf_flat_new.restype = ctypes.c_void_p - _lib.gpu_ivf_flat_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + _lib.gpu_ivf_flat_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] _lib.gpu_ivf_flat_load_file.restype = ctypes.c_void_p - _lib.gpu_ivf_flat_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_ivf_flat_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfFlatBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_ivf_flat_new_empty.restype = ctypes.c_void_p _lib.gpu_ivf_flat_destroy.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_ivf_flat_start.argtypes = [ctypes.c_void_p, ctypes.c_void_p] @@ -211,12 +225,12 @@ def _check_error(errmsg_ptr): _lib.gpu_ivf_flat_delete_id.argtypes = [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p] _lib.gpu_ivf_flat_search.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] _lib.gpu_ivf_flat_search.restype = IvfFlatSearchRes - _lib.gpu_ivf_flat_search_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] - _lib.gpu_ivf_flat_search_float.restype = IvfFlatSearchRes + _lib.gpu_ivf_flat_search_quantize.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] + _lib.gpu_ivf_flat_search_quantize.restype = IvfFlatSearchRes _lib.gpu_ivf_flat_search_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] _lib.gpu_ivf_flat_search_async.restype = ctypes.c_uint64 - _lib.gpu_ivf_flat_search_float_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] - _lib.gpu_ivf_flat_search_float_async.restype = ctypes.c_uint64 + _lib.gpu_ivf_flat_search_quantize_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_void_p] + _lib.gpu_ivf_flat_search_quantize_async.restype = ctypes.c_uint64 _lib.gpu_ivf_flat_search_wait.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_ivf_flat_search_wait.restype = IvfFlatSearchRes _lib.gpu_ivf_flat_get_neighbors.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64)] @@ -235,17 +249,17 @@ def _check_error(errmsg_ptr): _lib.gpu_ivf_flat_add_filter_chunk.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_ivf_flat_search_with_filter.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_char_p, ctypes.c_void_p] _lib.gpu_ivf_flat_search_with_filter.restype = IvfFlatSearchRes - _lib.gpu_ivf_flat_search_float_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_char_p, ctypes.c_void_p] - _lib.gpu_ivf_flat_search_float_with_filter.restype = IvfFlatSearchRes + _lib.gpu_ivf_flat_search_quantize_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfFlatSearchParams, ctypes.c_char_p, ctypes.c_void_p] + _lib.gpu_ivf_flat_search_quantize_with_filter.restype = IvfFlatSearchRes # IVF-PQ - _lib.gpu_ivf_pq_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_ivf_pq_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_ivf_pq_new.restype = ctypes.c_void_p - _lib.gpu_ivf_pq_new_from_data_file.argtypes = [ctypes.c_char_p, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + _lib.gpu_ivf_pq_new_from_data_file.argtypes = [ctypes.c_char_p, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] _lib.gpu_ivf_pq_new_from_data_file.restype = ctypes.c_void_p - _lib.gpu_ivf_pq_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + _lib.gpu_ivf_pq_load_file.argtypes = [ctypes.c_char_p, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_void_p] _lib.gpu_ivf_pq_load_file.restype = ctypes.c_void_p - _lib.gpu_ivf_pq_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_ivf_pq_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, IvfPqBuildParams, ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_ivf_pq_new_empty.restype = ctypes.c_void_p _lib.gpu_ivf_pq_destroy.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_ivf_pq_start.argtypes = [ctypes.c_void_p, ctypes.c_void_p] @@ -264,12 +278,12 @@ def _check_error(errmsg_ptr): _lib.gpu_ivf_pq_delete_id.argtypes = [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p] _lib.gpu_ivf_pq_search.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] _lib.gpu_ivf_pq_search.restype = IvfPqSearchRes - _lib.gpu_ivf_pq_search_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] - _lib.gpu_ivf_pq_search_float.restype = IvfPqSearchRes + _lib.gpu_ivf_pq_search_quantize.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] + _lib.gpu_ivf_pq_search_quantize.restype = IvfPqSearchRes _lib.gpu_ivf_pq_search_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] _lib.gpu_ivf_pq_search_async.restype = ctypes.c_uint64 - _lib.gpu_ivf_pq_search_float_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] - _lib.gpu_ivf_pq_search_float_async.restype = ctypes.c_uint64 + _lib.gpu_ivf_pq_search_quantize_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_void_p] + _lib.gpu_ivf_pq_search_quantize_async.restype = ctypes.c_uint64 _lib.gpu_ivf_pq_search_wait.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_ivf_pq_search_wait.restype = IvfPqSearchRes _lib.gpu_ivf_pq_get_neighbors.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64)] @@ -295,27 +309,28 @@ def _check_error(errmsg_ptr): _lib.gpu_ivf_pq_add_filter_chunk.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_uint32), ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_ivf_pq_search_with_filter.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_char_p, ctypes.c_void_p] _lib.gpu_ivf_pq_search_with_filter.restype = IvfPqSearchRes - _lib.gpu_ivf_pq_search_float_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_char_p, ctypes.c_void_p] - _lib.gpu_ivf_pq_search_float_with_filter.restype = IvfPqSearchRes + _lib.gpu_ivf_pq_search_quantize_with_filter.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, IvfPqSearchParams, ctypes.c_char_p, ctypes.c_void_p] + _lib.gpu_ivf_pq_search_quantize_with_filter.restype = IvfPqSearchRes # Brute Force - _lib.gpu_brute_force_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + # btype before qtype (base type, then storage type) + _lib.gpu_brute_force_new.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_brute_force_new.restype = ctypes.c_void_p - _lib.gpu_brute_force_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_brute_force_new_empty.argtypes = [ctypes.c_uint64, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_brute_force_new_empty.restype = ctypes.c_void_p _lib.gpu_brute_force_destroy.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_brute_force_start.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_brute_force_build.argtypes = [ctypes.c_void_p, ctypes.c_void_p] _lib.gpu_brute_force_add_chunk.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] - _lib.gpu_brute_force_add_chunk_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] + _lib.gpu_brute_force_add_chunk_quantize.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p] _lib.gpu_brute_force_search.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] _lib.gpu_brute_force_search.restype = ctypes.c_void_p - _lib.gpu_brute_force_search_float.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] - _lib.gpu_brute_force_search_float.restype = ctypes.c_void_p + _lib.gpu_brute_force_search_quantize.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] + _lib.gpu_brute_force_search_quantize.restype = ctypes.c_void_p _lib.gpu_brute_force_search_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] _lib.gpu_brute_force_search_async.restype = ctypes.c_uint64 - _lib.gpu_brute_force_search_float_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] - _lib.gpu_brute_force_search_float_async.restype = ctypes.c_uint64 + _lib.gpu_brute_force_search_quantize_async.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p] + _lib.gpu_brute_force_search_quantize_async.restype = ctypes.c_uint64 _lib.gpu_brute_force_search_wait.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_void_p] _lib.gpu_brute_force_search_wait.restype = ctypes.c_void_p _lib.gpu_brute_force_get_results.argtypes = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.POINTER(ctypes.c_int64), ctypes.POINTER(ctypes.c_float)] @@ -386,31 +401,31 @@ def __init__(self, handle, dimension): self.dimension = dimension @classmethod - def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = CagraBuildParams.default() - dataset = np.ascontiguousarray(dataset, dtype=np.float32) + dataset = np.ascontiguousarray(dataset, dtype=_np_dtype_for(btype)) count, dim = dataset.shape dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_cagra_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) - _check_error(errmsg); return cls(h, dim) + h = _lib.gpu_cagra_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) + _check_error(errmsg); idx = cls(h, dim); idx.btype = int(btype); return idx @classmethod - def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = CagraBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_cagra_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) + h = _lib.gpu_cagra_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) @classmethod - def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32): + def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32): if build_params is None: build_params = CagraBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) errmsg = ctypes.c_char_p() - h = _lib.gpu_cagra_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), ctypes.byref(errmsg)) + h = _lib.gpu_cagra_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) def start(self): @@ -442,7 +457,7 @@ def add_chunk(self, chunk, ids=None): id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p(); _lib.gpu_cagra_add_chunk_float(self.handle, chunk.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(chunk), id_ptr, ctypes.byref(errmsg)); _check_error(errmsg) def train_quantizer(self, train_data): - train_data = np.ascontiguousarray(train_data, dtype=np.float32) + train_data = np.ascontiguousarray(train_data, dtype=_np_dtype_for(getattr(self, 'btype', Quantization.F32))) errmsg = ctypes.c_char_p(); _lib.gpu_cagra_train_quantizer(self.handle, train_data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(train_data), ctypes.byref(errmsg)); _check_error(errmsg) def set_batch_window(self, window_us): @@ -474,10 +489,10 @@ def load_dir(self, directory, target_mode=DistributionMode.SINGLE_GPU): def search(self, queries, k, search_params=None): if search_params is None: search_params = CagraSearchParams.default() - queries = np.ascontiguousarray(queries, dtype=np.float32) + queries = np.ascontiguousarray(queries, dtype=_np_dtype_for(getattr(self, 'btype', Quantization.F32))) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - res = _lib.gpu_cagra_search_float(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + res = _lib.gpu_cagra_search_quantize(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -490,7 +505,7 @@ def search_async(self, queries, k, search_params=None): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - job_id = _lib.gpu_cagra_search_float_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + job_id = _lib.gpu_cagra_search_quantize_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) return job_id @@ -522,7 +537,7 @@ def search_with_filter(self, queries, k, preds_json, search_params=None): num_q, dim = queries.shape preds = preds_json.encode('utf-8') if preds_json else None errmsg = ctypes.c_char_p() - res = _lib.gpu_cagra_search_float_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) + res = _lib.gpu_cagra_search_quantize_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -546,31 +561,31 @@ def __init__(self, handle, dimension): self.dimension = dimension @classmethod - def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = IvfFlatBuildParams.default() dataset = np.ascontiguousarray(dataset, dtype=np.float32) count, dim = dataset.shape dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_flat_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) + h = _lib.gpu_ivf_flat_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dim) @classmethod - def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = IvfFlatBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_flat_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) + h = _lib.gpu_ivf_flat_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) @classmethod - def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32): + def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32): if build_params is None: build_params = IvfFlatBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_flat_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), ctypes.byref(errmsg)) + h = _lib.gpu_ivf_flat_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) def start(self): @@ -635,7 +650,7 @@ def search(self, queries, k, search_params=None): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - res = _lib.gpu_ivf_flat_search_float(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + res = _lib.gpu_ivf_flat_search_quantize(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -648,7 +663,7 @@ def search_async(self, queries, k, search_params=None): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - job_id = _lib.gpu_ivf_flat_search_float_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + job_id = _lib.gpu_ivf_flat_search_quantize_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) return job_id @@ -680,7 +695,7 @@ def search_with_filter(self, queries, k, preds_json, search_params=None): num_q, dim = queries.shape preds = preds_json.encode('utf-8') if preds_json else None errmsg = ctypes.c_char_p() - res = _lib.gpu_ivf_flat_search_float_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) + res = _lib.gpu_ivf_flat_search_quantize_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -716,39 +731,39 @@ def __init__(self, handle, dimension=None): self._dimension = dimension @classmethod - def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create(cls, dataset, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = IvfPqBuildParams.default() - dataset = np.ascontiguousarray(dataset, dtype=np.float32) + dataset = np.ascontiguousarray(dataset, dtype=_np_dtype_for(btype)) count, dim = dataset.shape dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_pq_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) - _check_error(errmsg); return cls(h, dim) + h = _lib.gpu_ivf_pq_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) + _check_error(errmsg); idx = cls(h, dim); idx.btype = int(btype); return idx @classmethod - def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32, ids=None): + def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32, ids=None): if build_params is None: build_params = IvfPqBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_pq_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), id_ptr, ctypes.byref(errmsg)) + h = _lib.gpu_ivf_pq_new_empty(total_count, dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) @classmethod - def create_from_data_file(cls, filename, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32): + def create_from_data_file(cls, filename, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32): if build_params is None: build_params = IvfPqBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_pq_new_from_data_file(filename.encode('utf-8'), int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), ctypes.byref(errmsg)) + h = _lib.gpu_ivf_pq_new_from_data_file(filename.encode('utf-8'), int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), ctypes.byref(errmsg)) _check_error(errmsg); return cls(h) @classmethod - def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, qtype=Quantization.F32): + def load_file(cls, filename, dimension, metric=DistanceType.L2Expanded, build_params=None, devices=[0], nthread=4, dist_mode=DistributionMode.SINGLE_GPU, btype=Quantization.F32, qtype=Quantization.F32): if build_params is None: build_params = IvfPqBuildParams.default() dev_arr = (ctypes.c_int * len(devices))(*devices) errmsg = ctypes.c_char_p() - h = _lib.gpu_ivf_pq_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(qtype), ctypes.byref(errmsg)) + h = _lib.gpu_ivf_pq_load_file(filename.encode('utf-8'), dimension, int(metric), build_params, dev_arr, len(devices), nthread, int(dist_mode), int(btype), int(qtype), ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) def start(self): @@ -778,7 +793,7 @@ def add_chunk(self, chunk, ids=None): errmsg = ctypes.c_char_p(); _lib.gpu_ivf_pq_add_chunk_float(self.handle, chunk.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(chunk), id_ptr, ctypes.byref(errmsg)); _check_error(errmsg) def train_quantizer(self, train_data): - train_data = np.ascontiguousarray(train_data, dtype=np.float32) + train_data = np.ascontiguousarray(train_data, dtype=_np_dtype_for(getattr(self, 'btype', Quantization.F32))) errmsg = ctypes.c_char_p(); _lib.gpu_ivf_pq_train_quantizer(self.handle, train_data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(train_data), ctypes.byref(errmsg)); _check_error(errmsg) def set_batch_window(self, window_us): @@ -810,10 +825,10 @@ def load_dir(self, directory, target_mode=DistributionMode.SINGLE_GPU): def search(self, queries, k, search_params=None): if search_params is None: search_params = IvfPqSearchParams.default() - queries = np.ascontiguousarray(queries, dtype=np.float32) + queries = np.ascontiguousarray(queries, dtype=_np_dtype_for(getattr(self, 'btype', Quantization.F32))) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - res = _lib.gpu_ivf_pq_search_float(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + res = _lib.gpu_ivf_pq_search_quantize(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -826,7 +841,7 @@ def search_async(self, queries, k, search_params=None): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - job_id = _lib.gpu_ivf_pq_search_float_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) + job_id = _lib.gpu_ivf_pq_search_quantize_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, ctypes.byref(errmsg)) _check_error(errmsg) return job_id @@ -858,7 +873,7 @@ def search_with_filter(self, queries, k, preds_json, search_params=None): num_q, dim = queries.shape preds = preds_json.encode('utf-8') if preds_json else None errmsg = ctypes.c_char_p() - res = _lib.gpu_ivf_pq_search_float_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) + res = _lib.gpu_ivf_pq_search_quantize_with_filter(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, search_params, preds, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -915,14 +930,16 @@ def create(cls, dataset, metric=DistanceType.L2Expanded, nthread=4, device_id=0, count, dim = dataset.shape id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_brute_force_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), nthread, device_id, int(qtype), id_ptr, ctypes.byref(errmsg)) + # btype before qtype; for the python tests base type == storage type. + h = _lib.gpu_brute_force_new(dataset.ctypes.data_as(ctypes.c_void_p), count, dim, int(metric), nthread, device_id, int(qtype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dim) @classmethod def create_empty(cls, total_count, dimension, metric=DistanceType.L2Expanded, nthread=4, device_id=0, qtype=Quantization.F32, ids=None): id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None errmsg = ctypes.c_char_p() - h = _lib.gpu_brute_force_new_empty(total_count, dimension, int(metric), nthread, device_id, int(qtype), id_ptr, ctypes.byref(errmsg)) + # btype before qtype; for the python tests base type == storage type. + h = _lib.gpu_brute_force_new_empty(total_count, dimension, int(metric), nthread, device_id, int(qtype), int(qtype), id_ptr, ctypes.byref(errmsg)) _check_error(errmsg); return cls(h, dimension) def start(self): @@ -932,13 +949,13 @@ def build(self): def add_chunk(self, chunk, ids=None): chunk = np.ascontiguousarray(chunk, dtype=np.float32) id_ptr = ids.ctypes.data_as(ctypes.POINTER(ctypes.c_int64)) if ids is not None else None - errmsg = ctypes.c_char_p(); _lib.gpu_brute_force_add_chunk_float(self.handle, chunk.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(chunk), id_ptr, ctypes.byref(errmsg)); _check_error(errmsg) + errmsg = ctypes.c_char_p(); _lib.gpu_brute_force_add_chunk_quantize(self.handle, chunk.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), len(chunk), id_ptr, ctypes.byref(errmsg)); _check_error(errmsg) def search(self, queries, k): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - res_ptr = _lib.gpu_brute_force_search_float(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, ctypes.byref(errmsg)) + res_ptr = _lib.gpu_brute_force_search_quantize(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, ctypes.byref(errmsg)) _check_error(errmsg) neighbors = np.zeros((num_q, k), dtype=np.int64) distances = np.zeros((num_q, k), dtype=np.float32) @@ -949,7 +966,7 @@ def search_async(self, queries, k): queries = np.ascontiguousarray(queries, dtype=np.float32) num_q, dim = queries.shape errmsg = ctypes.c_char_p() - job_id = _lib.gpu_brute_force_search_float_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, ctypes.byref(errmsg)) + job_id = _lib.gpu_brute_force_search_quantize_async(self.handle, queries.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), num_q, dim, k, ctypes.byref(errmsg)) _check_error(errmsg) return job_id diff --git a/cgo/cuvs/python/test/test_cuvs.py b/cgo/cuvs/python/test/test_cuvs.py index b5b1e051eade6..a0f786a0ea2c8 100644 --- a/cgo/cuvs/python/test/test_cuvs.py +++ b/cgo/cuvs/python/test/test_cuvs.py @@ -84,6 +84,35 @@ def test_ivf_pq(self): self.assertEqual(neighbors.shape, (5, self.k)) self.assertEqual(distances.shape, (5, self.k)) + def test_cagra_f16_quantize(self): + # vecf16 BASE quantized to int8/uint8 via the native half-source quantizer + # (btype=F16). Exercises the f16 data path: dataset + query stay half. + ds = np.random.random((self.n_rows, self.dim)).astype(np.float16) + q = ds[:5] + for qt in (cuvs.Quantization.INT8, cuvs.Quantization.UINT8): + index = cuvs.CagraIndex.create(ds, btype=cuvs.Quantization.F16, qtype=qt) + index.start() + index.train_quantizer(ds) # CAGRA from-dataset build does not auto-train the quantizer + index.build() + neighbors, distances = index.search(q, self.k) + self.assertEqual(neighbors.shape, (5, self.k)) + self.assertTrue(np.all(neighbors >= 0)) + self.assertTrue(np.all(neighbors < self.n_rows)) + + def test_ivf_pq_f16_quantize(self): + ds = np.random.random((self.n_rows, self.dim)).astype(np.float16) + q = ds[:5] + bp = cuvs.IvfPqBuildParams(n_lists=32, m=8, bits_per_code=8, add_data_on_build=True, kmeans_trainset_fraction=1.0) + for qt in (cuvs.Quantization.INT8, cuvs.Quantization.UINT8): + index = cuvs.IvfPqIndex.create(ds, build_params=bp, btype=cuvs.Quantization.F16, qtype=qt) + index.start() + index.train_quantizer(ds) + index.build() + neighbors, distances = index.search(q, self.k) + self.assertEqual(neighbors.shape, (5, self.k)) + self.assertTrue(np.all(neighbors >= 0)) + self.assertTrue(np.all(neighbors < self.n_rows)) + def test_kmeans(self): n_clusters = 5 kmeans = cuvs.KMeans(n_clusters=n_clusters, dimension=self.dim) diff --git a/cgo/cuvs/quantize.hpp b/cgo/cuvs/quantize.hpp index 0f5ced5cb5c6a..9667c3ce86116 100644 --- a/cgo/cuvs/quantize.hpp +++ b/cgo/cuvs/quantize.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,8 @@ #include #include #include +#include +#include #include #include #include @@ -99,22 +102,101 @@ class scalar_quantizer_t { auto out_view = raft::make_device_matrix_view(out_ptr, n_rows, n_cols); cuvs::preprocessing::quantize::scalar::transform(res, *quantizer_, src_view, out_view); } else { - // T is uint8_t, but cuVS transform expects int8_t output + // T is uint8_t. cuVS scalar transform only emits int8 [-128,127]; + // map it to uint8 [0,255] with a MONOTONIC +128 shift, NOT a raw + // cast (raft::copy would value-cast and wrap negatives: -1->255, + // -128->128, scrambling the L2 ordering for signed/zero-centered + // data). The shift is L2-invariant — base and query both pass through + // here, so the constant cancels in (a-b) — so uint8 recall matches int8. auto chunk_device_int8 = raft::make_device_matrix(res, n_rows, n_cols); cuvs::preprocessing::quantize::scalar::transform(res, *quantizer_, src_view, chunk_device_int8.view()); - auto out_view = raft::make_device_matrix_view(out_ptr, n_rows, n_cols); - raft::copy(res, out_view, chunk_device_int8.view()); + raft::linalg::unaryOp( + out_ptr, chunk_device_int8.data_handle(), n_rows * n_cols, + [] __device__(int8_t v) { return static_cast(static_cast(v) + 128); }, + raft::resource::get_cuda_stream(res)); } } else { - // For host pointers, we must use a temporary device buffer for the transform + // For host pointers, transform into a temporary device int8 buffer first. auto tmp_dev = raft::make_device_matrix(res, n_rows, n_cols); cuvs::preprocessing::quantize::scalar::transform(res, *quantizer_, src_view, tmp_dev.view()); - auto out_view = raft::make_host_matrix_view(out_ptr, n_rows, n_cols); - raft::copy(res, out_view, tmp_dev.view()); + if constexpr (std::is_same_v) { + // Monotonic int8->uint8 (+128) on device, then copy to host — see + // the device path above for why a raw cast is wrong. + auto tmp_u8 = raft::make_device_matrix(res, n_rows, n_cols); + raft::linalg::unaryOp( + tmp_u8.data_handle(), tmp_dev.data_handle(), n_rows * n_cols, + [] __device__(int8_t v) { return static_cast(static_cast(v) + 128); }, + raft::resource::get_cuda_stream(res)); + auto out_view = raft::make_host_matrix_view(out_ptr, n_rows, n_cols); + raft::copy(res, out_view, tmp_u8.view()); + } else { + auto out_view = raft::make_host_matrix_view(out_ptr, n_rows, n_cols); + raft::copy(res, out_view, tmp_dev.view()); + } raft::resource::sync_stream(res); } } + /** + * @brief Host (CPU) equivalent of transform(): quantizes a chunk of + * SOURCE-typed (S) elements into 1-byte T entirely on the CPU. + * + * Scalar quantization is a pure per-element affine map from the trained + * [min_, max_] range, so once the quantizer is trained no GPU is needed. + * This is a bit-for-bit port of cuVS' device quantize_op + * (cuvs/preprocessing/quantize/detail/scalar.cuh): the scale/offset are + * computed in `double` (the op's default TempT), the inner clamp uses the + * source-type comparison, ties round via lroundf, and uint8 storage applies + * the same monotonic +128 shift as the device path. Producing identical + * bytes to transform() keeps a CPU-built base consistent with a + * GPU-quantized query at search time. + * + * @tparam T Target storage type (int8_t or uint8_t). + * @param src Source elements, row-major, n_elements long. + * @param out Destination (host), n_elements long. + * @param n_elements Number of scalar elements (rows * dimension). + */ + template + void transform_host(const S* src, T* out, size_t n_elements) const { + if (!quantizer_) throw std::runtime_error("Quantizer not trained"); + static_assert(sizeof(T) == 1, "Quantization target must be 1-byte"); + + // cuVS maps the float interval onto the signed range [-128, 127]; + // uint8 is the same int8 result shifted by +128 (see transform()). + constexpr int q_type_min = std::numeric_limits::min(); // -128 + constexpr int q_type_max = std::numeric_limits::max(); // 127 + + const double dmin = static_cast(quantizer_->min_); + const double dmax = static_cast(quantizer_->max_); + const double scalar = (dmax > dmin) + ? (static_cast(q_type_max - q_type_min) / (dmax - dmin)) + : 1.0; + const double offset = static_cast(q_type_min) - dmin * scalar; + + // fp_lt() compares in the source type's float domain (half is cast to + // float; float compares natively) — replicate with a float compare. + const float fmin = static_cast(quantizer_->min_); + const float fmax = static_cast(quantizer_->max_); + + for (size_t i = 0; i < n_elements; ++i) { + const float xf = static_cast(src[i]); + int8_t q; + if (!(fmin < xf)) { + q = static_cast(q_type_min); + } else if (!(xf < fmax)) { + q = static_cast(q_type_max); + } else { + q = static_cast( + std::lroundf(static_cast(scalar * static_cast(src[i]) + offset))); + } + if constexpr (std::is_same_v) { + out[i] = static_cast(static_cast(q) + 128); + } else { + out[i] = static_cast(q); + } + } + } + bool is_trained() const { return quantizer_ != nullptr; } void reset() { quantizer_.reset(); } @@ -242,10 +324,18 @@ void load_matrix_chunked_ptr(const raft::resources& res, const std::string& file if constexpr (DoQuantize) { int64_t n_train = std::min(n_rows, static_cast(500)); std::vector train_host(n_train * n_cols); - std::streamsize train_wanted = static_cast(train_host.size() * sizeof(S)); - file.read(reinterpret_cast(train_host.data()), train_wanted); - if (file.gcount() != train_wanted) { - throw std::runtime_error("Truncated training-set read from: " + filename); + // Strided sample across ALL n_rows (not the first n_train contiguous + // rows) so the scalar quantizer learns the true [min,max] range even + // when the file is sorted/clustered — otherwise higher-magnitude rows + // past the prefix saturate to the storage extreme and recall collapses. + const int64_t stride = n_rows / n_train; // >= 1 since n_train <= n_rows + const std::streamsize row_bytes = static_cast(n_cols) * sizeof(S); + for (int64_t j = 0; j < n_train; ++j) { + file.seekg(sizeof(file_header_t) + static_cast(j) * stride * row_bytes); + file.read(reinterpret_cast(train_host.data() + j * n_cols), row_bytes); + if (file.gcount() != row_bytes) { + throw std::runtime_error("Truncated training-set read from: " + filename); + } } auto train_device = raft::make_device_matrix(res, n_train, n_cols); raft::copy(train_device.data_handle(), train_host.data(), train_host.size(), raft::resource::get_cuda_stream(res)); diff --git a/cgo/cuvs/test/batching_test.cu b/cgo/cuvs/test/batching_test.cu index f3bfbcc635bb2..e90f49949765f 100644 --- a/cgo/cuvs/test/batching_test.cu +++ b/cgo/cuvs/test/batching_test.cu @@ -34,7 +34,7 @@ TEST(DynamicBatchingTest, CagraConcurrentSearch) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); index.set_batch_window(100); index.start(); @@ -69,7 +69,7 @@ TEST(DynamicBatchingTest, IvfFlatConcurrentSearch) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); index.set_batch_window(100); index.start(); @@ -105,7 +105,7 @@ TEST(DynamicBatchingTest, IvfPqConcurrentSearch) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 8, DistributionMode_SINGLE_GPU); index.set_batch_window(100); index.start(); diff --git a/cgo/cuvs/test/benchmark_cuvs.cu b/cgo/cuvs/test/benchmark_cuvs.cu index cd36d81e4d0d5..85dcc79a65a35 100644 --- a/cgo/cuvs/test/benchmark_cuvs.cu +++ b/cgo/cuvs/test/benchmark_cuvs.cu @@ -98,7 +98,14 @@ template void run_benchmark(const std::string& index_name, distribution_mode_t mode, IndexT& index, const std::vector& recall_queries, const std::vector& recall_expected_ids, const benchmark_config_t& cfg, const SearchParamsT& sp) { - + + // float-input search dispatch: cagra/ivf_flat/ivf_pq and brute force all + // expose the base-typed search_quantize (B==float here; identity quantize + // when base == storage). + auto bench_search = [&](const float* q, uint64_t nq) { + return index.search_quantize(q, nq, cfg.dimension, cfg.limit, sp); + }; + for (int64_t window_us : {(int64_t)0, (int64_t)100}) { index.set_batch_window(window_us); @@ -108,7 +115,7 @@ void run_benchmark(const std::string& index_name, distribution_mode_t mode, // Warmup for (int i = 0; i < 5; ++i) { - index.search_float(queries.data(), 1, cfg.dimension, cfg.limit, sp); + bench_search(queries.data(), 1); } std::atomic total_completed{0}; @@ -119,7 +126,7 @@ void run_benchmark(const std::string& index_name, distribution_mode_t mode, for (uint32_t t = 0; t < cfg.n_threads; ++t) { threads.emplace_back([&, t, q_per_thread]() { for (uint32_t i = 0; i < q_per_thread; ++i) { - index.search_float(queries.data() + (t * q_per_thread + i) * cfg.dimension, 1, cfg.dimension, cfg.limit, sp); + bench_search(queries.data() + (t * q_per_thread + i) * cfg.dimension, 1); total_completed++; } }); @@ -132,7 +139,7 @@ void run_benchmark(const std::string& index_name, distribution_mode_t mode, double qps = total_completed.load() / diff.count(); // Self-recall - auto res = index.search_float(recall_queries.data(), cfg.n_queries, cfg.dimension, cfg.limit, sp); + auto res = bench_search(recall_queries.data(), cfg.n_queries); double recall = calculate_recall(res.neighbors, recall_expected_ids, cfg.n_queries, cfg.limit); std::cout << std::left << std::setw(45) << full_name @@ -174,14 +181,14 @@ void benchmark_all_indices(const std::vector& dataset, const benchmark_co if (mode != DistributionMode_SINGLE_GPU && active_devices.size() < 2) continue; - gpu_cagra_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + gpu_cagra_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); index.start(); index.build(); cagra_search_params_t sp = cagra_search_params_default(); sp.itopk_size = 128; sp.search_width = 1; - run_benchmark, cagra_search_params_t, T>("Cagra", mode, index, recall_queries, recall_expected_ids, cfg, sp); + run_benchmark, cagra_search_params_t, T>("Cagra", mode, index, recall_queries, recall_expected_ids, cfg, sp); index.destroy(); cudaDeviceSynchronize(); } @@ -198,13 +205,13 @@ void benchmark_all_indices(const std::vector& dataset, const benchmark_co if (mode != DistributionMode_SINGLE_GPU && active_devices.size() < 2) continue; - gpu_ivf_flat_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + gpu_ivf_flat_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); index.start(); index.build(); ivf_flat_search_params_t sp = ivf_flat_search_params_default(); sp.n_probes = 64; - run_benchmark, ivf_flat_search_params_t, T>("IvfFlat", mode, index, recall_queries, recall_expected_ids, cfg, sp); + run_benchmark, ivf_flat_search_params_t, T>("IvfFlat", mode, index, recall_queries, recall_expected_ids, cfg, sp); index.destroy(); } } @@ -221,13 +228,13 @@ void benchmark_all_indices(const std::vector& dataset, const benchmark_co if (mode != DistributionMode_SINGLE_GPU && active_devices.size() < 2) continue; - gpu_ivf_pq_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); + gpu_ivf_pq_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, active_devices, cfg.n_threads, mode); index.start(); index.build(); ivf_pq_search_params_t sp = ivf_pq_search_params_default(); sp.n_probes = 64; - run_benchmark, ivf_pq_search_params_t, T>("IvfPq", mode, index, recall_queries, recall_expected_ids, cfg, sp); + run_benchmark, ivf_pq_search_params_t, T>("IvfPq", mode, index, recall_queries, recall_expected_ids, cfg, sp); index.destroy(); } } @@ -237,12 +244,14 @@ void benchmark_all_indices(const std::vector& dataset, const benchmark_co distribution_mode_t mode = DistributionMode_SINGLE_GPU; std::vector active_devices = {cfg.devices[0]}; - gpu_brute_force_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, cfg.n_threads, active_devices[0]); + // Base type float (the benchmark queries with float32); storage T. This + // matches search_quantize(const B*=const float*) used in run_benchmark. + gpu_brute_force_t index(converted.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, cfg.n_threads, active_devices[0]); index.start(); index.build(); brute_force_search_params_t sp = brute_force_search_params_default(); - run_benchmark, brute_force_search_params_t, T>("BruteForce", mode, index, recall_queries, recall_expected_ids, cfg, sp); + run_benchmark, brute_force_search_params_t, T>("BruteForce", mode, index, recall_queries, recall_expected_ids, cfg, sp); index.destroy(); } } diff --git a/cgo/cuvs/test/benchmark_filter.cu b/cgo/cuvs/test/benchmark_filter.cu index 7739aff2ee841..735bdb51a4091 100644 --- a/cgo/cuvs/test/benchmark_filter.cu +++ b/cgo/cuvs/test/benchmark_filter.cu @@ -120,7 +120,7 @@ std::pair run_throughput(Index& index, pool.emplace_back([&, t, per_thread]() { for (uint32_t i = 0; i < per_thread; ++i) { auto t0 = std::chrono::high_resolution_clock::now(); - (void)index.search_float_with_filter( + (void)index.search_quantize_with_filter( queries.data() + (t * per_thread + i) * cfg.dimension, 1, cfg.dimension, cfg.limit, sp, preds_json); auto t1 = std::chrono::high_resolution_clock::now(); @@ -158,7 +158,7 @@ void sweep_selectivities(const std::string& tag, Index& index, const SP& sp, index.set_batch_window(window_us); for (uint32_t w = 0; w < cfg.warmup; ++w) { - (void)index.search_float_with_filter(throughput_queries.data(), 1, + (void)index.search_quantize_with_filter(throughput_queries.data(), 1, cfg.dimension, cfg.limit, sp, ""); } @@ -173,7 +173,7 @@ void sweep_selectivities(const std::string& tag, Index& index, const SP& sp, double qps = qt.first; double lat_us = qt.second; - auto res = index.search_float_with_filter( + auto res = index.search_quantize_with_filter( recall_queries.data(), cfg.n_queries, cfg.dimension, cfg.limit, sp, preds); auto r = self_recall(res.neighbors, recall_expected_ids, cats, k, cfg.n_queries, cfg.limit); @@ -241,7 +241,7 @@ int main() { { cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), cfg.n_vectors, cfg.dimension, + gpu_cagra_t index(dataset.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, devices, cfg.n_threads, DistributionMode_SINGLE_GPU); index.start(); @@ -261,7 +261,7 @@ int main() { { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 1024; - gpu_ivf_flat_t index(dataset.data(), cfg.n_vectors, cfg.dimension, + gpu_ivf_flat_t index(dataset.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, devices, cfg.n_threads, DistributionMode_SINGLE_GPU); index.start(); @@ -281,7 +281,7 @@ int main() { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 1024; bp.m = 64; - gpu_ivf_pq_t index(dataset.data(), cfg.n_vectors, cfg.dimension, + gpu_ivf_pq_t index(dataset.data(), cfg.n_vectors, cfg.dimension, DistanceType_L2Expanded, bp, devices, cfg.n_threads, DistributionMode_SINGLE_GPU); index.start(); diff --git a/cgo/cuvs/test/brute_force_test.cu b/cgo/cuvs/test/brute_force_test.cu index 26c8f361a6118..cf90fe3ff7aa2 100644 --- a/cgo/cuvs/test/brute_force_test.cu +++ b/cgo/cuvs/test/brute_force_test.cu @@ -39,7 +39,7 @@ TEST(GpuBruteForceTest, BasicLoadAndSearch) { const uint64_t count = 2; std::vector dataset = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -63,7 +63,7 @@ TEST(GpuBruteForceTest, BasicLoadAndSearchWithIds) { ids[i] = (int64_t)(i + 3000); } - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); index.start(); index.build(); @@ -94,7 +94,7 @@ TEST(GpuBruteForceTest, ParallelAddChunkWithOffset) { ids2[i] = (int64_t)(i + count_per_chunk); } - gpu_brute_force_t index(total_count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(total_count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); #include @@ -113,6 +113,35 @@ TEST(GpuBruteForceTest, ParallelAddChunkWithOffset) { index.destroy(); } +// f16 overflow path, NATIVE half add: empty gpu_brute_force_t -> +// add_chunk([]half) -> build -> native half search. This is the path +// IvfpqSearch.buildOverflow should use for a vecf16 base (feed native half, not +// add_chunk_float's f32->half cast). Confirms the native half overflow works. +TEST(GpuBruteForceTest, HalfEmptyAddChunkSearch) { + const uint32_t dimension = 8; + const uint64_t count = 50; + std::vector hdata(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) hdata[i * dimension + j] = __float2half((float)(i + 1)); + ids[i] = (int64_t)(i + 1); + } + + gpu_brute_force_t index(count, dimension, DistanceType_L2Expanded, 1, 0); + index.start(); + index.add_chunk(hdata.data(), count, -1, ids.data()); + index.build(); + + std::vector qh(dimension); + for (uint32_t j = 0; j < dimension; ++j) qh[j] = __float2half(25.0f); + auto result = index.search(qh.data(), 1, dimension, 3, brute_force_search_params_default()); + + ASSERT_EQ(result.neighbors.size(), (size_t)3); + ASSERT_EQ(result.neighbors[0], (int64_t)25); + + index.destroy(); +} + TEST(GpuBruteForceTest, SearchWithMultipleQueries) { const uint32_t dimension = 4; const uint64_t count = 4; @@ -123,7 +152,7 @@ TEST(GpuBruteForceTest, SearchWithMultipleQueries) { 0.0, 0.0, 0.0, 1.0 // ID 3 }; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -146,7 +175,7 @@ TEST(GpuBruteForceTest, SearchWithFloat16) { std::vector f_dataset = {1.0, 1.0, 2.0, 2.0}; std::vector h_dataset = float_to_half(f_dataset); - gpu_brute_force_t index(h_dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(h_dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -169,7 +198,7 @@ TEST(GpuBruteForceTest, SearchWithInnerProduct) { 0.0, 1.0 }; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_InnerProduct, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_InnerProduct, 1, 0); index.start(); index.build(); @@ -196,7 +225,7 @@ TEST(GpuBruteForceTest, EmptyDataset) { const uint32_t dimension = 128; const uint64_t count = 0; - gpu_brute_force_t index(nullptr, count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(nullptr, count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -229,7 +258,7 @@ TEST(GpuBruteForceTest, LargeLimit) { const uint64_t count = 5; std::vector dataset(count * dimension, 1.0); - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -263,7 +292,7 @@ TEST(GpuBruteForceTest, LargeLimitWithExplicitIds) { std::vector dataset(count * dimension, 1.0); std::vector ids = {1000, 1001, 1002, 1003, 1004}; - gpu_brute_force_t index(dataset.data(), count, dimension, + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); index.start(); index.build(); @@ -297,7 +326,7 @@ TEST(GpuBruteForceTest, SoftDeleteSearch) { 7.0, 8.0, 9.0 // ID 2 }; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -327,7 +356,7 @@ TEST(GpuBruteForceTest, SoftDeleteWithCustomIds) { std::vector dataset = {10, 10, 20, 20, 30, 30}; std::vector ids = {100, 200, 300}; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0, ids.data()); index.start(); index.build(); @@ -357,7 +386,7 @@ TEST(CuvsWorkerTest, BruteForceSearch) { std::vector dataset(count * dimension); for (size_t i = 0; i < dataset.size(); ++i) dataset[i] = (float)rand() / RAND_MAX; - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -382,7 +411,7 @@ TEST(CuvsWorkerTest, ConcurrentSearches) { } } - gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 4, 0); + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 4, 0); index.start(); index.build(); @@ -414,7 +443,7 @@ TEST(GpuBruteForceTest, KExceedsIndexSizeClampsAndPads) { } } - gpu_brute_force_t index(dataset.data(), count, dimension, + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -457,7 +486,7 @@ TEST(GpuBruteForceTest, MultiQueryKExceedsIndexSize) { } } - gpu_brute_force_t index(dataset.data(), count, dimension, + gpu_brute_force_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, 1, 0); index.start(); index.build(); @@ -490,3 +519,10 @@ TEST(GpuBruteForceTest, MultiQueryKExceedsIndexSize) { index.destroy(); } + +// NOTE: cuVS brute force does NOT support int8_t/uint8_t. cuvs::neighbors::brute_force::search +// only provides index and index overloads (verified: compiling +// gpu_brute_force_t/.search fails with "no matching search overload"). The +// header's "Supported T: ... int8_t, uint8_t" claim does not hold for search. For direct +// narrow-base ivfpq/cagra, the int8/uint8 overflow tier therefore uses the pure-Go brute force +// (pkg/vectorindex/brute_force, native int8/uint8 kernels), NOT a cuVS C++ brute force. diff --git a/cgo/cuvs/test/cagra_test.cu b/cgo/cuvs/test/cagra_test.cu index e32c3dba0265f..d032526f72678 100644 --- a/cgo/cuvs/test/cagra_test.cu +++ b/cgo/cuvs/test/cagra_test.cu @@ -21,9 +21,100 @@ #include #include #include +#include using namespace matrixone; +// Native half (f16) build + search — validates the direct vecf16-base path +// (gpu_cagra_t native add_chunk/search, no quantizer). Linking this proves +// cuVS supports cagra over half. +TEST(GpuCagraTest, BasicLoadAndSearchHalf) { + const uint32_t dimension = 16; + const uint64_t count = 1000; + std::vector dataset(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) + dataset[i * dimension + j] = __float2half((float)rand() / RAND_MAX); + ids[i] = (int64_t)(i + 1000); + } + + std::vector devices = {0}; + cagra_build_params_t bp = cagra_build_params_default(); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + index.start(); + index.build(); + + std::vector queries(dataset.begin(), dataset.begin() + dimension); + cagra_search_params_t sp = cagra_search_params_default(); + auto result = index.search(queries.data(), 1, dimension, 5, sp); + + ASSERT_EQ(result.neighbors.size(), (size_t)5); + ASSERT_EQ(result.neighbors[0], 1000LL); + + index.destroy(); +} + +// vecf16 base -> int8/uint8 storage via the native B(half)-source quantizer for +// CAGRA (gpu_cagra_t + add_chunk_quantize). Mirrors the ivf_pq +// HalfQuantizeToInt8Build coverage, which cagra previously lacked entirely. +// Verifies: train the half-source quantizer on the buffered vecf16 sample, +// transform half->T, build a CAGRA graph over the quantized codes, and search +// it (both with a native-T query and with a half query quantized via +// quantize_query). No f32 detour. +namespace { +template +void run_cagra_half_quantize_build(const char* label) { + TEST_LOG("CAGRA half-quantize build/search: " << label); + const uint32_t dimension = 16; + const uint64_t count = 2000; + std::vector dataset(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) + dataset[i * dimension + j] = __float2half((float)(rand() % 256) / 255.0f); + ids[i] = (int64_t)(i + 5000); + } + + std::vector devices = {0}; + cagra_build_params_t bp = cagra_build_params_default(); + gpu_cagra_t index(count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(dataset.data(), count, -1, ids.data()); + index.build(); + + cagra_search_params_t sp = cagra_search_params_default(); + + // 1) search with a native-T query (raw storage codes). + std::vector qnative(dimension, 0); + auto r1 = index.search(qnative.data(), 1, dimension, 5, sp); + ASSERT_EQ(r1.neighbors.size(), (size_t)5); + for (auto n : r1.neighbors) { + ASSERT_GE(n, (int64_t)5000); + ASSERT_LT(n, (int64_t)(5000 + count)); + } + + // 2) search with a half query quantized through the half-source quantizer + // (the production path) — the nearest neighbor of base[0] must be itself. + std::vector qhalf(dataset.begin(), dataset.begin() + dimension); + std::vector qcodes(dimension); + index.quantize_query(qhalf.data(), 1, qcodes.data()); + auto r2 = index.search(qcodes.data(), 1, dimension, 5, sp); + ASSERT_EQ(r2.neighbors.size(), (size_t)5); + ASSERT_EQ(r2.neighbors[0], (int64_t)5000); + + index.destroy(); +} +} // namespace + +TEST(GpuCagraTest, HalfQuantizeToInt8Build) { + run_cagra_half_quantize_build("f16->int8"); +} + +TEST(GpuCagraTest, HalfQuantizeToUint8Build) { + run_cagra_half_quantize_build("f16->uint8"); +} + TEST(GpuCagraTest, BasicLoadAndSearch) { const uint32_t dimension = 16; const uint64_t count = 1000; @@ -32,7 +123,7 @@ TEST(GpuCagraTest, BasicLoadAndSearch) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -58,7 +149,7 @@ TEST(GpuCagraTest, BasicLoadAndSearchWithIds) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); @@ -93,7 +184,7 @@ TEST(GpuCagraTest, ParallelAddChunkWithOffset) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); // Pre-allocate with total_count - gpu_cagra_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); // Add chunks in parallel threads @@ -128,7 +219,7 @@ TEST(GpuCagraTest, SaveAndLoadFromFile) { // 1. Build and Save { cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); index.save(filename); @@ -138,7 +229,7 @@ TEST(GpuCagraTest, SaveAndLoadFromFile) { // 2. Load and Search { cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.load(filename); @@ -169,7 +260,7 @@ TEST(GpuCagraTest, ReplicatedModeSimulation) { gpu_get_device_list(devices.data(), dev_count); cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); index.start(); index.build(); std::vector queries(dataset.begin(), dataset.begin() + dimension); @@ -198,7 +289,7 @@ TEST(GpuCagraTest, ManualShardedSearch) { gpu_get_device_list(devices.data(), dev_count); cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -230,7 +321,7 @@ TEST(GpuCagraTest, ManualShardedSearchWithIds) { gpu_get_device_list(devices.data(), dev_count); cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); @@ -260,7 +351,7 @@ TEST(GpuCagraTest, SoftDeleteSearch) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -303,7 +394,7 @@ TEST(GpuCagraTest, SoftDeleteWithCustomIds) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); @@ -342,7 +433,7 @@ TEST(GpuCagraTest, FilteredSearchIncludesOnlyAllowedCategories) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -393,7 +484,7 @@ TEST(GpuCagraTest, FilteredSearchCombinesWithDeleteBitset) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -433,7 +524,7 @@ TEST(GpuCagraTest, FilteredSearchEmptyPredsMatchesUnfiltered) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -472,7 +563,7 @@ TEST(GpuCagraTest, ExtendReplicatedWithHostIds) { } cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED, base_ids.data()); index.start(); @@ -523,7 +614,7 @@ TEST(GpuCagraTest, ExtendShardedThrows) { for (size_t i = 0; i < dataset.size(); ++i) dataset[i] = (float)rand() / RAND_MAX; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); @@ -552,7 +643,7 @@ TEST(GpuCagraTest, BuildParamsTooLargeForShardThrows) { for (size_t i = 0; i < dataset.size(); ++i) dataset[i] = (float)rand() / RAND_MAX; cagra_build_params_t bp = cagra_build_params_default(); // intermediate=128, graph=64 - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); @@ -572,7 +663,7 @@ TEST(GpuCagraTest, ExtendWithoutHostIds) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -615,7 +706,7 @@ TEST(GpuCagraTest, ExtendWithHostIds) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), n_base, dimension, + gpu_cagra_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, base_ids.data()); index.start(); @@ -663,7 +754,7 @@ TEST(GpuCagraTest, KExceedsIndexSizeClampsAndPads) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -708,7 +799,7 @@ TEST(GpuCagraTest, MultiQueryKExceedsIndexSize) { std::vector devices = {0}; cagra_build_params_t bp = cagra_build_params_default(); - gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_cagra_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); diff --git a/cgo/cuvs/test/filter_test.cu b/cgo/cuvs/test/filter_test.cu index a93e416928be5..a98b5cf1323b6 100644 --- a/cgo/cuvs/test/filter_test.cu +++ b/cgo/cuvs/test/filter_test.cu @@ -613,7 +613,7 @@ namespace { // Minimal derived index used as a stand-in for the real index types. We never // call start()/build()/search() — only the filter ingest + persistence methods. -struct test_index_t : public gpu_index_base_t { +struct test_index_t : public gpu_index_base_t { test_index_t() { // Populate the fields write_manifest reads so the file is valid JSON. this->dimension = 4; @@ -630,7 +630,7 @@ std::string make_tmp_dir(const std::string& tag) { // Best-effort cleanup from prior runs. std::string rm = "rm -rf " + path; ::system(rm.c_str()); - gpu_index_base_t::ensure_dir(path); + gpu_index_base_t::ensure_dir(path); return path; } diff --git a/cgo/cuvs/test/ivf_flat_test.cu b/cgo/cuvs/test/ivf_flat_test.cu index 785001dda9507..52a55e16f779f 100644 --- a/cgo/cuvs/test/ivf_flat_test.cu +++ b/cgo/cuvs/test/ivf_flat_test.cu @@ -37,7 +37,7 @@ TEST(GpuIvfFlatTest, BasicLoadSearchAndCenters) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -71,7 +71,7 @@ TEST(GpuIvfFlatTest, BasicLoadAndSearchWithIds) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 100; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); @@ -106,7 +106,7 @@ TEST(GpuIvfFlatTest, ParallelAddChunkWithOffset) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 100; - gpu_ivf_flat_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); std::thread t1([&]() { index.add_chunk(chunk1.data(), count_per_chunk, 0, ids1.data()); }); @@ -136,7 +136,7 @@ TEST(GpuIvfFlatTest, SaveAndLoadFromFile) { { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); index.save(filename); @@ -148,7 +148,7 @@ TEST(GpuIvfFlatTest, SaveAndLoadFromFile) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; // Construct without loading immediately - gpu_ivf_flat_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); // Start worker first index.load(filename); // Then load explicitly @@ -180,7 +180,7 @@ TEST(GpuIvfFlatTest, ReplicatedModeSimulation) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); index.start(); index.build(); @@ -207,7 +207,7 @@ TEST(GpuIvfFlatTest, ReplicatedLoadSearch) { { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, single_device, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, single_device, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); index.save(filename); @@ -224,7 +224,7 @@ TEST(GpuIvfFlatTest, ReplicatedLoadSearch) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); + gpu_ivf_flat_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); index.start(); index.load(filename); @@ -251,7 +251,7 @@ TEST(GpuIvfFlatTest, SetGetQuantizer) { bp.n_lists = 5; std::vector devices = {0}; - gpu_ivf_flat_t index(count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t index(count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); float min = -1.5f; float max = 2.5f; @@ -283,7 +283,7 @@ TEST(GpuIvfFlatTest, ManualShardedSearch) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 50; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -316,7 +316,7 @@ TEST(GpuIvfFlatTest, ManualShardedSearchWithIds) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 50; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); @@ -349,7 +349,7 @@ TEST(GpuIvfFlatTest, SimulatedReplicatedBuildSearch) { std::vector sim2 = {0, 0}; // 2 logical GPUs on physical device 0 ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); + gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); index.start(); index.build(); ASSERT_TRUE(index.info().find("\"ranks\": 2") != std::string::npos); // 2 replicas coexist @@ -373,7 +373,7 @@ TEST(GpuIvfFlatTest, SimulatedShardedBuildSearch) { std::vector sim2 = {0, 0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, ids.data()); + gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); ASSERT_TRUE(index.info().find("\"ranks\": 2") != std::string::npos); // 2 shards coexist @@ -405,7 +405,7 @@ TEST(GpuIvfFlatTest, SimulatedShardedDeleteSearch) { std::vector sim2 = {0, 0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, ids.data()); + gpu_ivf_flat_t index(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); ASSERT_TRUE(index.info().find("\"ranks\": 2") != std::string::npos); @@ -453,7 +453,7 @@ TEST(GpuIvfFlatTest, SimulatedReplicatedExtend) { std::vector sim2 = {0, 0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(ds.data(), base, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); + gpu_ivf_flat_t index(ds.data(), base, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); index.start(); index.build(); ASSERT_TRUE(index.info().find("\"ranks\": 2") != std::string::npos); @@ -484,7 +484,7 @@ TEST(GpuIvfFlatTest, SimulatedSaveLoadAcrossModes) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; ivf_flat_search_params_t sp = ivf_flat_search_params_default(); sp.n_probes = 4; - auto probe = [&](gpu_ivf_flat_t& idx, const std::vector& data, const std::vector rows) { + auto probe = [&](gpu_ivf_flat_t& idx, const std::vector& data, const std::vector rows) { for (int r : rows) { std::vector q(data.begin()+r*dim, data.begin()+(r+1)*dim); auto res = idx.search(q.data(), 1, dim, 1, sp); @@ -497,19 +497,19 @@ TEST(GpuIvfFlatTest, SimulatedSaveLoadAcrossModes) { std::string dirR = "/tmp/mo_sim_ivf_flat_rep"; system(("rm -rf " + dirR).c_str()); { - gpu_ivf_flat_t idx(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); + gpu_ivf_flat_t idx(ds.data(), count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED, ids.data()); idx.start(); idx.build(); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); idx.save_dir(dirR); idx.destroy(); } { - gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED); + gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED); idx.start(); idx.load_dir(dirR, DistributionMode_REPLICATED); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); probe(idx, ds, {0, 9, 15}); idx.destroy(); } { - gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, one, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, one, 1, DistributionMode_SINGLE_GPU); idx.start(); idx.load_dir(dirR, DistributionMode_SINGLE_GPU); probe(idx, ds, {0, 9, 15}); idx.destroy(); } @@ -518,11 +518,11 @@ TEST(GpuIvfFlatTest, SimulatedSaveLoadAcrossModes) { std::string dirS = "/tmp/mo_sim_ivf_flat_single"; system(("rm -rf " + dirS).c_str()); { - gpu_ivf_flat_t idx(ds.data(), count, dim, DistanceType_L2Expanded, bp, one, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_ivf_flat_t idx(ds.data(), count, dim, DistanceType_L2Expanded, bp, one, 1, DistributionMode_SINGLE_GPU, ids.data()); idx.start(); idx.build(); idx.save_dir(dirS); idx.destroy(); } { - gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED); + gpu_ivf_flat_t idx(count, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_REPLICATED); idx.start(); idx.load_dir(dirS, DistributionMode_REPLICATED); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); probe(idx, ds, {0, 9, 15}); idx.destroy(); @@ -535,13 +535,13 @@ TEST(GpuIvfFlatTest, SimulatedSaveLoadAcrossModes) { std::string dirSh = "/tmp/mo_sim_ivf_flat_shard"; system(("rm -rf " + dirSh).c_str()); { - gpu_ivf_flat_t idx(sds.data(), scount, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, sids.data()); + gpu_ivf_flat_t idx(sds.data(), scount, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED, sids.data()); idx.start(); idx.build(); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); idx.save_dir(dirSh); idx.destroy(); } { - gpu_ivf_flat_t idx(scount, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED); + gpu_ivf_flat_t idx(scount, dim, DistanceType_L2Expanded, bp, sim2, 2, DistributionMode_SHARDED); idx.start(); idx.load_dir(dirSh, DistributionMode_SHARDED); ASSERT_TRUE(idx.info().find("\"ranks\": 2") != std::string::npos); probe(idx, sds, {3, 40, 63}); idx.destroy(); @@ -563,7 +563,7 @@ TEST(GpuIvfFlatTest, ExtendWithoutHostIds) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -611,7 +611,7 @@ TEST(GpuIvfFlatTest, ExtendWithHostIds) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, base_ids.data()); index.start(); @@ -664,7 +664,7 @@ TEST(GpuIvfFlatTest, ExtendReplicatedWithHostIds) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED, base_ids.data()); index.start(); @@ -712,7 +712,7 @@ TEST(GpuIvfFlatTest, ExtendShardedWithHostIds) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, base_ids.data()); index.start(); @@ -759,7 +759,7 @@ TEST(GpuIvfFlatTest, ExtendShardedWithoutHostIds) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 10; - gpu_ivf_flat_t index(dataset.data(), n_base, dimension, + gpu_ivf_flat_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, nullptr); index.start(); @@ -802,7 +802,7 @@ TEST(GpuIvfFlatTest, ManualShardedGetCenters) { ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 50; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -833,7 +833,7 @@ TEST(GpuIvfFlatTest, FilteredSearchIncludesOnlyAllowedCategories) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(dataset.data(), count, dimension, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -880,7 +880,7 @@ TEST(GpuIvfFlatTest, FilteredSearchCombinesWithDeleteBitset) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(dataset.data(), count, dimension, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -920,7 +920,7 @@ TEST(GpuIvfFlatTest, FilteredSearchEmptyPredsMatchesUnfiltered) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 4; - gpu_ivf_flat_t index(dataset.data(), count, dimension, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -956,7 +956,7 @@ TEST(GpuIvfFlatTest, KExceedsIndexSizeClampsAndPads) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -1004,7 +1004,7 @@ TEST(GpuIvfFlatTest, MultiQueryKExceedsIndexSize) { std::vector devices = {0}; ivf_flat_build_params_t bp = ivf_flat_build_params_default(); bp.n_lists = 2; - gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_ivf_flat_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); diff --git a/cgo/cuvs/test/ivf_pq_test.cu b/cgo/cuvs/test/ivf_pq_test.cu index 53601988ff13e..0e7463fe1405b 100644 --- a/cgo/cuvs/test/ivf_pq_test.cu +++ b/cgo/cuvs/test/ivf_pq_test.cu @@ -21,6 +21,11 @@ #include #include #include +#include +#include +#include +#include +#include using namespace matrixone; @@ -38,7 +43,7 @@ TEST(GpuIvfPqTest, BasicLoadSearchAndCenters) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 2; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -73,7 +78,7 @@ TEST(GpuIvfPqTest, BasicLoadAndSearchWithIds) { std::vector devices = {0}; ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 100; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); index.start(); index.build(); @@ -87,6 +92,353 @@ TEST(GpuIvfPqTest, BasicLoadAndSearchWithIds) { index.destroy(); } +// Native half (f16) build + search — validates the direct vecf16-base path +// (gpu_ivf_pq_t native add_chunk/search, no quantizer). Linking this proves +// cuVS supports ivf_pq over half (unlike brute force over int8/uint8). +TEST(GpuIvfPqTest, BasicLoadAndSearchHalf) { + const uint32_t dimension = 16; + const uint64_t count = 1000; + std::vector dataset(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) + dataset[i * dimension + j] = __float2half((float)rand() / RAND_MAX); + ids[i] = (int64_t)(i + 2000); + } + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 100; + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, ids.data()); + index.start(); + index.build(); + + // Query == row 0, so the nearest neighbour must be its id (2000). + std::vector queries(dataset.begin(), dataset.begin() + dimension); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + auto result = index.search(queries.data(), 1, dimension, 5, sp); + + ASSERT_EQ(result.neighbors.size(), (size_t)5); + ASSERT_EQ(result.neighbors[0], 2000); + + index.destroy(); +} + +// vecf16 base -> int8 storage via the native B(half)-source quantizer +// (add_chunk_quantize). Verifies the quantize-build path: train the half-source +// quantizer on the buffered vecf16 sample, transform half->int8, store as int8, +// and build a searchable int8 index. No f32 detour. +TEST(GpuIvfPqTest, HalfQuantizeToInt8Build) { + const uint32_t dimension = 16; + const uint64_t count = 2000; + std::vector dataset(count * dimension); + std::vector ids(count); + for (size_t i = 0; i < count; ++i) { + for (size_t j = 0; j < dimension; ++j) + dataset[i * dimension + j] = __float2half((float)(rand() % 256) / 255.0f); + ids[i] = (int64_t)(i + 5000); + } + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 50; + gpu_ivf_pq_t index(count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(dataset.data(), count, -1, ids.data()); + index.build(); + + // The resulting int8 index is searchable with a native int8 query. + std::vector q(dimension, 0); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 50; + auto result = index.search(q.data(), 1, dimension, 5, sp); + ASSERT_EQ(result.neighbors.size(), (size_t)5); + for (auto n : result.neighbors) { + ASSERT_GE(n, (int64_t)5000); + ASSERT_LT(n, (int64_t)(5000 + count)); + } + + index.destroy(); +} + +// --------------------------------------------------------------------------- +// REPRODUCTION: f32 base -> int8 vs uint8 storage recall on SIGNED data. +// +// Isolates whether the uint8 quantization recall collapse (seen at 1M scale: +// int8 ~0.83, uint8 ~0.24) is a cuVS-layer bug or mo Go plumbing. This test +// builds BOTH indexes purely through the C++ cuVS wrapper (no mo storage/CDC), +// from the SAME signed dataset with the SAME scalar quantizer. transform +// differs from transform only by a monotonic +128 shift (asserted below) +// which is L2-invariant -- so cuVS uint8 recall MUST match int8 unless cuVS +// mishandles uint8 datasets. +namespace { +struct RecallData { + uint32_t dim; uint64_t count; uint64_t nq; + std::vector base, queries; + std::vector ids; + std::vector> gt; +}; + +RecallData make_signed_recall_data(uint32_t dim, uint64_t count, uint64_t nq, uint32_t k) { + RecallData d; d.dim = dim; d.count = count; d.nq = nq; + d.base.resize(count * dim); d.queries.resize(nq * dim); d.ids.resize(count); + srand(1234); + auto sgn = []() { return ((float)rand() / RAND_MAX) * 2.0f - 1.0f; }; // [-1,1], zero-centered + for (uint64_t i = 0; i < count; ++i) { + for (uint32_t j = 0; j < dim; ++j) d.base[i * dim + j] = sgn(); + d.ids[i] = (int64_t)i; + } + for (uint64_t q = 0; q < nq; ++q) + for (uint32_t j = 0; j < dim; ++j) d.queries[q * dim + j] = sgn(); + d.gt.resize(nq); + for (uint64_t q = 0; q < nq; ++q) { + std::vector> dist(count); + for (uint64_t i = 0; i < count; ++i) { + float s = 0; + for (uint32_t j = 0; j < dim; ++j) { float df = d.queries[q * dim + j] - d.base[i * dim + j]; s += df * df; } + dist[i] = {s, (int64_t)i}; + } + std::partial_sort(dist.begin(), dist.begin() + k, dist.end()); + d.gt[q].resize(k); + for (uint32_t r = 0; r < k; ++r) d.gt[q][r] = dist[r].second; + } + return d; +} + +template +double measure_quantize_recall(RecallData& d, uint32_t k) { + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 64; + gpu_ivf_pq_t index(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(d.base.data(), d.count, -1, d.ids.data()); + index.build(); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 64; + auto res = index.search_quantize(d.queries.data(), d.nq, d.dim, k, sp); + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < d.nq; ++q) { + std::set gt(d.gt[q].begin(), d.gt[q].end()); + for (uint32_t r = 0; r < k; ++r) { + int64_t n = res.neighbors[q * k + r]; + if (gt.count(n)) ++hit; + } + tot += k; + } + index.destroy(); + return (double)hit / (double)tot; +} +} // namespace + +TEST(GpuIvfPqRecall, Int8VsUint8SignedData) { + const uint32_t dim = 32, k = 10; + const uint64_t count = 4000, nq = 200; + RecallData d = make_signed_recall_data(dim, count, nq, k); + + double r_int8 = measure_quantize_recall(d, k); + double r_uint8 = measure_quantize_recall(d, k); + + // Confirm uint8 codes == int8 codes + 128 (monotonic, L2-invariant). If 0 + // mismatches, the quantization is identical up to a constant shift, so any + // recall gap is purely cuVS's uint8 dataset handling. + int mism = 0; + { + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 64; + gpu_ivf_pq_t qi(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qi.start(); qi.add_chunk_quantize(d.base.data(), d.count, -1, d.ids.data()); qi.build(); + gpu_ivf_pq_t qu(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qu.start(); qu.add_chunk_quantize(d.base.data(), d.count, -1, d.ids.data()); qu.build(); + std::vector ci(dim); std::vector cu(dim); + qi.quantize_query(d.queries.data(), 1, ci.data()); + qu.quantize_query(d.queries.data(), 1, cu.data()); + for (uint32_t j = 0; j < dim; ++j) if ((int)cu[j] != (int)ci[j] + 128) ++mism; + qi.destroy(); qu.destroy(); + } + + printf("[repro] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[repro] f32->int8 recall@%u = %.4f\n", k, r_int8); + printf("[repro] f32->uint8 recall@%u = %.4f\n", k, r_uint8); + printf("[repro] VERDICT: int8 high + uint8 low + 0 mismatches => cuVS uint8 bug (not mo plumbing)\n"); + + ASSERT_TRUE(r_int8 > 0.5); // int8 quantize recall should be reasonable + ASSERT_TRUE(mism == 0); // uint8 codes must equal int8 codes + 128 +} + +// --------------------------------------------------------------------------- +// PURE cuVS reproduction: calls cuvs::neighbors::ivf_pq::build/search DIRECTLY +// on raw int8 vs uint8 device matrices. No gpu_ivf_pq_t wrapper, no +// scalar_quantizer_t, no add_chunk/search_float -- zero matrixone code in the +// index path. The uint8 dataset is the int8 dataset + 128 (an exact, monotonic, +// L2-identical shift), so cuVS MUST return identical recall unless it mishandles +// uint8 ivf_pq datasets. This is the definitive cuVS-vs-ours test. +namespace { +// CPU scalar quantize float -> int8 via a global [lo,hi] -> [-128,127] map. +void cpu_quantize_int8(const std::vector& src, std::vector& out, float lo, float hi) { + out.resize(src.size()); + const float scale = 255.0f / (hi - lo); + for (size_t i = 0; i < src.size(); ++i) { + int iv = (int)lroundf((src[i] - lo) * scale - 128.0f); // [lo,hi] -> [-128,127] + iv = std::max(-128, std::min(127, iv)); + out[i] = (int8_t)iv; + } +} + +template +double pure_cuvs_ivfpq_recall(const std::vector& base_q, const std::vector& query_q, + uint64_t count, uint64_t nq, uint32_t dim, uint32_t k, + const std::vector>& gt) { + raft::resources res; + auto base_dev = raft::make_device_matrix(res, count, dim); + raft::copy(res, base_dev.view(), raft::make_host_matrix_view(base_q.data(), count, dim)); + auto query_dev = raft::make_device_matrix(res, nq, dim); + raft::copy(res, query_dev.view(), raft::make_host_matrix_view(query_q.data(), nq, dim)); + raft::resource::sync_stream(res); + + cuvs::neighbors::ivf_pq::index_params ip; + ip.metric = cuvs::distance::DistanceType::L2Expanded; + ip.n_lists = 64; + ip.pq_dim = dim / 2; + ip.pq_bits = 8; + auto index = cuvs::neighbors::ivf_pq::build(res, ip, raft::make_const_mdspan(base_dev.view())); + + cuvs::neighbors::ivf_pq::search_params sp; + sp.n_probes = 64; + auto neighbors = raft::make_device_matrix(res, nq, k); + auto distances = raft::make_device_matrix(res, nq, k); + cuvs::neighbors::ivf_pq::search(res, sp, index, raft::make_const_mdspan(query_dev.view()), + neighbors.view(), distances.view()); + raft::resource::sync_stream(res); + + std::vector nh(nq * k); + raft::copy(res, raft::make_host_matrix_view(nh.data(), nq, k), neighbors.view()); + raft::resource::sync_stream(res); + + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < nq; ++q) { + std::set g(gt[q].begin(), gt[q].end()); + for (uint32_t r = 0; r < k; ++r) if (g.count(nh[q * k + r])) ++hit; + tot += k; + } + return (double)hit / (double)tot; +} +} // namespace + +TEST(PureCuvsIvfPqRecall, Int8VsUint8SignedData) { + const uint32_t dim = 32, k = 10; + const uint64_t count = 4000, nq = 200; + RecallData d = make_signed_recall_data(dim, count, nq, k); // signed float data + float-L2 GT + + float lo = 1e30f, hi = -1e30f; + for (float v : d.base) { lo = std::min(lo, v); hi = std::max(hi, v); } + + std::vector base_i8, query_i8; + cpu_quantize_int8(d.base, base_i8, lo, hi); + cpu_quantize_int8(d.queries, query_i8, lo, hi); + + // uint8 dataset = int8 dataset + 128 (exact; L2(a-b) is identical). + std::vector base_u8(base_i8.size()), query_u8(query_i8.size()); + for (size_t i = 0; i < base_i8.size(); ++i) base_u8[i] = (uint8_t)((int)base_i8[i] + 128); + for (size_t i = 0; i < query_i8.size(); ++i) query_u8[i] = (uint8_t)((int)query_i8[i] + 128); + + double r_i8 = pure_cuvs_ivfpq_recall(base_i8, query_i8, count, nq, dim, k, d.gt); + double r_u8 = pure_cuvs_ivfpq_recall(base_u8, query_u8, count, nq, dim, k, d.gt); + + printf("[pure-cuvs] f32->int8 recall@%u = %.4f\n", k, r_i8); + printf("[pure-cuvs] f32->uint8 recall@%u = %.4f (uint8 = int8+128, L2-identical)\n", k, r_u8); + printf("[pure-cuvs] VERDICT: int8 high + uint8 low => cuVS uint8 ivf_pq bug (zero matrixone code in path)\n"); + ASSERT_TRUE(r_i8 > 0.3); +} + +// --------------------------------------------------------------------------- +// REPRODUCTION: f16 (half) base -> int8 vs uint8 storage recall on SIGNED data. +// +// Same isolation as GpuIvfPqRecall.Int8VsUint8SignedData, but the source +// element type is half (gpu_ivf_pq_t + the native half-source +// quantizer) instead of float. The base/query halves are derived from the SAME +// signed float dataset and graded against the SAME float-L2 ground truth. +// transform again differs from transform only by the monotonic +// +128 shift (asserted below), which is L2-invariant -- so cuVS uint8 recall +// MUST match int8 unless cuVS mishandles uint8 ivf_pq datasets. Proves the +// uint8 collapse is independent of the source element type (f32 vs f16). +namespace { +template +double measure_half_quantize_recall(RecallData& d, uint32_t k) { + // Convert the float base/queries to half — the native B source type. + std::vector base_h(d.base.size()), query_h(d.queries.size()); + for (size_t i = 0; i < d.base.size(); ++i) base_h[i] = __float2half(d.base[i]); + for (size_t i = 0; i < d.queries.size(); ++i) query_h[i] = __float2half(d.queries[i]); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 64; + gpu_ivf_pq_t index(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base_h.data(), d.count, -1, d.ids.data()); + index.build(); + + // Quantize the half queries to storage codes via the half-source quantizer, + // then run the native T search path (same as production). + std::vector qcodes(d.nq * d.dim); + index.quantize_query(query_h.data(), d.nq, qcodes.data()); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 64; + auto res = index.search(qcodes.data(), d.nq, d.dim, k, sp); + + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < d.nq; ++q) { + std::set gt(d.gt[q].begin(), d.gt[q].end()); + for (uint32_t r = 0; r < k; ++r) { + int64_t n = res.neighbors[q * k + r]; + if (gt.count(n)) ++hit; + } + tot += k; + } + index.destroy(); + return (double)hit / (double)tot; +} +} // namespace + +TEST(GpuIvfPqRecall, Int8VsUint8SignedDataHalf) { + const uint32_t dim = 32, k = 10; + const uint64_t count = 4000, nq = 200; + RecallData d = make_signed_recall_data(dim, count, nq, k); + + double r_int8 = measure_half_quantize_recall(d, k); + double r_uint8 = measure_half_quantize_recall(d, k); + + // Confirm uint8 query codes == int8 query codes + 128 (monotonic, + // L2-invariant), produced by the SAME half source. 0 mismatches => any + // recall gap is purely cuVS's uint8 dataset handling, not f16 conversion. + int mism = 0; + { + std::vector base_h(d.base.size()), query_h(d.queries.size()); + for (size_t i = 0; i < d.base.size(); ++i) base_h[i] = __float2half(d.base[i]); + for (size_t i = 0; i < d.queries.size(); ++i) query_h[i] = __float2half(d.queries[i]); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 64; + gpu_ivf_pq_t qi(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qi.start(); qi.add_chunk_quantize(base_h.data(), d.count, -1, d.ids.data()); qi.build(); + gpu_ivf_pq_t qu(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qu.start(); qu.add_chunk_quantize(base_h.data(), d.count, -1, d.ids.data()); qu.build(); + std::vector ci(dim); std::vector cu(dim); + qi.quantize_query(query_h.data(), 1, ci.data()); + qu.quantize_query(query_h.data(), 1, cu.data()); + for (uint32_t j = 0; j < dim; ++j) if ((int)cu[j] != (int)ci[j] + 128) ++mism; + qi.destroy(); qu.destroy(); + } + + printf("[repro-f16] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[repro-f16] f16->int8 recall@%u = %.4f\n", k, r_int8); + printf("[repro-f16] f16->uint8 recall@%u = %.4f\n", k, r_uint8); + printf("[repro-f16] VERDICT: int8 high + uint8 low + 0 mismatches => cuVS uint8 bug (f16 source)\n"); + + ASSERT_TRUE(r_int8 > 0.5); // f16->int8 quantize recall should be reasonable + ASSERT_TRUE(mism == 0); // uint8 codes must equal int8 codes + 128 +} + TEST(GpuIvfPqTest, ParallelAddChunkWithOffset) { const uint32_t dimension = 16; const uint64_t count_per_chunk = 500; @@ -108,7 +460,7 @@ TEST(GpuIvfPqTest, ParallelAddChunkWithOffset) { std::vector devices = {0}; ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 100; - gpu_ivf_pq_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(total_count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); std::thread t1([&]() { index.add_chunk(chunk1.data(), count_per_chunk, 0, ids1.data()); }); @@ -144,7 +496,7 @@ TEST(GpuIvfPqTest, SaveAndLoadFromFile) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 2; bp.m = 2; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); index.save(filename); @@ -156,7 +508,7 @@ TEST(GpuIvfPqTest, SaveAndLoadFromFile) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 2; bp.m = 2; - gpu_ivf_pq_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + gpu_ivf_pq_t index(filename, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.load(filename); @@ -192,7 +544,7 @@ TEST(GpuIvfPqTest, ManualShardedSearch) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 50; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -226,7 +578,7 @@ TEST(GpuIvfPqTest, ManualShardedSearchWithIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 50; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, ids.data()); index.start(); index.build(); @@ -258,7 +610,7 @@ TEST(GpuIvfPqTest, ManualShardedGetCenters) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 50; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED); index.start(); index.build(); @@ -287,7 +639,7 @@ TEST(GpuIvfPqTest, ReplicatedModeSimulation) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 100; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED); index.start(); index.build(); std::vector queries(dataset.begin(), dataset.begin() + dimension); @@ -315,7 +667,7 @@ TEST(GpuIvfPqTest, ExtendWithoutHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -383,7 +735,7 @@ TEST(GpuIvfPqTest, ExtendWithHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU, base_ids.data()); index.start(); @@ -439,7 +791,7 @@ TEST(GpuIvfPqTest, ExtendReplicatedWithHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; bp.m = 8; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_REPLICATED, base_ids.data()); index.start(); @@ -483,7 +835,7 @@ TEST(GpuIvfPqTest, ExtendShardedWithHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, base_ids.data()); index.start(); @@ -531,7 +883,7 @@ TEST(GpuIvfPqTest, ExtendShardedWithoutHostIds) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 10; - gpu_ivf_pq_t index(dataset.data(), n_base, dimension, + gpu_ivf_pq_t index(dataset.data(), n_base, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SHARDED, nullptr); index.start(); @@ -580,7 +932,7 @@ TEST(GpuIvfPqTest, FilteredSearchExcludesForbiddenCategory) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 4; bp.m = 4; - gpu_ivf_pq_t index(dataset.data(), count, dimension, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -627,7 +979,7 @@ TEST(GpuIvfPqTest, FilteredSearchCombinesWithDeleteBitset) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 4; bp.m = 4; - gpu_ivf_pq_t index(dataset.data(), count, dimension, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -689,7 +1041,7 @@ TEST(GpuIvfPqTest, FilteredSearchEmptyPredsMatchesUnfiltered) { ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 4; bp.m = 4; - gpu_ivf_pq_t index(dataset.data(), count, dimension, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); @@ -729,7 +1081,7 @@ TEST(GpuIvfPqTest, KExceedsIndexSizeClampsAndPads) { bp.n_lists = 2; bp.m = 2; bp.bits_per_code = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); @@ -790,7 +1142,7 @@ TEST(GpuIvfPqTest, MultiQueryKExceedsIndexSize) { bp.n_lists = 2; bp.m = 2; bp.bits_per_code = 8; - gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, + gpu_ivf_pq_t index(dataset.data(), count, dimension, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); index.start(); index.build(); diff --git a/cgo/cuvs/test/uint8_quant_bug.cu b/cgo/cuvs/test/uint8_quant_bug.cu new file mode 100644 index 0000000000000..a606475d7d2cc --- /dev/null +++ b/cgo/cuvs/test/uint8_quant_bug.cu @@ -0,0 +1,214 @@ +/* + * Copyright 2021 Matrix Origin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// --------------------------------------------------------------------------- +// STANDALONE repro for the uint8 quantization recall collapse. +// +// Isolates whether the uint8 quantization recall collapse (seen at 1M scale: +// int8 ~0.83, uint8 ~0.24) is a cuVS-layer bug or mo Go plumbing. It builds +// int8 vs uint8 indexes purely through the C++ cuVS wrapper (no mo storage/CDC) +// from the SAME signed dataset and the SAME scalar quantizer, for BOTH source +// element types: +// +// * f32 source (gpu_ivf_pq_t) via search_float (auto query quant) +// * f16 source (gpu_ivf_pq_t) via quantize_query + native search +// +// transform differs from transform only by a monotonic +128 shift +// (asserted: 0 mismatches), which is L2-invariant -- so cuVS uint8 recall MUST +// match int8 unless cuVS mishandles uint8 ivf_pq datasets. +// +// Build/run as its OWN executable (does not run the whole test suite): +// make uint8_quant_bug && ./uint8_quant_bug + +#include "cuvs_worker.hpp" +#include "ivf_pq.hpp" +#include "helper.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace matrixone; + +namespace { + +struct RecallData { + uint32_t dim; uint64_t count; uint64_t nq; + std::vector base, queries; + std::vector ids; + std::vector> gt; +}; + +RecallData make_signed_recall_data(uint32_t dim, uint64_t count, uint64_t nq, uint32_t k) { + RecallData d; d.dim = dim; d.count = count; d.nq = nq; + d.base.resize(count * dim); d.queries.resize(nq * dim); d.ids.resize(count); + srand(1234); + auto sgn = []() { return ((float)rand() / RAND_MAX) * 2.0f - 1.0f; }; // [-1,1], zero-centered + for (uint64_t i = 0; i < count; ++i) { + for (uint32_t j = 0; j < dim; ++j) d.base[i * dim + j] = sgn(); + d.ids[i] = (int64_t)i; + } + for (uint64_t q = 0; q < nq; ++q) + for (uint32_t j = 0; j < dim; ++j) d.queries[q * dim + j] = sgn(); + d.gt.resize(nq); + for (uint64_t q = 0; q < nq; ++q) { + std::vector> dist(count); + for (uint64_t i = 0; i < count; ++i) { + float s = 0; + for (uint32_t j = 0; j < dim; ++j) { float df = d.queries[q * dim + j] - d.base[i * dim + j]; s += df * df; } + dist[i] = {s, (int64_t)i}; + } + std::partial_sort(dist.begin(), dist.begin() + k, dist.end()); + d.gt[q].resize(k); + for (uint32_t r = 0; r < k; ++r) d.gt[q][r] = dist[r].second; + } + return d; +} + +double recall_at_k(const ivf_pq_search_result_t& res, const RecallData& d, uint32_t k) { + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < d.nq; ++q) { + std::set gt(d.gt[q].begin(), d.gt[q].end()); + for (uint32_t r = 0; r < k; ++r) { + int64_t n = res.neighbors[q * k + r]; + if (gt.count(n)) ++hit; + } + tot += k; + } + return (double)hit / (double)tot; +} + +// --- f32 source path: search_float auto-quantizes the float queries. --------- +template +double f32_quantize_recall(RecallData& d, uint32_t k) { + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 64; + gpu_ivf_pq_t index(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(d.base.data(), d.count, -1, d.ids.data()); + index.build(); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 64; + auto res = index.search_quantize(d.queries.data(), d.nq, d.dim, k, sp); + double r = recall_at_k(res, d, k); + index.destroy(); + return r; +} + +// --- f16 source path: quantize_query then native T search. ------------------- +template +double f16_quantize_recall(RecallData& d, uint32_t k) { + std::vector base_h(d.base.size()), query_h(d.queries.size()); + for (size_t i = 0; i < d.base.size(); ++i) base_h[i] = __float2half(d.base[i]); + for (size_t i = 0; i < d.queries.size(); ++i) query_h[i] = __float2half(d.queries[i]); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + bp.n_lists = 64; + gpu_ivf_pq_t index(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base_h.data(), d.count, -1, d.ids.data()); + index.build(); + + std::vector qcodes(d.nq * d.dim); + index.quantize_query(query_h.data(), d.nq, qcodes.data()); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = 64; + auto res = index.search(qcodes.data(), d.nq, d.dim, k, sp); + double r = recall_at_k(res, d, k); + index.destroy(); + return r; +} + +// Quantize one query through both int8 and uint8 quantizers (same source) and +// count dims where uint8 != int8 + 128. 0 => monotonic, L2-invariant shift. +template +int plus128_mismatches(RecallData& d, const std::vector& base_b, const std::vector& query_b) { + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); bp.n_lists = 64; + gpu_ivf_pq_t qi(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qi.start(); qi.add_chunk_quantize(base_b.data(), d.count, -1, d.ids.data()); qi.build(); + gpu_ivf_pq_t qu(d.count, d.dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qu.start(); qu.add_chunk_quantize(base_b.data(), d.count, -1, d.ids.data()); qu.build(); + std::vector ci(d.dim); std::vector cu(d.dim); + qi.quantize_query(query_b.data(), 1, ci.data()); + qu.quantize_query(query_b.data(), 1, cu.data()); + int mism = 0; + for (uint32_t j = 0; j < d.dim; ++j) if ((int)cu[j] != (int)ci[j] + 128) ++mism; + qi.destroy(); qu.destroy(); + return mism; +} + +} // namespace + +int main() { + const uint32_t dim = 32, k = 10; + const uint64_t count = 4000, nq = 200; + RecallData d = make_signed_recall_data(dim, count, nq, k); + + int failures = 0; + + // ---- f32 source ------------------------------------------------------- + { + double r_i8 = f32_quantize_recall(d, k); + double r_u8 = f32_quantize_recall(d, k); + int mism = plus128_mismatches(d, d.base, d.queries); + + printf("\n=== f32 source ===\n"); + printf("[f32] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[f32] f32->int8 recall@%u = %.4f\n", k, r_i8); + printf("[f32] f32->uint8 recall@%u = %.4f\n", k, r_u8); + // Data-driven verdict: with 0 mismatches the codes are identical up to a + // constant shift, so a large recall gap can only come from cuVS's uint8 + // dataset handling. A small gap => the collapse does NOT reproduce here. + printf("[f32] VERDICT: %s (gap=%.4f, mism=%d)\n", + (mism == 0 && r_i8 - r_u8 > 0.20) ? "uint8 COLLAPSE reproduced => cuVS uint8 bug" + : "no collapse at this scale (uint8 ~= int8)", + r_i8 - r_u8, mism); + if (!(r_i8 > 0.5)) { printf("[f32] FAIL: int8 recall too low (%.4f)\n", r_i8); ++failures; } + if (mism != 0) { printf("[f32] FAIL: %d +128 mismatches\n", mism); ++failures; } + } + + // ---- f16 source ------------------------------------------------------- + { + std::vector base_h(d.base.size()), query_h(d.queries.size()); + for (size_t i = 0; i < d.base.size(); ++i) base_h[i] = __float2half(d.base[i]); + for (size_t i = 0; i < d.queries.size(); ++i) query_h[i] = __float2half(d.queries[i]); + + double r_i8 = f16_quantize_recall(d, k); + double r_u8 = f16_quantize_recall(d, k); + int mism = plus128_mismatches(d, base_h, query_h); + + printf("\n=== f16 source ===\n"); + printf("[f16] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[f16] f16->int8 recall@%u = %.4f\n", k, r_i8); + printf("[f16] f16->uint8 recall@%u = %.4f\n", k, r_u8); + printf("[f16] VERDICT: %s (gap=%.4f, mism=%d)\n", + (mism == 0 && r_i8 - r_u8 > 0.20) ? "uint8 COLLAPSE reproduced => cuVS uint8 bug (f16 source)" + : "no collapse at this scale (uint8 ~= int8)", + r_i8 - r_u8, mism); + if (!(r_i8 > 0.5)) { printf("[f16] FAIL: int8 recall too low (%.4f)\n", r_i8); ++failures; } + if (mism != 0) { printf("[f16] FAIL: %d +128 mismatches\n", mism); ++failures; } + } + + printf("\n%s (%d failure(s))\n", failures == 0 ? "PASSED" : "FAILED", failures); + return failures == 0 ? 0 : 1; +} diff --git a/cgo/cuvs/test/wiki1m_uint8_bug.cu b/cgo/cuvs/test/wiki1m_uint8_bug.cu new file mode 100644 index 0000000000000..8c42e2c4722c2 --- /dev/null +++ b/cgo/cuvs/test/wiki1m_uint8_bug.cu @@ -0,0 +1,308 @@ +/* + * Copyright 2021 Matrix Origin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// --------------------------------------------------------------------------- +// REAL-DATASET repro for the uint8 quantization recall collapse, at 1M scale. +// +// The synthetic 4k repro (uint8_quant_bug.cu) did NOT reproduce the collapse, +// so this drives the SAME pure-C++ cuVS path (no mo Go plumbing) over the real +// wiki_all_1M dataset (1M x 768) with its published ground truth — the exact +// data/scale where the collapse (int8 ~0.83, uint8 ~0.24) was first seen. +// +// For each source element type it builds int8 and uint8 ivf_pq indexes from the +// SAME float base via the SAME scalar quantizer, then grades recall@k against +// the dataset's ground-truth neighbors. transform differs from +// transform only by a monotonic +128 shift (asserted: 0 mismatches), so +// uint8 recall MUST match int8 unless cuVS mishandles uint8 ivf_pq datasets. +// +// Build/run as its own executable: +// make wiki1m_uint8_bug && ./wiki1m_uint8_bug [base.fbin] [queries.fbin] [gt.ibin] +// Env knobs: MOQ=#queries (default 1000), MOK=k (default 10), +// MO_NLISTS (default 1024), MO_NPROBES (default 64), MO_F16=1. + +#include "cuvs_worker.hpp" +#include "ivf_pq.hpp" +#include "helper.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace matrixone; + +namespace { + +const char* kBase = "../../../vector_benchmark/wiki_all_1M/base.1M.fbin"; +const char* kQueries = "../../../vector_benchmark/wiki_all_1M/queries.fbin"; +const char* kGt = "../../../vector_benchmark/wiki_all_1M/groundtruth.1M.neighbors.ibin"; + +uint64_t env_u64(const char* k, uint64_t def) { + const char* v = getenv(k); + return v ? strtoull(v, nullptr, 10) : def; +} + +// .fbin: int32 n, int32 dim, then row-major float32[n*dim]. +// max_rows<=0 loads all; otherwise only the first max_rows rows. +std::vector load_fbin(const std::string& path, uint64_t& n, uint32_t& dim, int64_t max_rows = -1) { + FILE* f = fopen(path.c_str(), "rb"); + if (!f) throw std::runtime_error("cannot open " + path); + int32_t hn = 0, hd = 0; + if (fread(&hn, 4, 1, f) != 1 || fread(&hd, 4, 1, f) != 1) { fclose(f); throw std::runtime_error("bad header " + path); } + n = (uint64_t)hn; dim = (uint32_t)hd; + if (max_rows > 0 && (uint64_t)max_rows < n) n = (uint64_t)max_rows; + std::vector data(n * dim); + size_t got = fread(data.data(), sizeof(float), n * dim, f); + fclose(f); + if (got != n * dim) throw std::runtime_error("short read " + path); + return data; +} + +// .ibin: int32 n, int32 k, then row-major int32[n*k]. Returns first `keep` cols. +std::vector> load_ibin_gt(const std::string& path, uint64_t nq, uint32_t keep) { + FILE* f = fopen(path.c_str(), "rb"); + if (!f) throw std::runtime_error("cannot open " + path); + int32_t hn = 0, hk = 0; + if (fread(&hn, 4, 1, f) != 1 || fread(&hk, 4, 1, f) != 1) { fclose(f); throw std::runtime_error("bad header " + path); } + uint32_t k = (uint32_t)hk; + if (keep > k) throw std::runtime_error("gt k too small"); + std::vector row(k); + std::vector> gt(nq); + for (uint64_t q = 0; q < nq; ++q) { + if (fread(row.data(), 4, k, f) != k) { fclose(f); throw std::runtime_error("short gt read"); } + gt[q].resize(keep); + for (uint32_t r = 0; r < keep; ++r) gt[q][r] = (int64_t)row[r]; + } + fclose(f); + return gt; +} + +double recall_at_k(const ivf_pq_search_result_t& res, + const std::vector>& gt, uint64_t nq, uint32_t k) { + size_t hit = 0, tot = 0; + for (uint64_t q = 0; q < nq; ++q) { + std::set g(gt[q].begin(), gt[q].end()); + for (uint32_t r = 0; r < k; ++r) + if (g.count(res.neighbors[q * k + r])) ++hit; + tot += k; + } + return (double)hit / (double)tot; +} + +struct Cfg { uint32_t n_lists, n_probes, m, bits; }; + +inline void apply_cfg(ivf_pq_build_params_t& bp, const Cfg& c) { + bp.n_lists = c.n_lists; + if (c.m) bp.m = c.m; + if (c.bits) bp.bits_per_code = c.bits; +} + +template +double f32_recall(const std::vector& base, uint64_t count, uint32_t dim, + const std::vector& queries, uint64_t nq, + const std::vector>& gt, uint32_t k, const Cfg& c) { + std::vector ids(count); + std::iota(ids.begin(), ids.end(), (int64_t)0); // row index == gt id space + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + apply_cfg(bp, c); + gpu_ivf_pq_t index(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base.data(), count, -1, ids.data()); + index.build(); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = c.n_probes; + auto res = index.search_quantize(queries.data(), nq, dim, k, sp); + double r = recall_at_k(res, gt, nq, k); + index.destroy(); + return r; +} + +// Same as f32_recall but exercises the MO persist->load cycle: build, save_dir, +// destroy, reload a fresh index via load_dir, then search. This is what +// mo-service does (Pack/Unpack -> save_dir/load_dir). If uint8 recall is fine in +// f32_recall but collapses here, the bug is in cuVS serialize/deserialize of a +// uint8-built ivf_pq index (the quantizer.bin is byte-identical for int8/uint8). +template +double f32_recall_saveload(const std::vector& base, uint64_t count, uint32_t dim, + const std::vector& queries, uint64_t nq, + const std::vector>& gt, uint32_t k, const Cfg& c, + const std::string& dir) { + std::vector ids(count); + std::iota(ids.begin(), ids.end(), (int64_t)0); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + apply_cfg(bp, c); + { + gpu_ivf_pq_t index(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base.data(), count, -1, ids.data()); + index.build(); + index.save_dir(dir); // writes index.bin + quantizer.bin + manifest + index.destroy(); + } + // Fresh index, load from disk exactly like LoadIndex/Unpack does. + gpu_ivf_pq_t reloaded(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + reloaded.start(); + reloaded.load_dir(dir, DistributionMode_SINGLE_GPU); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = c.n_probes; + auto res = reloaded.search_quantize(queries.data(), nq, dim, k, sp); + double r = recall_at_k(res, gt, nq, k); + reloaded.destroy(); + return r; +} + +template +double f16_recall(const std::vector& base_h, uint64_t count, uint32_t dim, + const std::vector& query_h, uint64_t nq, + const std::vector>& gt, uint32_t k, const Cfg& c) { + std::vector ids(count); + std::iota(ids.begin(), ids.end(), (int64_t)0); + + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); + apply_cfg(bp, c); + gpu_ivf_pq_t index(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + index.start(); + index.add_chunk_quantize(base_h.data(), count, -1, ids.data()); + index.build(); + + std::vector qcodes(nq * dim); + index.quantize_query(query_h.data(), nq, qcodes.data()); + ivf_pq_search_params_t sp = ivf_pq_search_params_default(); + sp.n_probes = c.n_probes; + auto res = index.search(qcodes.data(), nq, dim, k, sp); + double r = recall_at_k(res, gt, nq, k); + index.destroy(); + return r; +} + +// Count dims where uint8 query code != int8 query code + 128, using the same +// B source quantizer trained on the same base. 0 => monotonic L2-invariant. +template +int plus128_mismatches(const std::vector& base_b, const std::vector& query_b, + uint64_t count, uint32_t dim, const Cfg& c) { + std::vector ids(count); + std::iota(ids.begin(), ids.end(), (int64_t)0); + std::vector devices = {0}; + ivf_pq_build_params_t bp = ivf_pq_build_params_default(); apply_cfg(bp, c); + gpu_ivf_pq_t qi(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qi.start(); qi.add_chunk_quantize(base_b.data(), count, -1, ids.data()); qi.build(); + gpu_ivf_pq_t qu(count, dim, DistanceType_L2Expanded, bp, devices, 1, DistributionMode_SINGLE_GPU); + qu.start(); qu.add_chunk_quantize(base_b.data(), count, -1, ids.data()); qu.build(); + std::vector ci(dim); std::vector cu(dim); + qi.quantize_query(query_b.data(), 1, ci.data()); + qu.quantize_query(query_b.data(), 1, cu.data()); + int mism = 0; + for (uint32_t j = 0; j < dim; ++j) if ((int)cu[j] != (int)ci[j] + 128) ++mism; + qi.destroy(); qu.destroy(); + return mism; +} + +} // namespace + +int main(int argc, char** argv) { + const char* base_path = argc > 1 ? argv[1] : kBase; + const char* qry_path = argc > 2 ? argv[2] : kQueries; + const char* gt_path = argc > 3 ? argv[3] : kGt; + + const uint32_t k = (uint32_t)env_u64("MOK", 10); + const uint64_t nq = env_u64("MOQ", 1000); + Cfg cfg{ (uint32_t)env_u64("MO_NLISTS", 1024), (uint32_t)env_u64("MO_NPROBES", 64), + (uint32_t)env_u64("MO_M", 0), (uint32_t)env_u64("MO_BITS", 0) }; // m/bits 0 => cuVS default + const bool do_f16 = env_u64("MO_F16", 0) != 0; + + printf("Loading base %s ...\n", base_path); + uint64_t count = 0; uint32_t dim = 0; + std::vector base = load_fbin(base_path, count, dim, -1); // FULL 1M (gt requires it) + printf(" base: count=%lu dim=%u\n", (unsigned long)count, dim); + + uint64_t qn_file = 0; uint32_t qdim = 0; + std::vector queries = load_fbin(qry_path, qn_file, qdim, (int64_t)nq); + if (qdim != dim) { printf("dim mismatch base=%u query=%u\n", dim, qdim); return 2; } + uint64_t use_nq = qn_file; + printf(" queries: nq=%lu dim=%u\n", (unsigned long)use_nq, qdim); + + auto gt = load_ibin_gt(gt_path, use_nq, k); + printf(" gt loaded for %lu queries, k=%u\n", (unsigned long)use_nq, k); + printf(" cfg: n_lists=%u n_probes=%u m=%u bits=%u (0=cuVS default)\n\n", + cfg.n_lists, cfg.n_probes, cfg.m, cfg.bits); + + const bool do_saveload = env_u64("MO_SAVELOAD", 0) != 0; + int failures = 0; + + // ---- f32 source ------------------------------------------------------- + { + double r_i8 = f32_recall(base, count, dim, queries, use_nq, gt, k, cfg); + double r_u8 = f32_recall(base, count, dim, queries, use_nq, gt, k, cfg); + int mism = plus128_mismatches(base, queries, count, dim, cfg); + + printf("\n=== f32 source (wiki_all_1M) ===\n"); + printf("[f32] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[f32] f32->int8 recall@%u = %.4f\n", k, r_i8); + printf("[f32] f32->uint8 recall@%u = %.4f\n", k, r_u8); + printf("[f32] VERDICT: %s (gap=%.4f, mism=%d)\n", + (mism == 0 && r_i8 - r_u8 > 0.20) ? "uint8 COLLAPSE reproduced => cuVS uint8 bug" + : "no collapse (uint8 ~= int8)", + r_i8 - r_u8, mism); + if (mism != 0) { printf("[f32] FAIL: %d +128 mismatches\n", mism); ++failures; } + + // The real mo path: build -> save_dir -> load_dir -> search. + if (do_saveload) { + double sl_i8 = f32_recall_saveload(base, count, dim, queries, use_nq, gt, k, cfg, "/tmp/ivfpq_sl_i8"); + double sl_u8 = f32_recall_saveload(base, count, dim, queries, use_nq, gt, k, cfg, "/tmp/ivfpq_sl_u8"); + printf("[f32+saveload] f32->int8 recall@%u = %.4f (in-proc %.4f)\n", k, sl_i8, r_i8); + printf("[f32+saveload] f32->uint8 recall@%u = %.4f (in-proc %.4f)\n", k, sl_u8, r_u8); + printf("[f32+saveload] VERDICT: %s\n", + (sl_i8 - sl_u8 > 0.20) + ? "uint8 COLLAPSE after save/load => cuVS serialize/deserialize bug for uint8 ivf_pq" + : "uint8 survives save/load (~= int8)"); + } + } + + // ---- f16 source (optional; converts the 3GB float base -> half) ------- + if (do_f16) { + std::vector base_h(base.size()), query_h(queries.size()); + for (size_t i = 0; i < base.size(); ++i) base_h[i] = __float2half(base[i]); + for (size_t i = 0; i < queries.size(); ++i) query_h[i] = __float2half(queries[i]); + + double r_i8 = f16_recall(base_h, count, dim, query_h, use_nq, gt, k, cfg); + double r_u8 = f16_recall(base_h, count, dim, query_h, use_nq, gt, k, cfg); + int mism = plus128_mismatches(base_h, query_h, count, dim, cfg); + + printf("\n=== f16 source (wiki_all_1M) ===\n"); + printf("[f16] +128 mismatches: %d / %u dims (0 => uint8==int8+128, L2-invariant)\n", mism, dim); + printf("[f16] f16->int8 recall@%u = %.4f\n", k, r_i8); + printf("[f16] f16->uint8 recall@%u = %.4f\n", k, r_u8); + printf("[f16] VERDICT: %s (gap=%.4f, mism=%d)\n", + (mism == 0 && r_i8 - r_u8 > 0.20) ? "uint8 COLLAPSE reproduced => cuVS uint8 bug (f16 source)" + : "no collapse (uint8 ~= int8)", + r_i8 - r_u8, mism); + if (mism != 0) { printf("[f16] FAIL: %d +128 mismatches\n", mism); ++failures; } + } + + printf("\n%s (%d hard failure(s); inspect recall gaps above)\n", + failures == 0 ? "DONE" : "FAILED", failures); + return failures == 0 ? 0 : 1; +} diff --git a/go.mod b/go.mod index 380da3b060a7a..63af5a3f83fb6 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/matrixorigin/matrixone // Minimum Go version required -go 1.25.4 +go 1.26.4 require ( github.com/BurntSushi/toml v1.2.1 @@ -23,7 +23,7 @@ require ( github.com/aws/smithy-go v1.22.1 github.com/axiomhq/hyperloglog v0.2.6 github.com/buger/jsonparser v1.1.1 - github.com/bytedance/sonic v1.15.0 + github.com/bytedance/sonic v1.15.2 github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 github.com/cespare/xxhash/v2 v2.3.0 github.com/charmbracelet/bubbletea v1.3.10 @@ -135,7 +135,7 @@ require ( github.com/bits-and-blooms/bitset v1.24.2 // indirect github.com/bufbuild/protocompile v0.6.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect - github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect github.com/charmbracelet/x/ansi v0.10.1 // indirect diff --git a/go.sum b/go.sum index 8ed20ff77bb42..c898a75ba8053 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,12 @@ github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic v1.15.2 h1:90H+rcF/FwLXwfB1cudOLq/je83n683Utf4Cbp0xHCo= +github.com/bytedance/sonic v1.15.2/go.mod h1:mT2NbXunuaEbnZ+mRIX/vYqKISmgEuHFDI4UzmKx2SA= github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI= +github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= diff --git a/optools/bvt_ut/Dockerfile b/optools/bvt_ut/Dockerfile index a1d96c631b2d8..69e24710e65c2 100644 --- a/optools/bvt_ut/Dockerfile +++ b/optools/bvt_ut/Dockerfile @@ -1,4 +1,4 @@ -FROM matrixorigin/tester:go1.25.4-jdk8 +FROM matrixorigin/tester:go1.26.4-jdk8 ARG GOPROXY="https://proxy.golang.org,direct" diff --git a/optools/compose_bvt/Dockerfile.tester b/optools/compose_bvt/Dockerfile.tester index 1361d1f10a14d..43e0fdcf650b3 100644 --- a/optools/compose_bvt/Dockerfile.tester +++ b/optools/compose_bvt/Dockerfile.tester @@ -1,4 +1,4 @@ -FROM matrixorigin/tester:go1.25.4-jdk8 +FROM matrixorigin/tester:go1.26.4-jdk8 WORKDIR / diff --git a/optools/images/Dockerfile b/optools/images/Dockerfile index 34885b49a48eb..ef13afa2cacd6 100644 --- a/optools/images/Dockerfile +++ b/optools/images/Dockerfile @@ -1,4 +1,4 @@ -FROM matrixorigin/golang:1.25-ubuntu22.04 AS builder +FROM matrixorigin/golang:1.26.4-ubuntu22.04 AS builder # goproxy ARG GOPROXY="https://proxy.golang.org,direct" diff --git a/optools/images/Dockerfile.dev b/optools/images/Dockerfile.dev index 836a5d78acf27..06aa5df647777 100644 --- a/optools/images/Dockerfile.dev +++ b/optools/images/Dockerfile.dev @@ -4,7 +4,7 @@ # - Binary is built inside container using local make build # - Much faster iteration cycle for development -FROM matrixorigin/golang:1.25-ubuntu22.04 +FROM matrixorigin/golang:1.26.4-ubuntu22.04 # Install build dependencies and runtime libraries RUN apt-get update && apt-get install -y \ diff --git a/pkg/catalog/types.go b/pkg/catalog/types.go index c7ee8630790f3..1dba87e027f28 100644 --- a/pkg/catalog/types.go +++ b/pkg/catalog/types.go @@ -385,6 +385,13 @@ const ( SystemSI_IVFFLAT_TblCol_Metadata_key = "__mo_index_key" SystemSI_IVFFLAT_TblCol_Metadata_val = "__mo_index_val" + // IVF_FLAT MetadataTable - well-known keys (rows in the key/val metadata table) + // QuantizeMin/QuantizeMax store the trained int8 scalar-quantizer bounds + // (cuVS-style asymmetric): [min,max] is mapped to the full int8 range [-128,127] + // via q(x)=round(x*mul+add). Entries and the query use the same transform. + SystemSI_IVFFLAT_Metadata_QuantizeMin = "quantize_min" + SystemSI_IVFFLAT_Metadata_QuantizeMax = "quantize_max" + // IVF_FLAT Centroids - Column names SystemSI_IVFFLAT_TblCol_Centroids_version = "__mo_index_centroid_version" SystemSI_IVFFLAT_TblCol_Centroids_id = "__mo_index_centroid_id" diff --git a/pkg/compare/arraycompare.go b/pkg/compare/arraycompare.go index 885278268cf1c..3ac0695297ba8 100644 --- a/pkg/compare/arraycompare.go +++ b/pkg/compare/arraycompare.go @@ -59,6 +59,14 @@ func (c arrayCompare) Compare(veci, vecj int, vi, vj int64) int { return types.CompareArrayFromBytes[float32](_x, _y, c.desc) case types.T_array_float64: return types.CompareArrayFromBytes[float64](_x, _y, c.desc) + case types.T_array_bf16: + return types.CompareArrayElementFromBytes[types.BF16](_x, _y, c.desc) + case types.T_array_float16: + return types.CompareArrayElementFromBytes[types.Float16](_x, _y, c.desc) + case types.T_array_int8: + return types.CompareArrayElementFromBytes[int8](_x, _y, c.desc) + case types.T_array_uint8: + return types.CompareArrayElementFromBytes[uint8](_x, _y, c.desc) default: panic("Compare Not supported") } diff --git a/pkg/compare/compare.go b/pkg/compare/compare.go index 0ab353f96f642..ce547f19da34b 100644 --- a/pkg/compare/compare.go +++ b/pkg/compare/compare.go @@ -156,7 +156,8 @@ func New(typ types.Type, desc, nullsLast bool) Compare { vs: make([]*vector.Vector, 2), isConstNull: make([]bool, 2), } - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: //NOTE: Used by merge_order, merge_top, top agg operators. return &arrayCompare{ desc: desc, diff --git a/pkg/container/types/array.go b/pkg/container/types/array.go index 22a1091516e91..d25310f5c4c33 100644 --- a/pkg/container/types/array.go +++ b/pkg/container/types/array.go @@ -33,21 +33,21 @@ const ( ) // BytesToArray bytes should be of little-endian format -func BytesToArray[T RealNumbers](input []byte) (res []T) { +func BytesToArray[T ArrayElement](input []byte) (res []T) { return DecodeSlice[T](input) } -func ArrayToBytes[T RealNumbers](input []T) []byte { +func ArrayToBytes[T ArrayElement](input []T) []byte { return EncodeSlice(input) } // ArrayToBase64 encodes a vector as base64 of its raw little-endian bytes. // ~22x faster than ArrayToString for 768-dim float32 (no per-element float formatting). -func ArrayToBase64[T RealNumbers](input []T) string { +func ArrayToBase64[T ArrayElement](input []T) string { return base64.StdEncoding.EncodeToString(EncodeSlice(input)) } -func ArrayToString[T RealNumbers](input []T) string { +func ArrayToString[T ArrayElement](input []T) string { var buffer bytes.Buffer _, _ = io.WriteString(&buffer, "[") for i, value := range input { @@ -64,13 +64,21 @@ func ArrayToString[T RealNumbers](input []T) string { _, _ = io.WriteString(&buffer, strconv.FormatFloat(float64(value), 'f', -1, 32)) case float64: _, _ = io.WriteString(&buffer, strconv.FormatFloat(value, 'f', -1, 64)) + case BF16: + _, _ = io.WriteString(&buffer, strconv.FormatFloat(float64(value.ToFloat32()), 'f', -1, 32)) + case Float16: + _, _ = io.WriteString(&buffer, strconv.FormatFloat(float64(value.ToFloat32()), 'f', -1, 32)) + case int8: + _, _ = io.WriteString(&buffer, strconv.FormatInt(int64(value), 10)) + case uint8: + _, _ = io.WriteString(&buffer, strconv.FormatUint(uint64(value), 10)) } } _, _ = io.WriteString(&buffer, "]") return buffer.String() } -func ArraysToString[T RealNumbers](input [][]T, sep string) string { +func ArraysToString[T ArrayElement](input [][]T, sep string) string { strValues := make([]string, len(input)) for i, row := range input { strValues[i] = ArrayToString(row) @@ -78,7 +86,7 @@ func ArraysToString[T RealNumbers](input [][]T, sep string) string { return strings.Join(strValues, sep) } -func StringToArray[T RealNumbers](str string) ([]T, error) { +func StringToArray[T ArrayElement](str string) ([]T, error) { input := strings.ReplaceAll(str, " ", "") if !(strings.HasPrefix(input, "[") && strings.HasSuffix(input, "]")) { @@ -113,7 +121,7 @@ func StringToArray[T RealNumbers](str string) ([]T, error) { } // StringToArrayToBytes convert "[1,2,3]" --> []float32{1.0,2.0,3.0} --> []bytes{11,33...} -func StringToArrayToBytes[T RealNumbers](input string) ([]byte, error) { +func StringToArrayToBytes[T ArrayElement](input string) ([]byte, error) { // Convert "[1,2,3]" --> []float32{1.0, 2.0, 3.0} a, err := StringToArray[T](input) if err != nil { @@ -123,7 +131,7 @@ func StringToArrayToBytes[T RealNumbers](input string) ([]byte, error) { return ArrayToBytes(a), nil } -func BytesToArrayToString[T RealNumbers](input []byte) string { +func BytesToArrayToString[T ArrayElement](input []byte) string { // Convert []byte{11, 33, 45, 56,.....} --> []float32{1.0, 2.0, 3.0} a := BytesToArray[T](input) diff --git a/pkg/container/types/array_str.go b/pkg/container/types/array_str.go index c95f2a1cf0279..160657943abc8 100644 --- a/pkg/container/types/array_str.go +++ b/pkg/container/types/array_str.go @@ -261,7 +261,7 @@ func indexFrom(str, substr string, start int) int { } // stringToT convert str to T -func stringToT[T RealNumbers](str string) (t T, err error) { +func stringToT[T ArrayElement](str string) (t T, err error) { switch any(t).(type) { case float32: num, err := strconv.ParseFloat(str, 32) @@ -277,6 +277,42 @@ func stringToT[T RealNumbers](str string) (t T, err error) { return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_float64.String()) } return *(*T)(unsafe.Pointer(&num)), nil + case BF16: + num, err := strconv.ParseFloat(str, 32) + if err != nil { + return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_array_bf16.String()) + } + bf := BF16FromFloat32(float32(num)) + return *(*T)(unsafe.Pointer(&bf)), nil + case Float16: + num, err := strconv.ParseFloat(str, 32) + if err != nil { + return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_array_float16.String()) + } + h := Float16FromFloat32(float32(num)) + return *(*T)(unsafe.Pointer(&h)), nil + case int8: + // Strict: a vecint8 string literal must be an integer in [-128,127]. + // Non-integer ("1.4") or out-of-range ("200") values error rather than + // silently rounding/clamping. (The vecf32 -> vecint8 CAST path does + // round+clamp; only direct string parsing is strict.) + num, err := strconv.ParseInt(str, 10, 8) + if err != nil { + return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_array_int8.String()) + } + i8 := int8(num) + return *(*T)(unsafe.Pointer(&i8)), nil + case uint8: + // Strict: a vecuint8 string literal must be an integer in [0,255]. + // Non-integer ("1.4") or out-of-range ("300") values error rather than + // silently rounding/clamping. (The vecf32 -> vecuint8 CAST path does + // round+clamp; only direct string parsing is strict.) + num, err := strconv.ParseUint(str, 10, 8) + if err != nil { + return t, moerr.NewInternalErrorNoCtxf("error while casting %s to %s", str, T_array_uint8.String()) + } + u8 := uint8(num) + return *(*T)(unsafe.Pointer(&u8)), nil default: panic(moerr.NewInternalErrorNoCtx("not implemented")) } diff --git a/pkg/container/types/bytes.go b/pkg/container/types/bytes.go index da3766afeb686..12b34d410fc97 100644 --- a/pkg/container/types/bytes.go +++ b/pkg/container/types/bytes.go @@ -110,7 +110,7 @@ func (v *Varlena) GetByteSlice(area []byte) []byte { // GetArray Returns []T from Varlena. If the Varlena size is less than Inline size, // it returns the value from the Varlena header. // Else, it returns the value from the area. -func GetArray[T RealNumbers](v *Varlena, area []byte) []T { +func GetArray[T ArrayElement](v *Varlena, area []byte) []T { svlen := (*v)[0] if svlen <= VarlenaInlineSize { return BytesToArray[T](v.ByteSlice()) diff --git a/pkg/container/types/compare.go b/pkg/container/types/compare.go index bbf2861d39685..5d9150e51fb61 100644 --- a/pkg/container/types/compare.go +++ b/pkg/container/types/compare.go @@ -107,6 +107,14 @@ func GenericDescCompare[T OrderedT](x, y T) int { // Compare returns an integer comparing two arrays/vectors lexicographically. // TODO: this function might not be correct. we need to compare using tolerance for float values. // TODO: need to check if we need len(v1)==len(v2) check. +// ArrayElementCompare orders two narrow-typed vectors by upcasting to float32. +// Direct uint16/int8 comparison would be wrong for bf16/f16 (the sign bit makes +// bit order disagree with value order), so all element types route through the +// float32 bridge. Exact for int8/float32; lossless for the stored bf16/f16 values. +func ArrayElementCompare[T ArrayElement](v1, v2 []T) int { + return ArrayCompare[float32](ToFloat32Array(v1), ToFloat32Array(v2)) +} + func ArrayCompare[T RealNumbers](v1, v2 []T) int { minLen := len(v1) if len(v2) < minLen { @@ -138,3 +146,15 @@ func CompareArrayFromBytes[T RealNumbers](_x, _y []byte, desc bool) int { } return ArrayCompare[T](x, y) } + +// CompareArrayElementFromBytes is the narrow-type (bf16/f16/int8) counterpart of +// CompareArrayFromBytes; it orders through the float32 bridge. +func CompareArrayElementFromBytes[T ArrayElement](_x, _y []byte, desc bool) int { + x := BytesToArray[T](_x) + y := BytesToArray[T](_y) + + if desc { + return ArrayElementCompare[T](y, x) + } + return ArrayElementCompare[T](x, y) +} diff --git a/pkg/container/types/encoding.go b/pkg/container/types/encoding.go index 73bca3f1550f9..7bf708279223c 100644 --- a/pkg/container/types/encoding.go +++ b/pkg/container/types/encoding.go @@ -380,7 +380,7 @@ func DecodeValue(val []byte, t T) any { return DecodeFixed[TS](val) case T_Rowid: return DecodeFixed[Rowid](val) - case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_datalink, T_geometry, T_geometry32: + case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8, T_datalink, T_geometry, T_geometry32: return val case T_enum: return DecodeFixed[Enum](val) @@ -553,7 +553,7 @@ func EncodeValue(val any, t T) []byte { case T_Rowid: return EncodeFixed(val.(Rowid)) case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, - T_array_float32, T_array_float64, T_datalink, T_geometry, T_geometry32: + T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8, T_datalink, T_geometry, T_geometry32: // Mainly used by Zonemap, which receives val input from DN batch/vector. // This val is mostly []bytes and not []float32 or []float64 return val.([]byte) diff --git a/pkg/container/types/float16.go b/pkg/container/types/float16.go new file mode 100644 index 0000000000000..9a61fb9166995 --- /dev/null +++ b/pkg/container/types/float16.go @@ -0,0 +1,325 @@ +// Copyright 2021 - 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// This file defines the narrow element types used by the vecbf16 / vecf16 / +// veci8 vector column types. They participate ONLY in the storage / +// serialization / accessor / display / cast-plumbing layer (the ArrayElement +// constraint). All arithmetic (distance, normalize, ...) is performed by +// upcasting to []float32, running the existing float32 kernels, and (for +// vector-returning ops) converting back. uint16/int8 arithmetic kernels are +// never written. + +// BF16 is the bfloat16 floating-point format: the top 16 bits of an IEEE +// float32 (1 sign bit, 8 exponent bits, 7 mantissa bits). Conversion to +// float32 is a left shift; conversion from float32 truncates the low 16 bits +// with round-to-nearest-even. +type BF16 uint16 + +// Float16 is the IEEE 754 binary16 (half precision) format: 1 sign bit, 5 +// exponent bits, 10 mantissa bits. +type Float16 uint16 + +// ---------------------------------------------------------------------------- +// BF16 +// ---------------------------------------------------------------------------- + +// ToFloat32 widens a bfloat16 to float32 by placing its bits in the high half +// of the float32 representation. +func (b BF16) ToFloat32() float32 { + return math.Float32frombits(uint32(b) << 16) +} + +// BF16FromFloat32 narrows a float32 to bfloat16 using round-to-nearest-even. +// NaN inputs are preserved as a (quiet) NaN. +func BF16FromFloat32(f float32) BF16 { + x := math.Float32bits(f) + if (x>>23)&0xff == 0xff && x&0x7fffff != 0 { + // NaN: truncating the low 16 bits could zero the mantissa and turn it + // into an Inf, so force a non-zero mantissa bit. + return BF16(x>>16 | 0x0040) + } + // Round to nearest even: add 0x7fff plus the LSB of the surviving mantissa. + rounding := uint32(0x7fff) + ((x >> 16) & 1) + return BF16((x + rounding) >> 16) +} + +// ---------------------------------------------------------------------------- +// Float16 (IEEE binary16) +// +// The conversion routines below follow the vetted scalar algorithm used by +// github.com/x448/float16 (Apache-2.0), with full subnormal / Inf / NaN +// handling and round-to-nearest-even. +// ---------------------------------------------------------------------------- + +// ToFloat32 widens an IEEE half to float32. +func (h Float16) ToFloat32() float32 { + return math.Float32frombits(f16bitsToF32bits(uint16(h))) +} + +// Float16FromFloat32 narrows a float32 to an IEEE half using +// round-to-nearest-even, with overflow to Inf and subnormal handling. +func Float16FromFloat32(f float32) Float16 { + return Float16(f32bitsToF16bits(math.Float32bits(f))) +} + +func f16bitsToF32bits(in uint16) uint32 { + sign := uint32(in&0x8000) << 16 // sign bit, shifted to float32 position + exp := uint32(in&0x7c00) >> 10 // 5-bit exponent + coef := uint32(in&0x03ff) << 13 // 10-bit mantissa, shifted to float32 position + + if exp == 0x1f { + if coef == 0 { + // Infinity + return sign | 0x7f800000 + } + // NaN + return sign | 0x7fc00000 | coef + } + + if exp == 0 { + if coef == 0 { + // signed zero + return sign + } + // normalize the subnormal + exp++ + for coef&0x7f800000 == 0 { + coef <<= 1 + exp-- + } + coef &= 0x007fffff + } + + return sign | ((exp + (0x7f - 0xf)) << 23) | coef +} + +func f32bitsToF16bits(u32 uint32) uint16 { + sign := u32 & 0x80000000 + exp := u32 & 0x7f800000 + coef := u32 & 0x007fffff + + if exp == 0x7f800000 { + // NaN or Infinity + nanBit := uint32(0) + if coef != 0 { + nanBit = uint32(0x0200) + } + return uint16((sign >> 16) | uint32(0x7c00) | nanBit | (coef >> 13)) + } + + halfSign := sign >> 16 + + unbiasedExp := int32(exp>>23) - 127 + halfExp := unbiasedExp + 15 + + if halfExp >= 0x1f { + // overflow -> Inf + return uint16(halfSign | uint32(0x7c00)) + } + + if halfExp <= 0 { + if 14-halfExp > 24 { + // too small -> signed zero + return uint16(halfSign) + } + coef := coef | uint32(0x00800000) + halfCoef := coef >> uint32(14-halfExp) + roundBit := uint32(1) << uint32(13-halfExp) + if (coef&roundBit) != 0 && (coef&(3*roundBit-1)) != 0 { + halfCoef++ + } + return uint16(halfSign | halfCoef) + } + + halfExp2 := uint32(halfExp) << 10 + halfCoef := coef >> 13 + roundBit := uint32(0x00001000) + if (coef&roundBit) != 0 && (coef&(3*roundBit-1)) != 0 { + return uint16((halfSign | halfExp2 | halfCoef) + 1) + } + return uint16(halfSign | halfExp2 | halfCoef) +} + +// ---------------------------------------------------------------------------- +// Batch converters (hot path). These power the float32 bridge used by the +// distance / cast wrappers; keep them allocation-light. +// ---------------------------------------------------------------------------- + +func BF16ToFloat32Slice(src []BF16) []float32 { + dst := make([]float32, len(src)) + for i, v := range src { + dst[i] = v.ToFloat32() + } + return dst +} + +func Float16ToFloat32Slice(src []Float16) []float32 { + dst := make([]float32, len(src)) + for i, v := range src { + dst[i] = v.ToFloat32() + } + return dst +} + +func Int8ToFloat32Slice(src []int8) []float32 { + dst := make([]float32, len(src)) + for i, v := range src { + dst[i] = float32(v) + } + return dst +} + +func Uint8ToFloat32Slice(src []uint8) []float32 { + dst := make([]float32, len(src)) + for i, v := range src { + dst[i] = float32(v) + } + return dst +} + +func Float32ToBF16Slice(src []float32) []BF16 { + dst := make([]BF16, len(src)) + for i, v := range src { + dst[i] = BF16FromFloat32(v) + } + return dst +} + +func Float32ToFloat16Slice(src []float32) []Float16 { + dst := make([]Float16, len(src)) + for i, v := range src { + dst[i] = Float16FromFloat32(v) + } + return dst +} + +// Float32ToInt8Slice rounds to nearest and clamps to the int8 range +// [-128, 127]. NaN maps to 0. +func Float32ToInt8Slice(src []float32) []int8 { + dst := make([]int8, len(src)) + for i, v := range src { + dst[i] = Float32ToInt8(v) + } + return dst +} + +// Float32ToInt8 rounds to nearest (ties away from zero, via math.Round) and +// clamps to [-128, 127]. NaN maps to 0. +func Float32ToInt8(v float32) int8 { + if v != v { // NaN + return 0 + } + r := math.Round(float64(v)) + if r > 127 { + return 127 + } + if r < -128 { + return -128 + } + return int8(r) +} + +// Float32ToUint8Slice rounds to nearest and clamps to the uint8 range +// [0, 255]. NaN maps to 0. +func Float32ToUint8Slice(src []float32) []uint8 { + dst := make([]uint8, len(src)) + for i, v := range src { + dst[i] = Float32ToUint8(v) + } + return dst +} + +// Float32ToUint8 rounds to nearest (ties away from zero, via math.Round) and +// clamps to [0, 255]. NaN maps to 0. +func Float32ToUint8(v float32) uint8 { + if v != v { // NaN + return 0 + } + r := math.Round(float64(v)) + if r > 255 { + return 255 + } + if r < 0 { + return 0 + } + return uint8(r) +} + +// ---------------------------------------------------------------------------- +// Generic float32 bridge. This is THE boundary between the storage tier +// (ArrayElement) and the compute tier (RealNumbers). Any math on a narrow type +// upcasts here, runs the float32 kernel, and (for vector results) converts back. +// ---------------------------------------------------------------------------- + +// ToFloat32Array upcasts any ArrayElement slice to []float32. For []float32 it +// returns the input unchanged (no copy); callers must not mutate the result in +// place when T is float32 unless they own the input. +func ToFloat32Array[T ArrayElement](in []T) []float32 { + switch v := any(in).(type) { + case []float32: + return v + case []float64: + out := make([]float32, len(v)) + for i, x := range v { + out[i] = float32(x) + } + return out + case []BF16: + return BF16ToFloat32Slice(v) + case []Float16: + return Float16ToFloat32Slice(v) + case []int8: + return Int8ToFloat32Slice(v) + case []uint8: + return Uint8ToFloat32Slice(v) + default: + panic(moerr.NewInternalErrorNoCtx("ToFloat32Array: unsupported element type")) + } +} + +// FromFloat32Array narrows a []float32 back to the target ArrayElement type. +// int8 rounds-to-nearest and clamps to [-128,127]; bf16/f16 round-to-nearest-even. +func FromFloat32Array[T ArrayElement](in []float32) []T { + var zero T + switch any(zero).(type) { + case float32: + out := make([]float32, len(in)) + copy(out, in) + return any(out).([]T) + case float64: + out := make([]float64, len(in)) + for i, x := range in { + out[i] = float64(x) + } + return any(out).([]T) + case BF16: + return any(Float32ToBF16Slice(in)).([]T) + case Float16: + return any(Float32ToFloat16Slice(in)).([]T) + case int8: + return any(Float32ToInt8Slice(in)).([]T) + case uint8: + return any(Float32ToUint8Slice(in)).([]T) + default: + panic(moerr.NewInternalErrorNoCtx("FromFloat32Array: unsupported element type")) + } +} diff --git a/pkg/container/types/float16_test.go b/pkg/container/types/float16_test.go new file mode 100644 index 0000000000000..53e586da7cca2 --- /dev/null +++ b/pkg/container/types/float16_test.go @@ -0,0 +1,250 @@ +// Copyright 2021 - 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "math" + "testing" +) + +func TestFloat16ReferenceValues(t *testing.T) { + // (float32 input, expected IEEE half bits, expected float32 after round-trip) + cases := []struct { + in float32 + bits uint16 + out float32 + }{ + {0.0, 0x0000, 0.0}, + {1.0, 0x3c00, 1.0}, + {-1.0, 0xbc00, -1.0}, + {2.0, 0x4000, 2.0}, + {0.5, 0x3800, 0.5}, + {-0.5, 0xb800, -0.5}, + {65504.0, 0x7bff, 65504.0}, // largest normal half + {0.00006103515625, 0x0400, 0.00006103515625}, // smallest normal half (2^-14) + {0.00006097555, 0x03ff, 0.000060975552}, // largest subnormal half + {5.9604645e-08, 0x0001, 5.9604645e-08}, // smallest positive subnormal (2^-24) + } + for _, c := range cases { + got := Float16FromFloat32(c.in) + if uint16(got) != c.bits { + t.Errorf("Float16FromFloat32(%v) bits = 0x%04x, want 0x%04x", c.in, uint16(got), c.bits) + } + back := Float16(c.bits).ToFloat32() + if math.Abs(float64(back-c.out)) > 1e-9 { + t.Errorf("Float16(0x%04x).ToFloat32() = %v, want %v", c.bits, back, c.out) + } + } +} + +func TestFloat16InfNaN(t *testing.T) { + // +Inf + if got := Float16FromFloat32(float32(math.Inf(1))); uint16(got) != 0x7c00 { + t.Errorf("+Inf -> 0x%04x, want 0x7c00", uint16(got)) + } + if got := Float16FromFloat32(float32(math.Inf(-1))); uint16(got) != 0xfc00 { + t.Errorf("-Inf -> 0x%04x, want 0xfc00", uint16(got)) + } + // overflow to +Inf + if got := Float16FromFloat32(70000.0); uint16(got) != 0x7c00 { + t.Errorf("70000 -> 0x%04x, want 0x7c00 (overflow to Inf)", uint16(got)) + } + // NaN stays NaN + nan := Float16FromFloat32(float32(math.NaN())) + if !math.IsNaN(float64(nan.ToFloat32())) { + t.Errorf("NaN did not survive round-trip: got %v", nan.ToFloat32()) + } + // Inf round-trip + if v := Float16(0x7c00).ToFloat32(); !math.IsInf(float64(v), 1) { + t.Errorf("0x7c00 -> %v, want +Inf", v) + } +} + +func TestBF16ReferenceValues(t *testing.T) { + cases := []struct { + in float32 + bits uint16 + }{ + {0.0, 0x0000}, + {1.0, 0x3f80}, + {-1.0, 0xbf80}, + {2.0, 0x4000}, + {0.5, 0x3f00}, + {3.14159265, 0x4049}, // pi truncated/rounded to bf16 + } + for _, c := range cases { + got := BF16FromFloat32(c.in) + if uint16(got) != c.bits { + t.Errorf("BF16FromFloat32(%v) = 0x%04x, want 0x%04x", c.in, uint16(got), c.bits) + } + } + // bf16 keeps full float32 exponent range: round-trip is close + for _, v := range []float32{1.0, -2.5, 100.0, 0.001, 12345.0} { + back := BF16FromFloat32(v).ToFloat32() + rel := math.Abs(float64((back - v) / v)) + if rel > 0.01 { // bf16 has ~7 mantissa bits -> ~0.4% worst case + t.Errorf("BF16 round-trip %v -> %v, rel err %v too high", v, back, rel) + } + } + // NaN survives + if !math.IsNaN(float64(BF16FromFloat32(float32(math.NaN())).ToFloat32())) { + t.Errorf("BF16 NaN did not survive round-trip") + } + // Inf survives + if !math.IsInf(float64(BF16FromFloat32(float32(math.Inf(1))).ToFloat32()), 1) { + t.Errorf("BF16 +Inf did not survive round-trip") + } +} + +func TestInt8Clamp(t *testing.T) { + cases := []struct { + in float32 + out int8 + }{ + {0.0, 0}, + {1.4, 1}, + {1.6, 2}, + {-1.6, -2}, + {127.0, 127}, + {128.0, 127}, // clamp high + {200.0, 127}, // clamp high + {-128.0, -128}, + {-129.0, -128}, // clamp low + {-500.0, -128}, // clamp low + } + for _, c := range cases { + if got := Float32ToInt8(c.in); got != c.out { + t.Errorf("Float32ToInt8(%v) = %d, want %d", c.in, got, c.out) + } + } + if Float32ToInt8(float32(math.NaN())) != 0 { + t.Errorf("NaN -> int8 should be 0") + } +} + +func TestFloat32Bridge(t *testing.T) { + // ToFloat32Array for each element type + if got := ToFloat32Array([]float32{1, 2, 3}); got[0] != 1 || got[2] != 3 { + t.Errorf("f32 bridge = %v", got) + } + if got := ToFloat32Array([]float64{1, 2, 3}); got[1] != 2 { + t.Errorf("f64 bridge = %v", got) + } + if got := ToFloat32Array([]BF16{BF16FromFloat32(1.5)}); got[0] != 1.5 { + t.Errorf("bf16 bridge = %v", got) + } + if got := ToFloat32Array([]Float16{Float16FromFloat32(2.5)}); got[0] != 2.5 { + t.Errorf("f16 bridge = %v", got) + } + if got := ToFloat32Array([]int8{-5, 7}); got[0] != -5 || got[1] != 7 { + t.Errorf("int8 bridge = %v", got) + } + // FromFloat32Array narrows correctly + src := []float32{1.0, 2.0, -3.0} + if out := FromFloat32Array[float32](src); out[2] != -3.0 { + t.Errorf("from f32 = %v", out) + } + if out := FromFloat32Array[float64](src); out[0] != 1.0 { + t.Errorf("from f64 = %v", out) + } + if out := FromFloat32Array[BF16](src); out[0].ToFloat32() != 1.0 { + t.Errorf("from bf16 = %v", out) + } + if out := FromFloat32Array[Float16](src); out[1].ToFloat32() != 2.0 { + t.Errorf("from f16 = %v", out) + } + if out := FromFloat32Array[int8]([]float32{1.4, 130, -200}); out[0] != 1 || out[1] != 127 || out[2] != -128 { + t.Errorf("from int8 = %v", out) + } +} + +func TestArrayElementCompare(t *testing.T) { + // bf16/f16 must order by value, not raw bits (negative has high bit set) + neg := []BF16{BF16FromFloat32(-1.0)} + pos := []BF16{BF16FromFloat32(1.0)} + if ArrayElementCompare(neg, pos) >= 0 { + t.Errorf("bf16 compare: -1 should be < 1") + } + negh := []Float16{Float16FromFloat32(-2.0)} + posh := []Float16{Float16FromFloat32(0.5)} + if ArrayElementCompare(negh, posh) >= 0 { + t.Errorf("f16 compare: -2 should be < 0.5") + } + if ArrayElementCompare([]int8{-5}, []int8{3}) >= 0 { + t.Errorf("int8 compare: -5 should be < 3") + } +} + +func TestStringToArrayNarrow(t *testing.T) { + // int8: strict integer parse. Valid integers in range round-trip exactly. + i8, err := StringToArray[int8]("[1, -2, 127, -128, 0]") + if err != nil { + t.Fatalf("int8 parse: %v", err) + } + want := []int8{1, -2, 127, -128, 0} + for i := range want { + if i8[i] != want[i] { + t.Errorf("int8[%d] = %d, want %d", i, i8[i], want[i]) + } + } + // int8: non-integer and out-of-range literals error (no silent round/clamp). + if _, err := StringToArray[int8]("[1.4]"); err == nil { + t.Errorf("int8 parse of non-integer should error") + } + if _, err := StringToArray[int8]("[200]"); err == nil { + t.Errorf("int8 parse of out-of-range should error") + } + if _, err := StringToArray[int8]("[-129]"); err == nil { + t.Errorf("int8 parse of out-of-range (low) should error") + } + // bf16 / f16: small integers round-trip exactly. + bf, err := StringToArray[BF16]("[1, 2, 3]") + if err != nil || bf[0].ToFloat32() != 1 || bf[2].ToFloat32() != 3 { + t.Errorf("bf16 parse: %v %v", bf, err) + } + h, err := StringToArray[Float16]("[0.5, -2, 4]") + if err != nil || h[0].ToFloat32() != 0.5 || h[1].ToFloat32() != -2 { + t.Errorf("f16 parse: %v %v", h, err) + } + // ArrayToString round-trips the narrow types. + if s := ArrayToString[int8]([]int8{1, -2, 127}); s != "[1, -2, 127]" { + t.Errorf("int8 ArrayToString = %q", s) + } + if s := ArrayToString[BF16](Float32ToBF16Slice([]float32{1, 2, 3})); s != "[1, 2, 3]" { + t.Errorf("bf16 ArrayToString = %q", s) + } +} + +func TestFloat16SliceRoundTrip(t *testing.T) { + src := []float32{1.0, 2.0, 0.5, -3.0, 0.0} + f16 := Float32ToFloat16Slice(src) + back := Float16ToFloat32Slice(f16) + for i := range src { + if back[i] != src[i] { + t.Errorf("f16 slice round-trip[%d]: %v != %v", i, back[i], src[i]) + } + } + bf := Float32ToBF16Slice(src) + bback := BF16ToFloat32Slice(bf) + for i := range src { + if math.Abs(float64(bback[i]-src[i])) > math.Abs(float64(src[i]))*0.01+1e-6 { + t.Errorf("bf16 slice round-trip[%d]: %v vs %v", i, bback[i], src[i]) + } + } + i8 := Float32ToInt8Slice([]float32{1.2, 130.0, -200.0}) + if i8[0] != 1 || i8[1] != 127 || i8[2] != -128 { + t.Errorf("int8 slice = %v", i8) + } +} diff --git a/pkg/container/types/types.go b/pkg/container/types/types.go index d55b3027dcd6e..adf410252c715 100644 --- a/pkg/container/types/types.go +++ b/pkg/container/types/types.go @@ -98,10 +98,47 @@ const ( // Array/Vector family T_array_float32 T = 224 // In SQL , it is vecf32 T_array_float64 T = 225 // In SQL , it is vecf64 + T_array_bf16 T = 226 // In SQL , it is vecbf16 (bfloat16) + T_array_float16 T = 227 // In SQL , it is vecf16 (IEEE fp16/half) + T_array_int8 T = 228 // In SQL , it is vecint8 (int8) + T_array_uint8 T = 229 // In SQL , it is vecuint8 (uint8) //note: max value of uint8 is 255 ) +// Canonical lowercase SQL type names for the array/vector types — the spelling +// used in DDL and CAST (`col vecf32(4)`, `cast(x as vecint8(4))`) and recognized +// by the parser keyword table. T.String() returns the uppercase display form; +// these are the single source of truth for the lowercase SQL spelling. +const ( + ArrayFloat32SQLName = "vecf32" + ArrayFloat64SQLName = "vecf64" + ArrayBF16SQLName = "vecbf16" + ArrayFloat16SQLName = "vecf16" + ArrayInt8SQLName = "vecint8" + ArrayUint8SQLName = "vecuint8" +) + +// ArraySQLName returns the lowercase SQL type name for an array element type +// (e.g. T_array_float32 -> "vecf32"), or "" if t is not an array/vector type. +func (t T) ArraySQLName() string { + switch t { + case T_array_float32: + return ArrayFloat32SQLName + case T_array_float64: + return ArrayFloat64SQLName + case T_array_bf16: + return ArrayBF16SQLName + case T_array_float16: + return ArrayFloat16SQLName + case T_array_int8: + return ArrayInt8SQLName + case T_array_uint8: + return ArrayUint8SQLName + } + return "" +} + const ( TxnTsSize = 12 SegmentidSize = 16 @@ -365,6 +402,16 @@ type RealNumbers interface { constraints.Float } +// ArrayElement is the set of element types that can back a vector column. +// It is used ONLY by the storage / serialization / accessor / display / +// cast-plumbing layer (pure byte reinterpretation + formatting). All math +// kernels stay on RealNumbers; narrow types reach them via a float32 bridge. +// Do NOT widen RealNumbers to include these — int8 is not a float and +// BF16/Float16 have no native arithmetic. +type ArrayElement interface { + ~float32 | ~float64 | BF16 | Float16 | int8 | uint8 +} + type FixedSizeTExceptStrType interface { bool | OrderedT | Decimal | TS | Rowid | Uuid | Blockid } @@ -426,6 +473,10 @@ var Types = map[string]T{ "array float32": T_array_float32, "array float64": T_array_float64, + "array bf16": T_array_bf16, + "array float16": T_array_float16, + "array int8": T_array_int8, + "array uint8": T_array_uint8, } func New(oid T, width, scale int32) Type { @@ -569,6 +620,14 @@ func (t Type) DescString() string { return fmt.Sprintf("VECF32(%d)", t.Width) case T_array_float64: return fmt.Sprintf("VECF64(%d)", t.Width) + case T_array_bf16: + return fmt.Sprintf("VECBF16(%d)", t.Width) + case T_array_float16: + return fmt.Sprintf("VECF16(%d)", t.Width) + case T_array_int8: + return fmt.Sprintf("VECINT8(%d)", t.Width) + case T_array_uint8: + return fmt.Sprintf("VECUINT8(%d)", t.Width) } return t.Oid.String() } @@ -579,6 +638,14 @@ func (t Type) GetArrayElementSize() int { return 4 case T_array_float64: return 8 + case T_array_bf16: + return 2 + case T_array_float16: + return 2 + case T_array_int8: + return 1 + case T_array_uint8: + return 1 } panic(moerr.NewInternalErrorNoCtx(fmt.Sprintf("unknown array type %d", t))) } @@ -651,7 +718,7 @@ func (t T) ToType() Type { case T_varchar: typ.Size = VarlenaSize typ.Width = MaxVarcharLen - case T_array_float32, T_array_float64: + case T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8: typ.Size = VarlenaSize typ.Width = MaxArrayDimension case T_binary: @@ -759,6 +826,14 @@ func (t T) String() string { return "VECF32" case T_array_float64: return "VECF64" + case T_array_bf16: + return "VECBF16" + case T_array_float16: + return "VECF16" + case T_array_int8: + return "VECINT8" + case T_array_uint8: + return "VECUINT8" case T_enum: return "ENUM" } @@ -844,6 +919,14 @@ func (t T) OidString() string { return "T_array_float32" case T_array_float64: return "T_array_float64" + case T_array_bf16: + return "T_array_bf16" + case T_array_float16: + return "T_array_float16" + case T_array_int8: + return "T_array_int8" + case T_array_uint8: + return "T_array_uint8" } return "unknown_type" } @@ -877,7 +960,7 @@ func (t T) TypeLen() int { return 4 case T_float64: return 8 - case T_char, T_varchar, T_json, T_blob, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_datalink, T_geometry, T_geometry32: + case T_char, T_varchar, T_json, T_blob, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8, T_datalink, T_geometry, T_geometry32: return VarlenaSize case T_decimal64: return 8 @@ -932,7 +1015,7 @@ func (t T) FixedLength() int { return RowidSize case T_Blockid: return BlockidSize - case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_datalink, T_geometry, T_geometry32: + case T_char, T_varchar, T_blob, T_json, T_text, T_binary, T_varbinary, T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8, T_datalink, T_geometry, T_geometry32: return -24 case T_enum: return 2 @@ -1013,7 +1096,8 @@ func (t T) IsDateRelate() bool { } func (t T) IsArrayRelate() bool { - if t == T_array_float32 || t == T_array_float64 { + if t == T_array_float32 || t == T_array_float64 || + t == T_array_bf16 || t == T_array_float16 || t == T_array_int8 || t == T_array_uint8 { return true } return false diff --git a/pkg/container/types/types_test.go b/pkg/container/types/types_test.go index e77c027f2cf66..efa8421e4419a 100644 --- a/pkg/container/types/types_test.go +++ b/pkg/container/types/types_test.go @@ -417,3 +417,21 @@ func BenchmarkTypesCompare(b *testing.B) { } }) } + +func TestArraySQLName(t *testing.T) { + // every array/vector type maps to its lowercase SQL name. + arrayTypes := []T{T_array_float32, T_array_float64, T_array_bf16, T_array_float16, T_array_int8, T_array_uint8} + wantNames := []string{"vecf32", "vecf64", "vecbf16", "vecf16", "vecint8", "vecuint8"} + for i, at := range arrayTypes { + require.Equal(t, wantNames[i], at.ArraySQLName()) + } + + // constants stay in sync with the method (and with the literal spellings). + require.Equal(t, ArrayFloat32SQLName, T_array_float32.ArraySQLName()) + require.Equal(t, ArrayInt8SQLName, T_array_int8.ArraySQLName()) + require.Equal(t, "vecbf16", ArrayBF16SQLName) + + // non-array types -> "". + require.Equal(t, "", T_int32.ArraySQLName()) + require.Equal(t, "", T_varchar.ArraySQLName()) +} diff --git a/pkg/container/vector/tools.go b/pkg/container/vector/tools.go index 4bb9254a90c97..8920131cf729e 100644 --- a/pkg/container/vector/tools.go +++ b/pkg/container/vector/tools.go @@ -122,7 +122,7 @@ func InefficientMustStrCol(v *Vector) []string { } // MustArrayCol Converts Vector<[]T> to [][]T -func MustArrayCol[T types.RealNumbers](v *Vector) [][]T { +func MustArrayCol[T types.ArrayElement](v *Vector) [][]T { if v.GetType().Oid == types.T_any || len(v.data) == 0 { return nil } diff --git a/pkg/container/vector/utils.go b/pkg/container/vector/utils.go index d553e5070b7a6..2d6f45420f4f1 100644 --- a/pkg/container/vector/utils.go +++ b/pkg/container/vector/utils.go @@ -331,6 +331,33 @@ func ArrayGetMinMax[T types.RealNumbers](vec *Vector) (minv, maxv []T) { return } +// ArrayElementGetMinMax mirrors ArrayGetMinMax for the narrow vector element +// types (bf16/f16/int8). Ordering goes through the float32 bridge via +// ArrayElementCompare so that bf16/f16 sign bits don't corrupt the comparison. +// The returned min/max are original stored values (no reconversion). +func ArrayElementGetMinMax[T types.ArrayElement](vec *Vector) (minv, maxv []T) { + col, area := MustVarlenaRawData(vec) + first := true + for i, j := 0, vec.Length(); i < j; i++ { + if vec.HasNull() && vec.IsNull(uint64(i)) { + continue + } + val := types.GetArray[T](&col[i], area) + if first { + minv, maxv = val, val + first = false + continue + } + if types.ArrayElementCompare[T](minv, val) > 0 { + minv = val + } + if types.ArrayElementCompare[T](maxv, val) < 0 { + maxv = val + } + } + return +} + func typeCompatible[T any](typ types.Type) bool { var t T switch (any)(t).(type) { diff --git a/pkg/container/vector/vector.go b/pkg/container/vector/vector.go index ef282697ac991..e0e7e05997b65 100644 --- a/pkg/container/vector/vector.go +++ b/pkg/container/vector/vector.go @@ -379,7 +379,7 @@ func (v *Vector) GetStringAt(i int) string { } // GetArrayAt Returns []T at the specific index of the vector -func GetArrayAt[T types.RealNumbers](v *Vector, i int) []T { +func GetArrayAt[T types.ArrayElement](v *Vector, i int) []T { if v.IsConst() { i = 0 } @@ -388,7 +388,7 @@ func GetArrayAt[T types.RealNumbers](v *Vector, i int) []T { return types.GetArray[T](&bs[i], v.area) } -func GetArrayAt2[T types.RealNumbers](v *Vector, bs []types.Varlena, i int) []T { +func GetArrayAt2[T types.ArrayElement](v *Vector, bs []types.Varlena, i int) []T { if v.IsConst() { i = 0 } @@ -451,7 +451,7 @@ func GetAny(vec *Vector, i int, deepCopy bool) any { case types.T_Blockid: return GetFixedAtNoTypeCheck[types.Blockid](vec, i) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: ret := vec.GetBytesAt(i) if deepCopy { copied := make([]byte, len(ret)) @@ -545,7 +545,7 @@ func NewConstBytes(typ types.Type, val []byte, length int, mp *mpool.MPool) (vec } // NewConstArray Creates a Const_Array Vector -func NewConstArray[T types.RealNumbers](typ types.Type, val []T, length int, mp *mpool.MPool) (vec *Vector, err error) { +func NewConstArray[T types.ArrayElement](typ types.Type, val []T, length int, mp *mpool.MPool) (vec *Vector, err error) { vec = NewVecFromReuse() vec.typ = typ vec.class = CONSTANT @@ -1050,7 +1050,7 @@ func (v *Vector) Shrink(sels []int64, negate bool) { case types.T_float64: shrinkFixed[float64](v, sels, negate) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: // XXX shrink varlena, but did not shrink area. For our vector, this // may well be the right thing. If want to shrink area as well, we // have to copy each varlena value and swizzle pointer. @@ -1122,7 +1122,7 @@ func (v *Vector) ShrinkByMask(sels *bitmap.Bitmap, negate bool, offset uint64) { case types.T_float64: shrinkFixedByMask[float64](v, sels, negate, offset) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: // XXX shrink varlena, but did not shrink area. For our vector, this // may well be the right thing. If want to shrink area as well, we // have to copy each varlena value and swizzle pointer. @@ -1190,7 +1190,7 @@ func (v *Vector) Shuffle(sels []int64, mp *mpool.MPool) (err error) { case types.T_float64: err = shuffleFixedNoTypeCheck[float64](v, sels, mp) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: err = shuffleFixedNoTypeCheck[types.Varlena](v, sels, mp) case types.T_date: err = shuffleFixedNoTypeCheck[types.Date](v, sels, mp) @@ -1264,7 +1264,7 @@ func (v *Vector) ShuffleWithBuf(sels []int64, mp *mpool.MPool, buf *[]byte) (err case types.T_float64: err = shuffleFixedNoTypeCheckWithBuf[float64](v, sels, buf) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: err = shuffleFixedNoTypeCheckWithBuf[types.Varlena](v, sels, buf) case types.T_date: err = shuffleFixedNoTypeCheckWithBuf[types.Date](v, sels, buf) @@ -2067,7 +2067,7 @@ func GetUnionAllFunction(typ types.Type, mp *mpool.MPool) func(v, w *Vector) err } case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: return func(v, w *Vector) error { if w.IsConstNull() { if err := appendMultiFixed(v, 0, true, w.length, mp); err != nil { @@ -2426,7 +2426,7 @@ func GetConstSetFunction(typ types.Type, mp *mpool.MPool) func(v, w *Vector, sel return SetConstFixed(v, ws[sel], length, mp) } case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, - types.T_json, types.T_blob, types.T_text, types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_json, types.T_blob, types.T_text, types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: return func(v, w *Vector, sel int64, length int) error { if w.IsConstNull() || w.nsp.Contains(uint64(sel)) { return SetConstNull(v, length, mp) @@ -3133,6 +3133,46 @@ func (v *Vector) String() string { return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) } return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) + case types.T_array_bf16: + col := MustArrayCol[types.BF16](v) + if len(col) == 1 { + if nulls.Contains(&v.nsp, 0) { + return "null" + } + return types.ArrayToString[types.BF16](col[0]) + } + str := types.ArraysToString[types.BF16](col, types.DefaultArraysToStringSep) + return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) + case types.T_array_float16: + col := MustArrayCol[types.Float16](v) + if len(col) == 1 { + if nulls.Contains(&v.nsp, 0) { + return "null" + } + return types.ArrayToString[types.Float16](col[0]) + } + str := types.ArraysToString[types.Float16](col, types.DefaultArraysToStringSep) + return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) + case types.T_array_uint8: + col := MustArrayCol[uint8](v) + if len(col) == 1 { + if nulls.Contains(&v.nsp, 0) { + return "null" + } + return types.ArrayToString[uint8](col[0]) + } + str := types.ArraysToString[uint8](col, types.DefaultArraysToStringSep) + return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) + case types.T_array_int8: + col := MustArrayCol[int8](v) + if len(col) == 1 { + if nulls.Contains(&v.nsp, 0) { + return "null" + } + return types.ArrayToString[int8](col[0]) + } + str := types.ArraysToString[int8](col, types.DefaultArraysToStringSep) + return fmt.Sprintf("%v-%s", str, v.nsp.GetBitmap().String()) default: panic("vec to string unknown types.") } @@ -3219,7 +3259,7 @@ func implDecimalRowToString[T types.DecimalWithFormat](v *Vector, idx int) strin } } -func implArrayRowToString[T types.RealNumbers](v *Vector, idx int) string { +func implArrayRowToString[T types.ArrayElement](v *Vector, idx int) string { if v.IsConstNull() { return "null" } @@ -3309,6 +3349,14 @@ func (v *Vector) RowToString(idx int) string { return implArrayRowToString[float32](v, idx) case types.T_array_float64: return implArrayRowToString[float64](v, idx) + case types.T_array_bf16: + return implArrayRowToString[types.BF16](v, idx) + case types.T_array_float16: + return implArrayRowToString[types.Float16](v, idx) + case types.T_array_int8: + return implArrayRowToString[int8](v, idx) + case types.T_array_uint8: + return implArrayRowToString[uint8](v, idx) default: panic("vec to string unknown types.") } @@ -3362,7 +3410,7 @@ func SetConstByteJson(vec *Vector, bj bytejson.ByteJson, length int, mp *mpool.M } // SetConstArray set current vector as Constant_Array vector of given length. -func SetConstArray[T types.RealNumbers](vec *Vector, val []T, length int, mp *mpool.MPool) error { +func SetConstArray[T types.ArrayElement](vec *Vector, val []T, length int, mp *mpool.MPool) error { var err error if err := extend(vec, 1, mp); err != nil { @@ -3445,7 +3493,7 @@ func AppendAny(vec *Vector, val any, isNull bool, mp *mpool.MPool) error { case types.T_Blockid: return appendOneFixed(vec, val.(types.Blockid), false, mp) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry, types.T_geometry32: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32: return appendOneBytes(vec, val.([]byte), false, mp) } return nil @@ -3486,7 +3534,7 @@ func AppendByteJson(vec *Vector, bj bytejson.ByteJson, isNull bool, mp *mpool.MP } // AppendArray mainly used in tests -func AppendArray[T types.RealNumbers](vec *Vector, val []T, isNull bool, mp *mpool.MPool) error { +func AppendArray[T types.ArrayElement](vec *Vector, val []T, isNull bool, mp *mpool.MPool) error { if vec.IsConst() { panic(moerr.NewInternalErrorNoCtx("append to const vector")) } @@ -3556,7 +3604,7 @@ func AppendStringList(vec *Vector, ws []string, isNulls []bool, mp *mpool.MPool) } // AppendArrayList mainly used in unit tests -func AppendArrayList[T types.RealNumbers](vec *Vector, ws [][]T, isNulls []bool, mp *mpool.MPool) error { +func AppendArrayList[T types.ArrayElement](vec *Vector, ws [][]T, isNulls []bool, mp *mpool.MPool) error { if vec.IsConst() { panic(moerr.NewInternalErrorNoCtx("append to const vector")) } @@ -3620,7 +3668,7 @@ func appendOneByteJson(vec *Vector, bj bytejson.ByteJson, isNull bool, mp *mpool } // appendOneArray mainly used for unit tests -func appendOneArray[T types.RealNumbers](vec *Vector, val []T, isNull bool, mp *mpool.MPool) error { +func appendOneArray[T types.ArrayElement](vec *Vector, val []T, isNull bool, mp *mpool.MPool) error { var err error var va types.Varlena @@ -3738,7 +3786,7 @@ func appendStringList(vec *Vector, vals []string, isNulls []bool, mp *mpool.MPoo } // appendArrayList mainly used for unit tests -func appendArrayList[T types.RealNumbers](vec *Vector, vals [][]T, isNulls []bool, mp *mpool.MPool) error { +func appendArrayList[T types.ArrayElement](vec *Vector, vals [][]T, isNulls []bool, mp *mpool.MPool) error { var err error if err = extend(vec, len(vals), mp); err != nil { @@ -4437,6 +4485,22 @@ func (v *Vector) GetMinMaxValue() (ok bool, minv, maxv []byte) { _minv, _maxv := ArrayGetMinMax[float64](v) minv = types.ArrayToBytes[float64](_minv) maxv = types.ArrayToBytes[float64](_maxv) + case types.T_array_bf16: + _minv, _maxv := ArrayElementGetMinMax[types.BF16](v) + minv = types.ArrayToBytes[types.BF16](_minv) + maxv = types.ArrayToBytes[types.BF16](_maxv) + case types.T_array_float16: + _minv, _maxv := ArrayElementGetMinMax[types.Float16](v) + minv = types.ArrayToBytes[types.Float16](_minv) + maxv = types.ArrayToBytes[types.Float16](_maxv) + case types.T_array_int8: + _minv, _maxv := ArrayElementGetMinMax[int8](v) + minv = types.ArrayToBytes[int8](_minv) + maxv = types.ArrayToBytes[int8](_maxv) + case types.T_array_uint8: + _minv, _maxv := ArrayElementGetMinMax[uint8](v) + minv = types.ArrayToBytes[uint8](_minv) + maxv = types.ArrayToBytes[uint8](_maxv) default: panic(fmt.Sprintf("unsupported type %s", v.GetType().String())) } @@ -4804,6 +4868,36 @@ func (v *Vector) InplaceSortAndCompact() { cleanDataNotResetArea() appendList(v, newCol, nil, nil) } + case types.T_array_bf16: + inplaceSortAndCompactArrayElement[types.BF16](v, cleanDataNotResetArea) + case types.T_array_float16: + inplaceSortAndCompactArrayElement[types.Float16](v, cleanDataNotResetArea) + case types.T_array_int8: + inplaceSortAndCompactArrayElement[int8](v, cleanDataNotResetArea) + case types.T_array_uint8: + inplaceSortAndCompactArrayElement[uint8](v, cleanDataNotResetArea) + } +} + +// inplaceSortAndCompactArrayElement sorts+dedups a narrow-typed vector using the +// float32-bridged comparator (so bf16/f16 order by value, not by raw bits). +func inplaceSortAndCompactArrayElement[T types.ArrayElement](v *Vector, cleanDataNotResetArea func()) { + col, area := MustVarlenaRawData(v) + sort.Slice(col, func(i, j int) bool { + return types.ArrayElementCompare[T]( + types.GetArray[T](&col[i], area), + types.GetArray[T](&col[j], area), + ) < 0 + }) + newCol := slices.CompactFunc(col, func(a, b types.Varlena) bool { + return types.ArrayElementCompare[T]( + types.GetArray[T](&a, area), + types.GetArray[T](&b, area), + ) == 0 + }) + if len(newCol) != len(col) { + cleanDataNotResetArea() + appendList(v, newCol, nil, nil) } } @@ -4975,9 +5069,29 @@ func (v *Vector) InplaceSort() { types.GetArray[float64](&col[j], area), ) < 0 }) + case types.T_array_bf16: + sortArrayElement[types.BF16](v) + case types.T_array_float16: + sortArrayElement[types.Float16](v) + case types.T_array_int8: + sortArrayElement[int8](v) + case types.T_array_uint8: + sortArrayElement[uint8](v) } } +// sortArrayElement sorts a narrow-typed vector in place using the +// float32-bridged comparator. +func sortArrayElement[T types.ArrayElement](v *Vector) { + col, area := MustVarlenaRawData(v) + sort.Slice(col, func(i, j int) bool { + return types.ArrayElementCompare[T]( + types.GetArray[T](&col[i], area), + types.GetArray[T](&col[j], area), + ) < 0 + }) +} + func BuildVarlenaInline(v1, v2 *types.Varlena) { // use three dword operation to improve performance p1 := v1.UnsafePtr() @@ -5074,7 +5188,7 @@ func BuildVarlenaFromByteJson(vec *Vector, v *types.Varlena, bj bytejson.ByteJso } // BuildVarlenaFromArray convert array to Varlena so that it can be stored in the vector -func BuildVarlenaFromArray[T types.RealNumbers](vec *Vector, v *types.Varlena, array *[]T, m *mpool.MPool) error { +func BuildVarlenaFromArray[T types.ArrayElement](vec *Vector, v *types.Varlena, array *[]T, m *mpool.MPool) error { _bs := types.ArrayToBytes[T](*array) bs := &_bs vlen := len(*bs) diff --git a/pkg/cuvs/brute_force.go b/pkg/cuvs/brute_force.go index 2b1c13a2bba97..3d35db8c39d39 100644 --- a/pkg/cuvs/brute_force.go +++ b/pkg/cuvs/brute_force.go @@ -28,18 +28,24 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" ) -// GpuBruteForce represents the C++ gpu_brute_force_t object -type GpuBruteForce[T VectorType] struct { +// GpuBruteForce represents the C++ gpu_brute_force_t object. +// B is the base/query element type, Q is the storage element type. The native +// dataset/chunks are storage-typed ([]Q); the quantize search entry points take +// base-typed ([]B) queries and quantize B->Q inside cuVS. cuVS brute force only +// supports (B,Q) combos (float,float),(float,half),(half,half); int8/uint8 +// storage is not supported and throws at runtime. +type GpuBruteForce[B, Q VectorType] struct { cIndex C.gpu_brute_force_c } // NewGpuBruteForce creates a new GpuBruteForce instance -func NewGpuBruteForce[T VectorType](dataset []T, countVectors uint64, dimension uint32, metric DistanceType, nthread uint32, deviceID int) (*GpuBruteForce[T], error) { +func NewGpuBruteForce[B, Q VectorType](dataset []Q, countVectors uint64, dimension uint32, metric DistanceType, nthread uint32, deviceID int) (*GpuBruteForce[B, Q], error) { if len(dataset) == 0 || countVectors == 0 || dimension == 0 { return nil, moerr.NewInternalErrorNoCtx("dataset, count_vectors, and dimension cannot be zero") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cIndex := C.gpu_brute_force_new( unsafe.Pointer(&dataset[0]), @@ -48,6 +54,7 @@ func NewGpuBruteForce[T VectorType](dataset []T, countVectors uint64, dimension C.distance_type_t(metric), C.uint32_t(nthread), C.int(deviceID), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -63,14 +70,15 @@ func NewGpuBruteForce[T VectorType](dataset []T, countVectors uint64, dimension if cIndex == nil { return nil, moerr.NewInternalErrorNoCtx("failed to create GpuBruteForce") } - return &GpuBruteForce[T]{cIndex: cIndex}, nil + return &GpuBruteForce[B, Q]{cIndex: cIndex}, nil } // NewGpuBruteForceEmpty creates a new GpuBruteForce instance with pre-allocated buffer but no data yet. -func NewGpuBruteForceEmpty[T VectorType](totalCount uint64, dimension uint32, metric DistanceType, - nthread uint32, deviceID int) (*GpuBruteForce[T], error) { +func NewGpuBruteForceEmpty[B, Q VectorType](totalCount uint64, dimension uint32, metric DistanceType, + nthread uint32, deviceID int) (*GpuBruteForce[B, Q], error) { - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cBruteForce := C.gpu_brute_force_new_empty( @@ -79,6 +87,7 @@ func NewGpuBruteForceEmpty[T VectorType](totalCount uint64, dimension uint32, me C.distance_type_t(metric), C.uint32_t(nthread), C.int(deviceID), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -94,11 +103,11 @@ func NewGpuBruteForceEmpty[T VectorType](totalCount uint64, dimension uint32, me return nil, moerr.NewInternalErrorNoCtx("failed to create GpuBruteForce") } - return &GpuBruteForce[T]{cIndex: cBruteForce}, nil + return &GpuBruteForce[B, Q]{cIndex: cBruteForce}, nil } // Start initializes the worker and resources -func (gb *GpuBruteForce[T]) Start() error { +func (gb *GpuBruteForce[B, Q]) Start() error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -113,7 +122,7 @@ func (gb *GpuBruteForce[T]) Start() error { } // Build triggers the dataset loading to GPU -func (gb *GpuBruteForce[T]) Build() error { +func (gb *GpuBruteForce[B, Q]) Build() error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -131,7 +140,7 @@ func (gb *GpuBruteForce[T]) Build() error { // If ids is non-nil it must have length chunkCount and supplies external int64 // ids (e.g. pkids) that the brute-force search will return in `neighbors` // instead of the internal 0..N-1 row index. -func (gb *GpuBruteForce[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (gb *GpuBruteForce[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -165,9 +174,10 @@ func (gb *GpuBruteForce[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) return nil } -// AddChunkFloat adds a chunk of float32 data, performing on-the-fly conversion if needed. -// See AddChunk for the meaning of ids. -func (gb *GpuBruteForce[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize adds a chunk of base-typed (B) vectors, converting them to the +// storage type Q on the C++ side (native store when B==Q, f32->f16 cast, or learned +// SQ for 1-byte Q) — the add counterpart of SearchQuantize. See AddChunk for ids. +func (gb *GpuBruteForce[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -183,9 +193,9 @@ func (gb *GpuBruteForce[T]) AddChunkFloat(chunk []float32, chunkCount uint64, id if ids != nil { idsPtr = (*C.int64_t)(&ids[0]) } - C.gpu_brute_force_add_chunk_float( + C.gpu_brute_force_add_chunk_quantize( gb.cIndex, - (*C.float)(&chunk[0]), + unsafe.Pointer(&chunk[0]), C.uint64_t(chunkCount), idsPtr, unsafe.Pointer(&errmsg), @@ -203,7 +213,7 @@ func (gb *GpuBruteForce[T]) AddChunkFloat(chunk []float32, chunkCount uint64, id // SearchInto performs a search and writes results into caller-provided slices (no internal allocation). // neighbors and distances must be pre-allocated to at least numQueries*limit elements. -func (gb *GpuBruteForce[T]) SearchInto(queries []T, numQueries uint64, queryDimension uint32, limit uint32, neighbors []int64, distances []float32) error { +func (gb *GpuBruteForce[B, Q]) SearchInto(queries []Q, numQueries uint64, queryDimension uint32, limit uint32, neighbors []int64, distances []float32) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -239,7 +249,7 @@ func (gb *GpuBruteForce[T]) SearchInto(queries []T, numQueries uint64, queryDime } // Search performs a search operation -func (gb *GpuBruteForce[T]) Search(queries []T, numQueries uint64, queryDimension uint32, limit uint32) ([]int64, []float32, error) { +func (gb *GpuBruteForce[B, Q]) Search(queries []Q, numQueries uint64, queryDimension uint32, limit uint32) ([]int64, []float32, error) { neighbors := make([]int64, numQueries*uint64(limit)) distances := make([]float32, numQueries*uint64(limit)) if err := gb.SearchInto(queries, numQueries, queryDimension, limit, neighbors, distances); err != nil { @@ -248,9 +258,10 @@ func (gb *GpuBruteForce[T]) Search(queries []T, numQueries uint64, queryDimensio return neighbors, distances, nil } -// SearchFloatInto performs a search with float32 queries and writes results into caller-provided slices. +// SearchQuantizeInto performs a search with base-typed (B) queries and writes +// results into caller-provided slices. cuVS quantizes B -> storage Q internally. // neighbors and distances must be pre-allocated to at least numQueries*limit elements. -func (gb *GpuBruteForce[T]) SearchFloatInto(queries []float32, numQueries uint64, queryDimension uint32, limit uint32, neighbors []int64, distances []float32) error { +func (gb *GpuBruteForce[B, Q]) SearchQuantizeInto(queries []B, numQueries uint64, queryDimension uint32, limit uint32, neighbors []int64, distances []float32) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -259,9 +270,9 @@ func (gb *GpuBruteForce[T]) SearchFloatInto(queries []float32, numQueries uint64 } var errmsg *C.char - cResult := C.gpu_brute_force_search_float( + cResult := C.gpu_brute_force_search_quantize( gb.cIndex, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(queryDimension), C.uint32_t(limit), @@ -285,18 +296,19 @@ func (gb *GpuBruteForce[T]) SearchFloatInto(queries []float32, numQueries uint64 return nil } -// SearchFloat performs a search operation with float32 queries -func (gb *GpuBruteForce[T]) SearchFloat(queries []float32, numQueries uint64, queryDimension uint32, limit uint32) ([]int64, []float32, error) { +// SearchQuantize performs a search operation with base-typed (B) queries; +// cuVS quantizes B -> storage Q internally. +func (gb *GpuBruteForce[B, Q]) SearchQuantize(queries []B, numQueries uint64, queryDimension uint32, limit uint32) ([]int64, []float32, error) { neighbors := make([]int64, numQueries*uint64(limit)) distances := make([]float32, numQueries*uint64(limit)) - if err := gb.SearchFloatInto(queries, numQueries, queryDimension, limit, neighbors, distances); err != nil { + if err := gb.SearchQuantizeInto(queries, numQueries, queryDimension, limit, neighbors, distances); err != nil { return nil, nil, err } return neighbors, distances, nil } // SearchAsync performs a K-Nearest Neighbor search asynchronously. -func (gb *GpuBruteForce[T]) SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +func (gb *GpuBruteForce[B, Q]) SearchAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { if gb.cIndex == nil { return 0, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -324,8 +336,9 @@ func (gb *GpuBruteForce[T]) SearchAsync(queries []T, numQueries uint64, dimensio return uint64(jobID), nil } -// SearchFloat32Async performs a K-Nearest Neighbor search with float32 queries asynchronously. -func (gb *GpuBruteForce[T]) SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +// SearchQuantizeAsync performs a K-Nearest Neighbor search with base-typed (B) +// queries asynchronously; cuVS quantizes B -> storage Q internally. +func (gb *GpuBruteForce[B, Q]) SearchQuantizeAsync(queries []B, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { if gb.cIndex == nil { return 0, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -334,9 +347,9 @@ func (gb *GpuBruteForce[T]) SearchFloat32Async(queries []float32, numQueries uin } var errmsg *C.char - jobID := C.gpu_brute_force_search_float_async( + jobID := C.gpu_brute_force_search_quantize_async( gb.cIndex, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -354,7 +367,7 @@ func (gb *GpuBruteForce[T]) SearchFloat32Async(queries []float32, numQueries uin } // SearchWait waits for an asynchronous search to complete and returns the results. -func (gb *GpuBruteForce[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { +func (gb *GpuBruteForce[B, Q]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { if gb.cIndex == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -386,7 +399,7 @@ func (gb *GpuBruteForce[T]) SearchWait(jobID uint64, numQueries uint64, limit ui } // Cap returns the capacity of the index buffer -func (gb *GpuBruteForce[T]) Cap() uint64 { +func (gb *GpuBruteForce[B, Q]) Cap() uint64 { if gb.cIndex == nil { return 0 } @@ -394,7 +407,7 @@ func (gb *GpuBruteForce[T]) Cap() uint64 { } // Len returns current number of vectors in index -func (gb *GpuBruteForce[T]) Len() uint64 { +func (gb *GpuBruteForce[B, Q]) Len() uint64 { if gb.cIndex == nil { return 0 } @@ -402,7 +415,7 @@ func (gb *GpuBruteForce[T]) Len() uint64 { } // Info returns detailed information about the index as a JSON string. -func (gb *GpuBruteForce[T]) Info() (string, error) { +func (gb *GpuBruteForce[B, Q]) Info() (string, error) { if gb.cIndex == nil { return "", moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -425,7 +438,7 @@ func (gb *GpuBruteForce[T]) Info() (string, error) { } // Destroy frees the C++ GpuBruteForce instance -func (gb *GpuBruteForce[T]) Destroy() error { +func (gb *GpuBruteForce[B, Q]) Destroy() error { if gb.cIndex == nil { return nil } @@ -441,7 +454,7 @@ func (gb *GpuBruteForce[T]) Destroy() error { } // SetFilterColumns registers filter-column metadata. See GpuCagra.SetFilterColumns. -func (gb *GpuBruteForce[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { +func (gb *GpuBruteForce[B, Q]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -458,7 +471,7 @@ func (gb *GpuBruteForce[T]) SetFilterColumns(colMetaJSON string, totalCount uint } // AddFilterChunk appends raw filter-column bytes. See GpuCagra.AddFilterChunk. -func (gb *GpuBruteForce[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (gb *GpuBruteForce[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if gb.cIndex == nil { return moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -489,7 +502,7 @@ func (gb *GpuBruteForce[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitma } // SearchWithFilter runs a filtered K-NN search. predsJSON="" = unfiltered. -func (gb *GpuBruteForce[T]) SearchWithFilter(queries []T, numQueries uint64, dimension uint32, limit uint32, predsJSON string) ([]int64, []float32, error) { +func (gb *GpuBruteForce[B, Q]) SearchWithFilter(queries []Q, numQueries uint64, dimension uint32, limit uint32, predsJSON string) ([]int64, []float32, error) { if gb.cIndex == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -531,8 +544,9 @@ func (gb *GpuBruteForce[T]) SearchWithFilter(queries []T, numQueries uint64, dim return neighbors, distances, nil } -// SearchFloatWithFilter runs a filtered K-NN search with float32 queries. -func (gb *GpuBruteForce[T]) SearchFloatWithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, predsJSON string) ([]int64, []float32, error) { +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) queries; +// cuVS quantizes B -> storage Q internally. +func (gb *GpuBruteForce[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, predsJSON string) ([]int64, []float32, error) { if gb.cIndex == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -544,9 +558,9 @@ func (gb *GpuBruteForce[T]) SearchFloatWithFilter(queries []float32, numQueries cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - cResult := C.gpu_brute_force_search_float_with_filter( + cResult := C.gpu_brute_force_search_quantize_with_filter( gb.cIndex, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -574,12 +588,47 @@ func (gb *GpuBruteForce[T]) SearchFloatWithFilter(queries []float32, numQueries return neighbors, distances, nil } -// SearchFloatWithFilterAsync submits a filtered float32 K-NN search and +// SearchQuantizeWithFilterAsync submits a filtered base-typed (B) K-NN search and // returns a job_id; collect the result with SearchWait. Mirrors -// SearchFloat32Async + the predicate-eval semantics of SearchFloatWithFilter. -// Used by the multi-index brute-force fallback so it runs in parallel with -// the primary IVF/CAGRA shards. -func (gb *GpuBruteForce[T]) SearchFloatWithFilterAsync(queries []float32, numQueries uint64, dimension uint32, limit uint32, predsJSON string) (uint64, error) { +// SearchQuantizeAsync + the predicate-eval semantics of SearchQuantizeWithFilter. +// cuVS quantizes B -> storage Q internally. Used by the multi-index brute-force +// fallback so it runs in parallel with the primary IVF/CAGRA shards. +func (gb *GpuBruteForce[B, Q]) SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, predsJSON string) (uint64, error) { + if gb.cIndex == nil { + return 0, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") + } + if len(queries) == 0 || numQueries == 0 { + return 0, nil + } + + var errmsg *C.char + cPreds := C.CString(predsJSON) + defer C.free(unsafe.Pointer(cPreds)) + + jobID := C.gpu_brute_force_search_quantize_with_filter_async( + gb.cIndex, + unsafe.Pointer(&queries[0]), + C.uint64_t(numQueries), + C.uint32_t(dimension), + C.uint32_t(limit), + cPreds, + unsafe.Pointer(&errmsg), + ) + runtime.KeepAlive(queries) + + if errmsg != nil { + errStr := C.GoString(errmsg) + C.free(unsafe.Pointer(errmsg)) + return 0, moerr.NewInternalErrorNoCtx(errStr) + } + return uint64(jobID), nil +} + +// SearchWithFilterAsync submits a filtered K-NN search with native-typed (T) +// queries and returns a job_id; collect the result with SearchWait. Native +// counterpart of SearchFloatWithFilterAsync (no widening) — lets the filtered +// overflow stay in the base element type T (e.g. half). +func (gb *GpuBruteForce[B, Q]) SearchWithFilterAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32, predsJSON string) (uint64, error) { if gb.cIndex == nil { return 0, moerr.NewInternalErrorNoCtx("GpuBruteForce is not initialized") } @@ -591,9 +640,9 @@ func (gb *GpuBruteForce[T]) SearchFloatWithFilterAsync(queries []float32, numQue cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - jobID := C.gpu_brute_force_search_float_with_filter_async( + jobID := C.gpu_brute_force_search_with_filter_async( gb.cIndex, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), diff --git a/pkg/cuvs/brute_force_test.go b/pkg/cuvs/brute_force_test.go index 91c5086bc548f..83ece6e810a8e 100644 --- a/pkg/cuvs/brute_force_test.go +++ b/pkg/cuvs/brute_force_test.go @@ -30,7 +30,7 @@ func TestGpuBruteForce(t *testing.T) { dataset[i*uint64(dimension)+1] = float32(i) } - index, err := NewGpuBruteForce[float32](dataset, n_vectors, dimension, L2Expanded, 1, 0) + index, err := NewGpuBruteForce[float32, float32](dataset, n_vectors, dimension, L2Expanded, 1, 0) if err != nil { t.Fatalf("Failed to create GpuBruteForce: %v", err) } @@ -62,7 +62,7 @@ func TestGpuBruteForceChunked(t *testing.T) { totalCount := uint64(100) // Create empty index (target type half) - index, err := NewGpuBruteForceEmpty[Float16](totalCount, dimension, L2Expanded, 1, 0) + index, err := NewGpuBruteForceEmpty[float32, Float16](totalCount, dimension, L2Expanded, 1, 0) if err != nil { t.Fatalf("Failed to create GpuBruteForceEmpty: %v", err) } @@ -88,7 +88,7 @@ func TestGpuBruteForceChunked(t *testing.T) { for j := range chunk { chunk[j] = val } - err = index.AddChunkFloat(chunk, chunkSize, nil) + err = index.AddChunkQuantize(chunk, chunkSize, nil) if err != nil { t.Fatalf("AddChunkFloat failed at offset %d: %v", i, err) } @@ -132,7 +132,7 @@ func TestGpuBruteForceFloat16(t *testing.T) { t.Fatalf("Failed to convert dataset to F16: %v", err) } - index, err := NewGpuBruteForce(hDataset, count, dimension, L2Expanded, 1, 0) + index, err := NewGpuBruteForce[Float16, Float16](hDataset, count, dimension, L2Expanded, 1, 0) if err != nil { t.Fatalf("Failed to create F16 GpuBruteForce: %v", err) } @@ -179,7 +179,7 @@ func TestGpuBruteForceFilter(t *testing.T) { pkids[i] = int64(1000 + i) } - idx, err := NewGpuBruteForceEmpty[float32](nVectors, dimension, L2Expanded, 1, 0) + idx, err := NewGpuBruteForceEmpty[float32, float32](nVectors, dimension, L2Expanded, 1, 0) if err != nil { t.Fatalf("NewGpuBruteForceEmpty: %v", err) } @@ -192,7 +192,7 @@ func TestGpuBruteForceFilter(t *testing.T) { if err = idx.SetFilterColumns(colMetaJSON, nVectors); err != nil { t.Fatalf("SetFilterColumns: %v", err) } - if err = idx.AddChunkFloat(dataset, nVectors, pkids); err != nil { + if err = idx.AddChunkQuantize(dataset, nVectors, pkids); err != nil { t.Fatalf("AddChunkFloat: %v", err) } // One column of int64; row i value = i. No nulls. @@ -214,7 +214,7 @@ func TestGpuBruteForceFilter(t *testing.T) { // Query closest to row 0; without filter NN would be pkid 1000 (row 0). queries := []float32{0.0, 0.0} predsJSON := `[{"col":0,"op":">","val":50}]` - jobID, err := idx.SearchFloatWithFilterAsync(queries, 1, dimension, 1, predsJSON) + jobID, err := idx.SearchQuantizeWithFilterAsync(queries, 1, dimension, 1, predsJSON) if err != nil { t.Fatalf("SearchFloatWithFilterAsync: %v", err) } @@ -231,7 +231,7 @@ func TestGpuBruteForceFilter(t *testing.T) { } // Sanity: empty preds JSON falls through to unfiltered NN (pkid 1000). - jobID2, err := idx.SearchFloatWithFilterAsync(queries, 1, dimension, 1, "") + jobID2, err := idx.SearchQuantizeWithFilterAsync(queries, 1, dimension, 1, "") if err != nil { t.Fatalf("SearchFloatWithFilterAsync (no preds): %v", err) } @@ -254,8 +254,8 @@ func BenchmarkGpuAddChunkAndSearchBruteForceF16(b *testing.B) { dataset[i] = rand.Float32() } - // Use Float16 as internal type - index, err := NewGpuBruteForceEmpty[Float16](uint64(totalCount), dimension, L2Expanded, 8, 0) + // Use Float16 storage with float32 base/query (quantize f32 -> half). + index, err := NewGpuBruteForceEmpty[float32, Float16](uint64(totalCount), dimension, L2Expanded, 8, 0) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -268,7 +268,7 @@ func BenchmarkGpuAddChunkAndSearchBruteForceF16(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -286,7 +286,7 @@ func BenchmarkGpuAddChunkAndSearchBruteForceF16(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, _, err := index.SearchFloat(queries, 1, dimension, 10) + _, _, err := index.SearchQuantize(queries, 1, dimension, 10) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -294,7 +294,7 @@ func BenchmarkGpuAddChunkAndSearchBruteForceF16(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - neighbors, _, err := index.SearchFloat(queries, numQueries, dimension, limit) + neighbors, _, err := index.SearchQuantize(queries, numQueries, dimension, limit) if err != nil { return nil, err } @@ -311,7 +311,7 @@ func BenchmarkGpuBruteForceF32(b *testing.B) { dataset[i] = rand.Float32() } - index, err := NewGpuBruteForce[float32](dataset, uint64(totalCount), dimension, L2Expanded, 8, 0) + index, err := NewGpuBruteForce[float32, float32](dataset, uint64(totalCount), dimension, L2Expanded, 8, 0) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -331,7 +331,7 @@ func BenchmarkGpuBruteForceF32(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, _, err := index.SearchFloat(queries, 1, dimension, 10) + _, _, err := index.SearchQuantize(queries, 1, dimension, 10) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -339,7 +339,7 @@ func BenchmarkGpuBruteForceF32(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - neighbors, _, err := index.SearchFloat(queries, numQueries, dimension, limit) + neighbors, _, err := index.SearchQuantize(queries, numQueries, dimension, limit) if err != nil { return nil, err } diff --git a/pkg/cuvs/cagra.go b/pkg/cuvs/cagra.go index e14a53871a769..8ca474c3a8000 100644 --- a/pkg/cuvs/cagra.go +++ b/pkg/cuvs/cagra.go @@ -32,7 +32,7 @@ import ( ) // GpuCagra represents the C++ gpu_cagra_t object. -type GpuCagra[T VectorType] struct { +type GpuCagra[B, Q VectorType] struct { cCagra C.gpu_cagra_c dimension uint32 nthread uint32 @@ -43,7 +43,7 @@ type GpuCagra[T VectorType] struct { // SetBatchWindow sets the batching window in microseconds for search operations. // A window of 0 disables batching; any positive value enables batching with that delay. -func (gi *GpuCagra[T]) SetBatchWindow(windowUs int64) error { +func (gi *GpuCagra[B, Q]) SetBatchWindow(windowUs int64) error { gi.batchWindowUs = windowUs if gi.cCagra != nil { var errmsg *C.char @@ -61,7 +61,7 @@ func (gi *GpuCagra[T]) SetBatchWindow(windowUs int64) error { // flag. false (default): dispatch eagerly at the full batch size. true: wait for // the batch to fill or the window to elapse, then dispatch at the real size. // Has no effect unless the batch window is > 0. -func (gi *GpuCagra[T]) SetDynbConservativeDispatch(enable bool) error { +func (gi *GpuCagra[B, Q]) SetDynbConservativeDispatch(enable bool) error { gi.dynbConservativeDispatch = enable if gi.cCagra != nil { var errmsg *C.char @@ -77,13 +77,14 @@ func (gi *GpuCagra[T]) SetDynbConservativeDispatch(enable bool) error { // NewGpuCagra creates a new GpuCagra instance from a dataset. // ids may be nil to use internal sequential IDs (0..count-1). -func NewGpuCagra[T VectorType](dataset []T, count uint64, dimension uint32, metric DistanceType, - bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuCagra[T], error) { +func NewGpuCagra[B, Q VectorType](dataset []Q, count uint64, dimension uint32, metric DistanceType, + bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuCagra[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -111,6 +112,7 @@ func NewGpuCagra[T VectorType](dataset []T, count uint64, dimension uint32, metr C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), cIds, unsafe.Pointer(&errmsg), @@ -129,7 +131,7 @@ func NewGpuCagra[T VectorType](dataset []T, count uint64, dimension uint32, metr return nil, moerr.NewInternalErrorNoCtx("failed to create GpuCagra") } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: dimension, nthread: nthread, @@ -138,13 +140,14 @@ func NewGpuCagra[T VectorType](dataset []T, count uint64, dimension uint32, metr } // NewGpuCagraFromFile creates a new GpuCagra instance by loading from a file. -func NewGpuCagraFromFile[T VectorType](filename string, dimension uint32, metric DistanceType, - bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[T], error) { +func NewGpuCagraFromFile[B, Q VectorType](filename string, dimension uint32, metric DistanceType, + bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cFilename := C.CString(filename) defer C.free(unsafe.Pointer(cFilename)) @@ -169,6 +172,7 @@ func NewGpuCagraFromFile[T VectorType](filename string, dimension uint32, metric C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), unsafe.Pointer(&errmsg), ) @@ -184,7 +188,7 @@ func NewGpuCagraFromFile[T VectorType](filename string, dimension uint32, metric return nil, moerr.NewInternalErrorNoCtx("failed to load GpuCagra from file") } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: dimension, nthread: nthread, @@ -196,8 +200,8 @@ func NewGpuCagraFromFile[T VectorType](filename string, dimension uint32, metric // For Sharded loads we peek manifest.json to learn the saved shard count and // truncate `devices` to that count, so the C++ worker only spawns threads / // RMM pools on devices that will actually host a shard. -func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, metric DistanceType, - bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[T], error) { +func NewGpuCagraFromDataDirectory[B, Q VectorType](dir string, dimension uint32, metric DistanceType, + bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } @@ -208,7 +212,8 @@ func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, me return nil, err } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() cDevices := make([]C.int, len(devices)) for i, d := range devices { cDevices[i] = C.int(d) @@ -230,6 +235,7 @@ func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, me C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -264,7 +270,7 @@ func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, me return nil, moerr.NewInternalErrorNoCtx(errStr) } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: dimension, nthread: nthread, @@ -273,7 +279,7 @@ func NewGpuCagraFromDataDirectory[T VectorType](dir string, dimension uint32, me } // Destroy frees the C++ gpu_cagra_t instance -func (gi *GpuCagra[T]) Destroy() error { +func (gi *GpuCagra[B, Q]) Destroy() error { if gi.cCagra == nil { return nil } @@ -289,7 +295,7 @@ func (gi *GpuCagra[T]) Destroy() error { } // Start initializes the worker and resources -func (gi *GpuCagra[T]) Start() error { +func (gi *GpuCagra[B, Q]) Start() error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -317,7 +323,7 @@ func (gi *GpuCagra[T]) Start() error { } // Build triggers the build or file loading process -func (gi *GpuCagra[T]) Build() error { +func (gi *GpuCagra[B, Q]) Build() error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -332,13 +338,14 @@ func (gi *GpuCagra[T]) Build() error { } // NewGpuCagraEmpty creates a new GpuCagra instance with pre-allocated buffer but no data yet. -func NewGpuCagraEmpty[T VectorType](totalCount uint64, dimension uint32, metric DistanceType, - bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[T], error) { +func NewGpuCagraEmpty[B, Q VectorType](totalCount uint64, dimension uint32, metric DistanceType, + bp CagraBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuCagra[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -360,6 +367,7 @@ func NewGpuCagraEmpty[T VectorType](totalCount uint64, dimension uint32, metric C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -376,7 +384,7 @@ func NewGpuCagraEmpty[T VectorType](totalCount uint64, dimension uint32, metric return nil, moerr.NewInternalErrorNoCtx("failed to create empty GpuCagra") } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: dimension, nthread: nthread, @@ -385,7 +393,7 @@ func NewGpuCagraEmpty[T VectorType](totalCount uint64, dimension uint32, metric } // AddChunk adds a chunk of data to the pre-allocated buffer. -func (gi *GpuCagra[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (gi *GpuCagra[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -416,8 +424,10 @@ func (gi *GpuCagra[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error return nil } -// AddChunkFloat adds a chunk of float32 data, performing on-the-fly quantization if needed. -func (gi *GpuCagra[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize adds a chunk of base-typed (B) data, quantizing natively to +// the storage type Q (int8/uint8) via the B-source quantizer. base_data is the +// raw bytes of chunkCount*dim B-typed elements. No f32 detour. +func (gi *GpuCagra[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -430,9 +440,9 @@ func (gi *GpuCagra[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []i if len(ids) > 0 { cIds = (*C.int64_t)(unsafe.Pointer(&ids[0])) } - C.gpu_cagra_add_chunk_float( + C.gpu_cagra_add_chunk_quantize( gi.cCagra, - (*C.float)(&chunk[0]), + unsafe.Pointer(&chunk[0]), C.uint64_t(chunkCount), cIds, unsafe.Pointer(&errmsg), @@ -448,8 +458,9 @@ func (gi *GpuCagra[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []i return nil } -// TrainQuantizer trains the scalar quantizer (if T is 1-byte) -func (gi *GpuCagra[T]) TrainQuantizer(trainData []float32, nSamples uint64) error { +// TrainQuantizer trains the scalar quantizer (if Q is 1-byte) from base-typed +// (B) training data. +func (gi *GpuCagra[B, Q]) TrainQuantizer(trainData []B, nSamples uint64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -460,7 +471,7 @@ func (gi *GpuCagra[T]) TrainQuantizer(trainData []float32, nSamples uint64) erro var errmsg *C.char C.gpu_cagra_train_quantizer( gi.cCagra, - (*C.float)(&trainData[0]), + unsafe.Pointer(&trainData[0]), C.uint64_t(nSamples), unsafe.Pointer(&errmsg), ) @@ -475,7 +486,7 @@ func (gi *GpuCagra[T]) TrainQuantizer(trainData []float32, nSamples uint64) erro } // SetQuantizer sets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuCagra[T]) SetQuantizer(min, max float32) error { +func (gi *GpuCagra[B, Q]) SetQuantizer(min, max float32) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -497,7 +508,7 @@ func (gi *GpuCagra[T]) SetQuantizer(min, max float32) error { } // GetQuantizer gets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuCagra[T]) GetQuantizer() (float32, float32, error) { +func (gi *GpuCagra[B, Q]) GetQuantizer() (float32, float32, error) { if gi.cCagra == nil { return 0, 0, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -520,7 +531,7 @@ func (gi *GpuCagra[T]) GetQuantizer() (float32, float32, error) { } // Save serializes the index to a file -func (gi *GpuCagra[T]) Save(filename string) error { +func (gi *GpuCagra[B, Q]) Save(filename string) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -538,7 +549,7 @@ func (gi *GpuCagra[T]) Save(filename string) error { } // Pack saves the index to a .tar or .tar.gz file using save_dir. -func (gi *GpuCagra[T]) Pack(filename string) error { +func (gi *GpuCagra[B, Q]) Pack(filename string) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -567,7 +578,7 @@ func (gi *GpuCagra[T]) Pack(filename string) error { // mode overrides the distribution mode at load time — pass Replicated to broadcast // a SINGLE_GPU .tar to all GPUs without rebuilding. // The index must already be initialized and started before calling Unpack. -func (gi *GpuCagra[T]) Unpack(filename string, mode DistributionMode) error { +func (gi *GpuCagra[B, Q]) Unpack(filename string, mode DistributionMode) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -596,7 +607,7 @@ func (gi *GpuCagra[T]) Unpack(filename string, mode DistributionMode) error { } // DeleteId removes an ID from the index (soft delete). -func (gi *GpuCagra[T]) DeleteId(id int64) error { +func (gi *GpuCagra[B, Q]) DeleteId(id int64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -614,7 +625,7 @@ func (gi *GpuCagra[T]) DeleteId(id int64) error { // path; if profiling shows the cgo crossing dominates we can swap to a // single batched cgo entry (the C++ side already does the host-side // id_to_index_ lookup; the loop is per-id). -func (gi *GpuCagra[T]) DeleteIds(ids []int64) error { +func (gi *GpuCagra[B, Q]) DeleteIds(ids []int64) error { for _, id := range ids { if err := gi.DeleteId(id); err != nil { return err @@ -623,8 +634,8 @@ func (gi *GpuCagra[T]) DeleteIds(ids []int64) error { return nil } -func (gi *GpuCagra[T]) adjustSearchParams(sp CagraSearchParams, limit uint32) CagraSearchParams { - qtype := GetQuantization[T]() +func (gi *GpuCagra[B, Q]) adjustSearchParams(sp CagraSearchParams, limit uint32) CagraSearchParams { + qtype := GetQuantization[Q]() isByteType := (qtype == INT8 || qtype == UINT8) if isByteType { @@ -641,7 +652,7 @@ func (gi *GpuCagra[T]) adjustSearchParams(sp CagraSearchParams, limit uint32) Ca } // Search performs a K-Nearest Neighbor search -func (gi *GpuCagra[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (SearchResult, error) { +func (gi *GpuCagra[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (SearchResult, error) { if gi.cCagra == nil { return SearchResult{}, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -696,7 +707,7 @@ func (gi *GpuCagra[T]) Search(queries []T, numQueries uint64, dimension uint32, } // SearchFloat performs a K-Nearest Neighbor search with float32 queries -func (gi *GpuCagra[T]) SearchFloat(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (SearchResult, error) { +func (gi *GpuCagra[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (SearchResult, error) { if gi.cCagra == nil { return SearchResult{}, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -712,9 +723,9 @@ func (gi *GpuCagra[T]) SearchFloat(queries []float32, numQueries uint64, dimensi search_width: C.size_t(sp.SearchWidth), } - res := C.gpu_cagra_search_float( + res := C.gpu_cagra_search_quantize( gi.cCagra, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -751,12 +762,12 @@ func (gi *GpuCagra[T]) SearchFloat(queries []float32, numQueries uint64, dimensi } // SearchAsync performs a K-Nearest Neighbor search asynchronously. -func (gi *GpuCagra[T]) SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +func (gi *GpuCagra[B, Q]) SearchAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { return gi.SearchAsyncWithParams(queries, numQueries, dimension, limit, DefaultCagraSearchParams()) } // SearchAsyncWithParams performs a K-Nearest Neighbor search asynchronously with custom parameters. -func (gi *GpuCagra[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (uint64, error) { +func (gi *GpuCagra[B, Q]) SearchAsyncWithParams(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (uint64, error) { if gi.cCagra == nil { return 0, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -792,13 +803,10 @@ func (gi *GpuCagra[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dim return uint64(jobID), nil } -// SearchFloat32Async performs a K-Nearest Neighbor search with float32 queries asynchronously. -func (gi *GpuCagra[T]) SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { - return gi.SearchFloat32AsyncWithParams(queries, numQueries, dimension, limit, DefaultCagraSearchParams()) -} - -// SearchFloat32AsyncWithParams performs a K-Nearest Neighbor search with float32 queries asynchronously with custom parameters. -func (gi *GpuCagra[T]) SearchFloat32AsyncWithParams(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (uint64, error) { +// SearchQuantizeAsyncWithParams submits an async KNN search with a base-typed (B) +// query; the index converts B to its storage type Q on device (B==Q copy, or the +// learned/cast quantizer for narrower Q). Unifies the former float32 and half query paths. +func (gi *GpuCagra[B, Q]) SearchQuantizeAsyncWithParams(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) (uint64, error) { if gi.cCagra == nil { return 0, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -814,9 +822,9 @@ func (gi *GpuCagra[T]) SearchFloat32AsyncWithParams(queries []float32, numQuerie search_width: C.size_t(sp.SearchWidth), } - jobID := C.gpu_cagra_search_float_async( + jobID := C.gpu_cagra_search_quantize_async( gi.cCagra, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -835,7 +843,7 @@ func (gi *GpuCagra[T]) SearchFloat32AsyncWithParams(queries []float32, numQuerie } // SearchWait waits for an asynchronous search to complete and returns the results. -func (gi *GpuCagra[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { +func (gi *GpuCagra[B, Q]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { if gi.cCagra == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -868,7 +876,7 @@ func (gi *GpuCagra[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) } // Cap returns the capacity of the index buffer -func (gi *GpuCagra[T]) Cap() uint64 { +func (gi *GpuCagra[B, Q]) Cap() uint64 { if gi.cCagra == nil { return 0 } @@ -876,7 +884,7 @@ func (gi *GpuCagra[T]) Cap() uint64 { } // Len returns current number of vectors in index -func (gi *GpuCagra[T]) Len() uint64 { +func (gi *GpuCagra[B, Q]) Len() uint64 { if gi.cCagra == nil { return 0 } @@ -886,7 +894,7 @@ func (gi *GpuCagra[T]) Len() uint64 { // GetFilterColMetaJSON returns the INCLUDE-column metadata of the loaded // index as a JSON string ready to be re-fed into SetFilterColumns. Returns // "" for indexes that were built without INCLUDE columns. -func (gi *GpuCagra[T]) GetFilterColMetaJSON() string { +func (gi *GpuCagra[B, Q]) GetFilterColMetaJSON() string { if gi.cCagra == nil { return "" } @@ -904,7 +912,7 @@ func (gi *GpuCagra[T]) GetFilterColMetaJSON() string { } // Info returns detailed information about the index as a JSON string. -func (gi *GpuCagra[T]) Info() (string, error) { +func (gi *GpuCagra[B, Q]) Info() (string, error) { if gi.cCagra == nil { return "", moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -928,7 +936,7 @@ func (gi *GpuCagra[T]) Info() (string, error) { // Extend adds more vectors to the index (single-GPU only). // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuCagra[T]) Extend(additionalData []T, numVectors uint64, newIDs []int64) error { +func (gi *GpuCagra[B, Q]) Extend(additionalData []Q, numVectors uint64, newIDs []int64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -961,7 +969,7 @@ func (gi *GpuCagra[T]) Extend(additionalData []T, numVectors uint64, newIDs []in } // MergeGpuCagra combines multiple single-GPU GpuCagra indices into a new one. -func MergeGpuCagra[T VectorType](indices []*GpuCagra[T], nthread uint32, devices []int) (*GpuCagra[T], error) { +func MergeGpuCagra[B, Q VectorType](indices []*GpuCagra[B, Q], nthread uint32, devices []int) (*GpuCagra[B, Q], error) { if len(indices) == 0 { return nil, moerr.NewInternalErrorNoCtx("no indices to merge") } @@ -1001,7 +1009,7 @@ func MergeGpuCagra[T VectorType](indices []*GpuCagra[T], nthread uint32, devices return nil, moerr.NewInternalErrorNoCtx("failed to merge GpuCagra indices") } - return &GpuCagra[T]{ + return &GpuCagra[B, Q]{ cCagra: cCagra, dimension: indices[0].dimension, nthread: nthread, @@ -1017,7 +1025,7 @@ type SearchResult struct { // SaveToDir saves the index files to a directory using gpu_cagra_save_dir. // This is used by CagraModel to save to a directory before packing to tar. -func (gi *GpuCagra[T]) SaveToDir(dirPath string) error { +func (gi *GpuCagra[B, Q]) SaveToDir(dirPath string) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1036,7 +1044,7 @@ func (gi *GpuCagra[T]) SaveToDir(dirPath string) error { // LoadFromDir loads index components from a directory using gpu_cagra_load_dir. // mode overrides the distribution mode at load time. // The index must already be initialized and started before calling LoadFromDir. -func (gi *GpuCagra[T]) LoadFromDir(dirPath string, mode DistributionMode) error { +func (gi *GpuCagra[B, Q]) LoadFromDir(dirPath string, mode DistributionMode) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1056,7 +1064,7 @@ func (gi *GpuCagra[T]) LoadFromDir(dirPath string, mode DistributionMode) error // colMetaJSON is a JSON array of {"name":"...","type":N} entries, where N is // 0=int32, 1=int64, 2=float32, 3=float64, 4=uint64 (VARCHAR hash). // Must be called after Start() and before Build(). -func (gi *GpuCagra[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { +func (gi *GpuCagra[B, Q]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1078,7 +1086,7 @@ func (gi *GpuCagra[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) e // matching MO's null-mask convention) of ceil(nrows/32) entries, or nil when // the chunk has no nulls. // Ownership transfers to C++ at call return — the Go slice can be freed. -func (gi *GpuCagra[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (gi *GpuCagra[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if gi.cCagra == nil { return moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1110,7 +1118,7 @@ func (gi *GpuCagra[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []u // SearchWithFilter runs a filtered K-NN search. predsJSON is a JSON predicate // array; passing "" yields unfiltered behavior identical to Search(). -func (gi *GpuCagra[T]) SearchWithFilter(queries []T, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (SearchResult, error) { +func (gi *GpuCagra[B, Q]) SearchWithFilter(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (SearchResult, error) { if gi.cCagra == nil { return SearchResult{}, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1161,8 +1169,9 @@ func (gi *GpuCagra[T]) SearchWithFilter(queries []T, numQueries uint64, dimensio return SearchResult{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilter runs a filtered K-NN search with float32 queries. -func (gi *GpuCagra[T]) SearchFloatWithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (SearchResult, error) { +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) +// queries; the index converts B to storage T (copy / quantize / f32->f16 cast). +func (gi *GpuCagra[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (SearchResult, error) { if gi.cCagra == nil { return SearchResult{}, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1180,9 +1189,9 @@ func (gi *GpuCagra[T]) SearchFloatWithFilter(queries []float32, numQueries uint6 cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - res := C.gpu_cagra_search_float_with_filter( + res := C.gpu_cagra_search_quantize_with_filter( gi.cCagra, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -1213,12 +1222,12 @@ func (gi *GpuCagra[T]) SearchFloatWithFilter(queries []float32, numQueries uint6 return SearchResult{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilterAsync submits a filtered float32 K-NN search and -// returns a job_id; collect the result with SearchWait. Mirrors -// SearchFloat32AsyncWithParams + the predicate-eval semantics of -// SearchFloatWithFilter. Used by MultiGpuCagra to dispatch per-shard -// filtered searches in parallel. -func (gi *GpuCagra[T]) SearchFloatWithFilterAsync(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (uint64, error) { +// SearchQuantizeWithFilterAsync submits a filtered K-NN search with base-typed +// (B) queries and returns a job_id; collect the result with SearchWait. Mirrors +// SearchQuantizeAsyncWithParams + the predicate-eval semantics of +// SearchQuantizeWithFilter. Used by MultiGpuCagra to dispatch per-shard +// filtered searches in parallel. The index converts B to storage T. +func (gi *GpuCagra[B, Q]) SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) (uint64, error) { if gi.cCagra == nil { return 0, moerr.NewInternalErrorNoCtx("GpuCagra is not initialized") } @@ -1236,9 +1245,9 @@ func (gi *GpuCagra[T]) SearchFloatWithFilterAsync(queries []float32, numQueries cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - jobID := C.gpu_cagra_search_float_with_filter_async( + jobID := C.gpu_cagra_search_quantize_with_filter_async( gi.cCagra, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), diff --git a/pkg/cuvs/cagra_test.go b/pkg/cuvs/cagra_test.go index 7095aabd3ccc4..704ed0b71af4d 100644 --- a/pkg/cuvs/cagra_test.go +++ b/pkg/cuvs/cagra_test.go @@ -36,7 +36,7 @@ func TestGpuCagra(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -80,7 +80,7 @@ func TestGpuCagraSaveLoad(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -97,7 +97,7 @@ func TestGpuCagraSaveLoad(t *testing.T) { defer os.Remove(filename) index.Destroy() - index2, err := NewGpuCagraFromFile[float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuCagraFromFile[float32, float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuCagra from file: %v", err) } @@ -129,7 +129,7 @@ func TestGpuCagraPackUnpack(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -147,7 +147,7 @@ func TestGpuCagraPackUnpack(t *testing.T) { } defer os.Remove(filename) - index2, err := NewGpuCagraEmpty[float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuCagraEmpty[float32, float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuCagraEmpty failed: %v", err) } @@ -180,7 +180,7 @@ func TestGpuCagraFromDataDirectory(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -223,7 +223,7 @@ func TestGpuCagraFromDataDirectory(t *testing.T) { t.Fatalf("Unpack to dir failed: %v", err) } - index2, err := NewGpuCagraFromDataDirectory[float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuCagraFromDataDirectory[float32, float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuCagraFromDataDirectory failed: %v", err) } @@ -271,7 +271,7 @@ func TestGpuShardedCagra(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) if err != nil { t.Fatalf("Failed to create sharded CAGRA: %v", err) } @@ -306,7 +306,7 @@ func TestGpuCagraChunked(t *testing.T) { bp.GraphDegree = 128 // Create empty index (target type int8) - index, err := NewGpuCagraEmpty[int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index, err := NewGpuCagraEmpty[float32, int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuCagraEmpty: %v", err) } @@ -325,7 +325,7 @@ func TestGpuCagraChunked(t *testing.T) { for j := range chunk { chunk[j] = val } - err = index.AddChunkFloat(chunk, chunkSize, nil) + err = index.AddChunkQuantize(chunk, chunkSize, nil) if err != nil { t.Fatalf("AddChunkFloat failed at offset %d: %v", i, err) } @@ -380,7 +380,7 @@ func TestGpuCagraExtend(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -436,11 +436,11 @@ func TestGpuCagraMerge(t *testing.T) { bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - idx1, err := NewGpuCagra[float32](ds1, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + idx1, err := NewGpuCagra[float32, float32](ds1, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create idx1: %v", err) } - idx2, err := NewGpuCagra[float32](ds2, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + idx2, err := NewGpuCagra[float32, float32](ds2, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create idx2: %v", err) } @@ -455,7 +455,7 @@ func TestGpuCagraMerge(t *testing.T) { defer idx1.Destroy() defer idx2.Destroy() - merged, err := MergeGpuCagra([]*GpuCagra[float32]{idx1, idx2}, 1, devices) + merged, err := MergeGpuCagra([]*GpuCagra[float32, float32]{idx1, idx2}, 1, devices) if err != nil { t.Fatalf("Merge failed: %v", err) } @@ -514,14 +514,14 @@ func TestGpuCagraMergeWithIds(t *testing.T) { } bp := DefaultCagraBuildParams() - idx1, err := NewGpuCagra[float32](ds1, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, ids1) + idx1, err := NewGpuCagra[float32, float32](ds1, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, ids1) if err != nil { t.Fatalf("Failed to create idx1: %v", err) } idx1.Start() idx1.Build() - idx2, err := NewGpuCagra[float32](ds2, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, ids2) + idx2, err := NewGpuCagra[float32, float32](ds2, count, dimension, L2Expanded, bp, devices, 1, SingleGpu, ids2) if err != nil { t.Fatalf("Failed to create idx2: %v", err) } @@ -531,7 +531,7 @@ func TestGpuCagraMergeWithIds(t *testing.T) { defer idx1.Destroy() defer idx2.Destroy() - merged, err := MergeGpuCagra([]*GpuCagra[float32]{idx1, idx2}, 1, devices) + merged, err := MergeGpuCagra([]*GpuCagra[float32, float32]{idx1, idx2}, 1, devices) if err != nil { t.Fatalf("Merge failed: %v", err) } @@ -583,7 +583,7 @@ func TestGpuCagraDeleteId(t *testing.T) { } bp := DefaultCagraBuildParams() - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } @@ -643,7 +643,7 @@ func TestGpuReplicatedCagra(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) if err != nil { t.Fatalf("Failed to create replicated CAGRA: %v", err) } @@ -684,7 +684,7 @@ func BenchmarkGpuShardedCagra(b *testing.B) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 16, Sharded, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 16, Sharded, nil) if err != nil { b.Fatalf("Failed to create sharded CAGRA: %v", err) } @@ -720,7 +720,7 @@ func BenchmarkGpuShardedCagra(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -743,7 +743,7 @@ func BenchmarkGpuSingleCagra(b *testing.B) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) if err != nil { b.Fatalf("Failed to create single CAGRA: %v", err) } @@ -779,7 +779,7 @@ func BenchmarkGpuSingleCagra(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -805,7 +805,7 @@ func BenchmarkGpuReplicatedCagra(b *testing.B) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) if err != nil { b.Fatalf("Failed to create replicated CAGRA: %v", err) } @@ -841,7 +841,7 @@ func BenchmarkGpuReplicatedCagra(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -866,7 +866,7 @@ func BenchmarkGpuAddChunkAndSearchCagraF16(b *testing.B) { bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 // Use Float16 as internal type - index, err := NewGpuCagraEmpty[Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuCagraEmpty[float32, Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -879,7 +879,7 @@ func BenchmarkGpuAddChunkAndSearchCagraF16(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -899,7 +899,7 @@ func BenchmarkGpuAddChunkAndSearchCagraF16(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -907,7 +907,7 @@ func BenchmarkGpuAddChunkAndSearchCagraF16(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -930,7 +930,7 @@ func BenchmarkGpuAddChunkAndSearchCagraInt8(b *testing.B) { bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 // Use int8 as internal type - index, err := NewGpuCagraEmpty[int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuCagraEmpty[float32, int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -943,7 +943,7 @@ func BenchmarkGpuAddChunkAndSearchCagraInt8(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -963,7 +963,7 @@ func BenchmarkGpuAddChunkAndSearchCagraInt8(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -971,7 +971,7 @@ func BenchmarkGpuAddChunkAndSearchCagraInt8(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -990,7 +990,7 @@ func TestGpuCagraLargeTopK(t *testing.T) { devices := []int{0} bp := DefaultCagraBuildParams() - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuCagra: %v", err) } diff --git a/pkg/cuvs/consolidate_test.go b/pkg/cuvs/consolidate_test.go index 1be296620b5fc..2a89a4db8477b 100644 --- a/pkg/cuvs/consolidate_test.go +++ b/pkg/cuvs/consolidate_test.go @@ -147,7 +147,7 @@ func TestShardedLoadWithFewerSavedShards(t *testing.T) { // --- Save phase: 2 shards over devs[:2] --- saveDevs := devs[:2] - src, err := NewGpuIvfFlat[float32](dataset, nVectors, dimension, L2Expanded, + src, err := NewGpuIvfFlat[float32, float32](dataset, nVectors, dimension, L2Expanded, bp, saveDevs, uint32(len(saveDevs)), Sharded, nil) if err != nil { t.Fatalf("save-side build: %v", err) @@ -182,7 +182,7 @@ func TestShardedLoadWithFewerSavedShards(t *testing.T) { // --- Load phase: caller supplies ALL available devs; wrapper should // truncate to the saved 2. --- - dst, err := NewGpuIvfFlatFromDataDirectory[float32](extractDir, dimension, L2Expanded, + dst, err := NewGpuIvfFlatFromDataDirectory[float32, float32](extractDir, dimension, L2Expanded, bp, devs, uint32(len(devs)), Sharded) if err != nil { t.Fatalf("load with extra devices: %v", err) diff --git a/pkg/cuvs/get_centers_test.go b/pkg/cuvs/get_centers_test.go index 8e02bfb67f0be..b484812b918f5 100644 --- a/pkg/cuvs/get_centers_test.go +++ b/pkg/cuvs/get_centers_test.go @@ -33,7 +33,7 @@ func testIvfFlatGetCenters[T VectorType](t *testing.T, name string) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 16 - index, err := NewGpuIvfFlat[T](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, T](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } diff --git a/pkg/cuvs/helper.go b/pkg/cuvs/helper.go index 8406cfaba036d..9d0bb4b36d153 100644 --- a/pkg/cuvs/helper.go +++ b/pkg/cuvs/helper.go @@ -186,7 +186,6 @@ type GpuIndexBase interface { // GpuIndex is a generic interface for all GPU-accelerated indexes that support async search. type GpuIndex[T VectorType] interface { SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) - SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) Destroy() error Cap() uint64 diff --git a/pkg/cuvs/info_test.go b/pkg/cuvs/info_test.go index 595e77e7777ee..386fd3e58a6e4 100644 --- a/pkg/cuvs/info_test.go +++ b/pkg/cuvs/info_test.go @@ -105,16 +105,16 @@ func TestIndexInfoComprehensive(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err = NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-Flat": bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err = NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-PQ": bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 16 - index, err = NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) } case "Float16": dataset := make([]Float16, n_vectors*uint64(dimension)) @@ -127,16 +127,16 @@ func TestIndexInfoComprehensive(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err = NewGpuCagra[Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuCagra[Float16, Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-Flat": bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err = NewGpuIvfFlat[Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfFlat[float32, Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-PQ": bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 16 - index, err = NewGpuIvfPq[Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfPq[Float16, Float16](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) } case "int8": dataset := make([]int8, n_vectors*uint64(dimension)) @@ -149,16 +149,20 @@ func TestIndexInfoComprehensive(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err = NewGpuCagra[int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + // int8 is a STORAGE (quantization) type, not a base type: the [B,Q] + // model only supports a float base (f32/f16) quantized to int8/uint8. + // Base f32, storage int8 (the wired f32xint8 combo); dataset is the + // storage type Q (see NewGpuCagra signature). + index, err = NewGpuCagra[float32, int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-Flat": bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err = NewGpuIvfFlat[int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfFlat[float32, int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-PQ": bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 16 - index, err = NewGpuIvfPq[int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfPq[float32, int8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) } case "uint8": dataset := make([]uint8, n_vectors*uint64(dimension)) @@ -171,16 +175,17 @@ func TestIndexInfoComprehensive(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 256 bp.GraphDegree = 128 - index, err = NewGpuCagra[uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + // uint8 storage from a float base (wired f32xuint8 combo); see int8 note above. + index, err = NewGpuCagra[float32, uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-Flat": bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err = NewGpuIvfFlat[uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfFlat[float32, uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) case "IVF-PQ": bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 16 - index, err = NewGpuIvfPq[uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) + index, err = NewGpuIvfPq[float32, uint8](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, distMode, nil) } } diff --git a/pkg/cuvs/ivf_flat.go b/pkg/cuvs/ivf_flat.go index cdcd1afdbdf5a..5be75b7db9a32 100644 --- a/pkg/cuvs/ivf_flat.go +++ b/pkg/cuvs/ivf_flat.go @@ -32,7 +32,7 @@ import ( ) // GpuIvfFlat represents the C++ gpu_ivf_flat_t object. -type GpuIvfFlat[T VectorType] struct { +type GpuIvfFlat[B, Q VectorType] struct { cIvfFlat C.gpu_ivf_flat_c dimension uint32 nthread uint32 @@ -43,7 +43,7 @@ type GpuIvfFlat[T VectorType] struct { // SetBatchWindow sets the batching window in microseconds for search operations. // A window of 0 disables batching; any positive value enables batching with that delay. -func (gi *GpuIvfFlat[T]) SetBatchWindow(windowUs int64) error { +func (gi *GpuIvfFlat[B, Q]) SetBatchWindow(windowUs int64) error { gi.batchWindowUs = windowUs if gi.cIvfFlat != nil { var errmsg *C.char @@ -61,7 +61,7 @@ func (gi *GpuIvfFlat[T]) SetBatchWindow(windowUs int64) error { // flag. false (default): dispatch eagerly at the full batch size. true: wait for // the batch to fill or the window to elapse, then dispatch at the real size. // Has no effect unless the batch window is > 0. -func (gi *GpuIvfFlat[T]) SetDynbConservativeDispatch(enable bool) error { +func (gi *GpuIvfFlat[B, Q]) SetDynbConservativeDispatch(enable bool) error { gi.dynbConservativeDispatch = enable if gi.cIvfFlat != nil { var errmsg *C.char @@ -80,13 +80,14 @@ func (gi *GpuIvfFlat[T]) SetDynbConservativeDispatch(enable bool) error { // For Sharded mode the shard count is len(devices) (one shard per GPU); // to use fewer shards than the GPUs you have available, just pass a // shorter `devices` slice. -func NewGpuIvfFlat[T VectorType](dataset []T, count uint64, dimension uint32, metric DistanceType, - bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuIvfFlat[T], error) { +func NewGpuIvfFlat[B, Q VectorType](dataset []Q, count uint64, dimension uint32, metric DistanceType, + bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuIvfFlat[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -114,6 +115,7 @@ func NewGpuIvfFlat[T VectorType](dataset []T, count uint64, dimension uint32, me C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), cIds, unsafe.Pointer(&errmsg), @@ -132,7 +134,7 @@ func NewGpuIvfFlat[T VectorType](dataset []T, count uint64, dimension uint32, me return nil, moerr.NewInternalErrorNoCtx("failed to create GpuIvfFlat") } - return &GpuIvfFlat[T]{ + return &GpuIvfFlat[B, Q]{ cIvfFlat: cIvfFlat, dimension: dimension, nthread: nthread, @@ -141,13 +143,14 @@ func NewGpuIvfFlat[T VectorType](dataset []T, count uint64, dimension uint32, me } // NewGpuIvfFlatFromFile creates a new GpuIvfFlat instance by loading from a file. -func NewGpuIvfFlatFromFile[T VectorType](filename string, dimension uint32, metric DistanceType, - bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[T], error) { +func NewGpuIvfFlatFromFile[B, Q VectorType](filename string, dimension uint32, metric DistanceType, + bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cFilename := C.CString(filename) defer C.free(unsafe.Pointer(cFilename)) @@ -172,6 +175,7 @@ func NewGpuIvfFlatFromFile[T VectorType](filename string, dimension uint32, metr C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), unsafe.Pointer(&errmsg), ) @@ -187,7 +191,7 @@ func NewGpuIvfFlatFromFile[T VectorType](filename string, dimension uint32, metr return nil, moerr.NewInternalErrorNoCtx("failed to load GpuIvfFlat from file") } - return &GpuIvfFlat[T]{ + return &GpuIvfFlat[B, Q]{ cIvfFlat: cIvfFlat, dimension: dimension, nthread: nthread, @@ -199,8 +203,8 @@ func NewGpuIvfFlatFromFile[T VectorType](filename string, dimension uint32, metr // For Sharded loads we peek manifest.json to learn the saved shard count and // truncate `devices` to that count, so the C++ worker only spawns threads / // RMM pools on devices that will actually host a shard. -func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, metric DistanceType, - bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[T], error) { +func NewGpuIvfFlatFromDataDirectory[B, Q VectorType](dir string, dimension uint32, metric DistanceType, + bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } @@ -211,7 +215,8 @@ func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, return nil, err } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() cDevices := make([]C.int, len(devices)) for i, d := range devices { cDevices[i] = C.int(d) @@ -233,6 +238,7 @@ func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -267,7 +273,7 @@ func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, return nil, moerr.NewInternalErrorNoCtx(errStr) } - return &GpuIvfFlat[T]{ + return &GpuIvfFlat[B, Q]{ cIvfFlat: cIvfFlat, dimension: dimension, nthread: nthread, @@ -276,7 +282,7 @@ func NewGpuIvfFlatFromDataDirectory[T VectorType](dir string, dimension uint32, } // Destroy frees the C++ gpu_ivf_flat_t instance -func (gi *GpuIvfFlat[T]) Destroy() error { +func (gi *GpuIvfFlat[B, Q]) Destroy() error { if gi.cIvfFlat == nil { return nil } @@ -292,7 +298,7 @@ func (gi *GpuIvfFlat[T]) Destroy() error { } // Start initializes the worker and resources -func (gi *GpuIvfFlat[T]) Start() error { +func (gi *GpuIvfFlat[B, Q]) Start() error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -320,7 +326,7 @@ func (gi *GpuIvfFlat[T]) Start() error { } // Build triggers the build or file loading process -func (gi *GpuIvfFlat[T]) Build() error { +func (gi *GpuIvfFlat[B, Q]) Build() error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -335,13 +341,14 @@ func (gi *GpuIvfFlat[T]) Build() error { } // NewGpuIvfFlatEmpty creates a new GpuIvfFlat instance with pre-allocated buffer but no data yet. -func NewGpuIvfFlatEmpty[T VectorType](totalCount uint64, dimension uint32, metric DistanceType, - bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[T], error) { +func NewGpuIvfFlatEmpty[B, Q VectorType](totalCount uint64, dimension uint32, metric DistanceType, + bp IvfFlatBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfFlat[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -363,6 +370,7 @@ func NewGpuIvfFlatEmpty[T VectorType](totalCount uint64, dimension uint32, metri C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -379,7 +387,7 @@ func NewGpuIvfFlatEmpty[T VectorType](totalCount uint64, dimension uint32, metri return nil, moerr.NewInternalErrorNoCtx("failed to create empty GpuIvfFlat") } - return &GpuIvfFlat[T]{ + return &GpuIvfFlat[B, Q]{ cIvfFlat: cIvfFlat, dimension: dimension, nthread: nthread, @@ -388,7 +396,7 @@ func NewGpuIvfFlatEmpty[T VectorType](totalCount uint64, dimension uint32, metri } // AddChunk adds a chunk of data to the pre-allocated buffer. -func (gi *GpuIvfFlat[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (gi *GpuIvfFlat[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -419,8 +427,10 @@ func (gi *GpuIvfFlat[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) err return nil } -// AddChunkFloat adds a chunk of float32 data, performing on-the-fly quantization if needed. -func (gi *GpuIvfFlat[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize adds a chunk of base-typed (B) data, converting B -> the +// storage type Q on device (B==Q copy, or the learned/cast quantizer for a +// narrower Q). Mirrors GpuCagra/GpuIvfPq.AddChunkQuantize. +func (gi *GpuIvfFlat[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -433,9 +443,9 @@ func (gi *GpuIvfFlat[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids [ if len(ids) > 0 { cIds = (*C.int64_t)(unsafe.Pointer(&ids[0])) } - C.gpu_ivf_flat_add_chunk_float( + C.gpu_ivf_flat_add_chunk_quantize( gi.cIvfFlat, - (*C.float)(&chunk[0]), + unsafe.Pointer(&chunk[0]), C.uint64_t(chunkCount), cIds, unsafe.Pointer(&errmsg), @@ -451,8 +461,8 @@ func (gi *GpuIvfFlat[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids [ return nil } -// TrainQuantizer trains the scalar quantizer (if T is 1-byte) -func (gi *GpuIvfFlat[T]) TrainQuantizer(trainData []float32, nSamples uint64) error { +// TrainQuantizer trains the scalar quantizer (if Q is 1-byte) +func (gi *GpuIvfFlat[B, Q]) TrainQuantizer(trainData []float32, nSamples uint64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -477,8 +487,8 @@ func (gi *GpuIvfFlat[T]) TrainQuantizer(trainData []float32, nSamples uint64) er return nil } -// SetQuantizer sets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuIvfFlat[T]) SetQuantizer(min, max float32) error { +// SetQuantizer sets the scalar quantizer parameters (if Q is 1-byte) +func (gi *GpuIvfFlat[B, Q]) SetQuantizer(min, max float32) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -499,8 +509,8 @@ func (gi *GpuIvfFlat[T]) SetQuantizer(min, max float32) error { return nil } -// GetQuantizer gets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuIvfFlat[T]) GetQuantizer() (float32, float32, error) { +// GetQuantizer gets the scalar quantizer parameters (if Q is 1-byte) +func (gi *GpuIvfFlat[B, Q]) GetQuantizer() (float32, float32, error) { if gi.cIvfFlat == nil { return 0, 0, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -523,7 +533,7 @@ func (gi *GpuIvfFlat[T]) GetQuantizer() (float32, float32, error) { } // Save serializes the index to a file -func (gi *GpuIvfFlat[T]) Save(filename string) error { +func (gi *GpuIvfFlat[B, Q]) Save(filename string) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -541,7 +551,7 @@ func (gi *GpuIvfFlat[T]) Save(filename string) error { } // Pack saves the index to a .tar or .tar.gz file using save_dir. -func (gi *GpuIvfFlat[T]) Pack(filename string) error { +func (gi *GpuIvfFlat[B, Q]) Pack(filename string) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -570,7 +580,7 @@ func (gi *GpuIvfFlat[T]) Pack(filename string) error { // mode overrides the distribution mode at load time — pass Replicated to broadcast // a SINGLE_GPU .tar to all GPUs without rebuilding. // The index must already be initialized and started before calling Unpack. -func (gi *GpuIvfFlat[T]) Unpack(filename string, mode DistributionMode) error { +func (gi *GpuIvfFlat[B, Q]) Unpack(filename string, mode DistributionMode) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -599,7 +609,7 @@ func (gi *GpuIvfFlat[T]) Unpack(filename string, mode DistributionMode) error { } // DeleteId removes an ID from the index (soft delete). -func (gi *GpuIvfFlat[T]) DeleteId(id int64) error { +func (gi *GpuIvfFlat[B, Q]) DeleteId(id int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -614,7 +624,7 @@ func (gi *GpuIvfFlat[T]) DeleteId(id int64) error { } // Search performs a K-Nearest Neighbor search -func (gi *GpuIvfFlat[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (SearchResultIvfFlat, error) { +func (gi *GpuIvfFlat[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (SearchResultIvfFlat, error) { if gi.cIvfFlat == nil { return SearchResultIvfFlat{}, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -666,7 +676,7 @@ func (gi *GpuIvfFlat[T]) Search(queries []T, numQueries uint64, dimension uint32 } // SearchFloat performs a K-Nearest Neighbor search with float32 queries -func (gi *GpuIvfFlat[T]) SearchFloat(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (SearchResultIvfFlat, error) { +func (gi *GpuIvfFlat[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (SearchResultIvfFlat, error) { if gi.cIvfFlat == nil { return SearchResultIvfFlat{}, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -679,9 +689,9 @@ func (gi *GpuIvfFlat[T]) SearchFloat(queries []float32, numQueries uint64, dimen n_probes: C.uint32_t(sp.NProbes), } - res := C.gpu_ivf_flat_search_float( + res := C.gpu_ivf_flat_search_quantize( gi.cIvfFlat, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -718,12 +728,12 @@ func (gi *GpuIvfFlat[T]) SearchFloat(queries []float32, numQueries uint64, dimen } // SearchAsync performs a K-Nearest Neighbor search asynchronously. -func (gi *GpuIvfFlat[T]) SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +func (gi *GpuIvfFlat[B, Q]) SearchAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { return gi.SearchAsyncWithParams(queries, numQueries, dimension, limit, DefaultIvfFlatSearchParams()) } // SearchAsyncWithParams performs a K-Nearest Neighbor search asynchronously with custom parameters. -func (gi *GpuIvfFlat[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (uint64, error) { +func (gi *GpuIvfFlat[B, Q]) SearchAsyncWithParams(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (uint64, error) { if gi.cIvfFlat == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -756,13 +766,10 @@ func (gi *GpuIvfFlat[T]) SearchAsyncWithParams(queries []T, numQueries uint64, d return uint64(jobID), nil } -// SearchFloat32Async performs a K-Nearest Neighbor search with float32 queries asynchronously. -func (gi *GpuIvfFlat[T]) SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { - return gi.SearchFloat32AsyncWithParams(queries, numQueries, dimension, limit, DefaultIvfFlatSearchParams()) -} - -// SearchFloat32AsyncWithParams performs a K-Nearest Neighbor search with float32 queries asynchronously with custom parameters. -func (gi *GpuIvfFlat[T]) SearchFloat32AsyncWithParams(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (uint64, error) { +// SearchQuantizeAsyncWithParams submits an async KNN search with a base-typed (B) +// query; the index converts B to its storage type Q on device (B==Q copy, or the +// learned/cast quantizer for narrower Q). Unifies the former float32 and half query paths. +func (gi *GpuIvfFlat[B, Q]) SearchQuantizeAsyncWithParams(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) (uint64, error) { if gi.cIvfFlat == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -775,9 +782,9 @@ func (gi *GpuIvfFlat[T]) SearchFloat32AsyncWithParams(queries []float32, numQuer n_probes: C.uint32_t(sp.NProbes), } - jobID := C.gpu_ivf_flat_search_float_async( + jobID := C.gpu_ivf_flat_search_quantize_async( gi.cIvfFlat, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -796,7 +803,7 @@ func (gi *GpuIvfFlat[T]) SearchFloat32AsyncWithParams(queries []float32, numQuer } // SearchWait waits for an asynchronous search to complete and returns the results. -func (gi *GpuIvfFlat[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { +func (gi *GpuIvfFlat[B, Q]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { if gi.cIvfFlat == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -829,7 +836,7 @@ func (gi *GpuIvfFlat[T]) SearchWait(jobID uint64, numQueries uint64, limit uint3 } // Cap returns the capacity of the index buffer -func (gi *GpuIvfFlat[T]) Cap() uint64 { +func (gi *GpuIvfFlat[B, Q]) Cap() uint64 { if gi.cIvfFlat == nil { return 0 } @@ -837,7 +844,7 @@ func (gi *GpuIvfFlat[T]) Cap() uint64 { } // Len returns current number of vectors in index -func (gi *GpuIvfFlat[T]) Len() uint64 { +func (gi *GpuIvfFlat[B, Q]) Len() uint64 { if gi.cIvfFlat == nil { return 0 } @@ -845,7 +852,7 @@ func (gi *GpuIvfFlat[T]) Len() uint64 { } // Info returns detailed information about the index as a JSON string. -func (gi *GpuIvfFlat[T]) Info() (string, error) { +func (gi *GpuIvfFlat[B, Q]) Info() (string, error) { if gi.cIvfFlat == nil { return "", moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -868,11 +875,11 @@ func (gi *GpuIvfFlat[T]) Info() (string, error) { } // GetCenters retrieves the trained centroids. -func (gi *GpuIvfFlat[T]) GetCenters(nLists uint32) ([]T, error) { +func (gi *GpuIvfFlat[B, Q]) GetCenters(nLists uint32) ([]Q, error) { if gi.cIvfFlat == nil { return nil, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } - centers := make([]T, nLists*gi.dimension) + centers := make([]Q, nLists*gi.dimension) var errmsg *C.char C.gpu_ivf_flat_get_centers(gi.cIvfFlat, unsafe.Pointer(¢ers[0]), unsafe.Pointer(&errmsg)) runtime.KeepAlive(centers) @@ -886,7 +893,7 @@ func (gi *GpuIvfFlat[T]) GetCenters(nLists uint32) ([]T, error) { } // GetNList retrieves the number of lists (centroids) in the index. -func (gi *GpuIvfFlat[T]) GetNList() uint32 { +func (gi *GpuIvfFlat[B, Q]) GetNList() uint32 { if gi.cIvfFlat == nil { return 0 } @@ -895,7 +902,7 @@ func (gi *GpuIvfFlat[T]) GetNList() uint32 { // Extend adds new vectors to an already-built index without rebuilding. // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuIvfFlat[T]) Extend(newData []T, nRows uint64, newIDs []int64) error { +func (gi *GpuIvfFlat[B, Q]) Extend(newData []Q, nRows uint64, newIDs []int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -929,7 +936,7 @@ func (gi *GpuIvfFlat[T]) Extend(newData []T, nRows uint64, newIDs []int64) error // ExtendFloat adds new float32 vectors to an already-built index, quantizing on-the-fly if needed. // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuIvfFlat[T]) ExtendFloat(newData []float32, nRows uint64, newIDs []int64) error { +func (gi *GpuIvfFlat[B, Q]) ExtendFloat(newData []float32, nRows uint64, newIDs []int64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -968,7 +975,7 @@ type SearchResultIvfFlat struct { } // SetFilterColumns registers filter-column metadata. See GpuCagra.SetFilterColumns. -func (gi *GpuIvfFlat[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { +func (gi *GpuIvfFlat[B, Q]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -985,7 +992,7 @@ func (gi *GpuIvfFlat[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) } // AddFilterChunk appends raw filter-column bytes. See GpuCagra.AddFilterChunk. -func (gi *GpuIvfFlat[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (gi *GpuIvfFlat[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if gi.cIvfFlat == nil { return moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -1016,7 +1023,7 @@ func (gi *GpuIvfFlat[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap [ } // SearchWithFilter runs a filtered K-NN search. predsJSON="" = unfiltered. -func (gi *GpuIvfFlat[T]) SearchWithFilter(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (SearchResultIvfFlat, error) { +func (gi *GpuIvfFlat[B, Q]) SearchWithFilter(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (SearchResultIvfFlat, error) { if gi.cIvfFlat == nil { return SearchResultIvfFlat{}, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -1062,8 +1069,9 @@ func (gi *GpuIvfFlat[T]) SearchWithFilter(queries []T, numQueries uint64, dimens return SearchResultIvfFlat{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilter runs a filtered K-NN search with float32 queries. -func (gi *GpuIvfFlat[T]) SearchFloatWithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (SearchResultIvfFlat, error) { +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) +// queries; the index converts B to storage T (copy / quantize / f32->f16 cast). +func (gi *GpuIvfFlat[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (SearchResultIvfFlat, error) { if gi.cIvfFlat == nil { return SearchResultIvfFlat{}, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -1076,9 +1084,9 @@ func (gi *GpuIvfFlat[T]) SearchFloatWithFilter(queries []float32, numQueries uin cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - res := C.gpu_ivf_flat_search_float_with_filter( + res := C.gpu_ivf_flat_search_quantize_with_filter( gi.cIvfFlat, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -1109,12 +1117,12 @@ func (gi *GpuIvfFlat[T]) SearchFloatWithFilter(queries []float32, numQueries uin return SearchResultIvfFlat{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilterAsync submits a filtered float32 K-NN search and -// returns a job_id; collect the result with SearchWait. Mirrors -// SearchFloat32AsyncWithParams + the predicate-eval semantics of -// SearchFloatWithFilter. Used by MultiGpuIvfFlat to dispatch per-shard -// filtered searches in parallel. -func (gi *GpuIvfFlat[T]) SearchFloatWithFilterAsync(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (uint64, error) { +// SearchQuantizeWithFilterAsync submits a filtered K-NN search with base-typed +// (B) queries and returns a job_id; collect the result with SearchWait. Mirrors +// SearchQuantizeAsyncWithParams + the predicate-eval semantics of +// SearchQuantizeWithFilter. Used by MultiGpuIvfFlat to dispatch per-shard +// filtered searches in parallel. The index converts B to storage T. +func (gi *GpuIvfFlat[B, Q]) SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) (uint64, error) { if gi.cIvfFlat == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfFlat is not initialized") } @@ -1127,9 +1135,9 @@ func (gi *GpuIvfFlat[T]) SearchFloatWithFilterAsync(queries []float32, numQuerie cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - jobID := C.gpu_ivf_flat_search_float_with_filter_async( + jobID := C.gpu_ivf_flat_search_quantize_with_filter_async( gi.cIvfFlat, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), diff --git a/pkg/cuvs/ivf_flat_test.go b/pkg/cuvs/ivf_flat_test.go index 9799e79079cee..20345cfed8212 100644 --- a/pkg/cuvs/ivf_flat_test.go +++ b/pkg/cuvs/ivf_flat_test.go @@ -35,7 +35,7 @@ func TestGpuIvfFlat(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -81,7 +81,7 @@ func TestGpuIvfFlatSaveLoad(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 2 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -96,7 +96,7 @@ func TestGpuIvfFlatSaveLoad(t *testing.T) { defer os.Remove(filename) index.Destroy() - index2, err := NewGpuIvfFlatFromFile[float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfFlatFromFile[float32, float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuIvfFlat from file: %v", err) } @@ -131,7 +131,7 @@ func TestGpuIvfFlatPackUnpack(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -147,7 +147,7 @@ func TestGpuIvfFlatPackUnpack(t *testing.T) { } defer os.Remove(filename) - index2, err := NewGpuIvfFlatEmpty[float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfFlatEmpty[float32, float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuIvfFlatEmpty failed: %v", err) } @@ -185,7 +185,7 @@ func TestGpuIvfFlatFromDataDirectory(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -211,7 +211,7 @@ func TestGpuIvfFlatFromDataDirectory(t *testing.T) { t.Fatalf("Unpack to dir failed: %v", err) } - index2, err := NewGpuIvfFlatFromDataDirectory[float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfFlatFromDataDirectory[float32, float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuIvfFlatFromDataDirectory failed: %v", err) } @@ -244,7 +244,7 @@ func TestGpuShardedIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) if err != nil { t.Fatalf("Failed to create sharded IVF-Flat: %v", err) } @@ -281,7 +281,7 @@ func TestGpuReplicatedIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) if err != nil { t.Fatalf("Failed to create replicated IVF-Flat: %v", err) } @@ -316,7 +316,7 @@ func TestGpuIvfFlatExtend(t *testing.T) { devices := []int{0} bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -381,7 +381,7 @@ func TestGpuIvfFlatExtendFloat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 // Use Float16 so ExtendFloat exercises quantization - index, err := NewGpuIvfFlat[Float16](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, Float16](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat[Float16]: %v", err) } @@ -409,7 +409,7 @@ func TestGpuIvfFlatExtendFloat(t *testing.T) { sp := DefaultIvfFlatSearchParams() sp.NProbes = 10 - r, err := index.SearchFloat([]float32{500, 500}, 1, dimension, 1, sp) + r, err := index.SearchQuantize([]float32{500, 500}, 1, dimension, 1, sp) if err != nil { t.Fatalf("Search failed: %v", err) } @@ -435,7 +435,7 @@ func TestGpuIvfFlatDeleteId(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfFlat: %v", err) } @@ -492,7 +492,7 @@ func TestGpuShardedIvfFlatDeleteId(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 10 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) if err != nil { t.Fatalf("Failed to create sharded IvfFlat: %v", err) } @@ -572,7 +572,7 @@ func BenchmarkGpuShardedIvfFlat(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Sharded, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Sharded, nil) if err != nil { b.Fatalf("Failed to create sharded IVF-Flat: %v", err) } @@ -601,7 +601,7 @@ func BenchmarkGpuShardedIvfFlat(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -609,7 +609,7 @@ func BenchmarkGpuShardedIvfFlat(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -632,7 +632,7 @@ func BenchmarkGpuSingleIvfFlat(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) if err != nil { b.Fatalf("Failed to create single IVF-Flat: %v", err) } @@ -661,7 +661,7 @@ func BenchmarkGpuSingleIvfFlat(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -669,7 +669,7 @@ func BenchmarkGpuSingleIvfFlat(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -695,7 +695,7 @@ func BenchmarkGpuReplicatedIvfFlat(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 - index, err := NewGpuIvfFlat[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) if err != nil { b.Fatalf("Failed to create replicated IVF-Flat: %v", err) } @@ -724,7 +724,7 @@ func BenchmarkGpuReplicatedIvfFlat(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -732,7 +732,7 @@ func BenchmarkGpuReplicatedIvfFlat(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -757,7 +757,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatF16(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 // Use Float16 as internal type - index, err := NewGpuIvfFlatEmpty[Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuIvfFlatEmpty[float32, Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -770,7 +770,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatF16(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -791,7 +791,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatF16(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -799,7 +799,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatF16(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -822,7 +822,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatInt8(b *testing.B) { bp := DefaultIvfFlatBuildParams() bp.NLists = 1000 // Use int8 as internal type - index, err := NewGpuIvfFlatEmpty[int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuIvfFlatEmpty[float32, int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -835,7 +835,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatInt8(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -856,7 +856,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatInt8(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -864,7 +864,7 @@ func BenchmarkGpuAddChunkAndSearchIvfFlatInt8(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -881,7 +881,7 @@ func TestGpuIvfFlatChunked(t *testing.T) { bp.NLists = 10 // Create empty index (target type int8) - index, err := NewGpuIvfFlatEmpty[int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index, err := NewGpuIvfFlatEmpty[float32, int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuIvfFlatEmpty: %v", err) } @@ -900,7 +900,7 @@ func TestGpuIvfFlatChunked(t *testing.T) { for j := range chunk { chunk[j] = val } - err = index.AddChunkFloat(chunk, chunkSize, nil) + err = index.AddChunkQuantize(chunk, chunkSize, nil) if err != nil { t.Fatalf("AddChunkFloat failed at offset %d: %v", i, err) } diff --git a/pkg/cuvs/ivf_pq.go b/pkg/cuvs/ivf_pq.go index e4bf899140de5..a1cc2ffd5c147 100644 --- a/pkg/cuvs/ivf_pq.go +++ b/pkg/cuvs/ivf_pq.go @@ -32,7 +32,7 @@ import ( ) // GpuIvfPq represents the C++ gpu_ivf_pq_t object. -type GpuIvfPq[T VectorType] struct { +type GpuIvfPq[B, Q VectorType] struct { cIvfPq C.gpu_ivf_pq_c dimension uint32 nthread uint32 @@ -43,7 +43,7 @@ type GpuIvfPq[T VectorType] struct { // SetBatchWindow sets the batching window in microseconds for search operations. // A window of 0 disables batching; any positive value enables batching with that delay. -func (gi *GpuIvfPq[T]) SetBatchWindow(windowUs int64) error { +func (gi *GpuIvfPq[B, Q]) SetBatchWindow(windowUs int64) error { gi.batchWindowUs = windowUs if gi.cIvfPq != nil { var errmsg *C.char @@ -61,7 +61,7 @@ func (gi *GpuIvfPq[T]) SetBatchWindow(windowUs int64) error { // flag. false (default): dispatch eagerly at the full batch size. true: wait for // the batch to fill or the window to elapse, then dispatch at the real size. // Has no effect unless the batch window is > 0. -func (gi *GpuIvfPq[T]) SetDynbConservativeDispatch(enable bool) error { +func (gi *GpuIvfPq[B, Q]) SetDynbConservativeDispatch(enable bool) error { gi.dynbConservativeDispatch = enable if gi.cIvfPq != nil { var errmsg *C.char @@ -77,13 +77,14 @@ func (gi *GpuIvfPq[T]) SetDynbConservativeDispatch(enable bool) error { // NewGpuIvfPq creates a new GpuIvfPq instance from a dataset. // ids may be nil to use internal sequential IDs (0..count-1). -func NewGpuIvfPq[T VectorType](dataset []T, count uint64, dimension uint32, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuIvfPq[T], error) { +func NewGpuIvfPq[B, Q VectorType](dataset []Q, count uint64, dimension uint32, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode, ids []int64) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -113,6 +114,7 @@ func NewGpuIvfPq[T VectorType](dataset []T, count uint64, dimension uint32, metr C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), cIds, unsafe.Pointer(&errmsg), @@ -131,7 +133,7 @@ func NewGpuIvfPq[T VectorType](dataset []T, count uint64, dimension uint32, metr return nil, moerr.NewInternalErrorNoCtx("failed to create GpuIvfPq") } - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: dimension, nthread: nthread, @@ -140,13 +142,14 @@ func NewGpuIvfPq[T VectorType](dataset []T, count uint64, dimension uint32, metr } // NewGpuIvfPqFromDataFile creates a new GpuIvfPq instance from a MODF datafile. -func NewGpuIvfPqFromDataFile[T VectorType](datafilename string, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[T], error) { +func NewGpuIvfPqFromDataFile[B, Q VectorType](datafilename string, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cFilename := C.CString(datafilename) defer C.free(unsafe.Pointer(cFilename)) @@ -172,6 +175,7 @@ func NewGpuIvfPqFromDataFile[T VectorType](datafilename string, metric DistanceT C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), unsafe.Pointer(&errmsg), ) @@ -189,7 +193,7 @@ func NewGpuIvfPqFromDataFile[T VectorType](datafilename string, metric DistanceT // dimension will be updated when GetDim() is called, but we can set it to 0 for now // or ideally GetDim() should be used. - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: 0, nthread: nthread, @@ -198,13 +202,14 @@ func NewGpuIvfPqFromDataFile[T VectorType](datafilename string, metric DistanceT } // NewGpuIvfPqEmpty creates a new GpuIvfPq instance with pre-allocated buffer but no data yet. -func NewGpuIvfPqEmpty[T VectorType](totalCount uint64, dimension uint32, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[T], error) { +func NewGpuIvfPqEmpty[B, Q VectorType](totalCount uint64, dimension uint32, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cDevices := make([]C.int, len(devices)) for i, d := range devices { @@ -228,6 +233,7 @@ func NewGpuIvfPqEmpty[T VectorType](totalCount uint64, dimension uint32, metric C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -244,7 +250,7 @@ func NewGpuIvfPqEmpty[T VectorType](totalCount uint64, dimension uint32, metric return nil, moerr.NewInternalErrorNoCtx("failed to create empty GpuIvfPq") } - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: dimension, nthread: nthread, @@ -253,7 +259,7 @@ func NewGpuIvfPqEmpty[T VectorType](totalCount uint64, dimension uint32, metric } // AddChunk adds a chunk of data to the pre-allocated buffer. -func (gi *GpuIvfPq[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (gi *GpuIvfPq[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -284,8 +290,11 @@ func (gi *GpuIvfPq[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error return nil } -// AddChunkFloat adds a chunk of float32 data, performing on-the-fly quantization if needed. -func (gi *GpuIvfPq[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { + +// AddChunkQuantize adds a chunk of base-typed (B) data, quantizing natively to +// the storage type Q (int8/uint8) via the B-source quantizer. base_data is the +// raw bytes of chunkCount*dim B-typed elements. No f32 detour. +func (gi *GpuIvfPq[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -298,9 +307,9 @@ func (gi *GpuIvfPq[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []i if len(ids) > 0 { cIds = (*C.int64_t)(unsafe.Pointer(&ids[0])) } - C.gpu_ivf_pq_add_chunk_float( + C.gpu_ivf_pq_add_chunk_quantize( gi.cIvfPq, - (*C.float)(&chunk[0]), + unsafe.Pointer(&chunk[0]), C.uint64_t(chunkCount), cIds, unsafe.Pointer(&errmsg), @@ -316,8 +325,9 @@ func (gi *GpuIvfPq[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []i return nil } -// TrainQuantizer trains the scalar quantizer (if T is 1-byte) -func (gi *GpuIvfPq[T]) TrainQuantizer(trainData []float32, nSamples uint64) error { +// TrainQuantizer trains the scalar quantizer (if Q is 1-byte) from base-typed +// (B) training data. +func (gi *GpuIvfPq[B, Q]) TrainQuantizer(trainData []B, nSamples uint64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -328,7 +338,7 @@ func (gi *GpuIvfPq[T]) TrainQuantizer(trainData []float32, nSamples uint64) erro var errmsg *C.char C.gpu_ivf_pq_train_quantizer( gi.cIvfPq, - (*C.float)(&trainData[0]), + unsafe.Pointer(&trainData[0]), C.uint64_t(nSamples), unsafe.Pointer(&errmsg), ) @@ -343,7 +353,7 @@ func (gi *GpuIvfPq[T]) TrainQuantizer(trainData []float32, nSamples uint64) erro } // SetQuantizer sets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuIvfPq[T]) SetQuantizer(min, max float32) error { +func (gi *GpuIvfPq[B, Q]) SetQuantizer(min, max float32) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -365,7 +375,7 @@ func (gi *GpuIvfPq[T]) SetQuantizer(min, max float32) error { } // GetQuantizer gets the scalar quantizer parameters (if T is 1-byte) -func (gi *GpuIvfPq[T]) GetQuantizer() (float32, float32, error) { +func (gi *GpuIvfPq[B, Q]) GetQuantizer() (float32, float32, error) { if gi.cIvfPq == nil { return 0, 0, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -388,13 +398,14 @@ func (gi *GpuIvfPq[T]) GetQuantizer() (float32, float32, error) { } // NewGpuIvfPqFromFile creates a new GpuIvfPq instance by loading from a file. -func NewGpuIvfPqFromFile[T VectorType](filename string, dimension uint32, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[T], error) { +func NewGpuIvfPqFromFile[B, Q VectorType](filename string, dimension uint32, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() var errmsg *C.char cFilename := C.CString(filename) defer C.free(unsafe.Pointer(cFilename)) @@ -421,6 +432,7 @@ func NewGpuIvfPqFromFile[T VectorType](filename string, dimension uint32, metric C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), unsafe.Pointer(&errmsg), ) @@ -436,7 +448,7 @@ func NewGpuIvfPqFromFile[T VectorType](filename string, dimension uint32, metric return nil, moerr.NewInternalErrorNoCtx("failed to load GpuIvfPq from file") } - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: dimension, nthread: nthread, @@ -448,8 +460,8 @@ func NewGpuIvfPqFromFile[T VectorType](filename string, dimension uint32, metric // For Sharded loads we peek manifest.json to learn the saved shard count and // truncate `devices` to that count, so the C++ worker only spawns threads / // RMM pools on devices that will actually host a shard. -func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, metric DistanceType, - bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[T], error) { +func NewGpuIvfPqFromDataDirectory[B, Q VectorType](dir string, dimension uint32, metric DistanceType, + bp IvfPqBuildParams, devices []int, nthread uint32, mode DistributionMode) (*GpuIvfPq[B, Q], error) { if len(devices) == 0 { return nil, moerr.NewInternalErrorNoCtx("at least one device must be specified") } @@ -460,7 +472,8 @@ func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, me return nil, err } - qtype := GetQuantization[T]() + btype := GetQuantization[B]() + qtype := GetQuantization[Q]() cDevices := make([]C.int, len(devices)) for i, d := range devices { cDevices[i] = C.int(d) @@ -484,6 +497,7 @@ func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, me C.int(len(devices)), C.uint32_t(nthread), C.distribution_mode_t(mode), + C.quantization_t(btype), C.quantization_t(qtype), nil, unsafe.Pointer(&errmsg), @@ -518,7 +532,7 @@ func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, me return nil, moerr.NewInternalErrorNoCtx(errStr) } - return &GpuIvfPq[T]{ + return &GpuIvfPq[B, Q]{ cIvfPq: cIvfPq, dimension: dimension, nthread: nthread, @@ -527,7 +541,7 @@ func NewGpuIvfPqFromDataDirectory[T VectorType](dir string, dimension uint32, me } // Destroy frees the C++ gpu_ivf_pq_t instance -func (gi *GpuIvfPq[T]) Destroy() error { +func (gi *GpuIvfPq[B, Q]) Destroy() error { if gi.cIvfPq == nil { return nil } @@ -543,7 +557,7 @@ func (gi *GpuIvfPq[T]) Destroy() error { } // Start initializes the worker and resources -func (gi *GpuIvfPq[T]) Start() error { +func (gi *GpuIvfPq[B, Q]) Start() error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -571,7 +585,7 @@ func (gi *GpuIvfPq[T]) Start() error { } // Build triggers the build or file loading process -func (gi *GpuIvfPq[T]) Build() error { +func (gi *GpuIvfPq[B, Q]) Build() error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -586,7 +600,7 @@ func (gi *GpuIvfPq[T]) Build() error { } // Save serializes the index to a file -func (gi *GpuIvfPq[T]) Save(filename string) error { +func (gi *GpuIvfPq[B, Q]) Save(filename string) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -604,7 +618,7 @@ func (gi *GpuIvfPq[T]) Save(filename string) error { } // Pack saves the index to a .tar or .tar.gz file using save_dir. -func (gi *GpuIvfPq[T]) Pack(filename string) error { +func (gi *GpuIvfPq[B, Q]) Pack(filename string) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -633,7 +647,7 @@ func (gi *GpuIvfPq[T]) Pack(filename string) error { // mode overrides the distribution mode at load time — pass Replicated to broadcast // a SINGLE_GPU .tar to all GPUs without rebuilding. // The index must already be initialized and started before calling Unpack. -func (gi *GpuIvfPq[T]) Unpack(filename string, mode DistributionMode) error { +func (gi *GpuIvfPq[B, Q]) Unpack(filename string, mode DistributionMode) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -662,7 +676,7 @@ func (gi *GpuIvfPq[T]) Unpack(filename string, mode DistributionMode) error { } // DeleteId removes an ID from the index (soft delete). -func (gi *GpuIvfPq[T]) DeleteId(id int64) error { +func (gi *GpuIvfPq[B, Q]) DeleteId(id int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -678,7 +692,7 @@ func (gi *GpuIvfPq[T]) DeleteId(id int64) error { // DeleteIds applies DeleteId in a loop. See cagra.GpuCagra.DeleteIds for // the rationale. -func (gi *GpuIvfPq[T]) DeleteIds(ids []int64) error { +func (gi *GpuIvfPq[B, Q]) DeleteIds(ids []int64) error { for _, id := range ids { if err := gi.DeleteId(id); err != nil { return err @@ -688,7 +702,7 @@ func (gi *GpuIvfPq[T]) DeleteIds(ids []int64) error { } // Search performs a K-Nearest Neighbor search -func (gi *GpuIvfPq[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (SearchResultIvfPq, error) { +func (gi *GpuIvfPq[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (SearchResultIvfPq, error) { if gi.cIvfPq == nil { return SearchResultIvfPq{}, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -740,7 +754,7 @@ func (gi *GpuIvfPq[T]) Search(queries []T, numQueries uint64, dimension uint32, } // SearchFloat performs an IVF-PQ search operation with float32 queries -func (gi *GpuIvfPq[T]) SearchFloat(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (SearchResultIvfPq, error) { +func (gi *GpuIvfPq[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (SearchResultIvfPq, error) { if gi.cIvfPq == nil { return SearchResultIvfPq{}, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -753,9 +767,9 @@ func (gi *GpuIvfPq[T]) SearchFloat(queries []float32, numQueries uint64, dimensi n_probes: C.uint32_t(sp.NProbes), } - res := C.gpu_ivf_pq_search_float( + res := C.gpu_ivf_pq_search_quantize( gi.cIvfPq, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -792,12 +806,12 @@ func (gi *GpuIvfPq[T]) SearchFloat(queries []float32, numQueries uint64, dimensi } // SearchAsync performs a K-Nearest Neighbor search asynchronously. -func (gi *GpuIvfPq[T]) SearchAsync(queries []T, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { +func (gi *GpuIvfPq[B, Q]) SearchAsync(queries []Q, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { return gi.SearchAsyncWithParams(queries, numQueries, dimension, limit, DefaultIvfPqSearchParams()) } // SearchAsyncWithParams performs a K-Nearest Neighbor search asynchronously with custom parameters. -func (gi *GpuIvfPq[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (uint64, error) { +func (gi *GpuIvfPq[B, Q]) SearchAsyncWithParams(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (uint64, error) { if gi.cIvfPq == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -830,13 +844,10 @@ func (gi *GpuIvfPq[T]) SearchAsyncWithParams(queries []T, numQueries uint64, dim return uint64(jobID), nil } -// SearchFloat32Async performs a K-Nearest Neighbor search with float32 queries asynchronously. -func (gi *GpuIvfPq[T]) SearchFloat32Async(queries []float32, numQueries uint64, dimension uint32, limit uint32) (uint64, error) { - return gi.SearchFloat32AsyncWithParams(queries, numQueries, dimension, limit, DefaultIvfPqSearchParams()) -} - -// SearchFloat32AsyncWithParams performs a K-Nearest Neighbor search with float32 queries asynchronously with custom parameters. -func (gi *GpuIvfPq[T]) SearchFloat32AsyncWithParams(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (uint64, error) { +// SearchQuantizeAsyncWithParams submits an async KNN search with a base-typed (B) +// query; the index converts B to its storage type Q on device (B==Q copy, or the +// learned/cast quantizer for narrower Q). Unifies the former float32 and half query paths. +func (gi *GpuIvfPq[B, Q]) SearchQuantizeAsyncWithParams(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) (uint64, error) { if gi.cIvfPq == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -849,9 +860,9 @@ func (gi *GpuIvfPq[T]) SearchFloat32AsyncWithParams(queries []float32, numQuerie n_probes: C.uint32_t(sp.NProbes), } - jobID := C.gpu_ivf_pq_search_float_async( + jobID := C.gpu_ivf_pq_search_quantize_async( gi.cIvfPq, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -870,7 +881,7 @@ func (gi *GpuIvfPq[T]) SearchFloat32AsyncWithParams(queries []float32, numQuerie } // SearchWait waits for an asynchronous search to complete and returns the results. -func (gi *GpuIvfPq[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { +func (gi *GpuIvfPq[B, Q]) SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) { if gi.cIvfPq == nil { return nil, nil, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -903,7 +914,7 @@ func (gi *GpuIvfPq[T]) SearchWait(jobID uint64, numQueries uint64, limit uint32) } // Cap returns the capacity of the index buffer -func (gi *GpuIvfPq[T]) Cap() uint64 { +func (gi *GpuIvfPq[B, Q]) Cap() uint64 { if gi.cIvfPq == nil { return 0 } @@ -911,7 +922,7 @@ func (gi *GpuIvfPq[T]) Cap() uint64 { } // Len returns current number of vectors in index -func (gi *GpuIvfPq[T]) Len() uint64 { +func (gi *GpuIvfPq[B, Q]) Len() uint64 { if gi.cIvfPq == nil { return 0 } @@ -921,7 +932,7 @@ func (gi *GpuIvfPq[T]) Len() uint64 { // GetFilterColMetaJSON returns the INCLUDE-column metadata of the loaded // index as a JSON string ready to be re-fed into SetFilterColumns. Returns // "" for indexes that were built without INCLUDE columns. -func (gi *GpuIvfPq[T]) GetFilterColMetaJSON() string { +func (gi *GpuIvfPq[B, Q]) GetFilterColMetaJSON() string { if gi.cIvfPq == nil { return "" } @@ -939,7 +950,7 @@ func (gi *GpuIvfPq[T]) GetFilterColMetaJSON() string { } // Info returns detailed information about the index as a JSON string. -func (gi *GpuIvfPq[T]) Info() (string, error) { +func (gi *GpuIvfPq[B, Q]) Info() (string, error) { if gi.cIvfPq == nil { return "", moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -962,13 +973,13 @@ func (gi *GpuIvfPq[T]) Info() (string, error) { } // GetCenters retrieves the trained centroids. -func (gi *GpuIvfPq[T]) GetCenters() ([]T, error) { +func (gi *GpuIvfPq[B, Q]) GetCenters() ([]Q, error) { if gi.cIvfPq == nil { return nil, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } nList := gi.GetNList() dim := gi.GetRotDim() - centers := make([]T, nList*dim) + centers := make([]Q, nList*dim) var errmsg *C.char C.gpu_ivf_pq_get_centers(gi.cIvfPq, unsafe.Pointer(¢ers[0]), C.uint64_t(len(centers)), unsafe.Pointer(&errmsg)) runtime.KeepAlive(centers) @@ -982,7 +993,7 @@ func (gi *GpuIvfPq[T]) GetCenters() ([]T, error) { } // GetNList retrieves the number of lists (centroids) in the index. -func (gi *GpuIvfPq[T]) GetNList() uint32 { +func (gi *GpuIvfPq[B, Q]) GetNList() uint32 { if gi.cIvfPq == nil { return 0 } @@ -990,7 +1001,7 @@ func (gi *GpuIvfPq[T]) GetNList() uint32 { } // GetDim retrieves the dimension of the index. -func (gi *GpuIvfPq[T]) GetDim() uint32 { +func (gi *GpuIvfPq[B, Q]) GetDim() uint32 { if gi.cIvfPq == nil { return 0 } @@ -998,7 +1009,7 @@ func (gi *GpuIvfPq[T]) GetDim() uint32 { } // GetRotDim retrieves the rotated dimension of the index. -func (gi *GpuIvfPq[T]) GetRotDim() uint32 { +func (gi *GpuIvfPq[B, Q]) GetRotDim() uint32 { if gi.cIvfPq == nil { return 0 } @@ -1006,7 +1017,7 @@ func (gi *GpuIvfPq[T]) GetRotDim() uint32 { } // GetDimExt retrieves the extended dimension of the index (including norms and padding). -func (gi *GpuIvfPq[T]) GetDimExt() uint32 { +func (gi *GpuIvfPq[B, Q]) GetDimExt() uint32 { if gi.cIvfPq == nil { return 0 } @@ -1014,18 +1025,18 @@ func (gi *GpuIvfPq[T]) GetDimExt() uint32 { } // GetDataset retrieves the flattened host dataset (for debugging). -func (gi *GpuIvfPq[T]) GetDataset(totalElements uint64) []T { +func (gi *GpuIvfPq[B, Q]) GetDataset(totalElements uint64) []Q { if gi.cIvfPq == nil { return nil } - data := make([]T, totalElements) + data := make([]Q, totalElements) C.gpu_ivf_pq_get_dataset(gi.cIvfPq, unsafe.Pointer(&data[0])) return data } // Extend adds new vectors to an already-built index without rebuilding. // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuIvfPq[T]) Extend(newData []T, nRows uint64, newIDs []int64) error { +func (gi *GpuIvfPq[B, Q]) Extend(newData []Q, nRows uint64, newIDs []int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1059,7 +1070,7 @@ func (gi *GpuIvfPq[T]) Extend(newData []T, nRows uint64, newIDs []int64) error { // ExtendFloat adds new float32 vectors to an already-built index, quantizing on-the-fly if needed. // newIDs may be nil to auto-assign sequential IDs starting from the current index size. -func (gi *GpuIvfPq[T]) ExtendFloat(newData []float32, nRows uint64, newIDs []int64) error { +func (gi *GpuIvfPq[B, Q]) ExtendFloat(newData []float32, nRows uint64, newIDs []int64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1098,7 +1109,7 @@ type SearchResultIvfPq struct { } // SetFilterColumns registers filter-column metadata. See GpuCagra.SetFilterColumns. -func (gi *GpuIvfPq[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { +func (gi *GpuIvfPq[B, Q]) SetFilterColumns(colMetaJSON string, totalCount uint64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1115,7 +1126,7 @@ func (gi *GpuIvfPq[T]) SetFilterColumns(colMetaJSON string, totalCount uint64) e } // AddFilterChunk appends raw filter-column bytes. See GpuCagra.AddFilterChunk. -func (gi *GpuIvfPq[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (gi *GpuIvfPq[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if gi.cIvfPq == nil { return moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1146,7 +1157,7 @@ func (gi *GpuIvfPq[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []u } // SearchWithFilter runs a filtered K-NN search. predsJSON="" = unfiltered. -func (gi *GpuIvfPq[T]) SearchWithFilter(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (SearchResultIvfPq, error) { +func (gi *GpuIvfPq[B, Q]) SearchWithFilter(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (SearchResultIvfPq, error) { if gi.cIvfPq == nil { return SearchResultIvfPq{}, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1192,8 +1203,9 @@ func (gi *GpuIvfPq[T]) SearchWithFilter(queries []T, numQueries uint64, dimensio return SearchResultIvfPq{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilter runs a filtered K-NN search with float32 queries. -func (gi *GpuIvfPq[T]) SearchFloatWithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (SearchResultIvfPq, error) { +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) +// queries; the index converts B to storage T (copy / quantize / f32->f16 cast). +func (gi *GpuIvfPq[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (SearchResultIvfPq, error) { if gi.cIvfPq == nil { return SearchResultIvfPq{}, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1206,9 +1218,9 @@ func (gi *GpuIvfPq[T]) SearchFloatWithFilter(queries []float32, numQueries uint6 cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - res := C.gpu_ivf_pq_search_float_with_filter( + res := C.gpu_ivf_pq_search_quantize_with_filter( gi.cIvfPq, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), @@ -1239,12 +1251,12 @@ func (gi *GpuIvfPq[T]) SearchFloatWithFilter(queries []float32, numQueries uint6 return SearchResultIvfPq{Neighbors: neighbors, Distances: distances}, nil } -// SearchFloatWithFilterAsync submits a filtered float32 K-NN search and -// returns a job_id; collect the result with SearchWait. Mirrors -// SearchFloat32AsyncWithParams + the predicate-eval semantics of -// SearchFloatWithFilter. Used by MultiGpuIvfPq to dispatch per-shard -// filtered searches in parallel. -func (gi *GpuIvfPq[T]) SearchFloatWithFilterAsync(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (uint64, error) { +// SearchQuantizeWithFilterAsync submits a filtered K-NN search with base-typed +// (B) queries and returns a job_id; collect the result with SearchWait. Mirrors +// SearchQuantizeAsyncWithParams + the predicate-eval semantics of +// SearchQuantizeWithFilter. Used by MultiGpuIvfPq to dispatch per-shard +// filtered searches in parallel. The index converts B to storage T. +func (gi *GpuIvfPq[B, Q]) SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) (uint64, error) { if gi.cIvfPq == nil { return 0, moerr.NewInternalErrorNoCtx("GpuIvfPq is not initialized") } @@ -1257,9 +1269,9 @@ func (gi *GpuIvfPq[T]) SearchFloatWithFilterAsync(queries []float32, numQueries cPreds := C.CString(predsJSON) defer C.free(unsafe.Pointer(cPreds)) - jobID := C.gpu_ivf_pq_search_float_with_filter_async( + jobID := C.gpu_ivf_pq_search_quantize_with_filter_async( gi.cIvfPq, - (*C.float)(unsafe.Pointer(&queries[0])), + unsafe.Pointer(&queries[0]), C.uint64_t(numQueries), C.uint32_t(dimension), C.uint32_t(limit), diff --git a/pkg/cuvs/ivf_pq_test.go b/pkg/cuvs/ivf_pq_test.go index e6d79084c3102..8cd7301aa6c91 100644 --- a/pkg/cuvs/ivf_pq_test.go +++ b/pkg/cuvs/ivf_pq_test.go @@ -38,7 +38,7 @@ func TestGpuIvfPq(t *testing.T) { bp.NLists = 10 bp.M = 8 // dimension 16 is divisible by 8 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -92,7 +92,7 @@ func TestGpuIvfPqSaveLoad(t *testing.T) { bp.NLists = 10 bp.M = 2 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -107,7 +107,7 @@ func TestGpuIvfPqSaveLoad(t *testing.T) { defer os.Remove(filename) index.Destroy() - index2, err := NewGpuIvfPqFromFile[float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfPqFromFile[float32, float32](filename, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuIvfPq from file: %v", err) } @@ -150,7 +150,7 @@ func TestGpuIvfPqPackUnpack(t *testing.T) { bp.NLists = 10 bp.M = 2 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -166,7 +166,7 @@ func TestGpuIvfPqPackUnpack(t *testing.T) { } defer os.Remove(filename) - index2, err := NewGpuIvfPqEmpty[float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfPqEmpty[float32, float32](0, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuIvfPqEmpty failed: %v", err) } @@ -208,7 +208,7 @@ func TestGpuIvfPqFromDataDirectory(t *testing.T) { bp.NLists = 10 bp.M = 2 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -234,7 +234,7 @@ func TestGpuIvfPqFromDataDirectory(t *testing.T) { t.Fatalf("Unpack to dir failed: %v", err) } - index2, err := NewGpuIvfPqFromDataDirectory[float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index2, err := NewGpuIvfPqFromDataDirectory[float32, float32](tmpDir, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("NewGpuIvfPqFromDataDirectory failed: %v", err) } @@ -261,7 +261,7 @@ func TestGpuIvfPqChunked(t *testing.T) { bp.M = 4 // Create empty index (target type int8) - index, err := NewGpuIvfPqEmpty[int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) + index, err := NewGpuIvfPqEmpty[float32, int8](totalCount, dimension, L2Expanded, bp, devices, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create GpuIvfPqEmpty: %v", err) } @@ -280,7 +280,7 @@ func TestGpuIvfPqChunked(t *testing.T) { for j := range chunk { chunk[j] = val } - err = index.AddChunkFloat(chunk, chunkSize, nil) + err = index.AddChunkQuantize(chunk, chunkSize, nil) if err != nil { t.Fatalf("AddChunkFloat failed at offset %d: %v", i, err) } @@ -343,7 +343,7 @@ func TestGpuShardedIvfPq(t *testing.T) { bp := DefaultIvfPqBuildParams() bp.NLists = 10 bp.M = 2 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Sharded, nil) if err != nil { t.Fatalf("Failed to create sharded IVF-PQ: %v", err) } @@ -385,7 +385,7 @@ func TestGpuReplicatedIvfPq(t *testing.T) { bp := DefaultIvfPqBuildParams() bp.NLists = 10 bp.M = 2 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, Replicated, nil) if err != nil { t.Fatalf("Failed to create replicated IVF-PQ: %v", err) } @@ -423,7 +423,7 @@ func TestGpuIvfPqExtend(t *testing.T) { bp := DefaultIvfPqBuildParams() bp.NLists = 10 bp.M = 8 - index, err := NewGpuIvfPq[float32](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -498,7 +498,7 @@ func TestGpuIvfPqExtendFloat(t *testing.T) { bp.M = 8 bp.KmeansTrainsetFraction = 1.0 // Use Float16 so ExtendFloat exercises quantization - index, err := NewGpuIvfPq[Float16](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[Float16, Float16](dataset, nBase, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq[Float16]: %v", err) } @@ -532,14 +532,20 @@ func TestGpuIvfPqExtendFloat(t *testing.T) { sp := DefaultIvfPqSearchParams() sp.NProbes = 10 - // Query exactly at extended cluster; expect ID in [3000, 3050) - qExt := make([]float32, dimension) - for j := range qExt { - qExt[j] = extVal + // Query exactly at extended cluster; expect ID in [3000, 3050). This is a + // Float16-base index, so SearchQuantize takes a []Float16 query (the old + // SearchFloat's implicit f32->half is gone — convert explicitly). + qExtF32 := make([]float32, dimension) + for j := range qExtF32 { + qExtF32[j] = extVal } - r, err := index.SearchFloat(qExt, 1, dimension, 1, sp) + qExt := make([]Float16, dimension) + if err := GpuConvertF32ToF16(qExtF32, qExt, 0); err != nil { + t.Fatalf("convert query to f16: %v", err) + } + r, err := index.SearchQuantize(qExt, 1, dimension, 1, sp) if err != nil { - t.Fatalf("SearchFloat failed: %v", err) + t.Fatalf("SearchQuantize failed: %v", err) } if r.Neighbors[0] < 3000 || r.Neighbors[0] >= 3050 { t.Errorf("expected neighbor in [3000, 3050), got %d dist=%f", r.Neighbors[0], r.Distances[0]) @@ -565,7 +571,7 @@ func TestGpuIvfPqDeleteId(t *testing.T) { bp.NLists = 10 bp.M = 8 bp.KmeansTrainsetFraction = 1.0 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create GpuIvfPq: %v", err) } @@ -608,7 +614,7 @@ func TestGpuIvfPqDeleteId(t *testing.T) { } // 2. Test SearchFloat (this verifies the fix in search_float_internal) - r, err = index.SearchFloat(q50, 1, dimension, 1, sp) + r, err = index.SearchQuantize(q50, 1, dimension, 1, sp) if err != nil { t.Fatalf("SearchFloat failed: %v", err) } @@ -633,7 +639,7 @@ func BenchmarkGpuShardedIvfPq(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 128 // 1024 / 8 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Sharded, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Sharded, nil) if err != nil { b.Fatalf("Failed to create sharded IVF-PQ: %v", err) } @@ -662,7 +668,7 @@ func BenchmarkGpuShardedIvfPq(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -670,7 +676,7 @@ func BenchmarkGpuShardedIvfPq(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -694,7 +700,7 @@ func BenchmarkGpuSingleIvfPq(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 128 // 1024 / 8 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, SingleGpu, nil) if err != nil { b.Fatalf("Failed to create single IVF-PQ: %v", err) } @@ -723,7 +729,7 @@ func BenchmarkGpuSingleIvfPq(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -731,7 +737,7 @@ func BenchmarkGpuSingleIvfPq(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -758,7 +764,7 @@ func BenchmarkGpuReplicatedIvfPq(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 bp.M = 128 // 1024 / 8 - index, err := NewGpuIvfPq[float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) + index, err := NewGpuIvfPq[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, devices, 8, Replicated, nil) if err != nil { b.Fatalf("Failed to create replicated IVF-PQ: %v", err) } @@ -787,7 +793,7 @@ func BenchmarkGpuReplicatedIvfPq(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -795,7 +801,7 @@ func BenchmarkGpuReplicatedIvfPq(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(n_vectors), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -820,7 +826,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqF16(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 // Use Float16 as internal type - index, err := NewGpuIvfPqEmpty[Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuIvfPqEmpty[float32, Float16](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -833,7 +839,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqF16(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -854,7 +860,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqF16(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -862,7 +868,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqF16(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } @@ -884,7 +890,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqInt8(b *testing.B) { bp := DefaultIvfPqBuildParams() bp.NLists = 1000 // Use int8 as internal type - index, err := NewGpuIvfPqEmpty[int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) + index, err := NewGpuIvfPqEmpty[float32, int8](uint64(totalCount), dimension, L2Expanded, bp, devices, 8, SingleGpu) if err != nil { b.Fatalf("Failed to create index: %v", err) } @@ -897,7 +903,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqInt8(b *testing.B) { // Add data in chunks using AddChunkFloat for i := 0; i < totalCount; i += chunkSize { chunk := dataset[i*dimension : (i+chunkSize)*dimension] - if err := index.AddChunkFloat(chunk, uint64(chunkSize), nil); err != nil { + if err := index.AddChunkQuantize(chunk, uint64(chunkSize), nil); err != nil { b.Fatalf("AddChunkFloat failed at %d: %v", i, err) } } @@ -918,7 +924,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqInt8(b *testing.B) { queries[i] = rand.Float32() } for pb.Next() { - _, err := index.SearchFloat(queries, 1, dimension, 10, sp) + _, err := index.SearchQuantize(queries, 1, dimension, 10, sp) if err != nil { b.Fatalf("Search failed: %v", err) } @@ -926,7 +932,7 @@ func BenchmarkGpuAddChunkAndSearchIvfPqInt8(b *testing.B) { }) b.StopTimer() ReportRecall(b, dataset, uint64(totalCount), uint32(dimension), 10, func(queries []float32, numQueries uint64, limit uint32) ([]int64, error) { - res, err := index.SearchFloat(queries, numQueries, dimension, limit, sp) + res, err := index.SearchQuantize(queries, numQueries, dimension, limit, sp) if err != nil { return nil, err } diff --git a/pkg/cuvs/metric_support_test.go b/pkg/cuvs/metric_support_test.go index e3f124f69d189..688d4025c0b2b 100644 --- a/pkg/cuvs/metric_support_test.go +++ b/pkg/cuvs/metric_support_test.go @@ -104,7 +104,7 @@ func TestCagraMetricSupport(t *testing.T) { bp := DefaultCagraBuildParams() bp.IntermediateGraphDegree = 16 bp.GraphDegree = 8 - idx, err := NewGpuCagra[float32](ds, count, dim, mc.metric, bp, []int{0}, 1, SingleGpu, ids) + idx, err := NewGpuCagra[float32, float32](ds, count, dim, mc.metric, bp, []int{0}, 1, SingleGpu, ids) if err != nil { t.Fatalf("build CAGRA(%s): %v", mc.name, err) } @@ -140,7 +140,7 @@ func TestIvfPqMetricSupport(t *testing.T) { bp.M = 8 bp.BitsPerCode = 8 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfPq[float32](ds, count, dim, mc.metric, bp, []int{0}, 1, SingleGpu, ids) + idx, err := NewGpuIvfPq[float32, float32](ds, count, dim, mc.metric, bp, []int{0}, 1, SingleGpu, ids) if err != nil { t.Fatalf("build IVF-PQ(%s): %v", mc.name, err) } diff --git a/pkg/cuvs/multi_index.go b/pkg/cuvs/multi_index.go index b76bcbc4cc30e..c6586d1b1cc45 100644 --- a/pkg/cuvs/multi_index.go +++ b/pkg/cuvs/multi_index.go @@ -23,16 +23,30 @@ import ( "github.com/matrixorigin/matrixone/pkg/vectorindex" ) +// BruteForceOverflow is the type-erased CDC overflow: the storage type OB is +// hidden so one field/helper can hold *GpuBruteForce[B, OB] for any OB — the +// index storage Q when it is float/half, else the base B (for int8/uint8 storage, +// which cuVS brute force cannot store). Queries are always the base type B; the +// overflow quantizes B -> OB inside cuVS, so OB never appears in any signature. +type BruteForceOverflow[B VectorType] interface { + SearchQuantizeAsync(queries []B, numQueries uint64, dimension uint32, limit uint32) (uint64, error) + SearchQuantizeWithFilterAsync(queries []B, numQueries uint64, dimension uint32, limit uint32, predsJSON string) (uint64, error) + SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) + Cap() uint64 + Len() uint64 + Destroy() error +} + // MultiGpuIndex manages multiple GpuIndex instances and performs search across all of them using default parameters. type MultiGpuIndex[T VectorType] struct { indices []GpuIndex[T] - bruteForce *GpuBruteForce[T] + bruteForce *GpuBruteForce[T, T] dimension uint32 metric DistanceType } // NewMultiGpuIndex creates a new MultiGpuIndex instance. -func NewMultiGpuIndex[T VectorType](indices []GpuIndex[T], bruteForce *GpuBruteForce[T], dimension uint32, metric DistanceType) *MultiGpuIndex[T] { +func NewMultiGpuIndex[T VectorType](indices []GpuIndex[T], bruteForce *GpuBruteForce[T, T], dimension uint32, metric DistanceType) *MultiGpuIndex[T] { return &MultiGpuIndex[T]{ indices: indices, bruteForce: bruteForce, @@ -48,13 +62,6 @@ func (mi *MultiGpuIndex[T]) Search(queries []T, numQueries uint64, dimension uin }, nil, nil, nil) } -// SearchFloat32 performs a K-Nearest Neighbor search with float32 queries across all internal indices asynchronously. -func (mi *MultiGpuIndex[T]) SearchFloat32(queries []float32, numQueries uint64, dimension uint32, limit uint32) ([]int64, []float32, error) { - return multiGpuSearch(mi.indices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.SearchFloat32Async(q, nQ, d, l) - }, nil, nil) -} - // Destroy destroys all internal indices. func (mi *MultiGpuIndex[T]) Destroy() error { var firstErr error @@ -73,15 +80,15 @@ func (mi *MultiGpuIndex[T]) Destroy() error { // --- MultiGpuIvfFlat --- -type MultiGpuIvfFlat[T VectorType] struct { - indices []*GpuIvfFlat[T] - bruteForce *GpuBruteForce[T] +type MultiGpuIvfFlat[B VectorType, Q VectorType] struct { + indices []*GpuIvfFlat[B, Q] + bruteForce BruteForceOverflow[B] dimension uint32 metric DistanceType } -func NewMultiGpuIvfFlat[T VectorType](indices []*GpuIvfFlat[T], bruteForce *GpuBruteForce[T], dimension uint32, metric DistanceType) *MultiGpuIvfFlat[T] { - return &MultiGpuIvfFlat[T]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} +func NewMultiGpuIvfFlat[B VectorType, Q VectorType](indices []*GpuIvfFlat[B, Q], bruteForce BruteForceOverflow[B], dimension uint32, metric DistanceType) *MultiGpuIvfFlat[B, Q] { + return &MultiGpuIvfFlat[B, Q]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} } // All MultiIndex paths funnel through multiGpuSearch — every inner index @@ -90,90 +97,155 @@ func NewMultiGpuIvfFlat[T VectorType](indices []*GpuIvfFlat[T], bruteForce *GpuB // search_wait() (plan: effervescent-hatching-dewdrop.md), there is no // remaining reason to keep the sync fallbacks here; they bypassed dynamic // batching and serialized through main_thread_. -func (mi *MultiGpuIvfFlat[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// +// Storage-typed (Q) query path. When an overflow brute force is loaded it is +// base-typed (GpuBruteForce[B]), so it needs a []B query; we only have one when +// B==Q (i.e. F32/F16 storage, where storage type == base type). For the +// quantized combos (B=float/half, Q=int8/uint8) the []Q->[]B assertion fails, +// qB stays nil, and multiGpuSearchBQ's guard returns a "B/Q dispatch mismatch" +// error rather than searching — a storage-typed (already-quantized) query +// cannot be reconstructed into the base-typed query the overflow requires. +// Production code reaches the overflow via the base-typed query path +// (SearchQuantize), which is unaffected; callers needing the overflow with a +// quantized index should use SearchQuantize, not this typed entry point. +func (mi *MultiGpuIvfFlat[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, queries, nil, numQueries, dimension, limit, func(idx GpuIndex[T], q []T, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfFlat[T]).SearchAsyncWithParams(q, nQ, d, l, sp) - }, nil, nil, nil) + // Reinterpret the native Q query as []B for the base-typed overflow. Only + // succeeds when B==Q; nil otherwise (see the method doc above). + var qB []B + if mi.bruteForce != nil { + qB, _ = any(queries).([]B) + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, queries, nil, qB, nil, numQueries, dimension, limit, + func(idx GpuIndex[Q], q []Q, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfFlat[B, Q]).SearchAsyncWithParams(q, nQ, d, l, sp) + }, nil, nil, nil) } -func (mi *MultiGpuIvfFlat[T]) SearchFloat32(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantize — see MultiGpuIvfPq.SearchQuantize. +func (mi *MultiGpuIvfFlat[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfFlat[T]).SearchFloat32AsyncWithParams(q, nQ, d, l, sp) - }, nil, nil) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeAsync(q, nQ, d, l) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfFlat[B, Q]).SearchQuantizeAsyncWithParams(q, nQ, d, l, sp) + }}) } // --- MultiGpuIvfPq --- -type MultiGpuIvfPq[T VectorType] struct { - indices []*GpuIvfPq[T] - bruteForce *GpuBruteForce[T] +// MultiGpuIvfPq carries two element types: storage Q (the main cuVS ivf_pq +// indices) and base B (the CDC/overflow brute force). B==Q for a direct index; +// for a quantized index (e.g. vecf16 base -> int8 storage) B is the base type +// (Float16/float32) so the overflow brute force is cuVS-supported and lossless. +type MultiGpuIvfPq[B VectorType, Q VectorType] struct { + indices []*GpuIvfPq[B, Q] + bruteForce BruteForceOverflow[B] dimension uint32 metric DistanceType } -func NewMultiGpuIvfPq[T VectorType](indices []*GpuIvfPq[T], bruteForce *GpuBruteForce[T], dimension uint32, metric DistanceType) *MultiGpuIvfPq[T] { - return &MultiGpuIvfPq[T]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} +func NewMultiGpuIvfPq[B VectorType, Q VectorType](indices []*GpuIvfPq[B, Q], bruteForce BruteForceOverflow[B], dimension uint32, metric DistanceType) *MultiGpuIvfPq[B, Q] { + return &MultiGpuIvfPq[B, Q]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} } -func (mi *MultiGpuIvfPq[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +func (mi *MultiGpuIvfPq[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, queries, nil, numQueries, dimension, limit, func(idx GpuIndex[T], q []T, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfPq[T]).SearchAsyncWithParams(q, nQ, d, l, sp) - }, nil, nil, nil) + // Native Q query — the direct (B==Q) path; the overflow takes the same query + // reinterpreted as []B (B==Q here). + var qB []B + if mi.bruteForce != nil { + qB, _ = any(queries).([]B) + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, queries, nil, qB, nil, numQueries, dimension, limit, + func(idx GpuIndex[Q], q []Q, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfPq[B, Q]).SearchAsyncWithParams(q, nQ, d, l, sp) + }, nil, nil, nil) } -func (mi *MultiGpuIvfPq[T]) SearchFloat32(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantize searches with a base-typed (B) query: each main index converts +// B -> its storage type Q on device (B==Q copy for a direct index, learned/cast +// quantizer for narrower Q), and the base-typed overflow brute force takes the +// same B query. Unifies the former SearchFloat32 (B=float32) and SearchQuantizeHalf +// (B=half) paths; the non-filter twin of SearchQuantizeWithFilter. Works +// overflow-only (no main index, small data). +func (mi *MultiGpuIvfPq[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfPq[T]).SearchFloat32AsyncWithParams(q, nQ, d, l, sp) - }, nil, nil) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeAsync(q, nQ, d, l) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfPq[B, Q]).SearchQuantizeAsyncWithParams(q, nQ, d, l, sp) + }}) } // --- MultiGpuCagra --- -type MultiGpuCagra[T VectorType] struct { - indices []*GpuCagra[T] - bruteForce *GpuBruteForce[T] +// MultiGpuCagra carries base type B (overflow) and storage type Q (cagra +// indices) — see MultiGpuIvfPq. +type MultiGpuCagra[B VectorType, Q VectorType] struct { + indices []*GpuCagra[B, Q] + bruteForce BruteForceOverflow[B] dimension uint32 metric DistanceType } -func NewMultiGpuCagra[T VectorType](indices []*GpuCagra[T], bruteForce *GpuBruteForce[T], dimension uint32, metric DistanceType) *MultiGpuCagra[T] { - return &MultiGpuCagra[T]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} +func NewMultiGpuCagra[B VectorType, Q VectorType](indices []*GpuCagra[B, Q], bruteForce BruteForceOverflow[B], dimension uint32, metric DistanceType) *MultiGpuCagra[B, Q] { + return &MultiGpuCagra[B, Q]{indices: indices, bruteForce: bruteForce, dimension: dimension, metric: metric} } -func (mi *MultiGpuCagra[T]) Search(queries []T, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +func (mi *MultiGpuCagra[B, Q]) Search(queries []Q, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, queries, nil, numQueries, dimension, limit, func(idx GpuIndex[T], q []T, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuCagra[T]).SearchAsyncWithParams(q, nQ, d, l, sp) - }, nil, nil, nil) + var qB []B + if mi.bruteForce != nil { + qB, _ = any(queries).([]B) + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, queries, nil, qB, nil, numQueries, dimension, limit, + func(idx GpuIndex[Q], q []Q, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuCagra[B, Q]).SearchAsyncWithParams(q, nQ, d, l, sp) + }, nil, nil, nil) } -func (mi *MultiGpuCagra[T]) SearchFloat32(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantize — see MultiGpuIvfPq.SearchQuantize. +func (mi *MultiGpuCagra[B, Q]) SearchQuantize(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuCagra[T]).SearchFloat32AsyncWithParams(q, nQ, d, l, sp) - }, nil, nil) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeAsync(q, nQ, d, l) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuCagra[B, Q]).SearchQuantizeAsyncWithParams(q, nQ, d, l, sp) + }}) } // --- Helper search function --- @@ -186,7 +258,7 @@ func (mi *MultiGpuCagra[T]) SearchFloat32(queries []float32, numQueries uint64, // uses SearchFloatWithFilterAsync. func multiGpuSearch[T VectorType]( indices []GpuIndex[T], - bruteForce *GpuBruteForce[T], + bruteForce *GpuBruteForce[T, T], miDimension uint32, queries []T, queriesF32 []float32, @@ -195,8 +267,8 @@ func multiGpuSearch[T VectorType]( limit uint32, searchFn func(GpuIndex[T], []T, uint64, uint32, uint32) (uint64, error), searchF32Fn func(GpuIndex[T], []float32, uint64, uint32, uint32) (uint64, error), - bfSearchFn func(*GpuBruteForce[T], []T, uint64, uint32, uint32) (uint64, error), - bfSearchF32Fn func(*GpuBruteForce[T], []float32, uint64, uint32, uint32) (uint64, error), + bfSearchFn func(*GpuBruteForce[T, T], []T, uint64, uint32, uint32) (uint64, error), + bfSearchF32Fn func(*GpuBruteForce[T, T], []float32, uint64, uint32, uint32) (uint64, error), ) ([]int64, []float32, error) { if queryDimension != miDimension { return nil, nil, moerr.NewInternalErrorNoCtx("query dimension mismatch") @@ -212,7 +284,7 @@ func multiGpuSearch[T VectorType]( } type jobInfo struct { - index GpuIndex[T] + w searchWaiter jobID uint64 } jobs := make([]jobInfo, 0, numIndices) @@ -228,7 +300,7 @@ func multiGpuSearch[T VectorType]( if err != nil { return nil, nil, err } - jobs = append(jobs, jobInfo{index: idx, jobID: jobID}) + jobs = append(jobs, jobInfo{w: idx, jobID: jobID}) } if bruteForce != nil { @@ -238,26 +310,31 @@ func multiGpuSearch[T VectorType]( if bfSearchFn != nil { jobID, err = bfSearchFn(bruteForce, queries, numQueries, queryDimension, limit) } else { - jobID, err = bruteForce.SearchAsync(queries, numQueries, queryDimension, limit) + // Native query reinterpreted as the base type B (== T here) for the + // quantize entry; SearchQuantizeAsync quantizes B -> storage T. + qB, _ := any(queries).([]T) + jobID, err = bruteForce.SearchQuantizeAsync(qB, numQueries, queryDimension, limit) } } else { if bfSearchF32Fn != nil { jobID, err = bfSearchF32Fn(bruteForce, queriesF32, numQueries, queryDimension, limit) } else { - jobID, err = bruteForce.SearchFloat32Async(queriesF32, numQueries, queryDimension, limit) + // f32 query reinterpreted as the base type B (== T == float32 here). + qB, _ := any(queriesF32).([]T) + jobID, err = bruteForce.SearchQuantizeAsync(qB, numQueries, queryDimension, limit) } } if err != nil { return nil, nil, err } - jobs = append(jobs, jobInfo{index: bruteForce, jobID: jobID}) + jobs = append(jobs, jobInfo{w: bruteForce, jobID: jobID}) } allNeighbors := make([][]int64, len(jobs)) allDistances := make([][]float32, len(jobs)) for i, job := range jobs { - neighbors, distances, err := job.index.SearchWait(job.jobID, numQueries, limit) + neighbors, distances, err := job.w.SearchWait(job.jobID, numQueries, limit) if err != nil { return nil, nil, err } @@ -269,6 +346,137 @@ func multiGpuSearch[T VectorType]( return n, d, nil } +// searchWaiter is the post-submission contract shared by GpuIndex[Q] and +// *GpuBruteForce[B]: once a search job is submitted, collecting its result is +// type-agnostic (jobID -> []int64 neighbors, []float32 distances). This lets +// multiGpuSearchBQ merge index (storage type Q) and overflow (base type B) +// results without the two types leaking into the wait/merge. +type searchWaiter interface { + SearchWait(jobID uint64, numQueries uint64, limit uint32) ([]int64, []float32, error) +} + +// multiGpuSearchBQ is multiGpuSearch with the brute-force overflow typed by the +// BASE type B (f16/f32) independently of the index storage type Q — the [B,Q] +// design. Indices are searched with the []Q (e.g. quantized) query, the +// base-typed overflow with the []B (or f32) query; both are submitted async to +// the worker pool and the post-submission collect/merge is type-agnostic +// (searchWaiter). No extra goroutine. When B==Q this is equivalent to +// multiGpuSearch with the overflow carrying the base type. +// idxBaseQuery carries a base-typed (B) index query + its dispatch function for +// the quantize-with-filter path: the index is searched with the native base +// query (f32 or half) and converts it to storage T inside cuVS (the const-B* +// search_quantize entry). Passed as an optional variadic to multiGpuSearchBQ so +// the many unfiltered/storage-typed callers stay untouched; when present it +// takes precedence over queriesQ/queriesQF32 for the index loop. The overflow +// still uses the queriesB/queriesBF32 channels independently. +type idxBaseQuery[Q VectorType, B VectorType] struct { + queries []B + fn func(GpuIndex[Q], []B, uint64, uint32, uint32) (uint64, error) +} + +func multiGpuSearchBQ[Q VectorType, B VectorType]( + indices []GpuIndex[Q], + bruteForce BruteForceOverflow[B], + miDimension uint32, + queriesQ []Q, + queriesQF32 []float32, + queriesB []B, + queriesBF32 []float32, + numQueries uint64, + queryDimension uint32, + limit uint32, + idxFn func(GpuIndex[Q], []Q, uint64, uint32, uint32) (uint64, error), + idxF32Fn func(GpuIndex[Q], []float32, uint64, uint32, uint32) (uint64, error), + bfFn func(BruteForceOverflow[B], []B, uint64, uint32, uint32) (uint64, error), + bfF32Fn func(BruteForceOverflow[B], []float32, uint64, uint32, uint32) (uint64, error), + idxBase ...idxBaseQuery[Q, B], +) ([]int64, []float32, error) { + if queryDimension != miDimension { + return nil, nil, moerr.NewInternalErrorNoCtx("query dimension mismatch") + } + + n := len(indices) + if bruteForce != nil { + n++ + } + if n == 0 { + return nil, nil, moerr.NewInternalErrorNoCtx("no indices in MultiIndex") + } + + var ib *idxBaseQuery[Q, B] + if len(idxBase) > 0 { + ib = &idxBase[0] + } + + type jobInfo struct { + w searchWaiter + jobID uint64 + } + jobs := make([]jobInfo, 0, n) + + for _, idx := range indices { + var jobID uint64 + var err error + if ib != nil { + // Base-typed quantize query: index converts B -> storage T in cuVS. + jobID, err = ib.fn(idx, ib.queries, numQueries, queryDimension, limit) + } else if queriesQ != nil { + jobID, err = idxFn(idx, queriesQ, numQueries, queryDimension, limit) + } else { + jobID, err = idxF32Fn(idx, queriesQF32, numQueries, queryDimension, limit) + } + if err != nil { + return nil, nil, err + } + jobs = append(jobs, jobInfo{w: idx, jobID: jobID}) + } + + if bruteForce != nil { + // Guard against a dispatch mismatch: if the overflow brute force is + // live but neither a base-typed (B) nor an f32 query was supplied, the + // async search would submit an empty job (job id 0) and SearchWait(0) + // would block forever. Fail loudly instead — this means the [B,Q] + // instantiation disagrees with the decoded query type. + if len(queriesB) == 0 && len(queriesBF32) == 0 { + return nil, nil, moerr.NewInternalErrorNoCtx("multiGpuSearchBQ: brute force is loaded but no base/f32 query was provided (B/Q dispatch mismatch)") + } + // The overflow always takes the base-typed (B) query and quantizes B->OB + // inside cuVS. When only an f32 channel was supplied (the SearchFloat32 + // paths, where B==float32), reinterpret it as []B. + qB := queriesB + if qB == nil { + qB, _ = any(queriesBF32).([]B) + } + var jobID uint64 + var err error + if bfFn != nil { + jobID, err = bfFn(bruteForce, qB, numQueries, queryDimension, limit) + } else if bfF32Fn != nil { + jobID, err = bfF32Fn(bruteForce, queriesBF32, numQueries, queryDimension, limit) + } else { + jobID, err = bruteForce.SearchQuantizeAsync(qB, numQueries, queryDimension, limit) + } + if err != nil { + return nil, nil, err + } + jobs = append(jobs, jobInfo{w: bruteForce, jobID: jobID}) + } + + allNeighbors := make([][]int64, len(jobs)) + allDistances := make([][]float32, len(jobs)) + for i, job := range jobs { + neighbors, distances, err := job.w.SearchWait(job.jobID, numQueries, limit) + if err != nil { + return nil, nil, err + } + allNeighbors[i] = neighbors + allDistances[i] = distances + } + + n2, d := mergeMultiResults(allNeighbors, allDistances, numQueries, limit) + return n2, d, nil +} + // mergeMultiResults does a k-way merge of per-index top-k results into a single // top-k per query using a max-heap. Empty slots (neighbor == -1) are skipped. // Shared by both the async-dispatched multiGpuSearch and the synchronous @@ -310,48 +518,73 @@ func mergeMultiResults(allNeighbors [][]int64, allDistances [][]float32, numQuer // --- Filtered async search variants --- // -// Every per-index filtered search is dispatched via SearchFloatWithFilterAsync -// (which returns a job_id) and collected with SearchWait, matching the -// unfiltered SearchFloat32 path. Predicate evaluation, H2D, and GPU work for -// sibling indices overlap on their own worker threads, including the -// brute-force fallback when mi.bruteForce is non-nil. +// Each per-index filtered search is dispatched async (returns a job_id) and +// collected with SearchWait. cagra/ivf_pq/ivf_flat all use the base-typed +// SearchQuantizeWithFilterAsync (the const-B* quantize path); the brute-force +// overflow uses the base-typed SearchQuantizeWithFilterAsync. Predicate evaluation, H2D, +// and GPU work for sibling indices overlap on their own worker threads, +// including the brute-force fallback when mi.bruteForce is non-nil. // // SHARDED inner indices no longer get routed through main_thread_ — see the // C++ search_*_with_filter_async branches and plan // .claude/plans/effervescent-hatching-dewdrop.md. -func (mi *MultiGpuCagra[T]) SearchFloat32WithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantizeWithFilter runs a filtered K-NN search with base-typed (B) +// queries: each index quantizes B -> storage Q inside cuVS (the const-B* +// search_quantize_with_filter entry) and the base-typed overflow takes the same +// native B query. Covers both f32 base (B==float, query was []float32) and vecf16 +// base (B==half, query was []Float16) — they differ only in the concrete query +// slice the caller asserts to []B. Both async via the worker pool; works +// overflow-only (no main index, small data). +func (mi *MultiGpuCagra[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp CagraSearchParams, predsJSON string) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuCagra[T]).SearchFloatWithFilterAsync(q, nQ, d, l, sp, predsJSON) - }, nil, func(bf *GpuBruteForce[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return bf.SearchFloatWithFilterAsync(q, nQ, d, l, predsJSON) - }) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeWithFilterAsync(q, nQ, d, l, predsJSON) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuCagra[B, Q]).SearchQuantizeWithFilterAsync(q, nQ, d, l, sp, predsJSON) + }}) } -func (mi *MultiGpuIvfFlat[T]) SearchFloat32WithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantizeWithFilter — see MultiGpuCagra.SearchQuantizeWithFilter. +func (mi *MultiGpuIvfFlat[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfFlatSearchParams, predsJSON string) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfFlat[T]).SearchFloatWithFilterAsync(q, nQ, d, l, sp, predsJSON) - }, nil, func(bf *GpuBruteForce[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return bf.SearchFloatWithFilterAsync(q, nQ, d, l, predsJSON) - }) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeWithFilterAsync(q, nQ, d, l, predsJSON) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfFlat[B, Q]).SearchQuantizeWithFilterAsync(q, nQ, d, l, sp, predsJSON) + }}) } -func (mi *MultiGpuIvfPq[T]) SearchFloat32WithFilter(queries []float32, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) ([]int64, []float32, error) { - genericIndices := make([]GpuIndex[T], len(mi.indices)) +// SearchQuantizeWithFilter — see MultiGpuCagra.SearchQuantizeWithFilter. +func (mi *MultiGpuIvfPq[B, Q]) SearchQuantizeWithFilter(queries []B, numQueries uint64, dimension uint32, limit uint32, sp IvfPqSearchParams, predsJSON string) ([]int64, []float32, error) { + genericIndices := make([]GpuIndex[Q], len(mi.indices)) for i, idx := range mi.indices { genericIndices[i] = idx } - return multiGpuSearch(genericIndices, mi.bruteForce, mi.dimension, nil, queries, numQueries, dimension, limit, nil, func(idx GpuIndex[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return idx.(*GpuIvfPq[T]).SearchFloatWithFilterAsync(q, nQ, d, l, sp, predsJSON) - }, nil, func(bf *GpuBruteForce[T], q []float32, nQ uint64, d uint32, l uint32) (uint64, error) { - return bf.SearchFloatWithFilterAsync(q, nQ, d, l, predsJSON) - }) + var qOv []B + if mi.bruteForce != nil { + qOv = queries + } + return multiGpuSearchBQ(genericIndices, mi.bruteForce, mi.dimension, nil, nil, qOv, nil, numQueries, dimension, limit, + nil, nil, func(bf BruteForceOverflow[B], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return bf.SearchQuantizeWithFilterAsync(q, nQ, d, l, predsJSON) + }, nil, idxBaseQuery[Q, B]{queries: queries, fn: func(idx GpuIndex[Q], q []B, nQ uint64, d uint32, l uint32) (uint64, error) { + return idx.(*GpuIvfPq[B, Q]).SearchQuantizeWithFilterAsync(q, nQ, d, l, sp, predsJSON) + }}) } diff --git a/pkg/cuvs/multi_index_test.go b/pkg/cuvs/multi_index_test.go index bd73a53f8c854..61d349589f1b3 100644 --- a/pkg/cuvs/multi_index_test.go +++ b/pkg/cuvs/multi_index_test.go @@ -53,7 +53,7 @@ func TestMultiGpuIndex(t *testing.T) { // IVF-Flat bpIvf := DefaultIvfFlatBuildParams() - idx2, err := NewGpuIvfFlat[float32](dataset2, count2, dimension, metric, bpIvf, devices, nthread, SingleGpu, nil) + idx2, err := NewGpuIvfFlat[float32, float32](dataset2, count2, dimension, metric, bpIvf, devices, nthread, SingleGpu, nil) assert.NoError(t, err) err = idx2.Start() assert.NoError(t, err) @@ -61,7 +61,7 @@ func TestMultiGpuIndex(t *testing.T) { assert.NoError(t, err) // Brute Force - bf, err := NewGpuBruteForce[float32](dataset1, count1, dimension, metric, nthread, 0) + bf, err := NewGpuBruteForce[float32, float32](dataset1, count1, dimension, metric, nthread, 0) assert.NoError(t, err) err = bf.Start() assert.NoError(t, err) @@ -93,7 +93,7 @@ func TestMultiGpuIndex(t *testing.T) { // --- Test Specialized MultiGpuIvfFlat --- t.Run("SpecializedIvfFlat", func(t *testing.T) { - mivf := NewMultiGpuIvfFlat[float32]([]*GpuIvfFlat[float32]{idx2}, bf, dimension, metric) + mivf := NewMultiGpuIvfFlat[float32, float32]([]*GpuIvfFlat[float32, float32]{idx2}, bf, dimension, metric) numQueries := uint64(5) limit := uint32(10) diff --git a/pkg/cuvs/search_async_batch_test.go b/pkg/cuvs/search_async_batch_test.go index b713d70c0c18c..198e7facbb694 100644 --- a/pkg/cuvs/search_async_batch_test.go +++ b/pkg/cuvs/search_async_batch_test.go @@ -128,7 +128,7 @@ func TestGpuCagraSearchFloat32AsyncBatched(t *testing.T) { // result demuxing through submit_batched_async's per-request setter. runConcurrentAsync(t, 16 /*nGoroutines*/, 8 /*nPerGoroutine*/, func(qid int) (int64, error) { q := []float32{float32(qid), float32(qid)} - jobID, err := index.SearchFloat32AsyncWithParams(q, 1, dimension, 1, sp) + jobID, err := index.SearchQuantizeAsyncWithParams(q, 1, dimension, 1, sp) if err != nil { return -1, err } @@ -151,7 +151,7 @@ func TestGpuIvfFlatSearchFloat32AsyncBatched(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 16 - index, err := NewGpuIvfFlat[float32](dataset, nVectors, dimension, L2Expanded, bp, []int{0}, 4, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, float32](dataset, nVectors, dimension, L2Expanded, bp, []int{0}, 4, SingleGpu, nil) if err != nil { t.Fatalf("NewGpuIvfFlat: %v", err) } @@ -174,7 +174,7 @@ func TestGpuIvfFlatSearchFloat32AsyncBatched(t *testing.T) { runConcurrentAsync(t, 16 /*nGoroutines*/, 8 /*nPerGoroutine*/, func(qid int) (int64, error) { q := []float32{float32(qid), float32(qid)} - jobID, err := index.SearchFloat32AsyncWithParams(q, 1, dimension, 1, sp) + jobID, err := index.SearchQuantizeAsyncWithParams(q, 1, dimension, 1, sp) if err != nil { return -1, err } @@ -251,7 +251,7 @@ func ivfPqAsyncBatchedMatchesSync(t *testing.T, conservativeDispatch bool) { } want := make([][]int64, nQueries) for qid := 0; qid < nQueries; qid++ { - res, err := index.SearchFloat(queryOf(qid), 1, dimension, limit, sp) + res, err := index.SearchQuantize(queryOf(qid), 1, dimension, limit, sp) if err != nil { t.Fatalf("SearchFloat reference qid=%d: %v", qid, err) } @@ -269,7 +269,7 @@ func ivfPqAsyncBatchedMatchesSync(t *testing.T, conservativeDispatch bool) { wg.Add(1) go func(qid int) { defer wg.Done() - jobID, err := index.SearchFloat32AsyncWithParams(queryOf(qid), 1, dimension, limit, sp) + jobID, err := index.SearchQuantizeAsyncWithParams(queryOf(qid), 1, dimension, limit, sp) if err != nil { errCh <- err return @@ -353,7 +353,7 @@ func TestGpuCagraAsyncBatchedMatchesSync(t *testing.T) { want := make([]int64, nQueries) for qid := 0; qid < nQueries; qid++ { q := []float32{float32(qid * 10), float32(qid * 10)} - res, err := index.SearchFloat(q, 1, dimension, 1, sp) + res, err := index.SearchQuantize(q, 1, dimension, 1, sp) if err != nil { t.Fatalf("SearchFloat reference: %v", err) } @@ -375,7 +375,7 @@ func TestGpuCagraAsyncBatchedMatchesSync(t *testing.T) { go func(qid int) { defer wg.Done() q := []float32{float32(qid * 10), float32(qid * 10)} - jobID, err := index.SearchFloat32AsyncWithParams(q, 1, dimension, 1, sp) + jobID, err := index.SearchQuantizeAsyncWithParams(q, 1, dimension, 1, sp) if err != nil { errCh <- err return diff --git a/pkg/cuvs/search_f16quant_test.go b/pkg/cuvs/search_f16quant_test.go new file mode 100644 index 0000000000000..f567adac056e0 --- /dev/null +++ b/pkg/cuvs/search_f16quant_test.go @@ -0,0 +1,170 @@ +//go:build gpu + +// Copyright 2021 - 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cuvs + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "golang.org/x/exp/rand" +) + +// f16 converts a float32 to the cuVS half (Float16) bit pattern. types.Float16 +// and cuvs.Float16 are both uint16 IEEE-754 halfs, so the bits transfer directly. +func f16(f float32) Float16 { return Float16(types.Float16FromFloat32(f)) } + +// makeF16Dataset builds a deterministic random half dataset in [0,1). +func makeF16Dataset(n uint64, dim uint32) []Float16 { + r := rand.New(rand.NewSource(1)) + ds := make([]Float16, n*uint64(dim)) + for i := range ds { + ds[i] = f16(r.Float32()) + } + return ds +} + +// TestGpuF16QuantizeAll covers the vecf16-base -> int8/uint8 quantization path +// (the native half-source quantizer) for IVF-PQ and CAGRA: build from native +// Float16 input via AddChunkQuantize, then SearchQuantize with a Float16 query. +// This is the f16->int8 / f16->uint8 combination the float32-base info_test and +// search_float_test do not exercise. +// +// Correctness is graded as self-match RECALL: each probe query is an exact copy +// of a stored row, so a working quantized search must return that row's id in the +// top-k for the large majority of probes. (Exact top-1 is not asserted — int8/ +// uint8 + product quantization are lossy by design; recall is the right metric, +// matching the C++ Int8VsUint8SignedDataHalf test.) A broken search would score +// ~0, so the 0.8 floor is a real correctness check, not a shape check. +func TestGpuF16QuantizeAll(t *testing.T) { + const ( + dimension = uint32(16) + nVectors = uint64(2000) + k = uint32(10) + minRecall = 0.8 + ) + deviceID := 0 + ds := makeF16Dataset(nVectors, dimension) + probes := []int64{0, 1, 7, 100, 500, 999, 1500, 1999} // rows to self-query + + t.Run("IVF-PQ/f16-int8", func(t *testing.T) { + runIvfPqF16Quant[int8](t, ds, probes, nVectors, dimension, k, deviceID, minRecall) + }) + t.Run("IVF-PQ/f16-uint8", func(t *testing.T) { + runIvfPqF16Quant[uint8](t, ds, probes, nVectors, dimension, k, deviceID, minRecall) + }) + t.Run("CAGRA/f16-int8", func(t *testing.T) { + runCagraF16Quant[int8](t, ds, probes, nVectors, dimension, k, deviceID, minRecall) + }) + t.Run("CAGRA/f16-uint8", func(t *testing.T) { + runCagraF16Quant[uint8](t, ds, probes, nVectors, dimension, k, deviceID, minRecall) + }) +} + +// rowVec returns a copy of row id from ds (an exact self-query). +func rowVec(ds []Float16, id int64, dim uint32) []Float16 { + off := uint64(id) * uint64(dim) + return append([]Float16(nil), ds[off:off+uint64(dim)]...) +} + +func contains(xs []int64, want int64) bool { + for _, x := range xs { + if x == want { + return true + } + } + return false +} + +// gradeSelfMatch runs each probe as a self-query and returns the fraction whose +// own id appears in the top-k. +func gradeSelfMatch(t *testing.T, probes []int64, k uint32, search func(int64) ([]int64, error)) float64 { + t.Helper() + hits := 0 + for _, id := range probes { + neighbors, err := search(id) + if err != nil { + t.Fatalf("SearchQuantize(row %d): %v", id, err) + } + if uint32(len(neighbors)) != k { + t.Fatalf("row %d: expected %d neighbors, got %d", id, k, len(neighbors)) + } + if contains(neighbors, id) { + hits++ + } + } + return float64(hits) / float64(len(probes)) +} + +func runIvfPqF16Quant[Q VectorType](t *testing.T, ds []Float16, probes []int64, n uint64, dim, k uint32, dev int, minRecall float64) { + // Start from the defaults (which set KmeansTrainsetFraction etc.) and override + // only NLists — the default 1024 lists would be near-empty for 2000 vectors. + // A struct literal would zero-default the omitted fields, e.g. + // KmeansTrainsetFraction=0 => no kmeans training => near-zero recall. + bp := DefaultIvfPqBuildParams() + bp.NLists = 50 + index, err := NewGpuIvfPqEmpty[Float16, Q](n, dim, L2Expanded, bp, []int{dev}, 1, SingleGpu) + if err != nil { + t.Fatalf("NewGpuIvfPqEmpty[Float16,Q]: %v", err) + } + defer index.Destroy() + index.Start() + if err = index.TrainQuantizer(ds, n); err != nil { + t.Fatalf("TrainQuantizer: %v", err) + } + if err = index.AddChunkQuantize(ds, n, nil); err != nil { + t.Fatalf("AddChunkQuantize: %v", err) + } + if err = index.Build(); err != nil { + t.Fatalf("Build: %v", err) + } + sp := DefaultIvfPqSearchParams() + sp.NProbes = bp.NLists // probe every list for a deterministic exhaustive search + recall := gradeSelfMatch(t, probes, k, func(id int64) ([]int64, error) { + res, err := index.SearchQuantize(rowVec(ds, id, dim), 1, dim, k, sp) + return res.Neighbors, err + }) + if recall < minRecall { + t.Errorf("IVF-PQ f16-quant self-match recall %.2f < %.2f", recall, minRecall) + } +} + +func runCagraF16Quant[Q VectorType](t *testing.T, ds []Float16, probes []int64, n uint64, dim, k uint32, dev int, minRecall float64) { + bp := DefaultCagraBuildParams() + index, err := NewGpuCagraEmpty[Float16, Q](n, dim, L2Expanded, bp, []int{dev}, 1, SingleGpu) + if err != nil { + t.Fatalf("NewGpuCagraEmpty[Float16,Q]: %v", err) + } + defer index.Destroy() + index.Start() + if err = index.TrainQuantizer(ds, n); err != nil { + t.Fatalf("TrainQuantizer: %v", err) + } + if err = index.AddChunkQuantize(ds, n, nil); err != nil { + t.Fatalf("AddChunkQuantize: %v", err) + } + if err = index.Build(); err != nil { + t.Fatalf("Build: %v", err) + } + sp := DefaultCagraSearchParams() + recall := gradeSelfMatch(t, probes, k, func(id int64) ([]int64, error) { + res, err := index.SearchQuantize(rowVec(ds, id, dim), 1, dim, k, sp) + return res.Neighbors, err + }) + if recall < minRecall { + t.Errorf("CAGRA f16-quant self-match recall %.2f < %.2f", recall, minRecall) + } +} diff --git a/pkg/cuvs/search_float_test.go b/pkg/cuvs/search_float_test.go index 7ad9106e7a0df..7b288f661b624 100644 --- a/pkg/cuvs/search_float_test.go +++ b/pkg/cuvs/search_float_test.go @@ -33,7 +33,7 @@ func TestGpuSearchFloatAll(t *testing.T) { } bp := IvfPqBuildParams{NLists: 10, M: 4, BitsPerCode: 8, AddDataOnBuild: true} // Create empty index - index, err := NewGpuIvfPqEmpty[int8](n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu) + index, err := NewGpuIvfPqEmpty[float32, int8](n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu) if err != nil { t.Fatalf("Failed to create IVF-PQ: %v", err) } @@ -46,7 +46,7 @@ func TestGpuSearchFloatAll(t *testing.T) { t.Fatalf("TrainQuantizer failed: %v", err) } - err = index.AddChunkFloat(dataset, n_vectors, nil) + err = index.AddChunkQuantize(dataset, n_vectors, nil) if err != nil { t.Fatalf("AddChunkFloat failed: %v", err) } @@ -56,7 +56,7 @@ func TestGpuSearchFloatAll(t *testing.T) { for i := range queries { queries[i] = float32(i % 10) } - res, err := index.SearchFloat(queries, 2, dimension, 1, IvfPqSearchParams{NProbes: 1}) + res, err := index.SearchQuantize(queries, 2, dimension, 1, IvfPqSearchParams{NProbes: 1}) if err != nil { t.Fatalf("SearchFloat failed: %v", err) } @@ -69,7 +69,7 @@ func TestGpuSearchFloatAll(t *testing.T) { t.Run("IVF-Flat", func(t *testing.T) { dataset := make([]Float16, n_vectors*uint64(dimension)) bp := IvfFlatBuildParams{NLists: 10, AddDataOnBuild: true} - index, err := NewGpuIvfFlat[Float16](dataset, n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu, nil) + index, err := NewGpuIvfFlat[float32, Float16](dataset, n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create IVF-Flat: %v", err) } @@ -78,7 +78,7 @@ func TestGpuSearchFloatAll(t *testing.T) { index.Build() queries := make([]float32, uint64(dimension)) - res, err := index.SearchFloat(queries, 1, dimension, 1, IvfFlatSearchParams{NProbes: 1}) + res, err := index.SearchQuantize(queries, 1, dimension, 1, IvfFlatSearchParams{NProbes: 1}) if err != nil { t.Fatalf("SearchFloat failed: %v", err) } @@ -91,7 +91,7 @@ func TestGpuSearchFloatAll(t *testing.T) { t.Run("CAGRA", func(t *testing.T) { dataset := make([]float32, n_vectors*uint64(dimension)) bp := CagraBuildParams{IntermediateGraphDegree: 64, GraphDegree: 32, AttachDatasetOnBuild: true} - index, err := NewGpuCagra[float32](dataset, n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu, nil) + index, err := NewGpuCagra[float32, float32](dataset, n_vectors, dimension, L2Expanded, bp, []int{deviceID}, 1, SingleGpu, nil) if err != nil { t.Fatalf("Failed to create CAGRA: %v", err) } @@ -100,7 +100,7 @@ func TestGpuSearchFloatAll(t *testing.T) { index.Build() queries := make([]float32, uint64(dimension)) - res, err := index.SearchFloat(queries, 1, dimension, 1, CagraSearchParams{ItopkSize: 64, SearchWidth: 1}) + res, err := index.SearchQuantize(queries, 1, dimension, 1, CagraSearchParams{ItopkSize: 64, SearchWidth: 1}) if err != nil { t.Fatalf("SearchFloat failed: %v", err) } @@ -112,7 +112,7 @@ func TestGpuSearchFloatAll(t *testing.T) { // 4. Test Brute-Force SearchFloat (with half) t.Run("Brute-Force", func(t *testing.T) { dataset := make([]Float16, n_vectors*uint64(dimension)) - index, err := NewGpuBruteForce[Float16](dataset, n_vectors, dimension, L2Expanded, 1, deviceID) + index, err := NewGpuBruteForce[float32, Float16](dataset, n_vectors, dimension, L2Expanded, 1, deviceID) if err != nil { t.Fatalf("Failed to create Brute-Force: %v", err) } @@ -121,9 +121,9 @@ func TestGpuSearchFloatAll(t *testing.T) { index.Build() queries := make([]float32, uint64(dimension)) - neighbors, _, err := index.SearchFloat(queries, 1, dimension, 1) + neighbors, _, err := index.SearchQuantize(queries, 1, dimension, 1) if err != nil { - t.Fatalf("SearchFloat failed: %v", err) + t.Fatalf("SearchQuantize failed: %v", err) } if len(neighbors) != 1 { t.Errorf("Expected 1 neighbor, got %d", len(neighbors)) diff --git a/pkg/cuvs/simulation_test.go b/pkg/cuvs/simulation_test.go index 4a4f2ba05e4a7..3015bd696f848 100644 --- a/pkg/cuvs/simulation_test.go +++ b/pkg/cuvs/simulation_test.go @@ -107,7 +107,7 @@ func TestSimulatedReplicatedIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 4 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfFlat[float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuIvfFlat[float32, float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -142,7 +142,7 @@ func TestSimulatedShardedIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 4 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfFlat[float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Sharded, ids) + idx, err := NewGpuIvfFlat[float32, float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Sharded, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -185,7 +185,7 @@ func TestSimulatedShardedDeleteIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 4 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfFlat[float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Sharded, ids) + idx, err := NewGpuIvfFlat[float32, float32](ds, count, dim, L2Expanded, bp, simDevices(), simRanks, Sharded, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -249,7 +249,7 @@ func TestSimulatedReplicatedExtendIvfFlat(t *testing.T) { bp := DefaultIvfFlatBuildParams() bp.NLists = 4 bp.KmeansTrainsetFraction = 1.0 - idx, err := NewGpuIvfFlat[float32](ds, base, dim, L2Expanded, bp, simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuIvfFlat[float32, float32](ds, base, dim, L2Expanded, bp, simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -303,7 +303,7 @@ func TestSimulatedReplicatedIvfPq(t *testing.T) { count := uint64(64) ds, ids := simData(count, dim) - idx, err := NewGpuIvfPq[float32](ds, count, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuIvfPq[float32, float32](ds, count, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -335,7 +335,7 @@ func TestSimulatedShardedIvfPq(t *testing.T) { count := uint64(128) // 4 shards of 32 ds, ids := simData(count, dim) - idx, err := NewGpuIvfPq[float32](ds, count, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Sharded, ids) + idx, err := NewGpuIvfPq[float32, float32](ds, count, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Sharded, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -367,7 +367,7 @@ func TestSimulatedReplicatedExtendIvfPq(t *testing.T) { base := uint64(64) ds, ids := simData(base, dim) - idx, err := NewGpuIvfPq[float32](ds, base, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuIvfPq[float32, float32](ds, base, dim, L2Expanded, simIvfPqParams(), simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -441,7 +441,7 @@ func TestSimulatedReplicatedCagra(t *testing.T) { count := uint64(64) ds, ids := simData(count, dim) - idx, err := NewGpuCagra[float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuCagra[float32, float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -473,7 +473,7 @@ func TestSimulatedShardedCagra(t *testing.T) { count := uint64(128) // 4 shards of 32 ds, ids := simData(count, dim) - idx, err := NewGpuCagra[float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Sharded, ids) + idx, err := NewGpuCagra[float32, float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Sharded, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -517,7 +517,7 @@ func TestSimulatedCagraSaveLoadAcrossModes(t *testing.T) { // Build REPLICATED under simulation and save the index files. { - idx, err := NewGpuCagra[float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated, ids) + idx, err := NewGpuCagra[float32, float32](ds, count, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated, ids) if err != nil { t.Fatalf("new: %v", err) } @@ -540,7 +540,7 @@ func TestSimulatedCagraSaveLoadAcrossModes(t *testing.T) { // Reload as REPLICATED (4 ranks). { - idx, err := NewGpuCagraFromDataDirectory[float32](dir, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated) + idx, err := NewGpuCagraFromDataDirectory[float32, float32](dir, dim, L2Expanded, simCagraBuildParams(), simDevices(), simRanks, Replicated) if err != nil { t.Fatalf("load replicated: %v", err) } @@ -557,7 +557,7 @@ func TestSimulatedCagraSaveLoadAcrossModes(t *testing.T) { // Reload the same files as SINGLE. { - idx, err := NewGpuCagraFromDataDirectory[float32](dir, dim, L2Expanded, simCagraBuildParams(), []int{0}, 1, SingleGpu) + idx, err := NewGpuCagraFromDataDirectory[float32, float32](dir, dim, L2Expanded, simCagraBuildParams(), []int{0}, 1, SingleGpu) if err != nil { t.Fatalf("load single: %v", err) } diff --git a/pkg/frontend/export.go b/pkg/frontend/export.go index 5d1b77c5739c9..7b928c78c96ad 100644 --- a/pkg/frontend/export.go +++ b/pkg/frontend/export.go @@ -718,6 +718,20 @@ func exportDataFromResultSetToCSVFile(oq *ExportConfig) error { } else if arr, ok := value.([]float64); ok { // this is for T_array_float64 type value = []byte(types.ArrayToString[float64](arr)) + } else if arr, ok := value.([]types.BF16); ok { + // this is for T_array_bf16 type + value = []byte(types.ArrayToString[types.BF16](arr)) + } else if arr, ok := value.([]types.Float16); ok { + // this is for T_array_float16 type + value = []byte(types.ArrayToString[types.Float16](arr)) + } else if arr, ok := value.([]int8); ok { + // this is for T_array_int8 type + value = []byte(types.ArrayToString[int8](arr)) + } else if s, ok := value.(string); ok { + // this is for T_array_uint8 (stored as its display string in + // extractRowFromVector, since []uint8 is indistinguishable from + // raw []byte) and any other string-valued varchar column + value = []byte(s) } if err = formatOutputString(oq, value.([]byte), symbol[i], closeby, true, buffer); err != nil { diff --git a/pkg/frontend/export_test.go b/pkg/frontend/export_test.go index c840ae594a9aa..8e3197e23ae53 100644 --- a/pkg/frontend/export_test.go +++ b/pkg/frontend/export_test.go @@ -265,6 +265,46 @@ func Test_exportDataToCSVFile(t *testing.T) { convey.So(exportDataFromResultSetToCSVFile(ep), convey.ShouldBeNil) }) + // Guards the narrow-vector export path: bf16/f16/int8 are emitted as their + // distinct slice types and vecuint8 as its display string (see + // extractRowFromVector). Before the fix the VARCHAR branch only special-cased + // []float32/[]float64 and then did value.([]byte) — a panic for []types.BF16 / + // []types.Float16 / []int8 and raw-byte corruption for []uint8. + convey.Convey("exportDataFromResultSetToCSVFile narrow vectors", t, func() { + ep := &ExportConfig{ + userConfig: &tree.ExportParam{ + Lines: &tree.Lines{TerminatedBy: &tree.Terminated{}}, + Fields: &tree.Fields{Terminated: &tree.Terminated{}, EnclosedBy: &tree.EnclosedBy{}, EscapedBy: &tree.EscapedBy{}}, + Header: true, + FilePath: "test/export_narrow.csv", + }, + mrs: &MysqlResultSet{}, + } + col := make([]MysqlColumn, 4) + for i := range col { + col[i].SetColumnType(defines.MYSQL_TYPE_VARCHAR) + ep.mrs.AddColumn(&col[i]) + } + f32 := []float32{1, 2, 3} + data := make([]interface{}, len(col)) + data[0] = types.Float32ToBF16Slice(f32) // bf16 slice + data[1] = types.Float32ToFloat16Slice(f32) // f16 slice + data[2] = []int8{1, 2, 3} // int8 slice + data[3] = types.ArrayToString[uint8]([]uint8{1, 2, 3}) // uint8 display string + ep.mrs.AddRow(data) + ep.Symbol = make([][]byte, len(col)) + ep.ColumnFlag = make([]bool, len(col)) + + stubs := gostub.StubFunc(&Close, nil) + defer stubs.Reset() + stubs = gostub.StubFunc(&openNewFile, nil) + defer stubs.Reset() + stubs = gostub.StubFunc(&writeDataToCSVFile, nil) + defer stubs.Reset() + + convey.So(exportDataFromResultSetToCSVFile(ep), convey.ShouldBeNil) + }) + convey.Convey("exportDataToCSVFile fail", t, func() { ep := &ExportConfig{ userConfig: &tree.ExportParam{ diff --git a/pkg/frontend/mysql_cmd_executor.go b/pkg/frontend/mysql_cmd_executor.go index 00d315e541a33..1e9b9817d8cdc 100644 --- a/pkg/frontend/mysql_cmd_executor.go +++ b/pkg/frontend/mysql_cmd_executor.go @@ -3901,7 +3901,8 @@ func convertEngineTypeToMysqlType(ctx context.Context, engineType types.T, col * col.SetColumnType(defines.MYSQL_TYPE_STRING) case types.T_varchar: col.SetColumnType(defines.MYSQL_TYPE_VAR_STRING) - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: col.SetColumnType(defines.MYSQL_TYPE_VARCHAR) case types.T_datalink: col.SetColumnType(defines.MYSQL_TYPE_TEXT) diff --git a/pkg/frontend/output.go b/pkg/frontend/output.go index 3dbb54c80d463..ee9f7609467ba 100644 --- a/pkg/frontend/output.go +++ b/pkg/frontend/output.go @@ -117,6 +117,36 @@ func extractRowFromVector(ctx context.Context, ses FeSession, vec *vector.Vector } else { row[i] = append([]float64(nil), arr...) } + case types.T_array_bf16: + arr := vector.GetArrayAt[types.BF16](vec, rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]types.BF16(nil), arr...) + } + case types.T_array_float16: + arr := vector.GetArrayAt[types.Float16](vec, rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]types.Float16(nil), arr...) + } + case types.T_array_int8: + arr := vector.GetArrayAt[int8](vec, rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]int8(nil), arr...) + } + case types.T_array_uint8: + // vecuint8's element slice is []uint8, which is indistinguishable from a + // raw []byte (binary/varbinary) value once the column is mapped to + // MYSQL_TYPE_VARCHAR. Every value-based consumer of mrs.Data (GetString, + // the legacy row encoders, CSV export) would then treat it as raw bytes + // and emit corrupt output. Store the display string instead so it is + // unambiguous; the main SELECT path (extractRowFromVector2/GetStringBased) + // is oid-aware and renders directly from the vector, unaffected by this. + row[i] = types.ArrayToString[uint8](vector.GetArrayAt[uint8](vec, rowIndex)) case types.T_date: row[i] = vector.GetFixedAtNoTypeCheck[types.Date](vec, rowIndex) case types.T_datetime: @@ -248,6 +278,34 @@ func extractRowFromVector2(ctx context.Context, ses FeSession, vec *vector.Vecto } else { row[i] = append([]float64(nil), arr...) } + case types.T_array_bf16: + arr := vector.GetArrayAt2[types.BF16](vec, colSlices.arrVarlena[sliceIdx], rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]types.BF16(nil), arr...) + } + case types.T_array_float16: + arr := vector.GetArrayAt2[types.Float16](vec, colSlices.arrVarlena[sliceIdx], rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]types.Float16(nil), arr...) + } + case types.T_array_int8: + arr := vector.GetArrayAt2[int8](vec, colSlices.arrVarlena[sliceIdx], rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]int8(nil), arr...) + } + case types.T_array_uint8: + arr := vector.GetArrayAt2[uint8](vec, colSlices.arrVarlena[sliceIdx], rowIndex) + if safeRefSlice { + row[i] = arr + } else { + row[i] = append([]uint8(nil), arr...) + } case types.T_date: row[i] = colSlices.arrDate[sliceIdx][rowIndex] case types.T_datetime: @@ -570,6 +628,14 @@ func (slices *ColumnSlices) GetStringBased(r uint64, i uint64) (string, error) { return types.ArrayToString[float32](vector.GetArrayAt2[float32](vec, slices.arrVarlena[sliceIdx], int(r))), nil case types.T_array_float64: return types.ArrayToString[float64](vector.GetArrayAt2[float64](vec, slices.arrVarlena[sliceIdx], int(r))), nil + case types.T_array_bf16: + return types.ArrayToString[types.BF16](vector.GetArrayAt2[types.BF16](vec, slices.arrVarlena[sliceIdx], int(r))), nil + case types.T_array_float16: + return types.ArrayToString[types.Float16](vector.GetArrayAt2[types.Float16](vec, slices.arrVarlena[sliceIdx], int(r))), nil + case types.T_array_int8: + return types.ArrayToString[int8](vector.GetArrayAt2[int8](vec, slices.arrVarlena[sliceIdx], int(r))), nil + case types.T_array_uint8: + return types.ArrayToString[uint8](vector.GetArrayAt2[uint8](vec, slices.arrVarlena[sliceIdx], int(r))), nil case types.T_Rowid: return slices.arrRowid[sliceIdx][r].String(), nil case types.T_Blockid: @@ -780,6 +846,9 @@ func convertVectorToSlice(ctx context.Context, ses FeSession, vec *vector.Vector case types.T_array_float64: colSlices.colIdx2SliceIdx[i] = len(colSlices.arrVarlena) colSlices.arrVarlena = append(colSlices.arrVarlena, vector.ToSliceNoTypeCheck2[types.Varlena](vec)) + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: + colSlices.colIdx2SliceIdx[i] = len(colSlices.arrVarlena) + colSlices.arrVarlena = append(colSlices.arrVarlena, vector.ToSliceNoTypeCheck2[types.Varlena](vec)) case types.T_date: colSlices.colIdx2SliceIdx[i] = len(colSlices.arrDate) colSlices.arrDate = append(colSlices.arrDate, vector.ToSliceNoTypeCheck2[types.Date](vec)) diff --git a/pkg/frontend/output_test.go b/pkg/frontend/output_test.go index 5625d7efb1e4c..5b155ac5a1c26 100644 --- a/pkg/frontend/output_test.go +++ b/pkg/frontend/output_test.go @@ -45,6 +45,79 @@ func TestExtractRowFromVector(t *testing.T) { } } +// TestExtractRowFromVectorNarrowVec guards the row-based (GetValue) display path +// for the narrow vector types. vecuint8 in particular must NOT be stored as +// []uint8: that is the same Go type as a raw []byte (binary/varbinary) value once +// the column is mapped to MYSQL_TYPE_VARCHAR, so every value-based consumer +// (GetString, the legacy row encoders, CSV export) would emit raw bytes / corrupt +// output. It is stored as its display string instead. bf16/f16/int8 stay as their +// distinct slice types (GetString renders them via ArrayToString). Every type must +// render as the human-readable "[1, 2, 3]" form, never raw bytes. +func TestExtractRowFromVectorNarrowVec(t *testing.T) { + mp := mpool.MustNewZero() + + f32 := []float32{1, 2, 3} + bf16 := types.Float32ToBF16Slice(f32) + f16 := types.Float32ToFloat16Slice(f32) + i8 := []int8{1, 2, 3} + u8 := []uint8{1, 2, 3} + + cases := []struct { + name string + oid types.T + bytes []byte + display string + assert func(t *testing.T, v any) + }{ + { + name: "bf16", oid: types.T_array_bf16, + bytes: types.ArrayToBytes[types.BF16](bf16), + display: types.ArrayToString[types.BF16](bf16), + assert: func(t *testing.T, v any) { _, ok := v.([]types.BF16); require.Truef(t, ok, "got %T", v) }, + }, + { + name: "f16", oid: types.T_array_float16, + bytes: types.ArrayToBytes[types.Float16](f16), + display: types.ArrayToString[types.Float16](f16), + assert: func(t *testing.T, v any) { _, ok := v.([]types.Float16); require.Truef(t, ok, "got %T", v) }, + }, + { + name: "int8", oid: types.T_array_int8, + bytes: types.ArrayToBytes[int8](i8), + display: types.ArrayToString[int8](i8), + assert: func(t *testing.T, v any) { _, ok := v.([]int8); require.Truef(t, ok, "got %T", v) }, + }, + { + name: "uint8", oid: types.T_array_uint8, + bytes: types.ArrayToBytes[uint8](u8), + display: types.ArrayToString[uint8](u8), + assert: func(t *testing.T, v any) { + s, ok := v.(string) + require.Truef(t, ok, "vecuint8 must be stored as a string (not []uint8/[]byte), got %T", v) + require.Equal(t, types.ArrayToString[uint8](u8), s) + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + vec := vector.NewVec(c.oid.ToType()) + require.NoError(t, vector.AppendAny(vec, c.bytes, false, mp)) + + row := make([]any, 1) + require.NoError(t, extractRowFromVector(context.TODO(), nil, vec, 0, row, 0, false)) + c.assert(t, row[0]) + + mrs := &MysqlResultSet{} + mrs.Data = [][]any{{row[0]}} + mrs.Columns = make([]Column, 1) + got, err := mrs.GetString(context.TODO(), 0, 0) + require.NoError(t, err) + require.Equal(t, c.display, got) + }) + } +} + func BenchmarkName(b *testing.B) { mp := mpool.MustNewZero() diff --git a/pkg/frontend/resultset.go b/pkg/frontend/resultset.go index 934fc833f3473..7264675153763 100644 --- a/pkg/frontend/resultset.go +++ b/pkg/frontend/resultset.go @@ -471,6 +471,12 @@ func (mrs *MysqlResultSet) GetString(ctx context.Context, rindex, cindex uint64) return types.ArrayToString[float32](v), nil case []float64: return types.ArrayToString[float64](v), nil + case []types.BF16: + return types.ArrayToString[types.BF16](v), nil + case []types.Float16: + return types.ArrayToString[types.Float16](v), nil + case []int8: + return types.ArrayToString[int8](v), nil case int: return strconv.FormatInt(int64(v), 10), nil case uint: diff --git a/pkg/frontend/util.go b/pkg/frontend/util.go index a843a62572dc3..137fcd1538f65 100644 --- a/pkg/frontend/util.go +++ b/pkg/frontend/util.go @@ -361,6 +361,14 @@ func getValueFromVector(ctx context.Context, vec *vector.Vector, feSes FeSession return vector.GetArrayAt[float32](vec, 0), nil case types.T_array_float64: return vector.GetArrayAt[float64](vec, 0), nil + case types.T_array_bf16: + return vector.GetArrayAt[types.BF16](vec, 0), nil + case types.T_array_float16: + return vector.GetArrayAt[types.Float16](vec, 0), nil + case types.T_array_int8: + return vector.GetArrayAt[int8](vec, 0), nil + case types.T_array_uint8: + return vector.GetArrayAt[uint8](vec, 0), nil case types.T_decimal64: val := vector.GetFixedAtNoTypeCheck[types.Decimal64](vec, 0) return val.Format(expr.Typ.Scale), nil diff --git a/pkg/fulltext/plugin/runtime/runtime.go b/pkg/fulltext/plugin/runtime/runtime.go index fc984170bc1d4..b83beb8c482aa 100644 --- a/pkg/fulltext/plugin/runtime/runtime.go +++ b/pkg/fulltext/plugin/runtime/runtime.go @@ -88,6 +88,9 @@ func (CatalogHooks) SupportedVectorTypes() []types.T { return nil } // SupportedPrimaryKeyTypes: fulltext imposes no PK-type constraint. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return nil } +// ValidQuantization — full-text indexes have no quantization, so nothing to gate. +func (CatalogHooks) ValidQuantization(_, _ string) error { return nil } + // SupportedOpTypes — fulltext has no metric/op-type concept. // SupportedIncludeColumnTypes: this index has no INCLUDE-column support. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } diff --git a/pkg/indexplugin/catalog/hooks.go b/pkg/indexplugin/catalog/hooks.go index 4e446db7cac8b..3567ae3b16cad 100644 --- a/pkg/indexplugin/catalog/hooks.go +++ b/pkg/indexplugin/catalog/hooks.go @@ -83,6 +83,18 @@ type Hooks interface { // and fulltext return nil, so SupportsIncludeColumnType reports false. SupportedIncludeColumnTypes() []types.T + // ValidQuantization reports whether QUANTIZATION='quant' is usable by this + // algorithm under op_type 'op', returning a descriptive error when not + // (nil = valid). It is the single per-algorithm rule for the + // (quantization, op_type) pair, so CREATE (plan-side schema validation) and + // REINDEX (compile-side ValidateReindexParams) gate it identically instead + // of duplicating the check. An empty quant means "no quantization / default + // storage" (valid); an empty op means "no metric in play" (only the + // storage-type rule applies). Example: the cuvs (CAGRA / IVF-PQ) backend + // rejects int8/uint8 with inner-product / cosine because its affine scalar + // quantizer only preserves L2 geometry. + ValidQuantization(quant, op string) error + // ExperimentalFlag returns the experimental-feature flag name that // must be enabled (set to true via SET / system var) for this // algorithm to be usable. Returns "" for non-experimental diff --git a/pkg/iscp/cuvs_writer.go b/pkg/iscp/cuvs_writer.go index d79e18fd7ed3a..3a1099de16164 100644 --- a/pkg/iscp/cuvs_writer.go +++ b/pkg/iscp/cuvs_writer.go @@ -22,6 +22,7 @@ import ( "github.com/bytedance/sonic" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/pb/plan" cuvscdc "github.com/matrixorigin/matrixone/pkg/vectorindex/cuvs" @@ -64,6 +65,8 @@ type CuvsCdcWriter struct { pkPos int32 partsPos []int32 dimension int32 + baseType types.T // vector column element type: vecf32 or vecf16 + vecBytesPer int // dim * base element size (4*dim for f32, 2*dim for f16) dbName string tblName string indexName string @@ -78,8 +81,8 @@ type CuvsCdcWriter struct { // switch on it); dbName/tblName/indexName are typically pulled from // the ISCP ConsumerInfo at the call site. // -// Both CAGRA and IVF-PQ on cuvs are fp32-only with a bigint PK; this -// constructor enforces both shapes. +// Both CAGRA and IVF-PQ on cuvs take a vecf32 or vecf16 base column with +// a bigint PK; this constructor enforces both shapes. func NewCuvsCdcWriter(algoName, dbName, tblName, indexName string, tabledef *plan.TableDef, indexdefs []*plan.IndexDef) (*CuvsCdcWriter, error) { @@ -121,11 +124,23 @@ func NewCuvsCdcWriter(algoName, dbName, tblName, indexName string, w.partsPos[i] = tabledef.Name2ColIndex[part] } vecTyp := tabledef.Cols[w.partsPos[0]].Typ - if vecTyp.Id != int32(types.T_array_float32) { + // cuvs accepts a vecf32 or vecf16 base column. The CDC record carries the + // vector as raw native base-type bytes (4*dim for f32, 2*dim for f16); the + // search-side overflow replay reinterprets them back to the base type B. + var baseElemSize int + switch types.T(vecTyp.Id) { + case types.T_array_float32: + w.baseType = types.T_array_float32 + baseElemSize = 4 + case types.T_array_float16: + w.baseType = types.T_array_float16 + baseElemSize = 2 + default: return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf( - "%s cuvs writer: vector column must be vecf32 (cuvs is fp32-only)", algoName)) + "%s cuvs writer: vector column must be vecf32 or vecf16", algoName)) } w.dimension = vecTyp.Width + w.vecBytesPer = int(vecTyp.Width) * baseElemSize // Resolve INCLUDE columns from indexAlgoParams. Returns zero // values when no INCLUDE columns are configured. @@ -172,6 +187,7 @@ func (w *CuvsCdcWriter) IndexName() string { return w.indexName } func (w *CuvsCdcWriter) IndexDef() []*plan.IndexDef { return w.indexdef } func (w *CuvsCdcWriter) Dimension() int32 { return w.dimension } func (w *CuvsCdcWriter) ColMetaJSON() string { return w.colMetaJSON } +func (w *CuvsCdcWriter) BaseVectorType() types.T { return w.baseType } // IndexSqlWriter implementation // @@ -218,7 +234,7 @@ func (w *CuvsCdcWriter) ToSql() ([]byte, error) { func (w *CuvsCdcWriter) appendDelete(key int64) error { out, err := cuvscdc.EncodeEventRecord(w.pendingRecords, cuvscdc.CdcOpDelete, - key, nil, nil, int(w.dimension), w.includeBytesPer) + key, nil, nil, w.vecBytesPer, w.includeBytesPer) if err != nil { return err } @@ -238,25 +254,36 @@ func (w *CuvsCdcWriter) encodeInsertOrUpsert(ctx context.Context, row []any, op // has a vector to index). return w.appendDelete(key) } - v, ok := rawVec.([]float32) - if !ok { - // A non-nil value of the wrong type is a real schema/type error, not a - // NULL vector — surface it instead of silently dropping the row to a - // DELETE (mirrors the HNSW sinker in index_sqlwriter.go). + // Extract the native base-type bytes verbatim. A vecf32 column arrives as + // []float32 (4 bytes/element), a vecf16 column as []types.Float16 (2 + // bytes/element); EncodeEventRecord validates the byte length against + // w.vecBytesPer. A typed-nil slice is an actually-absent vector → DELETE; + // any other type is a real schema error (mirrors the HNSW sinker). + var vecBytes []byte + switch v := rawVec.(type) { + case []float32: + if v == nil { + return w.appendDelete(key) + } + vecBytes = util.UnsafeSliceToBytes(v) + case []types.Float16: + if v == nil { + return w.appendDelete(key) + } + vecBytes = util.UnsafeSliceToBytes(v) + default: return moerr.NewInternalError(ctx, fmt.Sprintf( - "%s cuvs writer: invalid vector type, expected []float32, got %T", w.algoName, rawVec)) - } - if v == nil { - // Typed-nil slice — an actually absent vector; encode as DELETE. - return w.appendDelete(key) + "%s cuvs writer: invalid vector type, expected []float32 or []types.Float16, got %T", + w.algoName, rawVec)) } includeBytes, err := cuvscdc.EncodeIncludeRow(w.includeBindings, row, w.includeBytesPer) if err != nil { return err } + // Pass the raw native base-type bytes (4*dim for f32, 2*dim for f16). out, err := cuvscdc.EncodeEventRecord(w.pendingRecords, op, - key, v, includeBytes, int(w.dimension), w.includeBytesPer) + key, vecBytes, includeBytes, w.vecBytesPer, w.includeBytesPer) if err != nil { return err } diff --git a/pkg/iscp/cuvs_writer_test.go b/pkg/iscp/cuvs_writer_test.go index 47f31af51e8af..797fb807b6cff 100644 --- a/pkg/iscp/cuvs_writer_test.go +++ b/pkg/iscp/cuvs_writer_test.go @@ -203,7 +203,7 @@ func TestNewCuvsCdcWriter_RejectsVecF64(t *testing.T) { td.Cols[1].Typ.Id = int32(types.T_array_float64) _, err := NewCuvsCdcWriter("ivfpq", "db", "tbl", "idx", td, newTestCuvsIndexDefs(td)) require.Error(t, err) - require.Contains(t, err.Error(), "fp32-only") + require.Contains(t, err.Error(), "vecf32 or vecf16") } // --------------------------------------------------------------------------- diff --git a/pkg/iscp/index_sqlwriter.go b/pkg/iscp/index_sqlwriter.go index 33157069f50a8..96384f58b0e88 100644 --- a/pkg/iscp/index_sqlwriter.go +++ b/pkg/iscp/index_sqlwriter.go @@ -28,6 +28,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/plan" "github.com/matrixorigin/matrixone/pkg/vectorindex" "github.com/matrixorigin/matrixone/pkg/vectorindex/hnsw" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" ) @@ -668,7 +669,36 @@ func (w *IvfflatSqlWriter) toIvfflatUpsert(upsert bool) ([]byte, error) { } cols := strings.Join(coldefs, ", ") - cnames_str := strings.Join(cnames, ", ") + + // Entry projection. The last src column is the vector that becomes the entry. + // For int8 QUANTIZATION the entry must be scaled by the trained quantizer + // (q(x)=x*mul+add, mul=255/(max-min), add=-min*mul-128) just like the + // synchronous build (compile.go) and search; otherwise the implicit + // vecf32->vecint8 cast on REPLACE does identity round+clamp and every + // CDC-maintained row gets wrong int8 codes. min/max come from the metadata + // table; COALESCE falls back to identity (mul=1,add=0) when they are absent + // (pure-async indexes that never trained bounds — search also uses identity + // there, so the two stay consistent). float16/bf16 narrow losslessly via the + // implicit cast, so only int8 needs this. + entryProj := cnames[len(cnames)-1] + if qt, ok := quantizer.ToVectorType(w.ivfparam.Quantization); ok && (qt == types.T_array_int8 || qt == types.T_array_uint8) { + metaTbl := sqlquote.QualifiedIdent(w.info.DBName, w.meta_tbl) + sub := func(k string) string { + return fmt.Sprintf("(SELECT CAST(`%s` AS DOUBLE) FROM %s WHERE `%s` = '%s')", + catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, metaTbl, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, k) + } + minS := sub(catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin) + maxS := sub(catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax) + if qt == types.T_array_uint8 { + entryProj = quantizer.Uint8EntrySQLFromBounds(cnames[len(cnames)-1], minS, maxS, w.partsType[0].Width) + } else { + entryProj = quantizer.Int8EntrySQLFromBounds(cnames[len(cnames)-1], minS, maxS, w.partsType[0].Width) + } + } + projCols := append([]string(nil), cnames...) + projCols[len(projCols)-1] = entryProj + cnames_str := strings.Join(projCols, ", ") if upsert { sql += fmt.Sprintf("REPLACE INTO %s ", sqlquote.QualifiedIdent(w.info.DBName, w.entries_tbl)) diff --git a/pkg/iscp/util.go b/pkg/iscp/util.go index 6cbca560d8a88..690bc4bd75975 100644 --- a/pkg/iscp/util.go +++ b/pkg/iscp/util.go @@ -116,6 +116,16 @@ func extractRowFromVector(ctx context.Context, vec *vector.Vector, i int, row [] //| �? @ @@ | //+------------------------------+ row[i] = vector.GetArrayAt[float32](vec, rowIndex) + case types.T_array_float16: + // vecf16: extract natively as []types.Float16 (2 bytes/element). The + // cuvs CDC writer reinterprets these bytes verbatim — no f32 widening. + row[i] = vector.GetArrayAt[types.Float16](vec, rowIndex) + case types.T_array_bf16: + row[i] = vector.GetArrayAt[types.BF16](vec, rowIndex) + case types.T_array_int8: + row[i] = vector.GetArrayAt[int8](vec, rowIndex) + case types.T_array_uint8: + row[i] = vector.GetArrayAt[uint8](vec, rowIndex) case types.T_array_float64: row[i] = vector.GetArrayAt[float64](vec, rowIndex) case types.T_date: @@ -257,6 +267,23 @@ func convertColIntoSql( value := data.([]float64) typstr := typ.DescString() sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typstr)) + case types.T_array_float16: + // Narrow base columns (vecf16/bf16/int8/uint8). ArrayToString renders the + // half/bf16 bit pattern back to its decimal value and the int8/uint8 codes + // to integers, so CAST('[...]' as vecXXX(n)) reconstructs the same vector + // the ivfflat entry projection expects (matches the synchronous build, + // which reads the base column directly in SQL). + value := data.([]types.Float16) + sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typ.DescString())) + case types.T_array_bf16: + value := data.([]types.BF16) + sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typ.DescString())) + case types.T_array_int8: + value := data.([]int8) + sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typ.DescString())) + case types.T_array_uint8: + value := data.([]uint8) + sqlBuff = appendString(sqlBuff, fmt.Sprintf("CAST('%s' as %s)", types.ArrayToString(value), typ.DescString())) case types.T_date: value := data.(types.Date) sqlBuff = appendByte(sqlBuff, '\'') diff --git a/pkg/iscp/util_test.go b/pkg/iscp/util_test.go index e3064c3ab4f30..b73e221e71f15 100644 --- a/pkg/iscp/util_test.go +++ b/pkg/iscp/util_test.go @@ -31,7 +31,7 @@ import ( func mockUtilVector(t *testing.T, proc *process.Process) (*batch.Batch, []string) { i := 0 - nvec := 15 + nvec := 19 bat := batch.NewWithSize(nvec) res := make([]string, nvec) @@ -75,6 +75,40 @@ func mockUtilVector(t *testing.T, proc *process.Process) (*batch.Batch, []string i += 1 } + { + // []float16 (narrow base column) + bat.Vecs[i] = vector.NewVec(types.New(types.T_array_float16, 3, 0)) + vf16 := types.Float32ToFloat16Slice([]float32{0, 1, 2}) + vector.AppendArray[types.Float16](bat.Vecs[i], vf16, false, proc.Mp()) + res[i] = "CAST('[0, 1, 2]' as VECF16(3))" + i += 1 + } + + { + // []bf16 (narrow base column) + bat.Vecs[i] = vector.NewVec(types.New(types.T_array_bf16, 3, 0)) + vbf16 := types.Float32ToBF16Slice([]float32{0, 1, 2}) + vector.AppendArray[types.BF16](bat.Vecs[i], vbf16, false, proc.Mp()) + res[i] = "CAST('[0, 1, 2]' as VECBF16(3))" + i += 1 + } + + { + // []int8 (narrow base column) + bat.Vecs[i] = vector.NewVec(types.New(types.T_array_int8, 3, 0)) + vector.AppendArray[int8](bat.Vecs[i], []int8{0, 1, 2}, false, proc.Mp()) + res[i] = "CAST('[0, 1, 2]' as VECINT8(3))" + i += 1 + } + + { + // []uint8 (narrow base column) + bat.Vecs[i] = vector.NewVec(types.New(types.T_array_uint8, 3, 0)) + vector.AppendArray[uint8](bat.Vecs[i], []uint8{0, 1, 2}, false, proc.Mp()) + res[i] = "CAST('[0, 1, 2]' as VECUINT8(3))" + i += 1 + } + { // date bat.Vecs[i] = vector.NewVec(types.New(types.T_date, 4, 0)) diff --git a/pkg/partition/partition.go b/pkg/partition/partition.go index 68e5a8d285e8f..6691d79730912 100644 --- a/pkg/partition/partition.go +++ b/pkg/partition/partition.go @@ -176,7 +176,9 @@ func Partition(sels []int64, diffs []bool, partitions []int64, vec *vector.Vecto return genericPartition[types.Blockid](sels, diffs, partitions, vec) case types.T_char, types.T_varchar, types.T_json, types.T_text, types.T_binary, types.T_varbinary, types.T_blob, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + types.T_datalink: return bytesPartition(sels, diffs, partitions, vec) //Used by ORDER_BY SQL clause. //Byte partition logic doesn't use byte.Compare or Str. diff --git a/pkg/sort/sort.go b/pkg/sort/sort.go index 2def8288ccb09..020fd66723804 100644 --- a/pkg/sort/sort.go +++ b/pkg/sort/sort.go @@ -40,7 +40,8 @@ type sortType interface { ~[]types.Time | ~[]types.Enum | ~[]types.MoYear | ~[]types.TS | ~[]types.Decimal64 | ~[]types.Decimal128 | ~[]types.Decimal256 | ~[]types.Rowid | ~[]types.Blockid | ~[]types.Uuid | - ~[][]float32 | ~[][]float64 + ~[][]float32 | ~[][]float64 | + ~[][]types.BF16 | ~[][]types.Float16 | ~[][]int8 | ~[][]uint8 } type xorshift uint64 @@ -287,6 +288,34 @@ func Sort(desc, nullsLast, hasNull bool, os []int64, vec *vector.Vector) { } else { genericSort(col, os, arrayGreater[float64]) } + case types.T_array_bf16: + col := vector.MustArrayCol[types.BF16](vec) + if !desc { + genericSort(col, os, arrayElementLess[types.BF16]) + } else { + genericSort(col, os, arrayElementGreater[types.BF16]) + } + case types.T_array_float16: + col := vector.MustArrayCol[types.Float16](vec) + if !desc { + genericSort(col, os, arrayElementLess[types.Float16]) + } else { + genericSort(col, os, arrayElementGreater[types.Float16]) + } + case types.T_array_int8: + col := vector.MustArrayCol[int8](vec) + if !desc { + genericSort(col, os, arrayElementLess[int8]) + } else { + genericSort(col, os, arrayElementGreater[int8]) + } + case types.T_array_uint8: + col := vector.MustArrayCol[uint8](vec) + if !desc { + genericSort(col, os, arrayElementLess[uint8]) + } else { + genericSort(col, os, arrayElementGreater[uint8]) + } case types.T_TS: col := vector.MustFixedColNoTypeCheck[types.TS](vec) if !desc { @@ -394,6 +423,16 @@ func arrayGreater[T types.RealNumbers](data [][]T, i, j int64) bool { return types.ArrayCompare[T](data[i], data[j]) > 0 } +// Narrow vector element types (bf16/f16/int8) order through the float32 bridge +// so bf16/f16 sign bits do not corrupt the ordering. +func arrayElementLess[T types.ArrayElement](data [][]T, i, j int64) bool { + return types.ArrayElementCompare[T](data[i], data[j]) < 0 +} + +func arrayElementGreater[T types.ArrayElement](data [][]T, i, j int64) bool { + return types.ArrayElementCompare[T](data[i], data[j]) > 0 +} + func genericLess[T types.OrderedT](data []T, i, j int64) bool { return data[i] < data[j] } diff --git a/pkg/sql/colexec/evalExpression.go b/pkg/sql/colexec/evalExpression.go index 6360df9cdb469..b96396385ea63 100644 --- a/pkg/sql/colexec/evalExpression.go +++ b/pkg/sql/colexec/evalExpression.go @@ -850,10 +850,19 @@ func generateConstExpressionExecutor(proc *process.Process, typ types.Type, con case *plan.Literal_EnumVal: vec, err = vector.NewConstFixed(constEnumType, types.Enum(val.EnumVal), 1, proc.Mp()) case *plan.Literal_VecVal: - if typ.Oid == types.T_array_float32 { + switch typ.Oid { + case types.T_array_float32: vec, err = vector.NewConstArray(typ, types.BytesToArray[float32]([]byte(val.VecVal)), 1, proc.Mp()) - } else if typ.Oid == types.T_array_float64 { + case types.T_array_float64: vec, err = vector.NewConstArray(typ, types.BytesToArray[float64]([]byte(val.VecVal)), 1, proc.Mp()) + case types.T_array_bf16: + vec, err = vector.NewConstArray(typ, types.BytesToArray[types.BF16]([]byte(val.VecVal)), 1, proc.Mp()) + case types.T_array_float16: + vec, err = vector.NewConstArray(typ, types.BytesToArray[types.Float16]([]byte(val.VecVal)), 1, proc.Mp()) + case types.T_array_int8: + vec, err = vector.NewConstArray(typ, types.BytesToArray[int8]([]byte(val.VecVal)), 1, proc.Mp()) + case types.T_array_uint8: + vec, err = vector.NewConstArray(typ, types.BytesToArray[uint8]([]byte(val.VecVal)), 1, proc.Mp()) } default: return nil, moerr.NewNYI(proc.Ctx, fmt.Sprintf("const expression %v", con.GetValue())) diff --git a/pkg/sql/colexec/external/external.go b/pkg/sql/colexec/external/external.go index 2a0109c670b33..c8e783fff50c9 100644 --- a/pkg/sql/colexec/external/external.go +++ b/pkg/sql/colexec/external/external.go @@ -735,6 +735,26 @@ func isLegalLine(param *tree.ExternParam, cols []*plan.ColDef, fields []csvparse if err != nil { return false } + case types.T_array_bf16: + _, err := types.StringToArrayToBytes[types.BF16](field.Val) + if err != nil { + return false + } + case types.T_array_float16: + _, err := types.StringToArrayToBytes[types.Float16](field.Val) + if err != nil { + return false + } + case types.T_array_int8: + _, err := types.StringToArrayToBytes[int8](field.Val) + if err != nil { + return false + } + case types.T_array_uint8: + _, err := types.StringToArrayToBytes[uint8](field.Val) + if err != nil { + return false + } case types.T_json: if param.Format == tree.CSV { field.Val = fmt.Sprintf("%v", strings.Trim(field.Val, "\"")) @@ -1294,6 +1314,50 @@ func getColData(bat *batch.Batch, line []csvparser.Field, rowIdx int, param *Ext if err = vector.AppendBytes(vec, types.ArrayToBytes[float64](arr), false, mp); err != nil { return err } + case types.T_array_bf16: + arr, err := types.StringToArray[types.BF16](field.Val) + if err != nil { + return err + } + if int(vec.GetType().Width) != types.MaxArrayDimension && int(vec.GetType().Width) != len(arr) { + return moerr.NewArrayDefMismatchNoCtx(int(vec.GetType().Width), len(arr)) + } + if err = vector.AppendBytes(vec, types.ArrayToBytes[types.BF16](arr), false, mp); err != nil { + return err + } + case types.T_array_float16: + arr, err := types.StringToArray[types.Float16](field.Val) + if err != nil { + return err + } + if int(vec.GetType().Width) != types.MaxArrayDimension && int(vec.GetType().Width) != len(arr) { + return moerr.NewArrayDefMismatchNoCtx(int(vec.GetType().Width), len(arr)) + } + if err = vector.AppendBytes(vec, types.ArrayToBytes[types.Float16](arr), false, mp); err != nil { + return err + } + case types.T_array_int8: + arr, err := types.StringToArray[int8](field.Val) + if err != nil { + return err + } + if int(vec.GetType().Width) != types.MaxArrayDimension && int(vec.GetType().Width) != len(arr) { + return moerr.NewArrayDefMismatchNoCtx(int(vec.GetType().Width), len(arr)) + } + if err = vector.AppendBytes(vec, types.ArrayToBytes[int8](arr), false, mp); err != nil { + return err + } + case types.T_array_uint8: + arr, err := types.StringToArray[uint8](field.Val) + if err != nil { + return err + } + if int(vec.GetType().Width) != types.MaxArrayDimension && int(vec.GetType().Width) != len(arr) { + return moerr.NewArrayDefMismatchNoCtx(int(vec.GetType().Width), len(arr)) + } + if err = vector.AppendBytes(vec, types.ArrayToBytes[uint8](arr), false, mp); err != nil { + return err + } case types.T_json: var jsonBytes []byte if param.Extern.Format != tree.CSV { diff --git a/pkg/sql/colexec/external/hive_partition_fill.go b/pkg/sql/colexec/external/hive_partition_fill.go index 41bbe4757a072..f4b779e379ed9 100644 --- a/pkg/sql/colexec/external/hive_partition_fill.go +++ b/pkg/sql/colexec/external/hive_partition_fill.go @@ -362,7 +362,9 @@ func fillConstantVector( } return vector.SetConstFixed(vec, v, rowCount, mp) - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, + types.T_array_int8, types.T_array_uint8: return moerr.NewNotSupportedf(proc.Ctx, "unsupported partition column type VECTOR for col=%s, path=%s", col.Name, filePath) diff --git a/pkg/sql/colexec/external/hive_partition_test.go b/pkg/sql/colexec/external/hive_partition_test.go index 5a8c8d96e289e..88642b6e01e70 100644 --- a/pkg/sql/colexec/external/hive_partition_test.go +++ b/pkg/sql/colexec/external/hive_partition_test.go @@ -1906,12 +1906,21 @@ func TestFillConstantVector_Bool(t *testing.T) { func TestFillConstantVector_UnsupportedVector(t *testing.T) { proc := testutil.NewProc(t) - vec := vector.NewVec(types.T_array_float32.ToType()) - col := &plan.ColDef{Name: "emb", Typ: plan.Type{Id: int32(types.T_array_float32)}} - - err := fillConstantVector(vec, "[1,2,3]", col, 1, proc, "/test") - require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported") + // Vectors (any element width) cannot be a hive partition column. + for _, id := range []types.T{ + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, + types.T_array_int8, types.T_array_uint8, + } { + t.Run(id.String(), func(t *testing.T) { + vec := vector.NewVec(id.ToType()) + col := &plan.ColDef{Name: "emb", Typ: plan.Type{Id: int32(id)}} + + err := fillConstantVector(vec, "[1,2,3]", col, 1, proc, "/test") + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported partition column type VECTOR") + }) + } } func TestFillPartitionColumns_DefaultPartNull(t *testing.T) { diff --git a/pkg/sql/colexec/external/parquet.go b/pkg/sql/colexec/external/parquet.go index df3127be1cc83..dec1f5045e295 100644 --- a/pkg/sql/colexec/external/parquet.go +++ b/pkg/sql/colexec/external/parquet.go @@ -313,7 +313,9 @@ func (h *ParquetHandler) prepare(param *ExternalParam) error { if !col.Leaf() { targetType := types.T(def.Typ.Id) switch targetType { - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, + types.T_array_int8, types.T_array_uint8: physicalCol, fn = h.getNestedListMapper(col, def.Typ) default: if !isNestedTargetTypeSupported(targetType) { @@ -434,6 +436,53 @@ func (*ParquetHandler) getNestedListMapper(sc *parquet.Column, dt plan.Type) (*p return v.Double(), nil }) } + case types.T_array_bf16: + // bf16/f16 vectors are stored in parquet as FLOAT leaves and narrowed on load. + if leaf.Type().Kind() != parquet.Float { + return nil, nil + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processParquetListToArray(proc.Ctx, mp, page, proc, vec, width, func(v parquet.Value) (types.BF16, error) { + return types.BF16FromFloat32(v.Float()), nil + }) + } + case types.T_array_float16: + if leaf.Type().Kind() != parquet.Float { + return nil, nil + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processParquetListToArray(proc.Ctx, mp, page, proc, vec, width, func(v parquet.Value) (types.Float16, error) { + return types.Float16FromFloat32(v.Float()), nil + }) + } + case types.T_array_int8: + // int8/uint8 vectors are stored in parquet as INT32 leaves; load is strict + // (out-of-range values are rejected, mirroring the int8 string parse). + if leaf.Type().Kind() != parquet.Int32 { + return nil, nil + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processParquetListToArray(proc.Ctx, mp, page, proc, vec, width, func(v parquet.Value) (int8, error) { + x := v.Int32() + if x < math.MinInt8 || x > math.MaxInt8 { + return 0, moerr.NewOutOfRangeNoCtxf("vecint8", "value %d out of range [-128,127]", x) + } + return int8(x), nil + }) + } + case types.T_array_uint8: + if leaf.Type().Kind() != parquet.Int32 { + return nil, nil + } + mp.mapper = func(mp *columnMapper, page parquet.Page, proc *process.Process, vec *vector.Vector) error { + return processParquetListToArray(proc.Ctx, mp, page, proc, vec, width, func(v parquet.Value) (uint8, error) { + x := v.Int32() + if x < 0 || x > math.MaxUint8 { + return 0, moerr.NewOutOfRangeNoCtxf("vecuint8", "value %d out of range [0,255]", x) + } + return uint8(x), nil + }) + } default: return nil, nil } @@ -1894,7 +1943,7 @@ func readParquetPageAllValues(ctx context.Context, page parquet.Page) ([]parquet return values, nil } -func processParquetListToArray[T types.RealNumbers]( +func processParquetListToArray[T types.ArrayElement]( ctx context.Context, mp *columnMapper, page parquet.Page, diff --git a/pkg/sql/colexec/external/parquet_test.go b/pkg/sql/colexec/external/parquet_test.go index f0fc6975a4073..20df5db777e4c 100644 --- a/pkg/sql/colexec/external/parquet_test.go +++ b/pkg/sql/colexec/external/parquet_test.go @@ -506,6 +506,149 @@ func TestParquetListToVectorMapping(t *testing.T) { vec := vector.NewVec(types.New(types.T_array_float32, 3, 0)) require.ErrorContains(t, mp.mapping(page, proc, vec), "parquet list NULL elements are not supported") }) + + // Narrow vector targets: bf16/f16 decode from FLOAT leaves, int8/uint8 from + // INT32 leaves. Values are chosen exactly representable so the round-trip is + // loss-free and can be asserted by exact equality. + t.Run("float list to vecbf16", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.FloatType), []parquet.Row{ + { + parquet.FloatValue(1).Level(0, 1, 0), + parquet.FloatValue(2).Level(1, 1, 0), + parquet.FloatValue(-0.5).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_bf16), Width: 3}) + require.NotNil(t, leaf) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_bf16, 3, 0)) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + require.Equal(t, []types.BF16{ + types.BF16FromFloat32(1), types.BF16FromFloat32(2), types.BF16FromFloat32(-0.5), + }, vector.GetArrayAt[types.BF16](vec, 0)) + }) + + t.Run("float list to vecf16", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.FloatType), []parquet.Row{ + { + parquet.FloatValue(0.5).Level(0, 1, 0), + parquet.FloatValue(0.25).Level(1, 1, 0), + parquet.FloatValue(4).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_float16), Width: 3}) + require.NotNil(t, leaf) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_float16, 3, 0)) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + require.Equal(t, []types.Float16{ + types.Float16FromFloat32(0.5), types.Float16FromFloat32(0.25), types.Float16FromFloat32(4), + }, vector.GetArrayAt[types.Float16](vec, 0)) + }) + + t.Run("int32 list to vecint8", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + { + parquet.Int32Value(-128).Level(0, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + parquet.Int32Value(127).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_int8), Width: 3}) + require.NotNil(t, leaf) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_int8, 3, 0)) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + require.Equal(t, []int8{-128, 0, 127}, vector.GetArrayAt[int8](vec, 0)) + }) + + t.Run("int32 list to vecuint8", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + { + parquet.Int32Value(0).Level(0, 1, 0), + parquet.Int32Value(128).Level(1, 1, 0), + parquet.Int32Value(255).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_uint8), Width: 3}) + require.NotNil(t, leaf) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_uint8, 3, 0)) + require.NoError(t, mp.mapping(page, proc, vec)) + require.Equal(t, 1, vec.Length()) + require.Equal(t, []uint8{0, 128, 255}, vector.GetArrayAt[uint8](vec, 0)) + }) + + t.Run("vecint8 out of range rejected", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + { + parquet.Int32Value(200).Level(0, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + _, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_int8), Width: 3}) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_int8, 3, 0)) + require.ErrorContains(t, mp.mapping(page, proc, vec), "out of range") + }) + + t.Run("vecuint8 out of range rejected", func(t *testing.T) { + f, page := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + { + parquet.Int32Value(-1).Level(0, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + parquet.Int32Value(0).Level(1, 1, 0), + }, + }) + + var h ParquetHandler + _, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_uint8), Width: 3}) + require.NotNil(t, mp) + + vec := vector.NewVec(types.New(types.T_array_uint8, 3, 0)) + require.ErrorContains(t, mp.mapping(page, proc, vec), "out of range") + }) + + t.Run("vecbf16 rejects int32 leaf", func(t *testing.T) { + f, _ := writeListAndGetPage(t, parquet.Leaf(parquet.Int32Type), []parquet.Row{ + {parquet.Int32Value(1).Level(0, 1, 0)}, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_bf16), Width: 1}) + require.Nil(t, leaf) + require.Nil(t, mp) + }) + + t.Run("vecint8 rejects float leaf", func(t *testing.T) { + f, _ := writeListAndGetPage(t, parquet.Leaf(parquet.FloatType), []parquet.Row{ + {parquet.FloatValue(1).Level(0, 1, 0)}, + }) + + var h ParquetHandler + leaf, mp := h.getNestedListMapper(f.Root().Column("c"), plan.Type{Id: int32(types.T_array_int8), Width: 1}) + require.Nil(t, leaf) + require.Nil(t, mp) + }) } func TestParquetCrossTypeMappings(t *testing.T) { diff --git a/pkg/sql/colexec/group/exec2.go b/pkg/sql/colexec/group/exec2.go index 8f8e9aae46758..9f17dc9d420a5 100644 --- a/pkg/sql/colexec/group/exec2.go +++ b/pkg/sql/colexec/group/exec2.go @@ -180,6 +180,11 @@ func GetKeyWidth(id types.T, width0 int32, nullable bool) (width int) { if id == types.T_array_float64 { width *= 8 } + if id == types.T_array_bf16 || id == types.T_array_float16 { + width *= 2 + } + // T_array_int8 / T_array_uint8 are 1 byte/element -> width unchanged + // (width0 already counts). } else { width = id.TypeLen() } diff --git a/pkg/sql/colexec/productl2/product_l2.go b/pkg/sql/colexec/productl2/product_l2.go index 05c31ae8e40e1..27decd51c840d 100644 --- a/pkg/sql/colexec/productl2/product_l2.go +++ b/pkg/sql/colexec/productl2/product_l2.go @@ -274,27 +274,55 @@ func newMat[T types.RealNumbers](ctr *container, ap *Productl2, probes [][]T, nu } } + // T is the centroid (index) element type. T==float32 covers f32 centroids, + // which a base of any type (f32/f64/narrow) is decoded to; T==float64 is the + // plain f64 index where the base is f64 and reinterpreted directly. + _, toF32 := any(*new(T)).(float32) + oid := tblColVec.GetType().Oid for j := 0; j < probeCount; j++ { if tblColVec.IsNull(uint64(j)) { probes[j] = nullvec continue } - v := types.BytesToArray[T](tblColVec.GetBytesAt(j)) - probes[j] = v + b := tblColVec.GetBytesAt(j) + if !toF32 { + probes[j] = types.BytesToArray[T](b) // f64 centroids: base is f64 + continue + } + var f32 []float32 + switch oid { + case types.T_array_float64: + f64 := types.BytesToArray[float64](b) + f32 = make([]float32, len(f64)) + for i, x := range f64 { + f32[i] = float32(x) + } + case types.T_array_bf16: + f32 = types.BF16ToFloat32Slice(types.BytesToArray[types.BF16](b)) + case types.T_array_float16: + f32 = types.Float16ToFloat32Slice(types.BytesToArray[types.Float16](b)) + case types.T_array_int8: + f32 = types.Int8ToFloat32Slice(types.BytesToArray[int8](b)) + case types.T_array_uint8: + f32 = types.Uint8ToFloat32Slice(types.BytesToArray[uint8](b)) + default: // T_array_float32 + f32 = types.BytesToArray[float32](b) + } + probes[j] = any(f32).([]T) } return probes, nil } func (ctr *container) probe(ap *Productl2, proc *process.Process, result *vm.CallResult) error { - tblColPos := ap.OnExpr.GetF().GetArgs()[1].GetCol().GetColPos() - switch ctr.inBat.Vecs[tblColPos].GetType().Oid { - case types.T_array_float32: - return probeRun[float32](ctr, ap, proc, result) - case types.T_array_float64: + // Dispatch on the CENTROID (index) type, not the base type: under QUANTIZATION + // an f64/narrow base is assigned against f32 centroids, so the base must be + // decoded to f32 (in newMat) to match. Only a plain f64 index keeps f64. + centroidColPos := ap.OnExpr.GetF().GetArgs()[0].GetCol().GetColPos() + if ctr.bat.Vecs[centroidColPos].GetType().Oid == types.T_array_float64 { return probeRun[float64](ctr, ap, proc, result) } - return nil + return probeRun[float32](ctr, ap, proc, result) } func (ctr *container) release() { diff --git a/pkg/sql/colexec/table_function/cagra_create_gpu.go b/pkg/sql/colexec/table_function/cagra_create_gpu.go index bbf52fcf54d68..8a3a531547d15 100644 --- a/pkg/sql/colexec/table_function/cagra_create_gpu.go +++ b/pkg/sql/colexec/table_function/cagra_create_gpu.go @@ -23,6 +23,7 @@ import ( "github.com/bytedance/sonic" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -48,16 +49,34 @@ var cagraCatalogHooks = cagrart.CatalogHooks{} var cagra_runSql = sqlexec.RunSql +// cagraBuilder is the (B, Q)-erased build interface the create state drives. +// *cagraPkg.CagraBuild[B, Q] satisfies it for every wired (base, storage) +// combo. GetIndexes is [B,Q]-typed and intentionally NOT on the interface — +// end() routes through ToInsertSql instead. +type cagraBuilder interface { + // AddRow takes the raw base-type bytes of one vector (4*dim for an f32 base, + // 2*dim for an f16 base); the concrete builder reinterprets them to its + // []B/[]Q with UnsafeSliceCast (the interface can't name B). Passing []byte + // rather than `any` keeps the per-row build hot path allocation-free. + AddRow(id int64, vecBytes []byte) error + SetFilterColumns(colMetaJSON string) + AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error + ToInsertSql(ts int64) ([]string, error) + Destroy() error +} + type cagraCreateState struct { - inited bool - buildf32 *cagraPkg.CagraBuild[float32] - buildf16 *cagraPkg.CagraBuild[cuvs.Float16] - buildi8 *cagraPkg.CagraBuild[int8] - buildui8 *cagraPkg.CagraBuild[uint8] - param vectorindex.CagraParam - tblcfg vectorindex.IndexTableConfig - idxcfg vectorindex.IndexConfig - offset int + inited bool + builder cagraBuilder + param vectorindex.CagraParam + tblcfg vectorindex.IndexTableConfig + idxcfg vectorindex.IndexConfig + offset int + + // baseOid is the base (source) vector column element type — f32 or f16. + // The storage/quantization type (which builder is non-nil) may differ: + // f16 base is stored as half (direct) or quantized to int8/uint8. + baseOid types.T // filterCols is the INCLUDE column metadata derived at start() from // param.IncludedColumns (names) + argVecs[3:] (types). Empty when the @@ -95,19 +114,11 @@ func (u *cagraCreateState) end(tf *TableFunction, proc *process.Process) error { ) ts := time.Now().UnixMicro() - switch { - case u.buildf32 != nil: - sqls, err = u.buildf32.ToInsertSql(ts) - case u.buildf16 != nil: - sqls, err = u.buildf16.ToInsertSql(ts) - case u.buildi8 != nil: - sqls, err = u.buildi8.ToInsertSql(ts) - case u.buildui8 != nil: - sqls, err = u.buildui8.ToInsertSql(ts) - default: - // No builder selected → init didn't set one. Nothing to do for - // the cuvs side; the CDC tail (if any) below still emits. + if u.builder != nil { + sqls, err = u.builder.ToInsertSql(ts) } + // No builder selected → init didn't set one. Nothing to do for the cuvs + // side; the CDC tail (if any) below still emits. if err != nil { return err } @@ -121,9 +132,14 @@ func (u *cagraCreateState) end(tf *TableFunction, proc *process.Process) error { // record 0. Search-side can recover the INCLUDE-column layout // for tag=1 replay even when no tag=0 sub-index exists. colMetaJSON := colMetaJSONFromCols(u.filterCols) + // vecBytesPerRow = dim * base element size (2 for vecf16, else 4). + elemSize := 4 + if u.baseOid == types.T_array_float16 { + elemSize = 2 + } + vecBytesPerRow := int(u.idxcfg.CuvsCagra.Dimensions) * elemSize tailSqls, err := cuvscdc.SaveSmallTailAsCdc( - u.tblcfg, u.cdcTail, - int(u.idxcfg.CuvsCagra.Dimensions), ibpr, colMetaJSON) + u.tblcfg, u.cdcTail, vecBytesPerRow, ibpr, colMetaJSON) if err != nil { return err } @@ -160,17 +176,8 @@ func (u *cagraCreateState) free(tf *TableFunction, proc *process.Process, pipeli if u.batch != nil { u.batch.Clean(proc.Mp()) } - if u.buildf32 != nil { - u.buildf32.Destroy() - } - if u.buildf16 != nil { - u.buildf16.Destroy() - } - if u.buildi8 != nil { - u.buildi8.Destroy() - } - if u.buildui8 != nil { - u.buildui8.Destroy() + if u.builder != nil { + u.builder.Destroy() } } @@ -340,7 +347,16 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo faVec := tf.ctr.argVecs[2] if !catalogplugin.SupportsVectorType(cagraCatalogHooks, faVec.GetType().Oid) { - return moerr.NewInvalidInput(proc.Ctx, "third argument (vector) must be a float32 array") + return moerr.NewInvalidInput(proc.Ctx, "third argument (vector) must be a float32 / float16 array") + } + u.baseOid = faVec.GetType().Oid + + // Derive the storage qtype from the base column type when no QUANTIZATION + // was given: a vecf16 base with no quantization is stored natively as half. + // (vecf16 + QUANTIZATION=int8/uint8 keeps qt = int8/uint8 — quantize path.) + if u.baseOid == types.T_array_float16 && qt == metric.Quantization_F32 { + qt = metric.Quantization_F16 + u.idxcfg.CuvsCagra.Quantization = uint16(qt) } // dimension @@ -357,15 +373,25 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo uid := fmt.Sprintf("%s:%d:%d", tf.CnAddr, tf.MaxParallel, tf.ParallelID) // ---- create builder ---- - switch qt { - case metric.Quantization_F16: - u.buildf16, err = cagraPkg.NewCagraBuild[cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) - case metric.Quantization_INT8: - u.buildi8, err = cagraPkg.NewCagraBuild[int8](uid, u.idxcfg, u.tblcfg, nthread, devices) - case metric.Quantization_UINT8: - u.buildui8, err = cagraPkg.NewCagraBuild[uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) + // One real [B, Q] builder keyed on (base column type, storage qtype). + // The 7 wired combos: f32 base × {f32, f16, int8, uint8}; f16 base × + // {f16, int8, uint8}. + isF16Base := u.baseOid == types.T_array_float16 + switch { + case isF16Base && qt == metric.Quantization_F16: + u.builder, err = cagraPkg.NewCagraBuild[cuvs.Float16, cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) + case isF16Base && qt == metric.Quantization_INT8: + u.builder, err = cagraPkg.NewCagraBuild[cuvs.Float16, int8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case isF16Base && qt == metric.Quantization_UINT8: + u.builder, err = cagraPkg.NewCagraBuild[cuvs.Float16, uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_F16: + u.builder, err = cagraPkg.NewCagraBuild[float32, cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_INT8: + u.builder, err = cagraPkg.NewCagraBuild[float32, int8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_UINT8: + u.builder, err = cagraPkg.NewCagraBuild[float32, uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) default: - u.buildf32, err = cagraPkg.NewCagraBuild[float32](uid, u.idxcfg, u.tblcfg, nthread, devices) + u.builder, err = cagraPkg.NewCagraBuild[float32, float32](uid, u.idxcfg, u.tblcfg, nthread, devices) } if err != nil { return err @@ -381,7 +407,7 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo if len(u.filterCols) > 0 { logutil.Infof("CAGRA create: INCLUDE columns = %v (from %d arg vectors)", u.filterCols, len(tf.ctr.argVecs)-3) - if err = initFilterColumns(u.activeBuilder(), u.filterCols); err != nil { + if err = initFilterColumns(u.builder, u.filterCols); err != nil { return err } } @@ -413,16 +439,28 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo u.rowsSeen++ id := vector.GetFixedAtNoTypeCheck[int64](tf.ctr.argVecs[1], nthRow) - fa := types.BytesToArray[float32](faVec.GetBytesAt(nthRow)) - if uint(len(fa)) != u.idxcfg.CuvsCagra.Dimensions { - return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + // Decode the base vector to its native type (see ivfpq_create_gpu.go for the + // rationale). f16 base -> native []cuvs.Float16 for both the direct (half) + // add and the CDC tail (stored as native half bytes — no f32 detour). + var fa []float32 + var hf []cuvs.Float16 + if u.baseOid == types.T_array_float16 { + h := types.BytesToArray[types.Float16](faVec.GetBytesAt(nthRow)) + if uint(len(h)) != u.idxcfg.CuvsCagra.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + } + hf = f16ToCuvs(h) + } else { + fa = types.BytesToArray[float32](faVec.GetBytesAt(nthRow)) + if uint(len(fa)) != u.idxcfg.CuvsCagra.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + } } // Trailing rows below the cuvs threshold route to the CDC tail // (search-side brute-force replay) instead of the cuvs builder. if srcPos >= u.cdcCutoff { - vecCopy := append([]float32(nil), fa...) var incBytes []byte if len(u.filterCols) > 0 { incBytes, err = encodeIncludeRowFromArgVecs(u.filterCols, tf.ctr.argVecs, 3, nthRow) @@ -430,50 +468,38 @@ func (u *cagraCreateState) start(tf *TableFunction, proc *process.Process, nthRo return err } } + // Buffer the tail row as raw native base-type bytes so a vecf16 base is + // stored as half (2 bytes/elem) in the CDC record — no f32 detour. + var vecBytes []byte + if u.baseOid == types.T_array_float16 { + vecBytes = append([]byte(nil), util.UnsafeSliceToBytes(hf)...) + } else { + vecBytes = append([]byte(nil), util.UnsafeSliceToBytes(fa)...) + } u.cdcTail = append(u.cdcTail, cuvscdc.PendingRecord{ Pkid: id, - Vec: vecCopy, + Vec: vecBytes, Include: incBytes, }) return nil } - switch { - case u.buildf32 != nil: - err = u.buildf32.AddFloat(id, fa) - case u.buildf16 != nil: - err = u.buildf16.AddFloat(id, fa) - case u.buildi8 != nil: - err = u.buildi8.AddFloat(id, fa) - case u.buildui8 != nil: - err = u.buildui8.AddFloat(id, fa) + // Pass the vector as raw base-type bytes (f32 base -> fa, f16 base -> hf), + // reinterpreted with UnsafeSliceToBytes (zero-copy); the concrete + // CagraBuild[B,Q] casts them back to its own []B/[]Q. No per-row alloc. + vecBytes := util.UnsafeSliceToBytes(fa) + if u.baseOid == types.T_array_float16 { + vecBytes = util.UnsafeSliceToBytes(hf) } - if err != nil { + if err = u.builder.AddRow(id, vecBytes); err != nil { return err } // ---- per-row: append filter column values (if any) ---- if len(u.filterCols) > 0 { - if err = appendFilterRow(u.activeBuilder(), u.filterCols, tf.ctr.argVecs, 3, nthRow); err != nil { + if err = appendFilterRow(u.builder, u.filterCols, tf.ctr.argVecs, 3, nthRow); err != nil { return err } } return nil } - -// activeBuilder returns whichever quantization-specialised builder is live, -// exposed through the narrow filterColumnBuilder interface. Exactly one of -// the four fields is non-nil after a successful NewCagraBuild dispatch. -func (u *cagraCreateState) activeBuilder() filterColumnBuilder { - switch { - case u.buildf32 != nil: - return u.buildf32 - case u.buildf16 != nil: - return u.buildf16 - case u.buildi8 != nil: - return u.buildi8 - case u.buildui8 != nil: - return u.buildui8 - } - return nil -} diff --git a/pkg/sql/colexec/table_function/cagra_search_gpu.go b/pkg/sql/colexec/table_function/cagra_search_gpu.go index e17e94738936c..5670d95f3ad62 100644 --- a/pkg/sql/colexec/table_function/cagra_search_gpu.go +++ b/pkg/sql/colexec/table_function/cagra_search_gpu.go @@ -61,15 +61,27 @@ func newCagraAlgoFn(idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTabl // test-only: mirror the build-side device simulation so search loads the same // SHARDED / REPLICATED topology. No-op when gpu_multi_simulation < 2. devices = vectorindex.SimulateDevices(devices, tblcfg.GpuMultiSimulation) - switch metric.QuantizationType(idxcfg.CuvsCagra.Quantization) { + // Dispatch on (base type B, storage type Q): indices store Q, overflow is B. + q := metric.QuantizationType(idxcfg.CuvsCagra.Quantization) + if types.T(tblcfg.KeyPartType) == types.T_array_float16 { + switch q { + case metric.Quantization_INT8: + return cagraPkg.NewCagraSearch[cuvs.Float16, int8](idxcfg, tblcfg, devices) + case metric.Quantization_UINT8: + return cagraPkg.NewCagraSearch[cuvs.Float16, uint8](idxcfg, tblcfg, devices) + default: // F16 (direct) + return cagraPkg.NewCagraSearch[cuvs.Float16, cuvs.Float16](idxcfg, tblcfg, devices) + } + } + switch q { case metric.Quantization_F16: - return cagraPkg.NewCagraSearch[cuvs.Float16](idxcfg, tblcfg, devices) + return cagraPkg.NewCagraSearch[float32, cuvs.Float16](idxcfg, tblcfg, devices) case metric.Quantization_INT8: - return cagraPkg.NewCagraSearch[int8](idxcfg, tblcfg, devices) + return cagraPkg.NewCagraSearch[float32, int8](idxcfg, tblcfg, devices) case metric.Quantization_UINT8: - return cagraPkg.NewCagraSearch[uint8](idxcfg, tblcfg, devices) + return cagraPkg.NewCagraSearch[float32, uint8](idxcfg, tblcfg, devices) default: // Quantization_F32 and unknown - return cagraPkg.NewCagraSearch[float32](idxcfg, tblcfg, devices) + return cagraPkg.NewCagraSearch[float32, float32](idxcfg, tblcfg, devices) } } @@ -216,6 +228,24 @@ func (u *cagraSearchState) start(tf *TableFunction, proc *process.Process, nthRo u.idxcfg.CuvsCagra.Dimensions = uint(faVec.GetType().Width) u.idxcfg.Type = vectorindex.CAGRA + // The query vector type must equal the index's base column type. The + // planner pushdown normally forces this, but the table function has no + // other guard: without it a mismatched query (e.g. a vecf16 query against + // an f32-base/f32-storage index) would drive the f32->f16 storage override + // below and runCagraSearchHalf off the QUERY type and deserialize the + // on-disk index with the wrong storage type. Mirrors the CPU ivf_search guard. + if int32(faVec.GetType().Oid) != u.tblcfg.KeyPartType { + return moerr.NewInvalidInput(proc.Ctx, "query vector type does not match the index base column type") + } + + // A vecf16 base with no QUANTIZATION stores natively as half: derive the + // storage qtype from the (f16) base type so newCagraAlgo dispatches + // NewCagraSearch[cuvs.Float16]. (vecf16 + QUANTIZATION keeps int8/uint8.) + if types.T(u.tblcfg.KeyPartType) == types.T_array_float16 && + metric.QuantizationType(u.idxcfg.CuvsCagra.Quantization) == metric.Quantization_F32 { + u.idxcfg.CuvsCagra.Quantization = uint16(metric.Quantization_F16) + } + u.batch = tf.createResultBatch() u.inited = true } @@ -245,6 +275,13 @@ func (u *cagraSearchState) start(tf *TableFunction, proc *process.Process, nthRo veccache.Cache.Once() + // A vecf16 query is decoded natively to half. CagraSearch.Search dispatches: + // f16-direct (T==Float16) searches the half index natively; a quantized + // f16->int8/uint8 index quantizes the half query to T via the half quantizer. + if faVec.GetType().Oid == types.T_array_float16 { + return runCagraSearchHalf(proc, u, faVec, nthRow) + } + return runCagraSearch[float32](proc, u, faVec, nthRow) } @@ -253,7 +290,20 @@ func runCagraSearch[T types.RealNumbers](proc *process.Process, u *cagraSearchSt if uint(len(fa)) != u.idxcfg.CuvsCagra.Dimensions { return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.CuvsCagra.Dimensions, len(fa))) } + return cagraRunSearchQuery(proc, u, fa) +} + +// runCagraSearchHalf decodes a vecf16 query natively to []cuvs.Float16 (no f32 +// detour) for a half-storage index. +func runCagraSearchHalf(proc *process.Process, u *cagraSearchState, faVec *vector.Vector, nthRow int) (err error) { + h := types.BytesToArray[types.Float16](faVec.GetBytesAt(nthRow)) + if uint(len(h)) != u.idxcfg.CuvsCagra.Dimensions { + return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.CuvsCagra.Dimensions, len(h))) + } + return cagraRunSearchQuery(proc, u, f16ToCuvs(h)) +} +func cagraRunSearchQuery(proc *process.Process, u *cagraSearchState, fa any) (err error) { algo := newCagraAlgo(u.idxcfg, u.tblcfg) rt := vectorindex.RuntimeConfig{ diff --git a/pkg/sql/colexec/table_function/ivf_create.go b/pkg/sql/colexec/table_function/ivf_create.go index 7106395a7b29b..820bbc6ef5f3e 100644 --- a/pkg/sql/colexec/table_function/ivf_create.go +++ b/pkg/sql/colexec/table_function/ivf_create.go @@ -36,6 +36,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/kmeans/device" ivfflatrt "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/plugin/runtime" "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/process" @@ -153,6 +154,32 @@ func clustering[T types.RealNumbers](u *ivfCreateState, tf *TableFunction, proc } logutil.Infof("IVFFLAT END: After Kmeans clustering, insert centroids to table") + // int8/uint8 QUANTIZATION (cuVS-style asymmetric scalar quantizer): train + // [min,max] over the sample and persist them in the metadata table. Entries + // (build) and the query (search) map [min,max] onto the full int8 range + // [-128,127] (or uint8 [0,255]) with the same q(x)=round(x*mul+add). Using both + // bounds (not a symmetric scale) uses the whole range for offset data. The + // percentile bound-training is identical for int8 and uint8 (only the target + // range differs, applied later by compile/search). (bf16/float16 are float + // formats and need none.) + if qt, ok := quantizer.ToVectorType(u.param.Quantization); ok && (qt == types.T_array_int8 || qt == types.T_array_uint8) { + qmin, qmax := quantizer.TrainInt8(data) + insSQL := fmt.Sprintf( + "INSERT INTO `%s`.`%s` (`%s`, `%s`) VALUES ('%s', '%.9g'), ('%s', '%.9g') "+ + "ON DUPLICATE KEY UPDATE `%s` = VALUES(`%s`)", + u.tblcfg.DbName, u.tblcfg.MetadataTable, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, + catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin, qmin, + catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax, qmax, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, catalog.SystemSI_IVFFLAT_TblCol_Metadata_val) + res, err := ivf_runSql(sqlexec.NewSqlProcess(proc), insSQL) + if err != nil { + return err + } + res.Close() + logutil.Infof("IVFFLAT: int8 quantizer min=%g max=%g stored", qmin, qmax) + } + return nil } @@ -329,10 +356,12 @@ func (u *ivfCreateState) start(tf *TableFunction, proc *process.Process, nthRow return moerr.NewInvalidInput(proc.Ctx, "Second argument (vector must be a vecf32 or vecf64 type") } - if embedvec.GetType().Oid == types.T_array_float32 { - u.data32 = make([][]float32, 0, u.nsample) - } else { + // kmeans always clusters in float32 (or float64). Narrow input types + // (bf16/f16/int8) decode to float32 -> data32; only native float64 uses data64. + if embedvec.GetType().Oid == types.T_array_float64 { u.data64 = make([][]float64, 0, u.nsample) + } else { + u.data32 = make([][]float32, 0, u.nsample) } // dimension @@ -343,20 +372,32 @@ func (u *ivfCreateState) start(tf *TableFunction, proc *process.Process, nthRow for _, bat := range res.Batches { evec := bat.Vecs[0] for i := 0; i < bat.RowCount(); i++ { + var f32a []float32 switch evec.GetType().Oid { case types.T_array_float32: - f32a := types.BytesToArray[float32](evec.GetBytesAt(i)) - if uint(len(f32a)) != u.idxcfg.Ivfflat.Dimensions { - return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") - } - u.data32 = append(u.data32, append(make([]float32, 0, len(f32a)), f32a...)) + f32a = types.BytesToArray[float32](evec.GetBytesAt(i)) case types.T_array_float64: f64a := types.BytesToArray[float64](evec.GetBytesAt(i)) if uint(len(f64a)) != u.idxcfg.Ivfflat.Dimensions { return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") } u.data64 = append(u.data64, append(make([]float64, 0, len(f64a)), f64a...)) + continue + case types.T_array_bf16: + f32a = types.BF16ToFloat32Slice(types.BytesToArray[types.BF16](evec.GetBytesAt(i))) + case types.T_array_float16: + f32a = types.Float16ToFloat32Slice(types.BytesToArray[types.Float16](evec.GetBytesAt(i))) + case types.T_array_int8: + f32a = types.Int8ToFloat32Slice(types.BytesToArray[int8](evec.GetBytesAt(i))) + case types.T_array_uint8: + f32a = types.Uint8ToFloat32Slice(types.BytesToArray[uint8](evec.GetBytesAt(i))) + default: + return moerr.NewInternalError(proc.Ctx, "unsupported ivfflat vector type") + } + if uint(len(f32a)) != u.idxcfg.Ivfflat.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") } + u.data32 = append(u.data32, append(make([]float32, 0, len(f32a)), f32a...)) } } diff --git a/pkg/sql/colexec/table_function/ivf_search.go b/pkg/sql/colexec/table_function/ivf_search.go index 41821744b25aa..4babeca96f542 100644 --- a/pkg/sql/colexec/table_function/ivf_search.go +++ b/pkg/sql/colexec/table_function/ivf_search.go @@ -30,6 +30,7 @@ import ( veccache "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat" "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" "github.com/matrixorigin/matrixone/pkg/vm" "github.com/matrixorigin/matrixone/pkg/vm/process" @@ -58,13 +59,21 @@ var ( ) func newIvfAlgoFn(idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig) (veccache.VectorIndexSearchIf, error) { - switch idxcfg.Ivfflat.VectorType { + // The centroid search index is typed by the CENTROID storage type, not the + // entry/input type. For narrow entries the centroids are f32 (decoupled), so + // this returns IvfflatSearch[float32]. CentroidType == 0 (old indexes) means + // "same as entry". + ct := idxcfg.Ivfflat.CentroidType + if ct == 0 { + ct = idxcfg.Ivfflat.VectorType + } + switch ct { case int32(types.T_array_float32): return ivfflat.NewIvfflatSearch[float32](idxcfg, tblcfg), nil case int32(types.T_array_float64): return ivfflat.NewIvfflatSearch[float64](idxcfg, tblcfg), nil default: - return nil, moerr.NewInternalErrorNoCtx("newIvfAlgoFn: invalid vector type") + return nil, moerr.NewInternalErrorNoCtx("newIvfAlgoFn: invalid centroid type") } } @@ -197,7 +206,26 @@ func (u *ivfSearchState) start(tf *TableFunction, proc *process.Process, nthRow return err } u.idxcfg.Ivfflat.Version = version // version from meta table - u.idxcfg.Ivfflat.VectorType = u.tblcfg.KeyPartType // array float32 or array float64 + u.idxcfg.Ivfflat.VectorType = u.tblcfg.KeyPartType // entry/input type + // Centroid type is decoupled: f32 for narrow entries (must match the f32 + // centroid hidden table from schema.go), else same as the entry type. + switch types.T(u.tblcfg.KeyPartType) { + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: + u.idxcfg.Ivfflat.CentroidType = int32(types.T_array_float32) + default: + u.idxcfg.Ivfflat.CentroidType = u.tblcfg.KeyPartType + } + // QUANTIZATION: entries are stored as the quantization (down-cast) type, + // independent of the base column. The centroids are forced to f32 (decoupled + // — accurate assignment, fast f32 search) for ANY base type, including f64; + // the query is decoded to f32 for the centroid search and to the entry type + // for the re-rank. VectorType = the entry/quantization type. + if u.param.Quantization != "" { + if qt, ok := quantizer.ToVectorType(u.param.Quantization); ok { + u.idxcfg.Ivfflat.VectorType = int32(qt) + u.idxcfg.Ivfflat.CentroidType = int32(types.T_array_float32) + } + } u.batch = tf.createResultBatch() u.inited = true @@ -216,22 +244,56 @@ func (u *ivfSearchState) start(tf *TableFunction, proc *process.Process, nthRow faVec := tf.ctr.argVecs[1] - switch faVec.GetType().Oid { - case types.T_array_float32: - return runIvfSearchVector[float32](tf, u, proc, faVec, nthRow) - case types.T_array_float64: + // Dispatch on the CENTROID type, not the base type. Only a plain f64 index + // (f64 base, no quantization) keeps f64 centroids; every other case — f32 base, + // narrow base, or any base under QUANTIZATION — searches f32 centroids, so the + // query is decoded to float32 regardless of its column type. + if u.idxcfg.Ivfflat.CentroidType == int32(types.T_array_float64) { return runIvfSearchVector[float64](tf, u, proc, faVec, nthRow) - default: - return moerr.NewInternalError(proc.Ctx, "vector is not array_float32 or array_float64") } + return runIvfSearchVectorToF32(tf, u, proc, faVec, nthRow) } func runIvfSearchVector[T types.RealNumbers](tf *TableFunction, u *ivfSearchState, proc *process.Process, faVec *vector.Vector, nthRow int) (err error) { if faVec.IsNull(uint64(nthRow)) { return nil } + return runIvfSearchQuery(tf, u, proc, types.BytesToArray[T](faVec.GetBytesAt(nthRow))) +} + +// runIvfSearchVectorToF32 decodes the query (of any vector column type: f32, f64, +// or narrow bf16/f16/int8) to float32 and runs the float32 centroid search. The +// SQL re-rank then encodes the query in the entry/quantization type. +func runIvfSearchVectorToF32(tf *TableFunction, u *ivfSearchState, proc *process.Process, faVec *vector.Vector, nthRow int) error { + if faVec.IsNull(uint64(nthRow)) { + return nil + } + b := faVec.GetBytesAt(nthRow) + var fa []float32 + switch faVec.GetType().Oid { + case types.T_array_float32: + fa = types.BytesToArray[float32](b) + case types.T_array_float64: + f64 := types.BytesToArray[float64](b) + fa = make([]float32, len(f64)) + for i, x := range f64 { + fa[i] = float32(x) + } + case types.T_array_bf16: + fa = types.BF16ToFloat32Slice(types.BytesToArray[types.BF16](b)) + case types.T_array_float16: + fa = types.Float16ToFloat32Slice(types.BytesToArray[types.Float16](b)) + case types.T_array_int8: + fa = types.Int8ToFloat32Slice(types.BytesToArray[int8](b)) + case types.T_array_uint8: + fa = types.Uint8ToFloat32Slice(types.BytesToArray[uint8](b)) + default: + return moerr.NewInternalError(proc.Ctx, "unsupported ivfflat vector type") + } + return runIvfSearchQuery(tf, u, proc, fa) +} - fa := types.BytesToArray[T](faVec.GetBytesAt(nthRow)) +func runIvfSearchQuery[T types.RealNumbers](tf *TableFunction, u *ivfSearchState, proc *process.Process, fa []T) (err error) { if uint(len(fa)) != u.idxcfg.Ivfflat.Dimensions { return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.Ivfflat.Dimensions, len(fa))) } diff --git a/pkg/sql/colexec/table_function/ivfpq_create_gpu.go b/pkg/sql/colexec/table_function/ivfpq_create_gpu.go index b748951e56245..d8a28bc49e67f 100644 --- a/pkg/sql/colexec/table_function/ivfpq_create_gpu.go +++ b/pkg/sql/colexec/table_function/ivfpq_create_gpu.go @@ -20,9 +20,11 @@ import ( "fmt" "strconv" "time" + "unsafe" "github.com/bytedance/sonic" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -48,16 +50,45 @@ var ivfpqCatalogHooks = ivfpqrt.CatalogHooks{} var ivfpq_runSql = sqlexec.RunSql +// f16ToCuvs reinterprets a []types.Float16 as []cuvs.Float16. Both are uint16 +// with identical layout; this is a zero-copy view (the caller does not retain it +// past the GPU add, which copies to device). Shared by the ivfpq/cagra GPU +// table functions for the native f16 (half) path. +func f16ToCuvs(s []types.Float16) []cuvs.Float16 { + if len(s) == 0 { + return nil + } + return unsafe.Slice((*cuvs.Float16)(unsafe.Pointer(&s[0])), len(s)) +} + +// ivfpqBuilder is the (B, Q)-erased build interface the create state drives. +// *ivfpqPkg.IvfpqBuild[B, Q] satisfies it for every wired (base, storage) +// combo. GetIndexes is [B,Q]-typed and intentionally NOT on the interface — +// end() routes through ToInsertSql instead. +type ivfpqBuilder interface { + // AddRow takes the raw base-type bytes of one vector (4*dim for an f32 base, + // 2*dim for an f16 base); the concrete builder reinterprets them to its + // []B/[]Q with UnsafeSliceCast (the interface can't name B). Passing []byte + // rather than `any` keeps the per-row build hot path allocation-free. + AddRow(id int64, vecBytes []byte) error + SetFilterColumns(colMetaJSON string) + AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error + ToInsertSql(ts int64) ([]string, error) + Destroy() error +} + type ivfpqCreateState struct { - inited bool - buildf32 *ivfpqPkg.IvfpqBuild[float32] - buildf16 *ivfpqPkg.IvfpqBuild[cuvs.Float16] - buildi8 *ivfpqPkg.IvfpqBuild[int8] - buildui8 *ivfpqPkg.IvfpqBuild[uint8] - param vectorindex.IvfpqParam - tblcfg vectorindex.IndexTableConfig - idxcfg vectorindex.IndexConfig - offset int + inited bool + builder ivfpqBuilder + param vectorindex.IvfpqParam + tblcfg vectorindex.IndexTableConfig + idxcfg vectorindex.IndexConfig + offset int + + // baseOid is the base (source) vector column element type — f32 or f16. + // The storage/quantization type (which builder is non-nil) may differ: + // f16 base is stored as half (direct) or quantized to int8/uint8. + baseOid types.T // filterCols is the INCLUDE column metadata derived at start() from // param.IncludedColumns (names) + argVecs[3:] (types). Empty when the @@ -72,7 +103,9 @@ type ivfpqCreateState struct { // records under vectorindex.CdcTailId. cdcCutoff int64 rowsSeen int64 - cdcTail []cuvscdc.PendingRecord + // CDC tail records, with each vector stored as raw native base-type bytes + // (f16 stays 2-byte — no f32 widening). vecBytesPerRow = dim * base elem size. + cdcTail []cuvscdc.PendingRecord // srcEmpty short-circuits the per-row code when SELECT COUNT(*) // at init time returned zero — nothing to build, nothing to CDC. @@ -93,19 +126,11 @@ func (u *ivfpqCreateState) end(tf *TableFunction, proc *process.Process) error { ) ts := time.Now().UnixMicro() - switch { - case u.buildf32 != nil: - sqls, err = u.buildf32.ToInsertSql(ts) - case u.buildf16 != nil: - sqls, err = u.buildf16.ToInsertSql(ts) - case u.buildi8 != nil: - sqls, err = u.buildi8.ToInsertSql(ts) - case u.buildui8 != nil: - sqls, err = u.buildui8.ToInsertSql(ts) - default: - // No builder selected → init didn't set one. Nothing to do for - // the cuvs side; the CDC tail (if any) below still emits. + if u.builder != nil { + sqls, err = u.builder.ToInsertSql(ts) } + // No builder selected → init didn't set one. Nothing to do for the cuvs + // side; the CDC tail (if any) below still emits. if err != nil { return err } @@ -119,9 +144,14 @@ func (u *ivfpqCreateState) end(tf *TableFunction, proc *process.Process) error { // record 0. Search-side can recover the INCLUDE-column layout // for tag=1 replay even when no tag=0 sub-index exists. colMetaJSON := colMetaJSONFromCols(u.filterCols) + // vecBytesPerRow = dim * base element size (2 for vecf16, else 4). + elemSize := 4 + if u.baseOid == types.T_array_float16 { + elemSize = 2 + } + vecBytesPerRow := int(u.idxcfg.CuvsIvfpq.Dimensions) * elemSize tailSqls, err := cuvscdc.SaveSmallTailAsCdc( - u.tblcfg, u.cdcTail, - int(u.idxcfg.CuvsIvfpq.Dimensions), ibpr, colMetaJSON) + u.tblcfg, u.cdcTail, vecBytesPerRow, ibpr, colMetaJSON) if err != nil { return err } @@ -158,17 +188,8 @@ func (u *ivfpqCreateState) free(tf *TableFunction, proc *process.Process, pipeli if u.batch != nil { u.batch.Clean(proc.Mp()) } - if u.buildf32 != nil { - u.buildf32.Destroy() - } - if u.buildf16 != nil { - u.buildf16.Destroy() - } - if u.buildi8 != nil { - u.buildi8.Destroy() - } - if u.buildui8 != nil { - u.buildui8.Destroy() + if u.builder != nil { + u.builder.Destroy() } } @@ -350,7 +371,16 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo faVec := tf.ctr.argVecs[2] if !catalogplugin.SupportsVectorType(ivfpqCatalogHooks, faVec.GetType().Oid) { - return moerr.NewInvalidInput(proc.Ctx, "third argument (vector) must be a float32 array") + return moerr.NewInvalidInput(proc.Ctx, "third argument (vector) must be a float32 / float16 array") + } + u.baseOid = faVec.GetType().Oid + + // Derive the storage qtype from the base column type when no QUANTIZATION + // was given: a vecf16 base with no quantization is stored natively as half. + // (vecf16 + QUANTIZATION=int8/uint8 keeps qt = int8/uint8 — quantize path.) + if u.baseOid == types.T_array_float16 && qt == metric.Quantization_F32 { + qt = metric.Quantization_F16 + u.idxcfg.CuvsIvfpq.Quantization = uint16(qt) } // dimension @@ -367,15 +397,25 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo uid := fmt.Sprintf("%s:%d:%d", tf.CnAddr, tf.MaxParallel, tf.ParallelID) // ---- create builder ---- - switch qt { - case metric.Quantization_F16: - u.buildf16, err = ivfpqPkg.NewIvfpqBuild[cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) - case metric.Quantization_INT8: - u.buildi8, err = ivfpqPkg.NewIvfpqBuild[int8](uid, u.idxcfg, u.tblcfg, nthread, devices) - case metric.Quantization_UINT8: - u.buildui8, err = ivfpqPkg.NewIvfpqBuild[uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) + // One real [B, Q] builder keyed on (base column type, storage qtype). + // The 7 wired combos: f32 base × {f32, f16, int8, uint8}; f16 base × + // {f16, int8, uint8}. + isF16Base := u.baseOid == types.T_array_float16 + switch { + case isF16Base && qt == metric.Quantization_F16: + u.builder, err = ivfpqPkg.NewIvfpqBuild[cuvs.Float16, cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) + case isF16Base && qt == metric.Quantization_INT8: + u.builder, err = ivfpqPkg.NewIvfpqBuild[cuvs.Float16, int8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case isF16Base && qt == metric.Quantization_UINT8: + u.builder, err = ivfpqPkg.NewIvfpqBuild[cuvs.Float16, uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_F16: + u.builder, err = ivfpqPkg.NewIvfpqBuild[float32, cuvs.Float16](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_INT8: + u.builder, err = ivfpqPkg.NewIvfpqBuild[float32, int8](uid, u.idxcfg, u.tblcfg, nthread, devices) + case qt == metric.Quantization_UINT8: + u.builder, err = ivfpqPkg.NewIvfpqBuild[float32, uint8](uid, u.idxcfg, u.tblcfg, nthread, devices) default: - u.buildf32, err = ivfpqPkg.NewIvfpqBuild[float32](uid, u.idxcfg, u.tblcfg, nthread, devices) + u.builder, err = ivfpqPkg.NewIvfpqBuild[float32, float32](uid, u.idxcfg, u.tblcfg, nthread, devices) } if err != nil { return err @@ -390,7 +430,7 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo if len(u.filterCols) > 0 { logutil.Infof("IVFPQ create: INCLUDE columns = %v (from %d arg vectors)", u.filterCols, len(tf.ctr.argVecs)-3) - if err = initFilterColumns(u.activeBuilder(), u.filterCols); err != nil { + if err = initFilterColumns(u.builder, u.filterCols); err != nil { return err } } @@ -422,17 +462,33 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo u.rowsSeen++ id := vector.GetFixedAtNoTypeCheck[int64](tf.ctr.argVecs[1], nthRow) - fa := types.BytesToArray[float32](faVec.GetBytesAt(nthRow)) - if uint(len(fa)) != u.idxcfg.CuvsIvfpq.Dimensions { - return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + // Decode the base vector to its native type. f32 base -> []float32 (used by + // the f32 path and the CDC tail). f16 base -> native []cuvs.Float16 for the + // direct (half-storage) add; the CDC tail still transports f32 (exact widen) + // until the CDC pipeline is made native (step 5). + var fa []float32 + var hf []cuvs.Float16 + if u.baseOid == types.T_array_float16 { + h := types.BytesToArray[types.Float16](faVec.GetBytesAt(nthRow)) + if uint(len(h)) != u.idxcfg.CuvsIvfpq.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + } + hf = f16ToCuvs(h) + if srcPos >= u.cdcCutoff { + fa = types.Float16ToFloat32Slice(h) + } + } else { + fa = types.BytesToArray[float32](faVec.GetBytesAt(nthRow)) + if uint(len(fa)) != u.idxcfg.CuvsIvfpq.Dimensions { + return moerr.NewInternalError(proc.Ctx, "vector dimension mismatch") + } } // Trailing rows below the cuvs k-means threshold (lists) route to // the CDC tail (search-side brute-force replay) instead of the // cuvs builder. if srcPos >= u.cdcCutoff { - vecCopy := append([]float32(nil), fa...) var incBytes []byte if len(u.filterCols) > 0 { incBytes, err = encodeIncludeRowFromArgVecs(u.filterCols, tf.ctr.argVecs, 3, nthRow) @@ -440,48 +496,37 @@ func (u *ivfpqCreateState) start(tf *TableFunction, proc *process.Process, nthRo return err } } + // Buffer the tail row as raw native base-type bytes so a vecf16 base is + // stored as half (2 bytes/elem) in the CDC record — no f32 detour. + var vecBytes []byte + if u.baseOid == types.T_array_float16 { + vecBytes = append([]byte(nil), util.UnsafeSliceToBytes(hf)...) + } else { + vecBytes = append([]byte(nil), util.UnsafeSliceToBytes(fa)...) + } u.cdcTail = append(u.cdcTail, cuvscdc.PendingRecord{ Pkid: id, - Vec: vecCopy, + Vec: vecBytes, Include: incBytes, }) return nil } - switch { - case u.buildf32 != nil: - err = u.buildf32.AddFloat(id, fa) - case u.buildf16 != nil: - err = u.buildf16.AddFloat(id, fa) - case u.buildi8 != nil: - err = u.buildi8.AddFloat(id, fa) - case u.buildui8 != nil: - err = u.buildui8.AddFloat(id, fa) + // Pass the vector as raw base-type bytes (f32 base -> fa, f16 base -> hf), + // reinterpreted with UnsafeSliceToBytes (zero-copy); the concrete + // IvfpqBuild[B,Q] casts them back to its own []B/[]Q. No per-row alloc. + vecBytes := util.UnsafeSliceToBytes(fa) + if u.baseOid == types.T_array_float16 { + vecBytes = util.UnsafeSliceToBytes(hf) } - if err != nil { + if err = u.builder.AddRow(id, vecBytes); err != nil { return err } if len(u.filterCols) > 0 { - if err = appendFilterRow(u.activeBuilder(), u.filterCols, tf.ctr.argVecs, 3, nthRow); err != nil { + if err = appendFilterRow(u.builder, u.filterCols, tf.ctr.argVecs, 3, nthRow); err != nil { return err } } return nil } - -// activeBuilder returns the live quantization-specialised builder through the -// filterColumnBuilder interface. See cagraCreateState.activeBuilder. -func (u *ivfpqCreateState) activeBuilder() filterColumnBuilder { - switch { - case u.buildf32 != nil: - return u.buildf32 - case u.buildf16 != nil: - return u.buildf16 - case u.buildi8 != nil: - return u.buildi8 - case u.buildui8 != nil: - return u.buildui8 - } - return nil -} diff --git a/pkg/sql/colexec/table_function/ivfpq_search_gpu.go b/pkg/sql/colexec/table_function/ivfpq_search_gpu.go index 165ea3f22fd77..6ad58b66938e5 100644 --- a/pkg/sql/colexec/table_function/ivfpq_search_gpu.go +++ b/pkg/sql/colexec/table_function/ivfpq_search_gpu.go @@ -61,15 +61,28 @@ func newIvfpqAlgoFn(idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTabl // test-only: mirror the build-side device simulation so search loads the same // SHARDED / REPLICATED topology. No-op when gpu_multi_simulation < 2. devices = vectorindex.SimulateDevices(devices, tblcfg.GpuMultiSimulation) - switch metric.QuantizationType(idxcfg.CuvsIvfpq.Quantization) { + // Dispatch on (base type B, storage type Q): the main indices store Q (the + // quantization), the CDC/overflow brute force is the base type B (f32/f16). + q := metric.QuantizationType(idxcfg.CuvsIvfpq.Quantization) + if types.T(tblcfg.KeyPartType) == types.T_array_float16 { + switch q { + case metric.Quantization_INT8: + return ivfpqPkg.NewIvfpqSearch[cuvs.Float16, int8](idxcfg, tblcfg, devices) + case metric.Quantization_UINT8: + return ivfpqPkg.NewIvfpqSearch[cuvs.Float16, uint8](idxcfg, tblcfg, devices) + default: // F16 (direct) + return ivfpqPkg.NewIvfpqSearch[cuvs.Float16, cuvs.Float16](idxcfg, tblcfg, devices) + } + } + switch q { case metric.Quantization_F16: - return ivfpqPkg.NewIvfpqSearch[cuvs.Float16](idxcfg, tblcfg, devices) + return ivfpqPkg.NewIvfpqSearch[float32, cuvs.Float16](idxcfg, tblcfg, devices) case metric.Quantization_INT8: - return ivfpqPkg.NewIvfpqSearch[int8](idxcfg, tblcfg, devices) + return ivfpqPkg.NewIvfpqSearch[float32, int8](idxcfg, tblcfg, devices) case metric.Quantization_UINT8: - return ivfpqPkg.NewIvfpqSearch[uint8](idxcfg, tblcfg, devices) + return ivfpqPkg.NewIvfpqSearch[float32, uint8](idxcfg, tblcfg, devices) default: - return ivfpqPkg.NewIvfpqSearch[float32](idxcfg, tblcfg, devices) + return ivfpqPkg.NewIvfpqSearch[float32, float32](idxcfg, tblcfg, devices) } } @@ -219,6 +232,24 @@ func (u *ivfpqSearchState) start(tf *TableFunction, proc *process.Process, nthRo u.idxcfg.CuvsIvfpq.Dimensions = uint(faVec.GetType().Width) u.idxcfg.Type = vectorindex.IVFPQ + // The query vector type must equal the index's base column type. The + // planner pushdown normally forces this, but the table function has no + // other guard: without it a mismatched query (e.g. a vecf16 query against + // an f32-base/f32-storage index) would drive the f32->f16 storage override + // below off the QUERY type and deserialize the on-disk index with the wrong + // storage type. Mirrors the CPU ivf_search guard. + if int32(faVec.GetType().Oid) != u.tblcfg.KeyPartType { + return moerr.NewInvalidInput(proc.Ctx, "query vector type does not match the index base column type") + } + + // A vecf16 base with no QUANTIZATION stores natively as half: derive the + // storage qtype from the (f16) base type so newIvfpqAlgo dispatches + // NewIvfpqSearch[cuvs.Float16]. (vecf16 + QUANTIZATION keeps int8/uint8.) + if types.T(u.tblcfg.KeyPartType) == types.T_array_float16 && + metric.QuantizationType(u.idxcfg.CuvsIvfpq.Quantization) == metric.Quantization_F32 { + u.idxcfg.CuvsIvfpq.Quantization = uint16(metric.Quantization_F16) + } + u.batch = tf.createResultBatch() u.inited = true } @@ -247,6 +278,13 @@ func (u *ivfpqSearchState) start(tf *TableFunction, proc *process.Process, nthRo veccache.Cache.Once() + // A vecf16 query is decoded natively to half. IvfpqSearch.Search dispatches: + // f16-direct (T==Float16) searches the half index natively; a quantized + // f16->int8/uint8 index quantizes the half query to T via the half quantizer. + if faVec.GetType().Oid == types.T_array_float16 { + return runIvfpqSearchHalf(proc, u, faVec, nthRow) + } + return runIvfpqSearch[float32](proc, u, faVec, nthRow) } @@ -255,7 +293,21 @@ func runIvfpqSearch[T types.RealNumbers](proc *process.Process, u *ivfpqSearchSt if uint(len(fa)) != u.idxcfg.CuvsIvfpq.Dimensions { return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.CuvsIvfpq.Dimensions, len(fa))) } + return ivfpqRunSearchQuery(proc, u, fa) +} + +// runIvfpqSearchHalf decodes a vecf16 query natively to []cuvs.Float16 (no f32 +// detour) for a half-storage index. IvfpqSearch.Search dispatches the native +// half path; a filtered query is half-cast to f32 there (exact). +func runIvfpqSearchHalf(proc *process.Process, u *ivfpqSearchState, faVec *vector.Vector, nthRow int) (err error) { + h := types.BytesToArray[types.Float16](faVec.GetBytesAt(nthRow)) + if uint(len(h)) != u.idxcfg.CuvsIvfpq.Dimensions { + return moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("vector ops between different dimensions (%d, %d) is not permitted.", u.idxcfg.CuvsIvfpq.Dimensions, len(h))) + } + return ivfpqRunSearchQuery(proc, u, f16ToCuvs(h)) +} +func ivfpqRunSearchQuery(proc *process.Process, u *ivfpqSearchState, fa any) (err error) { algo := newIvfpqAlgo(u.idxcfg, u.tblcfg) rt := vectorindex.RuntimeConfig{ diff --git a/pkg/sql/compile/ddl.go b/pkg/sql/compile/ddl.go index 61a4fa7df8664..567aaa209ed96 100644 --- a/pkg/sql/compile/ddl.go +++ b/pkg/sql/compile/ddl.go @@ -517,9 +517,13 @@ func reindexSpecifiedParams(stmt tree.Statement, indexName string) map[string]st addInt(catalog.IndexAlgoParamKmeansTrainPercent, opt.KmeansTrainPercent) addInt(catalog.IndexAlgoParamKmeansMaxIteration, opt.KmeansMaxIteration) addInt(catalog.IndexAlgoParamMaxIndexCapacity, opt.MaxIndexCapacity) - // NOTE: quantization is intentionally NOT handled by reindex. The vecf16 - // branch owns the quantization work (per-backend validity, BF16, ...), so - // reindex neither merges nor rejects it here — revisit once that lands. + // quantization is normalized to lowercase here (matching the CREATE INDEX + // path) so case-sensitive consumers (GPU build switch / quantizer) behave + // identically; the per-backend VALUE check (which names a given algorithm + // accepts) is done in each plugin's ValidateReindexParams. + if opt.Quantization != "" { + m[catalog.Quantization] = catalog.ToLower(opt.Quantization) + } return m } @@ -2558,6 +2562,38 @@ func (s *Scope) DropIndex(c *Compile) error { return err } + //6. Plugin-mediated drop hook — mirrors the HandleCreateIndex dispatch in + // CreateIndex. Vector-index plugins use it to evict their in-process search + // cache for the dropped index, so GPU/host resources are freed NOW instead of + // lingering until the 5-min VectorIndexCacheTTL housekeeping. Without this the + // hook (pkg/vectorindex/*/plugin/compile HandleDropIndex) was never invoked. + dropPluginIndexes := make(map[string]*MultiTableIndex) + for _, idef := range oldTableDef.Indexes { + if idef.IndexName != qry.IndexName || idef.Unique || !indexplugin.IsPluginAlgo(idef.IndexAlgo) { + continue + } + algo := catalog.ToLower(idef.IndexAlgo) + mti, ok := dropPluginIndexes[algo] + if !ok { + mti = &MultiTableIndex{IndexAlgo: algo, IndexDefs: make(map[string]*plan.IndexDef)} + dropPluginIndexes[algo] = mti + } + mti.IndexDefs[catalog.ToLower(idef.IndexAlgoTableType)] = idef + } + if len(dropPluginIndexes) > 0 { + dctx := newPluginCompileCtx(s, c, oldTableDef.TblId, nil, d, qry.Database, oldTableDef, nil) + for _, mti := range dropPluginIndexes { + if p, ok := indexplugin.Get(mti.IndexAlgo); ok { + // Best-effort cleanup: the 5-min TTL is the backstop, so don't + // fail the DROP if cache eviction errors — just log. + if e := p.Compile().HandleDropIndex(dctx, mti.IndexDefs); e != nil { + logutil.Warnf("[plugin] %s HandleDropIndex %s.%s/%s: %v", + mti.IndexAlgo, qry.Database, qry.Table, qry.IndexName, e) + } + } + } + } + return nil } diff --git a/pkg/sql/compile/plugin_context.go b/pkg/sql/compile/plugin_context.go index 729f6c3649876..2f7cab66b55a5 100644 --- a/pkg/sql/compile/plugin_context.go +++ b/pkg/sql/compile/plugin_context.go @@ -15,6 +15,7 @@ package compile import ( + "github.com/matrixorigin/matrixone/pkg/fileservice" compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -85,6 +86,27 @@ func (p *pluginCompileCtx) MainTableID() uint64 { return p.mainTabl func (p *pluginCompileCtx) MainExtra() *api.SchemaExtra { return p.mainExtra } func (p *pluginCompileCtx) RunSql(sql string) error { return p.c.runSql(sql) } +// RunWithSourceReadCacheSkip runs fn with SkipMemoryCacheWrites attached to the +// compile context, so block reads performed inside fn — notably an index build's +// source-table scans (kmeans sample + entry assignment) — do NOT populate the +// fileservice memory cache. The build reads the source once; queries never re-read +// it (re-rank fetches only a handful of rows), so caching it would just evict the +// index-entry blocks the queries actually hit. Mirrors the SkipAllCache policy +// compaction (mergeobjects) and LOAD DATA (external) already use for one-shot bulk +// reads. runSqlWithResultAndOptions reads c.proc.Ctx for the sub-execution, so +// attaching the policy here propagates to every read in the build; it is restored +// afterward. The build runs synchronously within this compile, so the temporary +// swap is single-threaded. +func (p *pluginCompileCtx) RunWithSourceReadCacheSkip(fn func() error) error { + if p.c == nil || p.c.proc == nil { + return fn() + } + prev := p.c.proc.Ctx + p.c.proc.Ctx = fileservice.WithFileServicePolicy(prev, fileservice.SkipMemoryCacheWrites) + defer func() { p.c.proc.Ctx = prev }() + return fn() +} + func (p *pluginCompileCtx) BuildIndexTable(def *plan.TableDef) error { return indexTableBuild(p.c, p.mainTableID, p.mainExtra, def, p.dbSource) } diff --git a/pkg/sql/compile/reindex_params_test.go b/pkg/sql/compile/reindex_params_test.go index 77df271471eb0..3ade308193f6c 100644 --- a/pkg/sql/compile/reindex_params_test.go +++ b/pkg/sql/compile/reindex_params_test.go @@ -38,6 +38,7 @@ func TestReindexSpecifiedParams(t *testing.T) { IntermediateGraphDegree: 128, GraphDegree: 64, ITopkSize: 256, + Quantization: "Float16", // mixed case -> normalized to lowercase KmeansTrainPercent: 5, KmeansMaxIteration: 30, MaxIndexCapacity: 2000, @@ -56,6 +57,7 @@ func TestReindexSpecifiedParams(t *testing.T) { catalog.IntermediateGraphDegree: "128", catalog.GraphDegree: "64", catalog.ITopkSize: "256", + catalog.Quantization: "float16", // normalized from "Float16" catalog.IndexAlgoParamKmeansTrainPercent: "5", catalog.IndexAlgoParamKmeansMaxIteration: "30", catalog.IndexAlgoParamMaxIndexCapacity: "2000", diff --git a/pkg/sql/parsers/dialect/mysql/keywords.go b/pkg/sql/parsers/dialect/mysql/keywords.go index dacefa55815df..256003a70e6a9 100644 --- a/pkg/sql/parsers/dialect/mysql/keywords.go +++ b/pkg/sql/parsers/dialect/mysql/keywords.go @@ -688,6 +688,10 @@ func init() { "array": ARRAY, "vecf32": VECF32, "vecf64": VECF64, + "vecbf16": VECBF16, + "vecf16": VECF16, + "vecint8": VECINT8, + "vecuint8": VECUINT8, "backup": BACKUP, "filesystem": FILESYSTEM, "handler": HANDLER, diff --git a/pkg/sql/parsers/dialect/mysql/mysql_sql.go b/pkg/sql/parsers/dialect/mysql/mysql_sql.go index d9b580e21ab98..5983cc4c87f62 100644 --- a/pkg/sql/parsers/dialect/mysql/mysql_sql.go +++ b/pkg/sql/parsers/dialect/mysql/mysql_sql.go @@ -261,517 +261,521 @@ const ENUM = 57549 const UUID = 57550 const VECF32 = 57551 const VECF64 = 57552 -const GEOMETRY = 57553 -const POINT = 57554 -const LINESTRING = 57555 -const POLYGON = 57556 -const GEOMETRYCOLLECTION = 57557 -const MULTIPOINT = 57558 -const MULTILINESTRING = 57559 -const MULTIPOLYGON = 57560 -const GEOMETRY32 = 57561 -const GEOGRAPHY = 57562 -const GEOGRAPHY32 = 57563 -const POINT32 = 57564 -const LINESTRING32 = 57565 -const POLYGON32 = 57566 -const GEOMETRYCOLLECTION32 = 57567 -const MULTIPOINT32 = 57568 -const MULTILINESTRING32 = 57569 -const MULTIPOLYGON32 = 57570 -const INT1 = 57571 -const INT2 = 57572 -const INT3 = 57573 -const INT4 = 57574 -const INT8 = 57575 -const S3OPTION = 57576 -const STAGEOPTION = 57577 -const SQL_SMALL_RESULT = 57578 -const SQL_BIG_RESULT = 57579 -const SQL_BUFFER_RESULT = 57580 -const SQL_CALC_FOUND_ROWS = 57581 -const LOW_PRIORITY = 57582 -const HIGH_PRIORITY = 57583 -const DELAYED = 57584 -const CREATE = 57585 -const ALTER = 57586 -const DROP = 57587 -const RENAME = 57588 -const REMOVE = 57589 -const ANALYZE = 57590 -const PHYPLAN = 57591 -const ADD = 57592 -const RETURNS = 57593 -const SCHEMA = 57594 -const TABLE = 57595 -const SEQUENCE = 57596 -const INDEX = 57597 -const VIEW = 57598 -const TO = 57599 -const IGNORE = 57600 -const IF = 57601 -const PRIMARY = 57602 -const COLUMN = 57603 -const CONSTRAINT = 57604 -const SPATIAL = 57605 -const FULLTEXT = 57606 -const FOREIGN = 57607 -const KEY_BLOCK_SIZE = 57608 -const SHOW = 57609 -const DESCRIBE = 57610 -const EXPLAIN = 57611 -const DATE = 57612 -const ESCAPE = 57613 -const REPAIR = 57614 -const OPTIMIZE = 57615 -const TRUNCATE = 57616 -const MAXVALUE = 57617 -const PARTITION = 57618 -const REORGANIZE = 57619 -const LESS = 57620 -const THAN = 57621 -const PROCEDURE = 57622 -const TRIGGER = 57623 -const STATUS = 57624 -const VARIABLES = 57625 -const ROLE = 57626 -const PROXY = 57627 -const AVG_ROW_LENGTH = 57628 -const STORAGE = 57629 -const DISK = 57630 -const MEMORY = 57631 -const CHECKSUM = 57632 -const COMPRESSION = 57633 -const DATA = 57634 -const DIRECTORY = 57635 -const DELAY_KEY_WRITE = 57636 -const ENCRYPTION = 57637 -const ENGINE = 57638 -const MAX_ROWS = 57639 -const MIN_ROWS = 57640 -const PACK_KEYS = 57641 -const ROW_FORMAT = 57642 -const STATS_AUTO_RECALC = 57643 -const STATS_PERSISTENT = 57644 -const STATS_SAMPLE_PAGES = 57645 -const DYNAMIC = 57646 -const COMPRESSED = 57647 -const REDUNDANT = 57648 -const COMPACT = 57649 -const FIXED = 57650 -const COLUMN_FORMAT = 57651 -const AUTO_RANDOM = 57652 -const ENGINE_ATTRIBUTE = 57653 -const SECONDARY_ENGINE_ATTRIBUTE = 57654 -const INSERT_METHOD = 57655 -const RESTRICT = 57656 -const CASCADE = 57657 -const ACTION = 57658 -const PARTIAL = 57659 -const SIMPLE = 57660 -const CHECK = 57661 -const ENFORCED = 57662 -const RANGE = 57663 -const LIST = 57664 -const ALGORITHM = 57665 -const LINEAR = 57666 -const PARTITIONS = 57667 -const SUBPARTITION = 57668 -const SUBPARTITIONS = 57669 -const CLUSTER = 57670 -const TYPE = 57671 -const ANY = 57672 -const SOME = 57673 -const EXTERNAL = 57674 -const LOCALFILE = 57675 -const URL = 57676 -const PREPARE = 57677 -const DEALLOCATE = 57678 -const RESET = 57679 -const EXTENSION = 57680 -const RETENTION = 57681 -const PERIOD = 57682 -const CLONE = 57683 -const BRANCH = 57684 -const LOG = 57685 -const REVERT = 57686 -const REBASE = 57687 -const DIFF = 57688 -const PICK = 57689 -const CONFLICT = 57690 -const CONFLICT_FAIL = 57691 -const CONFLICT_SKIP = 57692 -const CONFLICT_ACCEPT = 57693 -const OUTPUT = 57694 -const SUMMARY = 57695 -const INCREMENT = 57696 -const CYCLE = 57697 -const MINVALUE = 57698 -const PUBLICATION = 57699 -const SUBSCRIPTION = 57700 -const SUBSCRIPTIONS = 57701 -const PUBLICATIONS = 57702 -const SYNC_INTERVAL = 57703 -const SYNC = 57704 -const COVERAGE = 57705 -const CCPR = 57706 -const PROPERTIES = 57707 -const PARSER = 57708 -const VISIBLE = 57709 -const INVISIBLE = 57710 -const BTREE = 57711 -const HASH = 57712 -const RTREE = 57713 -const BSI = 57714 -const IVFFLAT = 57715 -const MASTER = 57716 -const HNSW = 57717 -const CAGRA = 57718 -const IVFPQ = 57719 -const ZONEMAP = 57720 -const LEADING = 57721 -const BOTH = 57722 -const TRAILING = 57723 -const UNKNOWN = 57724 -const LISTS = 57725 -const OP_TYPE = 57726 -const REINDEX = 57727 -const EF_SEARCH = 57728 -const EF_CONSTRUCTION = 57729 -const M = 57730 -const ASYNC = 57731 -const FORCE_SYNC = 57732 -const AUTO_UPDATE = 57733 -const INTERMEDIATE_GRAPH_DEGREE = 57734 -const GRAPH_DEGREE = 57735 -const QUANTIZATION = 57736 -const BITS_PER_CODE = 57737 -const DISTRIBUTION_MODE = 57738 -const ITOPK_SIZE = 57739 -const INCLUDE = 57740 -const KMEANS_TRAIN_PERCENT = 57741 -const KMEANS_MAX_ITERATION = 57742 -const MAX_INDEX_CAPACITY = 57743 -const EXPIRE = 57744 -const ACCOUNT = 57745 -const ACCOUNTS = 57746 -const UNLOCK = 57747 -const DAY = 57748 -const NEVER = 57749 -const PUMP = 57750 -const MYSQL_COMPATIBILITY_MODE = 57751 -const UNIQUE_CHECK_ON_AUTOINCR = 57752 -const MODIFY = 57753 -const CHANGE = 57754 -const SECOND = 57755 -const ASCII = 57756 -const COALESCE = 57757 -const COLLATION = 57758 -const HOUR = 57759 -const MICROSECOND = 57760 -const MINUTE = 57761 -const MONTH = 57762 -const QUARTER = 57763 -const REPEAT = 57764 -const REVERSE = 57765 -const ROW_COUNT = 57766 -const WEEK = 57767 -const REVOKE = 57768 -const FUNCTION = 57769 -const PRIVILEGES = 57770 -const TABLESPACE = 57771 -const EXECUTE = 57772 -const SUPER = 57773 -const GRANT = 57774 -const OPTION = 57775 -const REFERENCES = 57776 -const REPLICATION = 57777 -const SLAVE = 57778 -const CLIENT = 57779 -const USAGE = 57780 -const RELOAD = 57781 -const FILE = 57782 -const FILES = 57783 -const TEMPORARY = 57784 -const ROUTINE = 57785 -const EVENT = 57786 -const SHUTDOWN = 57787 -const NULLX = 57788 -const AUTO_INCREMENT = 57789 -const APPROXNUM = 57790 -const ENGINES = 57791 -const LOW_CARDINALITY = 57792 -const AUTOEXTEND_SIZE = 57793 -const ADMIN_NAME = 57794 -const RANDOM = 57795 -const SUSPEND = 57796 -const ATTRIBUTE = 57797 -const HISTORY = 57798 -const REUSE = 57799 -const CURRENT = 57800 -const OPTIONAL = 57801 -const FAILED_LOGIN_ATTEMPTS = 57802 -const PASSWORD_LOCK_TIME = 57803 -const UNBOUNDED = 57804 -const SECONDARY = 57805 -const RESTRICTED = 57806 -const USER = 57807 -const IDENTIFIED = 57808 -const CIPHER = 57809 -const ISSUER = 57810 -const X509 = 57811 -const SUBJECT = 57812 -const SAN = 57813 -const REQUIRE = 57814 -const SSL = 57815 -const NONE = 57816 -const PASSWORD = 57817 -const SHARED = 57818 -const EXCLUSIVE = 57819 -const MAX_QUERIES_PER_HOUR = 57820 -const MAX_UPDATES_PER_HOUR = 57821 -const MAX_CONNECTIONS_PER_HOUR = 57822 -const MAX_USER_CONNECTIONS = 57823 -const FORMAT = 57824 -const VERBOSE = 57825 -const CONNECTION = 57826 -const TRIGGERS = 57827 -const PROFILES = 57828 -const LOAD = 57829 -const INLINE = 57830 -const INFILE = 57831 -const TERMINATED = 57832 -const OPTIONALLY = 57833 -const ENCLOSED = 57834 -const ESCAPED = 57835 -const STARTING = 57836 -const LINES = 57837 -const ROWS = 57838 -const IMPORT = 57839 -const DISCARD = 57840 -const JSONTYPE = 57841 -const MODUMP = 57842 -const OVER = 57843 -const PRECEDING = 57844 -const FOLLOWING = 57845 -const GROUPS = 57846 -const DATABASES = 57847 -const TABLES = 57848 -const SEQUENCES = 57849 -const EXTENDED = 57850 -const FULL = 57851 -const PROCESSLIST = 57852 -const FIELDS = 57853 -const COLUMNS = 57854 -const OPEN = 57855 -const ERRORS = 57856 -const WARNINGS = 57857 -const INDEXES = 57858 -const SCHEMAS = 57859 -const NODE = 57860 -const LOCKS = 57861 -const ROLES = 57862 -const RULE = 57863 -const RULES = 57864 -const TABLE_NUMBER = 57865 -const COLUMN_NUMBER = 57866 -const TABLE_VALUES = 57867 -const TABLE_SIZE = 57868 -const TASKS = 57869 -const RUNS = 57870 -const NAMES = 57871 -const GLOBAL = 57872 -const PERSIST = 57873 -const SESSION = 57874 -const ISOLATION = 57875 -const LEVEL = 57876 -const READ = 57877 -const WRITE = 57878 -const ONLY = 57879 -const REPEATABLE = 57880 -const COMMITTED = 57881 -const UNCOMMITTED = 57882 -const SERIALIZABLE = 57883 -const LOCAL = 57884 -const EVENTS = 57885 -const PLUGINS = 57886 -const CURRENT_TIMESTAMP = 57887 -const DATABASE = 57888 -const CURRENT_TIME = 57889 -const LOCALTIME = 57890 -const LOCALTIMESTAMP = 57891 -const UTC_DATE = 57892 -const UTC_TIME = 57893 -const UTC_TIMESTAMP = 57894 -const REPLACE = 57895 -const CONVERT = 57896 -const SEPARATOR = 57897 -const TIMESTAMPDIFF = 57898 -const TIMESTAMPADD = 57899 -const CURRENT_DATE = 57900 -const CURRENT_USER = 57901 -const CURRENT_ROLE = 57902 -const SECOND_MICROSECOND = 57903 -const MINUTE_MICROSECOND = 57904 -const MINUTE_SECOND = 57905 -const HOUR_MICROSECOND = 57906 -const HOUR_SECOND = 57907 -const HOUR_MINUTE = 57908 -const DAY_MICROSECOND = 57909 -const DAY_SECOND = 57910 -const DAY_MINUTE = 57911 -const DAY_HOUR = 57912 -const YEAR_MONTH = 57913 -const SQL_TSI_HOUR = 57914 -const SQL_TSI_DAY = 57915 -const SQL_TSI_WEEK = 57916 -const SQL_TSI_MONTH = 57917 -const SQL_TSI_QUARTER = 57918 -const SQL_TSI_YEAR = 57919 -const SQL_TSI_SECOND = 57920 -const SQL_TSI_MINUTE = 57921 -const RECURSIVE = 57922 -const CONFIG = 57923 -const DRAINER = 57924 -const SOURCE = 57925 -const STREAM = 57926 -const HEADERS = 57927 -const CONNECTOR = 57928 -const CONNECTORS = 57929 -const DAEMON = 57930 -const PAUSE = 57931 -const CANCEL = 57932 -const RESUME = 57933 -const SCHEDULE = 57934 -const TIMEZONE = 57935 -const TIMEOUT = 57936 -const TASK = 57937 -const MATCH = 57938 -const AGAINST = 57939 -const BOOLEAN = 57940 -const LANGUAGE = 57941 -const QUERY = 57942 -const EXPANSION = 57943 -const WITHOUT = 57944 -const VALIDATION = 57945 -const UPGRADE = 57946 -const RETRY = 57947 -const ADDDATE = 57948 -const BIT_AND = 57949 -const BIT_OR = 57950 -const BIT_XOR = 57951 -const CAST = 57952 -const COUNT = 57953 -const APPROX_COUNT = 57954 -const APPROX_COUNT_DISTINCT = 57955 -const SERIAL_EXTRACT = 57956 -const APPROX_PERCENTILE = 57957 -const CURDATE = 57958 -const CURTIME = 57959 -const DATE_ADD = 57960 -const DATE_SUB = 57961 -const EXTRACT = 57962 -const GROUP_CONCAT = 57963 -const MAX = 57964 -const MID = 57965 -const MIN = 57966 -const NOW = 57967 -const POSITION = 57968 -const SESSION_USER = 57969 -const STD = 57970 -const STDDEV = 57971 -const MEDIAN = 57972 -const CLUSTER_CENTERS = 57973 -const KMEANS = 57974 -const STDDEV_POP = 57975 -const STDDEV_SAMP = 57976 -const SUBDATE = 57977 -const SUBSTR = 57978 -const SUBSTRING = 57979 -const SUM = 57980 -const SYSDATE = 57981 -const SYSTEM_USER = 57982 -const TRANSLATE = 57983 -const TRIM = 57984 -const VARIANCE = 57985 -const VAR_POP = 57986 -const VAR_SAMP = 57987 -const AVG = 57988 -const RANK = 57989 -const ROW_NUMBER = 57990 -const DENSE_RANK = 57991 -const CUME_DIST = 57992 -const BIT_CAST = 57993 -const LAG = 57994 -const LEAD = 57995 -const FIRST_VALUE = 57996 -const LAST_VALUE = 57997 -const NTH_VALUE = 57998 -const NTILE = 57999 -const PERCENT_RANK = 58000 -const BITMAP_BIT_POSITION = 58001 -const BITMAP_BUCKET_NUMBER = 58002 -const BITMAP_COUNT = 58003 -const BITMAP_CONSTRUCT_AGG = 58004 -const BITMAP_OR_AGG = 58005 -const GET_FORMAT = 58006 -const SRID = 58007 -const NEXTVAL = 58008 -const SETVAL = 58009 -const CURRVAL = 58010 -const LASTVAL = 58011 -const ROW = 58012 -const OUTFILE = 58013 -const HEADER = 58014 -const MAX_FILE_SIZE = 58015 -const FORCE_QUOTE = 58016 -const PARALLEL = 58017 -const STRICT = 58018 -const SPLITSIZE = 58019 -const UNUSED = 58020 -const BINDINGS = 58021 -const GENERATED = 58022 -const ALWAYS = 58023 -const STORED = 58024 -const VIRTUAL = 58025 -const DO = 58026 -const DECLARE = 58027 -const LOOP = 58028 -const WHILE = 58029 -const LEAVE = 58030 -const ITERATE = 58031 -const UNTIL = 58032 -const CALL = 58033 -const PREV = 58034 -const SLIDING = 58035 -const FILL = 58036 -const SPBEGIN = 58037 -const BACKEND = 58038 -const SERVERS = 58039 -const HANDLER = 58040 -const PERCENT = 58041 -const SAMPLE = 58042 -const MO_TS = 58043 -const PITR = 58044 -const RECOVERY_WINDOW = 58045 -const INTERNAL = 58046 -const CDC_TASK_NAME = 58047 -const CDC = 58048 -const GROUPING = 58049 -const SETS = 58050 -const CUBE = 58051 -const ROLLUP = 58052 -const LOGSERVICE = 58053 -const REPLICAS = 58054 -const STORES = 58055 -const SETTINGS = 58056 -const KILL = 58057 -const BACKUP = 58058 -const FILESYSTEM = 58059 -const PARALLELISM = 58060 -const RESTORE = 58061 -const QUERY_RESULT = 58062 -const ARRAY = 58063 +const VECBF16 = 57553 +const VECF16 = 57554 +const VECINT8 = 57555 +const VECUINT8 = 57556 +const GEOMETRY = 57557 +const POINT = 57558 +const LINESTRING = 57559 +const POLYGON = 57560 +const GEOMETRYCOLLECTION = 57561 +const MULTIPOINT = 57562 +const MULTILINESTRING = 57563 +const MULTIPOLYGON = 57564 +const GEOMETRY32 = 57565 +const GEOGRAPHY = 57566 +const GEOGRAPHY32 = 57567 +const POINT32 = 57568 +const LINESTRING32 = 57569 +const POLYGON32 = 57570 +const GEOMETRYCOLLECTION32 = 57571 +const MULTIPOINT32 = 57572 +const MULTILINESTRING32 = 57573 +const MULTIPOLYGON32 = 57574 +const INT1 = 57575 +const INT2 = 57576 +const INT3 = 57577 +const INT4 = 57578 +const INT8 = 57579 +const S3OPTION = 57580 +const STAGEOPTION = 57581 +const SQL_SMALL_RESULT = 57582 +const SQL_BIG_RESULT = 57583 +const SQL_BUFFER_RESULT = 57584 +const SQL_CALC_FOUND_ROWS = 57585 +const LOW_PRIORITY = 57586 +const HIGH_PRIORITY = 57587 +const DELAYED = 57588 +const CREATE = 57589 +const ALTER = 57590 +const DROP = 57591 +const RENAME = 57592 +const REMOVE = 57593 +const ANALYZE = 57594 +const PHYPLAN = 57595 +const ADD = 57596 +const RETURNS = 57597 +const SCHEMA = 57598 +const TABLE = 57599 +const SEQUENCE = 57600 +const INDEX = 57601 +const VIEW = 57602 +const TO = 57603 +const IGNORE = 57604 +const IF = 57605 +const PRIMARY = 57606 +const COLUMN = 57607 +const CONSTRAINT = 57608 +const SPATIAL = 57609 +const FULLTEXT = 57610 +const FOREIGN = 57611 +const KEY_BLOCK_SIZE = 57612 +const SHOW = 57613 +const DESCRIBE = 57614 +const EXPLAIN = 57615 +const DATE = 57616 +const ESCAPE = 57617 +const REPAIR = 57618 +const OPTIMIZE = 57619 +const TRUNCATE = 57620 +const MAXVALUE = 57621 +const PARTITION = 57622 +const REORGANIZE = 57623 +const LESS = 57624 +const THAN = 57625 +const PROCEDURE = 57626 +const TRIGGER = 57627 +const STATUS = 57628 +const VARIABLES = 57629 +const ROLE = 57630 +const PROXY = 57631 +const AVG_ROW_LENGTH = 57632 +const STORAGE = 57633 +const DISK = 57634 +const MEMORY = 57635 +const CHECKSUM = 57636 +const COMPRESSION = 57637 +const DATA = 57638 +const DIRECTORY = 57639 +const DELAY_KEY_WRITE = 57640 +const ENCRYPTION = 57641 +const ENGINE = 57642 +const MAX_ROWS = 57643 +const MIN_ROWS = 57644 +const PACK_KEYS = 57645 +const ROW_FORMAT = 57646 +const STATS_AUTO_RECALC = 57647 +const STATS_PERSISTENT = 57648 +const STATS_SAMPLE_PAGES = 57649 +const DYNAMIC = 57650 +const COMPRESSED = 57651 +const REDUNDANT = 57652 +const COMPACT = 57653 +const FIXED = 57654 +const COLUMN_FORMAT = 57655 +const AUTO_RANDOM = 57656 +const ENGINE_ATTRIBUTE = 57657 +const SECONDARY_ENGINE_ATTRIBUTE = 57658 +const INSERT_METHOD = 57659 +const RESTRICT = 57660 +const CASCADE = 57661 +const ACTION = 57662 +const PARTIAL = 57663 +const SIMPLE = 57664 +const CHECK = 57665 +const ENFORCED = 57666 +const RANGE = 57667 +const LIST = 57668 +const ALGORITHM = 57669 +const LINEAR = 57670 +const PARTITIONS = 57671 +const SUBPARTITION = 57672 +const SUBPARTITIONS = 57673 +const CLUSTER = 57674 +const TYPE = 57675 +const ANY = 57676 +const SOME = 57677 +const EXTERNAL = 57678 +const LOCALFILE = 57679 +const URL = 57680 +const PREPARE = 57681 +const DEALLOCATE = 57682 +const RESET = 57683 +const EXTENSION = 57684 +const RETENTION = 57685 +const PERIOD = 57686 +const CLONE = 57687 +const BRANCH = 57688 +const LOG = 57689 +const REVERT = 57690 +const REBASE = 57691 +const DIFF = 57692 +const PICK = 57693 +const CONFLICT = 57694 +const CONFLICT_FAIL = 57695 +const CONFLICT_SKIP = 57696 +const CONFLICT_ACCEPT = 57697 +const OUTPUT = 57698 +const SUMMARY = 57699 +const INCREMENT = 57700 +const CYCLE = 57701 +const MINVALUE = 57702 +const PUBLICATION = 57703 +const SUBSCRIPTION = 57704 +const SUBSCRIPTIONS = 57705 +const PUBLICATIONS = 57706 +const SYNC_INTERVAL = 57707 +const SYNC = 57708 +const COVERAGE = 57709 +const CCPR = 57710 +const PROPERTIES = 57711 +const PARSER = 57712 +const VISIBLE = 57713 +const INVISIBLE = 57714 +const BTREE = 57715 +const HASH = 57716 +const RTREE = 57717 +const BSI = 57718 +const IVFFLAT = 57719 +const MASTER = 57720 +const HNSW = 57721 +const CAGRA = 57722 +const IVFPQ = 57723 +const ZONEMAP = 57724 +const LEADING = 57725 +const BOTH = 57726 +const TRAILING = 57727 +const UNKNOWN = 57728 +const LISTS = 57729 +const OP_TYPE = 57730 +const REINDEX = 57731 +const EF_SEARCH = 57732 +const EF_CONSTRUCTION = 57733 +const M = 57734 +const ASYNC = 57735 +const FORCE_SYNC = 57736 +const AUTO_UPDATE = 57737 +const INTERMEDIATE_GRAPH_DEGREE = 57738 +const GRAPH_DEGREE = 57739 +const QUANTIZATION = 57740 +const BITS_PER_CODE = 57741 +const DISTRIBUTION_MODE = 57742 +const ITOPK_SIZE = 57743 +const INCLUDE = 57744 +const KMEANS_TRAIN_PERCENT = 57745 +const KMEANS_MAX_ITERATION = 57746 +const MAX_INDEX_CAPACITY = 57747 +const EXPIRE = 57748 +const ACCOUNT = 57749 +const ACCOUNTS = 57750 +const UNLOCK = 57751 +const DAY = 57752 +const NEVER = 57753 +const PUMP = 57754 +const MYSQL_COMPATIBILITY_MODE = 57755 +const UNIQUE_CHECK_ON_AUTOINCR = 57756 +const MODIFY = 57757 +const CHANGE = 57758 +const SECOND = 57759 +const ASCII = 57760 +const COALESCE = 57761 +const COLLATION = 57762 +const HOUR = 57763 +const MICROSECOND = 57764 +const MINUTE = 57765 +const MONTH = 57766 +const QUARTER = 57767 +const REPEAT = 57768 +const REVERSE = 57769 +const ROW_COUNT = 57770 +const WEEK = 57771 +const REVOKE = 57772 +const FUNCTION = 57773 +const PRIVILEGES = 57774 +const TABLESPACE = 57775 +const EXECUTE = 57776 +const SUPER = 57777 +const GRANT = 57778 +const OPTION = 57779 +const REFERENCES = 57780 +const REPLICATION = 57781 +const SLAVE = 57782 +const CLIENT = 57783 +const USAGE = 57784 +const RELOAD = 57785 +const FILE = 57786 +const FILES = 57787 +const TEMPORARY = 57788 +const ROUTINE = 57789 +const EVENT = 57790 +const SHUTDOWN = 57791 +const NULLX = 57792 +const AUTO_INCREMENT = 57793 +const APPROXNUM = 57794 +const ENGINES = 57795 +const LOW_CARDINALITY = 57796 +const AUTOEXTEND_SIZE = 57797 +const ADMIN_NAME = 57798 +const RANDOM = 57799 +const SUSPEND = 57800 +const ATTRIBUTE = 57801 +const HISTORY = 57802 +const REUSE = 57803 +const CURRENT = 57804 +const OPTIONAL = 57805 +const FAILED_LOGIN_ATTEMPTS = 57806 +const PASSWORD_LOCK_TIME = 57807 +const UNBOUNDED = 57808 +const SECONDARY = 57809 +const RESTRICTED = 57810 +const USER = 57811 +const IDENTIFIED = 57812 +const CIPHER = 57813 +const ISSUER = 57814 +const X509 = 57815 +const SUBJECT = 57816 +const SAN = 57817 +const REQUIRE = 57818 +const SSL = 57819 +const NONE = 57820 +const PASSWORD = 57821 +const SHARED = 57822 +const EXCLUSIVE = 57823 +const MAX_QUERIES_PER_HOUR = 57824 +const MAX_UPDATES_PER_HOUR = 57825 +const MAX_CONNECTIONS_PER_HOUR = 57826 +const MAX_USER_CONNECTIONS = 57827 +const FORMAT = 57828 +const VERBOSE = 57829 +const CONNECTION = 57830 +const TRIGGERS = 57831 +const PROFILES = 57832 +const LOAD = 57833 +const INLINE = 57834 +const INFILE = 57835 +const TERMINATED = 57836 +const OPTIONALLY = 57837 +const ENCLOSED = 57838 +const ESCAPED = 57839 +const STARTING = 57840 +const LINES = 57841 +const ROWS = 57842 +const IMPORT = 57843 +const DISCARD = 57844 +const JSONTYPE = 57845 +const MODUMP = 57846 +const OVER = 57847 +const PRECEDING = 57848 +const FOLLOWING = 57849 +const GROUPS = 57850 +const DATABASES = 57851 +const TABLES = 57852 +const SEQUENCES = 57853 +const EXTENDED = 57854 +const FULL = 57855 +const PROCESSLIST = 57856 +const FIELDS = 57857 +const COLUMNS = 57858 +const OPEN = 57859 +const ERRORS = 57860 +const WARNINGS = 57861 +const INDEXES = 57862 +const SCHEMAS = 57863 +const NODE = 57864 +const LOCKS = 57865 +const ROLES = 57866 +const RULE = 57867 +const RULES = 57868 +const TABLE_NUMBER = 57869 +const COLUMN_NUMBER = 57870 +const TABLE_VALUES = 57871 +const TABLE_SIZE = 57872 +const TASKS = 57873 +const RUNS = 57874 +const NAMES = 57875 +const GLOBAL = 57876 +const PERSIST = 57877 +const SESSION = 57878 +const ISOLATION = 57879 +const LEVEL = 57880 +const READ = 57881 +const WRITE = 57882 +const ONLY = 57883 +const REPEATABLE = 57884 +const COMMITTED = 57885 +const UNCOMMITTED = 57886 +const SERIALIZABLE = 57887 +const LOCAL = 57888 +const EVENTS = 57889 +const PLUGINS = 57890 +const CURRENT_TIMESTAMP = 57891 +const DATABASE = 57892 +const CURRENT_TIME = 57893 +const LOCALTIME = 57894 +const LOCALTIMESTAMP = 57895 +const UTC_DATE = 57896 +const UTC_TIME = 57897 +const UTC_TIMESTAMP = 57898 +const REPLACE = 57899 +const CONVERT = 57900 +const SEPARATOR = 57901 +const TIMESTAMPDIFF = 57902 +const TIMESTAMPADD = 57903 +const CURRENT_DATE = 57904 +const CURRENT_USER = 57905 +const CURRENT_ROLE = 57906 +const SECOND_MICROSECOND = 57907 +const MINUTE_MICROSECOND = 57908 +const MINUTE_SECOND = 57909 +const HOUR_MICROSECOND = 57910 +const HOUR_SECOND = 57911 +const HOUR_MINUTE = 57912 +const DAY_MICROSECOND = 57913 +const DAY_SECOND = 57914 +const DAY_MINUTE = 57915 +const DAY_HOUR = 57916 +const YEAR_MONTH = 57917 +const SQL_TSI_HOUR = 57918 +const SQL_TSI_DAY = 57919 +const SQL_TSI_WEEK = 57920 +const SQL_TSI_MONTH = 57921 +const SQL_TSI_QUARTER = 57922 +const SQL_TSI_YEAR = 57923 +const SQL_TSI_SECOND = 57924 +const SQL_TSI_MINUTE = 57925 +const RECURSIVE = 57926 +const CONFIG = 57927 +const DRAINER = 57928 +const SOURCE = 57929 +const STREAM = 57930 +const HEADERS = 57931 +const CONNECTOR = 57932 +const CONNECTORS = 57933 +const DAEMON = 57934 +const PAUSE = 57935 +const CANCEL = 57936 +const RESUME = 57937 +const SCHEDULE = 57938 +const TIMEZONE = 57939 +const TIMEOUT = 57940 +const TASK = 57941 +const MATCH = 57942 +const AGAINST = 57943 +const BOOLEAN = 57944 +const LANGUAGE = 57945 +const QUERY = 57946 +const EXPANSION = 57947 +const WITHOUT = 57948 +const VALIDATION = 57949 +const UPGRADE = 57950 +const RETRY = 57951 +const ADDDATE = 57952 +const BIT_AND = 57953 +const BIT_OR = 57954 +const BIT_XOR = 57955 +const CAST = 57956 +const COUNT = 57957 +const APPROX_COUNT = 57958 +const APPROX_COUNT_DISTINCT = 57959 +const SERIAL_EXTRACT = 57960 +const APPROX_PERCENTILE = 57961 +const CURDATE = 57962 +const CURTIME = 57963 +const DATE_ADD = 57964 +const DATE_SUB = 57965 +const EXTRACT = 57966 +const GROUP_CONCAT = 57967 +const MAX = 57968 +const MID = 57969 +const MIN = 57970 +const NOW = 57971 +const POSITION = 57972 +const SESSION_USER = 57973 +const STD = 57974 +const STDDEV = 57975 +const MEDIAN = 57976 +const CLUSTER_CENTERS = 57977 +const KMEANS = 57978 +const STDDEV_POP = 57979 +const STDDEV_SAMP = 57980 +const SUBDATE = 57981 +const SUBSTR = 57982 +const SUBSTRING = 57983 +const SUM = 57984 +const SYSDATE = 57985 +const SYSTEM_USER = 57986 +const TRANSLATE = 57987 +const TRIM = 57988 +const VARIANCE = 57989 +const VAR_POP = 57990 +const VAR_SAMP = 57991 +const AVG = 57992 +const RANK = 57993 +const ROW_NUMBER = 57994 +const DENSE_RANK = 57995 +const CUME_DIST = 57996 +const BIT_CAST = 57997 +const LAG = 57998 +const LEAD = 57999 +const FIRST_VALUE = 58000 +const LAST_VALUE = 58001 +const NTH_VALUE = 58002 +const NTILE = 58003 +const PERCENT_RANK = 58004 +const BITMAP_BIT_POSITION = 58005 +const BITMAP_BUCKET_NUMBER = 58006 +const BITMAP_COUNT = 58007 +const BITMAP_CONSTRUCT_AGG = 58008 +const BITMAP_OR_AGG = 58009 +const GET_FORMAT = 58010 +const SRID = 58011 +const NEXTVAL = 58012 +const SETVAL = 58013 +const CURRVAL = 58014 +const LASTVAL = 58015 +const ROW = 58016 +const OUTFILE = 58017 +const HEADER = 58018 +const MAX_FILE_SIZE = 58019 +const FORCE_QUOTE = 58020 +const PARALLEL = 58021 +const STRICT = 58022 +const SPLITSIZE = 58023 +const UNUSED = 58024 +const BINDINGS = 58025 +const GENERATED = 58026 +const ALWAYS = 58027 +const STORED = 58028 +const VIRTUAL = 58029 +const DO = 58030 +const DECLARE = 58031 +const LOOP = 58032 +const WHILE = 58033 +const LEAVE = 58034 +const ITERATE = 58035 +const UNTIL = 58036 +const CALL = 58037 +const PREV = 58038 +const SLIDING = 58039 +const FILL = 58040 +const SPBEGIN = 58041 +const BACKEND = 58042 +const SERVERS = 58043 +const HANDLER = 58044 +const PERCENT = 58045 +const SAMPLE = 58046 +const MO_TS = 58047 +const PITR = 58048 +const RECOVERY_WINDOW = 58049 +const INTERNAL = 58050 +const CDC_TASK_NAME = 58051 +const CDC = 58052 +const GROUPING = 58053 +const SETS = 58054 +const CUBE = 58055 +const ROLLUP = 58056 +const LOGSERVICE = 58057 +const REPLICAS = 58058 +const STORES = 58059 +const SETTINGS = 58060 +const KILL = 58061 +const BACKUP = 58062 +const FILESYSTEM = 58063 +const PARALLELISM = 58064 +const RESTORE = 58065 +const QUERY_RESULT = 58066 +const ARRAY = 58067 var yyToknames = [...]string{ "$end", @@ -1001,6 +1005,10 @@ var yyToknames = [...]string{ "UUID", "VECF32", "VECF64", + "VECBF16", + "VECF16", + "VECINT8", + "VECUINT8", "GEOMETRY", "POINT", "LINESTRING", @@ -1525,7 +1533,7 @@ const yyEofCode = 1 const yyErrCode = 2 const yyInitialStackSize = 16 -//line mysql_sql.y:14497 +//line mysql_sql.y:14553 //line yacctab:1 var yyExca = [...]int{ @@ -1537,499 +1545,515 @@ var yyExca = [...]int{ 24, 878, -2, 871, -1, 181, - 270, 1396, - 272, 1240, + 274, 1396, + 276, 1240, -2, 1313, -1, 211, 46, 689, - 272, 689, - 299, 696, - 300, 696, - 533, 689, + 276, 689, + 303, 696, + 304, 696, + 537, 689, -2, 727, -1, 251, - 742, 2259, + 746, 2267, -2, 576, - -1, 606, - 742, 2386, + -1, 610, + 746, 2394, -2, 436, - -1, 664, - 742, 2445, + -1, 668, + 746, 2453, -2, 434, - -1, 665, - 742, 2446, + -1, 669, + 746, 2454, -2, 435, - -1, 666, - 742, 2447, + -1, 670, + 746, 2455, -2, 437, - -1, 824, - 351, 201, - 505, 201, - 506, 201, - -2, 2130, - -1, 892, + -1, 828, + 355, 201, + 509, 201, + 510, 201, + -2, 2134, + -1, 896, 88, 1886, - -2, 2322, - -1, 893, - 88, 1904, - -2, 2291, + -2, 2330, -1, 897, + 88, 1904, + -2, 2299, + -1, 901, 88, 1905, - -2, 2321, - -1, 941, - 88, 1807, - -2, 2535, - -1, 942, - 88, 1808, - -2, 2534, - -1, 943, - 88, 1809, - -2, 2524, - -1, 944, - 88, 2497, - -2, 2517, + -2, 2329, -1, 945, - 88, 2498, - -2, 2518, + 88, 1807, + -2, 2543, -1, 946, - 88, 2499, - -2, 2526, + 88, 1808, + -2, 2542, -1, 947, - 88, 2500, - -2, 2506, + 88, 1809, + -2, 2532, -1, 948, - 88, 2501, - -2, 2515, + 88, 2505, + -2, 2525, -1, 949, - 88, 2502, - -2, 2528, + 88, 2506, + -2, 2526, -1, 950, - 88, 2503, - -2, 2533, + 88, 2507, + -2, 2534, -1, 951, - 88, 2504, - -2, 2538, + 88, 2508, + -2, 2514, -1, 952, - 88, 2505, - -2, 2539, + 88, 2509, + -2, 2523, -1, 953, - 88, 1882, - -2, 2360, + 88, 2510, + -2, 2536, -1, 954, - 88, 1883, - -2, 2110, + 88, 2511, + -2, 2541, -1, 955, - 88, 1884, - -2, 2369, + 88, 2512, + -2, 2546, -1, 956, - 88, 1885, - -2, 2123, + 88, 2513, + -2, 2547, + -1, 957, + 88, 1882, + -2, 2368, -1, 958, - 88, 1888, - -2, 2132, + 88, 1883, + -2, 2114, + -1, 959, + 88, 1884, + -2, 2377, -1, 960, - 88, 1890, - -2, 2394, + 88, 1885, + -2, 2127, -1, 962, - 88, 1892, - -2, 2154, + 88, 1888, + -2, 2136, -1, 964, + 88, 1890, + -2, 2402, + -1, 966, + 88, 1892, + -2, 2158, + -1, 968, 88, 1894, - -2, 2406, - -1, 965, + -2, 2414, + -1, 969, 88, 1895, - -2, 2405, - -1, 966, + -2, 2413, + -1, 970, 88, 1896, - -2, 2220, - -1, 967, + -2, 2228, + -1, 971, 88, 1897, - -2, 2317, - -1, 970, + -2, 2325, + -1, 974, 88, 1900, - -2, 2417, - -1, 972, + -2, 2425, + -1, 976, 88, 1902, - -2, 2420, - -1, 973, + -2, 2428, + -1, 977, 88, 1903, - -2, 2422, - -1, 974, + -2, 2430, + -1, 978, 88, 1906, - -2, 2429, - -1, 975, + -2, 2437, + -1, 979, 88, 1907, - -2, 2300, - -1, 976, + -2, 2308, + -1, 980, 88, 1908, - -2, 2347, - -1, 977, + -2, 2355, + -1, 981, 88, 1909, - -2, 2311, - -1, 978, + -2, 2319, + -1, 982, 88, 1910, - -2, 2337, - -1, 989, + -2, 2345, + -1, 993, 88, 1783, - -2, 2529, - -1, 990, + -2, 2537, + -1, 994, 88, 1784, - -2, 2530, - -1, 991, + -2, 2538, + -1, 995, 88, 1785, - -2, 2531, - -1, 1105, - 528, 727, - 529, 727, + -2, 2539, + -1, 1109, + 532, 727, + 533, 727, -2, 690, - -1, 1160, - 130, 2110, - 141, 2110, - 173, 2110, - -2, 2078, - -1, 1294, + -1, 1164, + 130, 2114, + 141, 2114, + 173, 2114, + -2, 2082, + -1, 1302, 24, 907, -2, 850, - -1, 1416, + -1, 1424, 11, 878, 24, 878, -2, 1645, - -1, 1512, + -1, 1520, 24, 907, -2, 850, - -1, 1897, + -1, 1909, 88, 1957, - -2, 2319, - -1, 1898, + -2, 2327, + -1, 1910, 88, 1958, - -2, 2320, - -1, 2596, + -2, 2328, + -1, 2608, 89, 1096, -2, 1102, - -1, 2613, + -1, 2625, 113, 1305, 160, 1305, 208, 1305, 211, 1305, - 312, 1305, + 316, 1305, -2, 1298, - -1, 2798, + -1, 2810, 11, 878, 24, 878, -2, 1023, - -1, 2835, - 89, 2064, - 174, 2064, - -2, 2302, - -1, 2836, - 89, 2064, - 174, 2064, - -2, 2301, - -1, 2837, + -1, 2847, + 89, 2068, + 174, 2068, + -2, 2310, + -1, 2848, + 89, 2068, + 174, 2068, + -2, 2309, + -1, 2849, 89, 2022, 174, 2022, - -2, 2288, - -1, 2838, + -2, 2296, + -1, 2850, 89, 2023, 174, 2023, - -2, 2293, - -1, 2839, + -2, 2301, + -1, 2851, 89, 2024, 174, 2024, - -2, 2208, - -1, 2840, + -2, 2216, + -1, 2852, 89, 2025, 174, 2025, - -2, 2201, - -1, 2841, + -2, 2209, + -1, 2853, 89, 2026, 174, 2026, - -2, 2097, - -1, 2842, + -2, 2101, + -1, 2854, 89, 2027, 174, 2027, - -2, 2290, - -1, 2843, + -2, 2298, + -1, 2855, 89, 2028, 174, 2028, - -2, 2206, - -1, 2844, + -2, 2214, + -1, 2856, 89, 2029, 174, 2029, - -2, 2200, - -1, 2845, + -2, 2208, + -1, 2857, 89, 2030, 174, 2030, - -2, 2185, - -1, 2846, - 89, 2064, - 174, 2064, - -2, 2186, - -1, 2847, - 89, 2064, - 174, 2064, - -2, 2187, - -1, 2849, - 89, 2035, - 174, 2035, - -2, 2337, - -1, 2850, + -2, 2189, + -1, 2858, + 89, 2068, + 174, 2068, + -2, 2190, + -1, 2859, + 89, 2068, + 174, 2068, + -2, 2191, + -1, 2860, + 89, 2068, + 174, 2068, + -2, 2192, + -1, 2861, + 89, 2068, + 174, 2068, + -2, 2193, + -1, 2862, + 89, 2068, + 174, 2068, + -2, 2194, + -1, 2863, + 89, 2068, + 174, 2068, + -2, 2195, + -1, 2865, + 89, 2039, + 174, 2039, + -2, 2345, + -1, 2866, 89, 2012, 174, 2012, - -2, 2322, - -1, 2851, - 89, 2062, - 174, 2062, - -2, 2291, - -1, 2852, - 89, 2062, - 174, 2062, - -2, 2321, - -1, 2853, - 89, 2062, - 174, 2062, - -2, 2133, - -1, 2854, - 89, 2060, - 174, 2060, - -2, 2311, - -1, 2855, + -2, 2330, + -1, 2867, + 89, 2066, + 174, 2066, + -2, 2299, + -1, 2868, + 89, 2066, + 174, 2066, + -2, 2329, + -1, 2869, + 89, 2066, + 174, 2066, + -2, 2137, + -1, 2870, + 89, 2064, + 174, 2064, + -2, 2319, + -1, 2871, 88, 1992, 89, 1992, 163, 1992, 164, 1992, 166, 1992, 174, 1992, - -2, 2096, - -1, 2856, + -2, 2100, + -1, 2872, 88, 1993, 89, 1993, 163, 1993, 164, 1993, 166, 1993, 174, 1993, - -2, 2098, - -1, 2857, + -2, 2102, + -1, 2873, 88, 1994, 89, 1994, 163, 1994, 164, 1994, 166, 1994, 174, 1994, - -2, 2365, - -1, 2858, + -2, 2373, + -1, 2874, 88, 1996, 89, 1996, 163, 1996, 164, 1996, 166, 1996, 174, 1996, - -2, 2292, - -1, 2859, + -2, 2300, + -1, 2875, 88, 1998, 89, 1998, 163, 1998, 164, 1998, 166, 1998, 174, 1998, - -2, 2269, - -1, 2860, + -2, 2277, + -1, 2876, 88, 2000, 89, 2000, 163, 2000, 164, 2000, 166, 2000, 174, 2000, - -2, 2207, - -1, 2861, + -2, 2215, + -1, 2877, 88, 2002, 89, 2002, 163, 2002, 164, 2002, 166, 2002, 174, 2002, - -2, 2179, - -1, 2862, + -2, 2183, + -1, 2878, 88, 2003, 89, 2003, 163, 2003, 164, 2003, 166, 2003, 174, 2003, - -2, 2180, - -1, 2863, + -2, 2184, + -1, 2879, 88, 2005, 89, 2005, 163, 2005, 164, 2005, 166, 2005, 174, 2005, - -2, 2095, - -1, 2864, - 89, 2067, - 163, 2067, - 164, 2067, - 166, 2067, - 174, 2067, - -2, 2138, - -1, 2865, - 89, 2067, - 163, 2067, - 164, 2067, - 166, 2067, - 174, 2067, - -2, 2155, - -1, 2866, - 89, 2070, - 163, 2070, - 164, 2070, - 166, 2070, - 174, 2070, - -2, 2134, - -1, 2867, - 89, 2070, - 163, 2070, - 164, 2070, - 166, 2070, - 174, 2070, - -2, 2223, - -1, 2868, - 89, 2067, - 163, 2067, - 164, 2067, - 166, 2067, - 174, 2067, - -2, 2251, - -1, 2869, - 89, 2040, - 174, 2040, + -2, 2099, + -1, 2880, + 89, 2071, + 163, 2071, + 164, 2071, + 166, 2071, + 174, 2071, + -2, 2142, + -1, 2881, + 89, 2071, + 163, 2071, + 164, 2071, + 166, 2071, + 174, 2071, -2, 2159, - -1, 2870, - 89, 2041, - 174, 2041, - -2, 2237, - -1, 2871, - 89, 2042, - 174, 2042, - -2, 2198, - -1, 2872, - 89, 2043, - 174, 2043, - -2, 2238, - -1, 2873, + -1, 2882, + 89, 2074, + 163, 2074, + 164, 2074, + 166, 2074, + 174, 2074, + -2, 2138, + -1, 2883, + 89, 2074, + 163, 2074, + 164, 2074, + 166, 2074, + 174, 2074, + -2, 2231, + -1, 2884, + 89, 2071, + 163, 2071, + 164, 2071, + 166, 2071, + 174, 2071, + -2, 2259, + -1, 2885, 89, 2044, 174, 2044, - -2, 2160, - -1, 2874, + -2, 2163, + -1, 2886, 89, 2045, 174, 2045, - -2, 2212, - -1, 2875, + -2, 2245, + -1, 2887, 89, 2046, 174, 2046, - -2, 2211, - -1, 2876, + -2, 2206, + -1, 2888, 89, 2047, 174, 2047, - -2, 2213, - -1, 2877, + -2, 2246, + -1, 2889, 89, 2048, 174, 2048, - -2, 2162, - -1, 2878, + -2, 2164, + -1, 2890, 89, 2049, 174, 2049, - -2, 2161, - -1, 2879, + -2, 2220, + -1, 2891, 89, 2050, 174, 2050, - -2, 2163, - -1, 2880, + -2, 2219, + -1, 2892, 89, 2051, 174, 2051, - -2, 2164, - -1, 2881, + -2, 2221, + -1, 2893, 89, 2052, 174, 2052, - -2, 2165, - -1, 2882, + -2, 2166, + -1, 2894, 89, 2053, 174, 2053, - -2, 2166, - -1, 2883, + -2, 2165, + -1, 2895, 89, 2054, 174, 2054, -2, 2167, - -1, 2884, + -1, 2896, 89, 2055, 174, 2055, -2, 2168, - -1, 2885, + -1, 2897, 89, 2056, 174, 2056, -2, 2169, - -1, 2886, + -1, 2898, 89, 2057, 174, 2057, -2, 2170, - -1, 3141, + -1, 2899, + 89, 2058, + 174, 2058, + -2, 2171, + -1, 2900, + 89, 2059, + 174, 2059, + -2, 2172, + -1, 2901, + 89, 2060, + 174, 2060, + -2, 2173, + -1, 2902, + 89, 2061, + 174, 2061, + -2, 2174, + -1, 3157, 113, 1305, 160, 1305, 208, 1305, 211, 1305, - 312, 1305, + 316, 1305, -2, 1299, - -1, 3168, + -1, 3184, 86, 792, 174, 792, -2, 1511, - -1, 3638, + -1, 3654, 211, 1305, - 336, 1608, + 340, 1608, -2, 1574, - -1, 3683, + -1, 3699, 11, 878, 24, 878, -2, 1645, - -1, 3874, + -1, 3890, 113, 1305, 160, 1305, 208, 1305, 211, 1305, -2, 1452, - -1, 3878, + -1, 3894, 113, 1305, 160, 1305, 208, 1305, 211, 1305, -2, 1452, - -1, 3893, + -1, 3909, 86, 792, 174, 792, -2, 1511, - -1, 3914, + -1, 3930, 211, 1305, - 336, 1608, + 340, 1608, -2, 1575, - -1, 4112, + -1, 4128, 113, 1305, 160, 1305, 208, 1305, 211, 1305, -2, 1453, - -1, 4140, + -1, 4156, 89, 1414, 174, 1414, -2, 1305, - -1, 4341, + -1, 4357, 89, 1414, 174, 1414, -2, 1305, - -1, 4561, + -1, 4577, 89, 1418, 174, 1418, -2, 1305, - -1, 4616, + -1, 4632, 89, 1419, 174, 1419, -2, 1305, @@ -2037,6850 +2061,6915 @@ var yyExca = [...]int{ const yyPrivate = 57344 -const yyLast = 68018 +const yyLast = 68689 var yyAct = [...]int{ - 858, 834, 4665, 860, 4639, 3198, 240, 4657, 1802, 2210, - 4571, 4565, 3899, 4013, 1877, 4575, 3661, 4564, 4576, 3960, - 4341, 3624, 4465, 2823, 4522, 843, 4414, 3518, 3749, 3928, - 4237, 4171, 4319, 1710, 3192, 836, 4279, 1873, 4405, 4008, - 3520, 3750, 1454, 4340, 4442, 3747, 4099, 3845, 1943, 889, - 717, 3091, 1636, 1295, 4309, 3195, 38, 1159, 4018, 1642, - 3853, 227, 3, 4415, 4417, 3389, 1945, 2149, 736, 1930, - 3915, 2675, 747, 3859, 4120, 3318, 3633, 747, 760, 769, - 3171, 1880, 769, 4109, 3589, 3572, 4080, 3879, 1300, 3547, - 4114, 832, 2923, 154, 1927, 2315, 3317, 3319, 3287, 3576, - 2277, 2333, 3221, 3653, 3642, 3018, 3635, 3881, 1168, 70, - 3843, 3314, 3807, 2792, 70, 3099, 2396, 3680, 225, 2357, - 3799, 2428, 787, 3731, 2830, 2678, 782, 1949, 3709, 3127, - 1926, 3349, 3554, 3641, 766, 3552, 1703, 2312, 3550, 3537, - 778, 2930, 1297, 3600, 2635, 3549, 3545, 3548, 3500, 3142, - 37, 2394, 826, 2560, 1779, 1786, 831, 2057, 2462, 2405, - 2559, 2424, 2404, 2397, 2904, 1795, 2308, 1030, 3305, 2362, - 2793, 2168, 1790, 1807, 1791, 2281, 2423, 3223, 2775, 3115, - 3109, 2770, 1596, 747, 1068, 3203, 70, 2676, 3158, 2634, - 2200, 236, 8, 6, 2278, 2120, 2613, 235, 7, 2828, - 1944, 1223, 2458, 833, 835, 1871, 1561, 2425, 1752, 1719, - 1688, 735, 1682, 2604, 2141, 1153, 717, 2403, 2391, 2671, - 825, 2400, 2562, 1913, 844, 2607, 1937, 1318, 1862, 2379, - 24, 1759, 2167, 1152, 2115, 1625, 775, 1870, 1687, 2800, - 240, 1611, 240, 1539, 1213, 1214, 1684, 1067, 2119, 993, - 751, 747, 1742, 1637, 1950, 2771, 1534, 785, 1621, 226, - 784, 1193, 25, 1047, 26, 716, 218, 17, 1065, 1116, - 1053, 222, 744, 28, 10, 1510, 1455, 781, 1876, 2432, - 768, 1100, 4427, 16, 1645, 1381, 1382, 1383, 1380, 1381, - 1382, 1383, 1380, 1381, 1382, 1383, 1380, 2081, 2802, 1210, - 4305, 3063, 1607, 1241, 995, 996, 3063, 3063, 3896, 3763, - 3612, 3510, 3509, 1165, 3412, 3411, 2442, 1535, 1301, 4063, - 3862, 1302, 3017, 14, 3742, 1646, 2968, 2910, 2908, 1536, - 2905, 2907, 2070, 70, 1766, 15, 1762, 742, 1206, 1205, - 224, 737, 2558, 1529, 1603, 1604, 1605, 1686, 70, 4392, - 70, 765, 1495, 2824, 4047, 3511, 3507, 34, 1209, 2573, - 1211, 761, 773, 1206, 2565, 2077, 1206, 1538, 3493, 3495, - 1167, 3492, 4651, 1803, 1821, 1662, 3055, 3053, 5, 2064, - 1525, 1764, 754, 1301, 1381, 1382, 1383, 1380, 1381, 1382, - 1383, 1380, 4006, 3385, 1241, 3383, 2367, 4573, 4572, 4164, - 1017, 763, 1014, 3756, 4400, 4244, 4238, 4009, 3490, 3748, - 2390, 4419, 1449, 764, 2399, 994, 8, 2928, 1204, 3464, - 3057, 816, 7, 3535, 818, 2386, 2750, 2716, 4671, 817, - 4413, 1005, 1928, 1929, 4648, 762, 4252, 1259, 1260, 1226, - 4411, 4291, 4250, 816, 3834, 2995, 818, 2580, 4478, 1540, - 1727, 817, 1546, 1544, 1543, 1063, 3829, 3538, 2594, 2260, - 1249, 1253, 1255, 1257, 1262, 1018, 1267, 1263, 1264, 1265, - 1266, 1138, 1015, 1244, 1245, 1246, 1247, 1224, 1225, 1250, - 1169, 1227, 780, 1229, 1230, 1231, 1232, 1228, 1233, 1234, - 1235, 1236, 1237, 1240, 1242, 1238, 1239, 1268, 1269, 1270, - 1271, 1272, 1273, 1274, 1275, 1277, 1276, 1278, 1279, 1280, - 1281, 1282, 1283, 1284, 1285, 1252, 1254, 1256, 1258, 1261, - 1819, 3462, 2440, 1163, 1164, 1241, 827, 1588, 1259, 1260, - 1226, 3312, 4052, 2608, 1215, 2820, 1658, 2091, 1378, 1659, - 1985, 1818, 2821, 2161, 3090, 2089, 4050, 1006, 3357, 3358, - 1553, 1249, 1253, 1255, 1257, 1262, 1243, 1267, 1263, 1264, - 1265, 1266, 4293, 2325, 1244, 1245, 1246, 1247, 1224, 1225, - 1250, 3494, 1227, 2291, 1229, 1230, 1231, 1232, 1228, 1233, - 1234, 1235, 1236, 1237, 1240, 1242, 1238, 1239, 1268, 1269, - 1270, 1271, 1272, 1273, 1274, 1275, 1277, 1276, 1278, 1279, - 1280, 1281, 1282, 1283, 1284, 1285, 1252, 1254, 1256, 1258, - 1261, 3491, 2292, 2293, 984, 1570, 983, 985, 986, 3088, - 987, 988, 2096, 2097, 1018, 3356, 1015, 827, 1672, 1863, - 2807, 1602, 1867, 2806, 2756, 1351, 2808, 3628, 1353, 1568, - 1661, 1131, 1129, 2755, 1130, 1765, 1763, 1243, 3517, 1689, - 1358, 1691, 3058, 1359, 1012, 1633, 1866, 1643, 1644, 1259, - 1260, 1226, 3626, 1641, 2924, 1125, 1354, 1640, 1643, 1644, - 2182, 1879, 1134, 4035, 183, 223, 182, 214, 184, 4579, - 4580, 1361, 1249, 1253, 1255, 1257, 1262, 2708, 1267, 1263, - 1264, 1265, 1266, 2159, 3087, 1244, 1245, 1246, 1247, 1224, - 1225, 1250, 1376, 1227, 1371, 1229, 1230, 1231, 1232, 1228, - 1233, 1234, 1235, 1236, 1237, 1240, 1242, 1238, 1239, 1268, - 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1277, 1276, 1278, - 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1252, 1254, 1256, - 1258, 1261, 1016, 1162, 1013, 1139, 747, 1161, 219, 4422, - 3111, 747, 1306, 4421, 183, 223, 182, 214, 184, 3338, - 3112, 2784, 2785, 4422, 4536, 183, 223, 182, 214, 184, - 3086, 769, 769, 1332, 4420, 747, 4421, 4535, 1243, 1868, - 183, 223, 182, 214, 184, 1347, 1569, 4605, 4548, 4420, - 4534, 1135, 4643, 4644, 2537, 183, 223, 182, 214, 184, - 1356, 3751, 3056, 1865, 1675, 4524, 4527, 4403, 3390, 3110, - 1216, 1349, 766, 766, 766, 4524, 4241, 2441, 1305, 3391, - 1828, 3392, 1883, 3751, 1352, 1355, 1528, 2949, 219, 1660, - 2092, 3395, 1307, 2303, 2299, 4438, 2160, 3089, 2090, 219, - 2444, 2080, 3766, 1424, 1858, 3083, 1313, 1348, 4034, 4426, - 1321, 1324, 1165, 1137, 219, 2309, 4036, 3844, 2323, 2324, - 2765, 1310, 1357, 3568, 70, 70, 70, 4304, 3769, 219, - 3242, 4578, 2436, 3429, 3062, 1302, 3851, 1302, 4054, 3118, - 3306, 1631, 4091, 1059, 1251, 1302, 4295, 4296, 4260, 816, - 4261, 1306, 818, 1970, 748, 2758, 2258, 817, 4406, 4407, - 4408, 4409, 2602, 3096, 3566, 1009, 1338, 3943, 3425, 1167, - 3413, 2749, 2959, 2752, 3410, 1374, 1375, 4550, 2467, 3758, - 3084, 1325, 1864, 3423, 2751, 1373, 1458, 210, 1350, 2714, - 2431, 1165, 1346, 4007, 1136, 3384, 1206, 2761, 2762, 1206, - 1206, 1206, 1571, 1360, 3300, 3562, 2760, 4301, 1206, 4088, - 1302, 1206, 4048, 1369, 1370, 3573, 4263, 1459, 3563, 3564, - 3574, 3065, 2443, 2825, 2768, 4051, 1368, 734, 3959, 2906, - 1889, 1892, 1893, 1767, 3565, 1251, 1882, 1881, 4368, 4251, - 1010, 1890, 3955, 1133, 1653, 3846, 4262, 1747, 1167, 4232, - 1326, 3657, 1545, 3658, 3660, 3659, 2447, 2449, 2450, 819, - 820, 821, 822, 823, 1656, 1657, 1542, 1221, 4430, 1531, - 1533, 994, 1537, 4282, 4115, 4064, 1316, 3868, 1536, 1294, - 3054, 819, 820, 821, 822, 823, 1541, 3630, 1557, 765, - 765, 765, 1560, 2259, 1335, 3735, 3587, 1567, 1536, 761, - 761, 761, 1420, 1421, 1422, 1423, 1508, 1293, 1164, 1513, - 4331, 1330, 1331, 3114, 1820, 1011, 3601, 1337, 4294, 4458, - 3655, 3656, 4453, 3085, 3159, 747, 3654, 1068, 3811, 3813, - 1425, 1019, 2615, 3560, 771, 770, 4323, 1643, 1644, 763, - 763, 763, 3310, 2610, 1323, 1322, 4460, 3948, 1552, 2694, - 1132, 764, 764, 764, 3501, 2674, 2697, 4443, 1221, 4056, - 4057, 4058, 1643, 1644, 3625, 3574, 1251, 3900, 4466, 1966, - 3197, 3907, 1620, 762, 762, 762, 1963, 3663, 2591, 3825, - 1965, 1962, 1964, 1968, 1969, 4288, 3193, 3194, 1967, 3197, - 4437, 4072, 747, 2772, 1671, 3822, 4053, 1677, 1309, 1311, - 1314, 747, 1632, 3530, 2748, 717, 717, 4260, 1061, 4261, - 1062, 2726, 1639, 2696, 4159, 717, 717, 3964, 4677, 1714, - 1714, 2725, 747, 1363, 767, 4255, 1364, 4021, 1328, 3124, - 2779, 2783, 2784, 2785, 2780, 2789, 2781, 2787, 1470, 1471, - 2782, 2681, 2788, 2825, 769, 1743, 736, 3824, 2746, 2747, - 1618, 2154, 1755, 1716, 1366, 4154, 1712, 1712, 1008, 1699, - 2681, 2684, 4660, 1698, 1635, 1634, 1592, 240, 1336, 3574, - 1617, 1418, 1616, 4467, 2786, 4263, 717, 1721, 4310, 2695, - 3634, 4148, 4563, 1298, 3484, 4345, 2717, 3882, 2674, 1221, - 71, 1415, 1414, 183, 223, 4332, 4004, 2764, 3569, 2691, - 1563, 1564, 1565, 1562, 767, 4262, 1574, 1576, 1577, 1578, - 1579, 1315, 1581, 2310, 3307, 767, 3117, 1673, 1587, 4256, - 1891, 4324, 4297, 4416, 4549, 780, 1312, 1575, 1514, 4521, - 767, 1512, 3808, 3888, 1685, 1973, 1974, 1975, 1976, 1977, - 1978, 1971, 1972, 153, 3426, 767, 2448, 1799, 1343, 2302, - 2300, 3684, 1804, 1708, 1709, 3650, 2955, 2812, 2436, 3631, - 1859, 2754, 1817, 2712, 2563, 3351, 3353, 219, 1676, 1573, - 71, 3121, 3122, 1362, 4092, 70, 2433, 3243, 1601, 3244, - 3245, 71, 3655, 3656, 1613, 3561, 3120, 1580, 1841, 1627, - 1628, 2298, 2275, 1844, 3662, 1595, 71, 1593, 1559, 3974, - 3367, 3368, 3699, 1714, 2680, 1714, 1306, 1806, 3686, 2682, - 3428, 71, 1586, 1367, 1813, 4661, 3837, 3072, 2685, 2459, - 1693, 1695, 1572, 2680, 2674, 2679, 2073, 2677, 2682, 766, - 1706, 1707, 766, 766, 1585, 1365, 1548, 1663, 1664, 1777, - 4344, 1780, 1781, 1610, 1647, 1584, 1852, 1650, 1342, 2684, - 1583, 1619, 1024, 1782, 1783, 1774, 1140, 1744, 1629, 183, - 223, 1788, 1789, 2683, 3154, 1697, 1648, 1649, 774, 1651, - 1652, 2614, 1878, 1654, 3651, 1550, 1714, 1622, 1626, 1626, - 1626, 70, 2683, 3150, 70, 70, 4562, 3294, 1069, 1060, - 2592, 1768, 1797, 1306, 1947, 742, 1722, 1599, 70, 1794, - 4162, 1735, 1798, 3240, 1622, 1622, 3800, 1979, 1980, 2946, - 1998, 1984, 1793, 1931, 1756, 1028, 4155, 4156, 3080, 1999, - 1026, 1025, 2445, 2446, 1741, 2099, 1757, 819, 820, 821, - 822, 823, 2006, 3148, 2008, 2690, 2009, 2010, 2011, 2688, - 2016, 2018, 2017, 2584, 1321, 1324, 1899, 1900, 1901, 1902, - 1903, 1904, 1905, 1906, 1907, 1908, 1909, 1910, 4150, 4658, - 4659, 3352, 4149, 1556, 1924, 1925, 1875, 2779, 2783, 2784, - 2785, 2780, 2789, 2781, 2787, 2586, 2585, 2782, 4256, 2788, - 2072, 2100, 4257, 3151, 1562, 1126, 1031, 1306, 1948, 3815, - 1165, 3889, 2583, 1983, 1071, 1072, 1073, 1126, 1856, 2082, - 2078, 1826, 2083, 1894, 1829, 2086, 2685, 1027, 2098, 1809, - 747, 747, 747, 2264, 2262, 1325, 2007, 1020, 2263, 2101, - 2103, 2055, 2104, 2015, 2106, 2107, 2108, 1792, 1982, 736, - 1743, 2738, 3583, 3262, 3263, 2116, 1021, 1714, 2122, 2123, - 4121, 2125, 1677, 747, 2711, 4231, 765, 1167, 747, 765, - 765, 1714, 1850, 2066, 1846, 1068, 761, 1849, 2150, 761, - 761, 1874, 1869, 1851, 1845, 1554, 1555, 2058, 3073, 1997, - 1911, 1912, 4673, 1848, 1922, 1923, 1714, 2074, 1547, 1126, - 1128, 1872, 1677, 1127, 4667, 2786, 4654, 760, 1838, 4679, - 4618, 3102, 1128, 1915, 1612, 1127, 763, 4531, 4591, 763, - 763, 3706, 3652, 1141, 1835, 1836, 3611, 2181, 764, 2143, - 1379, 764, 764, 1847, 1677, 1612, 3169, 1024, 1296, 2190, - 2190, 3487, 1677, 2430, 1677, 1677, 3103, 3104, 747, 747, - 762, 2257, 2825, 762, 762, 2116, 2268, 2933, 4588, 1714, - 2272, 2273, 3705, 1612, 4587, 2288, 2061, 717, 1381, 1382, - 1383, 1380, 2430, 2503, 1296, 1827, 2502, 2438, 1830, 1831, - 3701, 717, 3271, 1714, 1343, 2606, 2124, 3840, 2790, 4668, - 2185, 4619, 3261, 3768, 1128, 4619, 2552, 1127, 1323, 1322, - 1023, 1549, 1551, 4592, 3584, 1026, 1025, 2012, 2013, 2330, - 2332, 747, 2116, 1714, 3170, 2338, 2146, 747, 747, 747, - 778, 778, 4581, 3153, 4559, 1379, 3488, 2348, 1840, 2350, - 2351, 2352, 2356, 2126, 828, 2358, 3485, 1839, 2430, 2056, - 2212, 1861, 240, 4589, 2954, 240, 240, 2062, 240, 2438, - 3667, 4514, 2266, 3665, 2326, 3541, 4513, 2112, 2113, 2114, - 2110, 1379, 3499, 1381, 1382, 1383, 1380, 2071, 2791, 2075, - 2128, 2129, 2130, 2131, 2079, 4172, 4173, 4174, 4178, 4176, - 4177, 4179, 4180, 4181, 4175, 998, 999, 1000, 1001, 3706, - 1418, 3170, 2156, 2157, 4488, 1998, 1998, 2407, 1988, 1989, - 1990, 1379, 2111, 1343, 2414, 2186, 2304, 2477, 1860, 4560, - 2295, 2004, 2297, 2193, 2005, 2605, 3497, 3706, 2340, 2341, - 2342, 3486, 2147, 2316, 2317, 2151, 2318, 2319, 2791, 2155, - 4461, 4449, 2954, 2024, 2025, 2121, 1379, 1509, 2150, 2389, - 2337, 1379, 1714, 2427, 2311, 2791, 2174, 2192, 2376, 2137, - 70, 2173, 1340, 70, 70, 2164, 70, 2170, 2179, 4390, - 2366, 2054, 3370, 2369, 2370, 2938, 2372, 2180, 1725, 766, - 2183, 2184, 3059, 2929, 2162, 2429, 2194, 2195, 2290, 2477, - 2667, 2557, 2289, 2551, 4389, 2550, 2408, 2354, 1165, 2265, - 2169, 4360, 2171, 2172, 2189, 2191, 1343, 1341, 2429, 2512, - 2165, 2166, 4359, 4358, 2276, 2421, 2178, 2270, 2511, 4357, - 2510, 2294, 2420, 2296, 2305, 2438, 4450, 2175, 2176, 2321, - 2274, 70, 1622, 3131, 3137, 3138, 3139, 3132, 3136, 3133, - 3135, 3134, 1594, 4335, 4334, 1934, 1626, 2271, 2187, 1700, - 4087, 1341, 2786, 3850, 4391, 1167, 2335, 2383, 1626, 2476, - 2336, 1003, 2329, 4686, 2402, 4307, 2343, 2344, 2654, 3272, - 3274, 3275, 3276, 3273, 183, 223, 182, 214, 184, 2632, - 2363, 4276, 1207, 1208, 4273, 2375, 2477, 1212, 4669, 1381, - 1382, 1383, 1380, 1201, 1202, 1203, 4218, 2477, 2477, 3969, - 1165, 1872, 2456, 2457, 2477, 2381, 2019, 2020, 2021, 2022, - 2681, 2684, 2026, 2027, 2028, 2029, 2031, 2032, 2033, 2034, - 2035, 2036, 2037, 2038, 2039, 2040, 2041, 1200, 2438, 2438, - 1197, 4387, 2150, 1395, 1394, 1404, 1405, 1406, 1407, 1397, - 1398, 1399, 1400, 1401, 1402, 1403, 1396, 2475, 219, 2416, - 2477, 2418, 940, 4220, 1843, 3896, 3375, 1167, 3172, 1381, - 1382, 1383, 1380, 2564, 2505, 2566, 1379, 2568, 2569, 2632, - 3068, 2572, 1381, 1382, 1383, 1380, 2957, 2956, 2465, 2422, - 747, 1677, 747, 1677, 2825, 2948, 2661, 1381, 1382, 1383, - 1380, 3909, 2435, 2587, 2498, 2481, 765, 3870, 2419, 2535, - 826, 2361, 2346, 747, 747, 747, 761, 3792, 2479, 2603, - 2451, 2076, 2454, 2455, 3788, 1823, 2460, 2653, 1433, 747, - 747, 747, 747, 1327, 1291, 3675, 2991, 2992, 1286, 3967, - 3459, 2453, 1915, 2985, 3448, 1998, 1998, 2320, 2536, 2538, - 2539, 2540, 2636, 2542, 2639, 3458, 763, 1022, 3346, 3162, - 2641, 2642, 2643, 2469, 2646, 1677, 2464, 2463, 764, 1396, - 1267, 1263, 1264, 1265, 1266, 4325, 3036, 2990, 2685, 2989, - 2988, 2986, 3024, 2680, 2674, 2679, 3910, 2677, 2682, 2545, - 762, 3616, 3871, 1677, 3016, 1381, 1382, 1383, 1380, 2669, - 1772, 1771, 3793, 2143, 2970, 1381, 1382, 1383, 1380, 3789, - 2703, 998, 999, 1000, 1001, 2417, 1381, 1382, 1383, 1380, - 3676, 1194, 1195, 1196, 1199, 2952, 1198, 4454, 2543, 1379, - 181, 212, 221, 213, 2940, 861, 871, 3420, 1165, 2935, - 2920, 2918, 2683, 2791, 2936, 862, 2916, 863, 867, 870, - 866, 864, 865, 2549, 211, 1381, 1382, 1383, 1380, 2625, - 2987, 2632, 2577, 2473, 2579, 4326, 2914, 1379, 2631, 2553, - 2710, 2519, 2658, 4455, 2546, 747, 2190, 2640, 2660, 1379, - 2662, 2554, 2518, 2501, 2795, 2795, 2288, 2795, 2492, 1379, - 2452, 1415, 1414, 2491, 1623, 1167, 4025, 2490, 2567, 1655, - 2478, 4680, 2571, 2437, 2402, 1832, 4122, 717, 717, 1702, - 2632, 4327, 868, 2544, 4647, 1306, 1608, 2709, 3602, 2941, - 1609, 1714, 747, 1029, 2936, 2921, 2919, 1987, 1986, 3885, - 2595, 2915, 2083, 4428, 1399, 1400, 1401, 1402, 1403, 1396, - 747, 3883, 2673, 869, 1987, 1986, 1306, 2887, 736, 2672, - 1458, 2915, 4123, 2632, 2552, 1755, 1379, 2288, 2818, 1704, - 2895, 2629, 2897, 2628, 2663, 240, 2626, 1379, 1379, 4382, - 1705, 4306, 4248, 1379, 4190, 3886, 2891, 1003, 1379, 1165, - 2666, 1459, 1379, 4152, 2753, 2477, 2799, 3884, 2438, 4151, - 1833, 4137, 4095, 2513, 2514, 3861, 2516, 2647, 2797, 3603, - 2801, 3707, 1608, 2523, 3697, 2905, 1609, 2648, 2649, 3689, - 3677, 2943, 2833, 1701, 3578, 1624, 3303, 2651, 2652, 3302, - 2950, 2659, 4024, 2427, 2686, 2687, 2803, 2692, 3129, 3064, - 1714, 2967, 1714, 2809, 1714, 2810, 1167, 2939, 2814, 1306, - 2832, 2827, 2570, 2030, 2411, 3604, 2410, 2969, 2409, 1590, - 1381, 1382, 1383, 1380, 2815, 2816, 1589, 2894, 1308, 2650, - 2023, 3743, 3740, 2977, 2656, 2899, 1938, 2657, 2470, 2960, - 1760, 2364, 2364, 70, 3519, 3094, 1938, 1714, 1306, 3376, - 1921, 2655, 2998, 2900, 2763, 3522, 3522, 2769, 1397, 1398, - 1399, 1400, 1401, 1402, 1403, 1396, 1918, 1920, 1917, 3007, - 1919, 2804, 1383, 1380, 1714, 2964, 1381, 1382, 1383, 1380, - 2993, 1626, 2105, 4533, 1712, 4275, 4274, 2909, 1380, 4167, - 4166, 3605, 1693, 1695, 1381, 1382, 1383, 1380, 3232, 3230, - 3209, 2819, 3207, 3741, 4503, 4504, 4143, 3008, 4362, 4363, - 4089, 1712, 4096, 4097, 3128, 2822, 1394, 1404, 1405, 1406, - 1407, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1396, 2888, - 1435, 3521, 3848, 2893, 3519, 2927, 3066, 3013, 3014, 3283, - 1822, 3070, 3281, 1434, 3074, 2002, 1853, 220, 1854, 3002, - 4676, 747, 747, 747, 1384, 1381, 1382, 1383, 1380, 3046, - 2003, 3047, 1417, 4596, 2979, 2892, 4558, 3009, 1306, 3279, - 4090, 1427, 4557, 2980, 2966, 2982, 1714, 4506, 2925, 1677, - 2961, 1381, 1382, 1383, 1380, 1677, 2268, 4505, 2975, 3268, - 2901, 2996, 3849, 4502, 4501, 2951, 4500, 1437, 4499, 3282, - 2953, 3450, 3280, 3165, 3168, 2958, 1387, 1388, 1389, 1390, - 1391, 1392, 1393, 1385, 3174, 4675, 4497, 3038, 3050, 3039, - 2494, 3041, 4496, 3043, 3044, 2931, 2932, 2971, 2972, 3278, - 4495, 4494, 3184, 2974, 2833, 4493, 1381, 1382, 1383, 1380, - 4492, 3854, 1306, 4490, 3149, 1761, 4489, 2984, 2994, 3267, - 3206, 4456, 3092, 1381, 1382, 1383, 1380, 1306, 1306, 1306, - 2190, 1760, 2832, 1306, 3449, 3216, 3217, 3218, 3219, 1306, - 3226, 4348, 3227, 3228, 3143, 3229, 4338, 3231, 1872, 1696, - 4328, 4300, 3146, 4621, 4272, 4239, 4161, 3160, 3226, 4678, - 2493, 1381, 1382, 1383, 1380, 4125, 4124, 3901, 3051, 3887, - 2795, 1381, 1382, 1383, 1380, 3860, 3847, 3830, 3201, 70, - 1381, 1382, 1383, 1380, 3284, 3567, 3416, 1381, 1382, 1383, - 1380, 3388, 3185, 3201, 3212, 3213, 3387, 3292, 2212, 3215, - 3187, 717, 3266, 3265, 3264, 3222, 3256, 3250, 3249, 2268, - 3248, 3247, 3175, 1306, 2288, 2288, 2288, 2288, 2288, 2288, - 3125, 3106, 3173, 3108, 3060, 2922, 3144, 2811, 4568, 2556, - 2385, 1306, 2288, 2384, 2382, 2795, 2378, 3105, 2377, 2327, - 3289, 3123, 4475, 3204, 2088, 2085, 1824, 3204, 4042, 1527, - 3553, 3354, 4672, 1714, 3152, 1381, 1382, 1383, 1380, 3200, - 4670, 4039, 8, 3167, 3164, 4014, 747, 747, 7, 1381, - 1382, 1383, 1380, 4645, 3211, 1381, 1382, 1383, 1380, 3320, - 1289, 3019, 3020, 4298, 4299, 4038, 4611, 3025, 1381, 1382, - 1383, 1380, 4028, 3186, 3006, 4545, 2999, 3320, 3189, 4027, - 4543, 4280, 4519, 3202, 4487, 4440, 3342, 3208, 4100, 4434, - 4425, 3214, 1381, 1382, 1383, 1380, 4423, 3295, 4410, 1381, - 1382, 1383, 1380, 3435, 2121, 3205, 1381, 1382, 1383, 1380, - 4401, 3372, 3177, 4377, 4376, 3246, 4367, 3180, 4366, 1288, - 3258, 4352, 240, 4347, 4346, 2715, 4303, 240, 2718, 2719, - 2720, 2721, 2722, 2723, 2724, 4287, 4285, 2727, 2728, 2729, - 2730, 2731, 2732, 2733, 2734, 2735, 2736, 2737, 3371, 2739, - 2740, 2741, 2742, 2743, 3308, 2744, 4271, 1998, 3298, 1998, - 4240, 3355, 3409, 3304, 3183, 4145, 1754, 4104, 4093, 3415, - 3176, 1381, 1382, 1383, 1380, 1714, 4026, 4077, 3422, 3181, - 3182, 1381, 1382, 1383, 1380, 3345, 3339, 3343, 3321, 3322, - 3323, 3324, 3325, 3326, 4076, 3344, 3301, 3952, 4074, 4069, - 4067, 3362, 4625, 1381, 1382, 1383, 1380, 4046, 4045, 3359, - 4044, 1781, 3404, 4041, 3774, 4040, 3363, 4016, 4012, 3489, - 4010, 1782, 1783, 3980, 1381, 1382, 1383, 1380, 1788, 1789, - 70, 3977, 2485, 3971, 3199, 70, 3288, 3460, 3842, 3832, - 3377, 1381, 1382, 1383, 1380, 3381, 1381, 1382, 1383, 1380, - 2058, 3454, 3817, 1797, 3801, 3408, 1381, 1382, 1383, 1380, - 1794, 3780, 3778, 1798, 1381, 1382, 1383, 1380, 3772, 3757, - 3718, 3695, 3694, 3692, 3406, 3691, 3678, 3673, 1381, 1382, - 1383, 1380, 3672, 3579, 3539, 3533, 3505, 3523, 3379, 3508, - 3513, 3506, 3378, 3504, 3512, 2561, 747, 1677, 3430, 3427, - 3414, 3397, 3419, 3424, 3386, 3524, 3526, 3527, 3529, 3361, - 3531, 3532, 3402, 3453, 3296, 3293, 3405, 3400, 3341, 3393, - 3290, 3407, 1306, 3277, 3269, 3259, 3257, 3253, 1306, 1381, - 1382, 1383, 1380, 3252, 3556, 3558, 3418, 3251, 3095, 3081, - 1381, 1382, 1383, 1380, 3069, 3571, 3061, 940, 939, 3431, - 3432, 747, 2947, 2926, 1884, 1885, 1886, 1887, 1888, 3447, - 3438, 3439, 2889, 2588, 2575, 3441, 3586, 2574, 3590, 1306, - 3443, 3444, 747, 2388, 747, 2268, 1306, 1306, 2380, 2188, - 2118, 3440, 2087, 3442, 2084, 2069, 2068, 1825, 3542, 3451, - 2288, 2636, 1466, 3615, 3201, 1462, 1461, 1292, 1007, 1935, - 4633, 4473, 4469, 1939, 1940, 1941, 1942, 3498, 4277, 4267, - 4266, 1417, 2703, 4253, 1981, 4249, 1381, 1382, 1383, 1380, - 3035, 3582, 4075, 1992, 3640, 4043, 3643, 3593, 3643, 3643, - 4022, 3502, 3575, 1306, 3599, 3201, 3991, 3503, 3972, 3607, - 3878, 3877, 3201, 3201, 183, 223, 2474, 1381, 1382, 1383, - 1380, 3668, 3143, 3585, 3874, 3034, 3664, 183, 223, 1714, - 1714, 3839, 1165, 3797, 2145, 3623, 3795, 3794, 3515, 183, - 223, 3146, 3791, 3627, 3629, 2046, 3790, 2048, 2049, 2050, - 2051, 2052, 1381, 1382, 1383, 1380, 2059, 3779, 3777, 3618, - 3613, 3761, 3669, 3670, 2142, 3746, 1712, 1712, 3745, 3201, - 3608, 3730, 3729, 3609, 3559, 3543, 747, 3606, 3540, 3496, - 3113, 3581, 3033, 3456, 2472, 3592, 3638, 3032, 2144, 1167, - 3556, 3445, 3597, 3598, 1381, 1382, 1383, 1380, 3639, 3610, - 3437, 219, 4632, 1677, 3436, 3614, 2268, 2268, 3648, 1381, - 1382, 1383, 1380, 219, 1381, 1382, 1383, 1380, 3434, 2673, - 3622, 873, 155, 3369, 2917, 2913, 2672, 155, 3465, 3466, - 2912, 3644, 3645, 2911, 3467, 3468, 3469, 3470, 2524, 3471, - 3472, 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, 3481, - 3649, 3031, 183, 223, 3666, 2517, 2509, 2158, 2508, 2507, - 2506, 1306, 1381, 1382, 1383, 1380, 2998, 3030, 223, 182, - 214, 184, 1815, 2504, 3744, 3682, 2500, 3674, 1381, 1382, - 1383, 1380, 3029, 2177, 2499, 2286, 2497, 2488, 3238, 3239, - 183, 223, 743, 2484, 1381, 1382, 1383, 1380, 2483, 155, - 2387, 2047, 1812, 3254, 3255, 3028, 3646, 2045, 3617, 1381, - 1382, 1383, 1380, 3619, 3620, 2044, 2043, 3702, 3703, 747, - 3679, 2042, 3688, 2001, 3687, 2000, 1814, 2833, 3690, 3696, - 223, 3027, 1381, 1382, 1383, 1380, 3299, 1991, 1726, 1724, - 3403, 219, 3714, 4595, 3715, 4512, 4474, 1456, 2059, 4485, - 3026, 183, 223, 2059, 2059, 2832, 4468, 746, 1381, 1382, - 1383, 1380, 749, 3693, 219, 4396, 4393, 3726, 3727, 3728, - 3723, 4375, 4356, 4349, 3700, 4234, 3621, 1381, 1382, 1383, - 1380, 4233, 4185, 4165, 4163, 3733, 1404, 1405, 1406, 1407, - 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1396, 3803, 4483, - 3023, 153, 3804, 219, 2358, 2365, 4158, 3754, 2368, 4136, - 3762, 2371, 4119, 3992, 2373, 3022, 3818, 3989, 3820, 3950, - 3949, 3946, 3945, 3826, 3021, 219, 3781, 1381, 1382, 1383, - 1380, 3765, 3908, 3905, 3764, 3903, 3863, 3816, 3814, 3812, - 3536, 1166, 1381, 1382, 1383, 1380, 155, 3827, 3446, 1776, - 3704, 1381, 1382, 1383, 1380, 2395, 3770, 779, 1787, 1778, - 1793, 155, 1796, 155, 747, 2268, 1784, 1773, 746, 1597, - 3821, 3331, 3823, 3291, 3722, 3285, 4481, 3015, 3869, 3210, - 3156, 3783, 3003, 3785, 3155, 3787, 3147, 3876, 3107, 3037, - 2934, 3838, 2813, 2745, 3809, 2630, 2597, 2596, 3841, 2555, - 1916, 2795, 2288, 3893, 1381, 1382, 1383, 1380, 219, 1381, - 1382, 1383, 1380, 3802, 2345, 2065, 1857, 3798, 1816, 1785, - 3682, 2997, 1526, 3806, 1511, 3911, 3858, 3836, 1306, 1507, - 1506, 1505, 1504, 1503, 1502, 1501, 749, 3640, 1500, 1499, - 1498, 1306, 1497, 1496, 3831, 2976, 1495, 3835, 1381, 1382, - 1383, 1380, 1494, 1493, 1492, 1491, 1306, 1490, 3966, 1489, - 1488, 1487, 1714, 1486, 1485, 3855, 1484, 1483, 3961, 3962, - 3963, 1482, 1381, 1382, 1383, 1380, 3857, 3975, 1481, 2466, - 1480, 1479, 1478, 2471, 1477, 1476, 3895, 1475, 3867, 1474, - 747, 2480, 2268, 1473, 3912, 3968, 2288, 1306, 3875, 1712, - 3944, 1472, 3892, 1469, 1468, 3947, 2548, 3954, 3902, 1467, - 3904, 1465, 1464, 3890, 3891, 1463, 1460, 1453, 1452, 3935, - 3898, 1450, 3222, 2547, 1449, 1448, 3998, 2541, 1447, 1446, - 2489, 1445, 240, 1381, 1382, 1383, 1380, 1444, 2496, 2645, - 1933, 3951, 1443, 1442, 1441, 3956, 3953, 1440, 3984, 3981, - 1381, 1382, 1383, 1380, 1381, 1382, 1383, 1380, 3965, 1439, - 3997, 1438, 1432, 3320, 1431, 3970, 2515, 1381, 1382, 1383, - 1380, 2520, 2521, 2522, 1430, 1429, 2525, 2526, 2527, 2528, - 2529, 2530, 2531, 2532, 2533, 2534, 3979, 1428, 3983, 3976, - 1345, 3982, 1290, 3987, 3986, 2612, 3985, 3973, 3710, 3711, - 1333, 4623, 4577, 2150, 3713, 3685, 4059, 3297, 3130, 2826, - 4065, 2624, 1606, 1344, 3329, 4020, 4071, 3336, 3978, 3334, - 3994, 3332, 3337, 3721, 3335, 3328, 3333, 3720, 3719, 3716, - 3995, 1306, 3340, 3327, 138, 73, 72, 4532, 4015, 4412, - 70, 4141, 3163, 69, 2937, 3894, 1591, 2139, 2140, 3577, - 4005, 3759, 3760, 3897, 1306, 1714, 1714, 3957, 3636, 4105, - 3637, 3399, 3590, 2713, 4068, 3734, 4070, 2134, 2135, 2136, - 4055, 2249, 1769, 3161, 4113, 3234, 2931, 2932, 4113, 1306, - 3993, 1808, 3235, 3236, 3237, 2965, 2582, 2581, 4102, 1805, - 2589, 2347, 1712, 1931, 1306, 4130, 1306, 3201, 4107, 4108, - 4062, 2261, 4101, 1339, 4353, 738, 739, 740, 4133, 4073, - 4135, 3551, 4049, 1714, 741, 3544, 3188, 4082, 4084, 3157, - 4083, 4110, 4079, 2665, 2622, 4103, 2148, 2109, 1987, 1986, - 4636, 4094, 1522, 1523, 747, 4351, 1306, 1306, 1520, 1521, - 1306, 1306, 1518, 1519, 3671, 3320, 4106, 1516, 1517, 4117, - 1931, 2766, 4118, 2759, 4126, 2269, 1666, 1665, 4339, 4187, - 1878, 1372, 1878, 4129, 3895, 4219, 2412, 3732, 4189, 3725, - 4182, 2590, 2408, 2415, 2153, 4142, 3944, 4139, 1615, 1614, - 2150, 1582, 4146, 4226, 4169, 4170, 1638, 4086, 4183, 4184, - 4138, 2638, 2963, 4602, 4600, 3935, 4085, 4235, 4236, 4551, - 4144, 2962, 4529, 4528, 4526, 4444, 4397, 4229, 4228, 4131, - 4011, 1714, 1395, 1394, 1404, 1405, 1406, 1407, 1397, 1398, - 1399, 1400, 1401, 1402, 1403, 1396, 2247, 4222, 3782, 3753, - 3752, 3738, 2392, 2698, 183, 223, 2668, 4188, 4268, 4269, - 2059, 747, 2059, 4221, 4247, 1810, 3737, 4224, 1712, 4259, - 3374, 1612, 4627, 4626, 4626, 4281, 4066, 4283, 4111, 3819, - 3805, 2059, 2059, 3417, 2249, 3076, 3075, 3067, 2890, 4246, - 2486, 4242, 1329, 1303, 4627, 4160, 3996, 155, 155, 155, - 1166, 4284, 4606, 4286, 4081, 3880, 3396, 4254, 4258, 2616, - 4029, 1801, 4030, 998, 999, 1000, 1001, 1296, 1296, 1754, - 1630, 81, 4000, 2, 4649, 4650, 4315, 1, 219, 3052, - 4320, 2063, 4313, 1524, 4289, 1002, 997, 1690, 2224, 2805, - 2322, 1718, 4017, 4127, 4128, 4290, 2067, 1306, 1004, 3347, - 3348, 1299, 3724, 3350, 4264, 4265, 1304, 2328, 1600, 3082, - 4343, 4337, 2434, 4308, 3309, 4302, 4037, 2757, 2601, 3570, - 1598, 1070, 1993, 1837, 2942, 1320, 2945, 1834, 4311, 1416, - 1334, 1319, 4314, 4317, 1317, 4020, 4316, 1936, 2014, 875, - 2398, 4329, 3286, 3260, 4225, 4635, 1306, 4333, 4664, 4594, - 4061, 4638, 1855, 859, 4520, 3755, 3394, 4402, 4598, 4404, - 4245, 2439, 1377, 1878, 3401, 1096, 919, 887, 2240, 1451, - 4350, 1811, 3463, 3461, 886, 3852, 3119, 4230, 1714, 3366, - 4322, 4388, 1097, 2374, 4399, 2978, 4243, 4223, 2981, 1770, - 1775, 2664, 4330, 4464, 4361, 4140, 3632, 3196, 1800, 4459, - 3000, 3001, 3906, 4033, 4031, 4032, 786, 2301, 715, 3004, - 3005, 4385, 1150, 4186, 2623, 1712, 2644, 4191, 4355, 1044, - 3833, 2611, 1045, 1037, 3141, 3010, 3011, 3012, 3140, 1895, - 3864, 3865, 3866, 1386, 1914, 4424, 3482, 3483, 3872, 3873, - 1426, 4418, 830, 4429, 2468, 3116, 3929, 3360, 4398, 80, - 79, 78, 4436, 77, 248, 878, 247, 4278, 4098, 3040, - 4515, 3042, 2228, 4640, 3045, 856, 1884, 2059, 855, 4431, - 854, 4432, 853, 2234, 852, 851, 2777, 2778, 2776, 2774, - 2773, 2283, 4445, 2282, 3373, 3736, 2353, 4441, 1515, 2355, - 3588, 3225, 3958, 2222, 2256, 3220, 2201, 2223, 2225, 2227, - 4433, 2229, 2230, 2231, 2235, 2236, 2237, 2239, 2242, 2243, - 2244, 4463, 4439, 2199, 1681, 2693, 2700, 1306, 2232, 2241, - 2233, 2198, 4448, 4447, 4574, 3771, 4023, 4476, 4477, 4491, - 4157, 3270, 4019, 2133, 2689, 2218, 1306, 4480, 4482, 4484, - 4486, 3241, 4457, 2215, 2214, 4462, 3233, 1714, 4508, 4153, - 4498, 4147, 4509, 4471, 2246, 4318, 4112, 4516, 3913, 3914, - 3920, 1248, 2621, 1222, 1217, 4479, 4196, 1219, 3178, 3179, - 1220, 4517, 2248, 1218, 2983, 3698, 2670, 3546, 3101, 3100, - 4507, 3098, 3097, 1566, 1712, 4435, 4547, 4078, 2831, 2829, - 1287, 4544, 3712, 3708, 3516, 1532, 1530, 2406, 4518, 3717, - 3330, 2393, 1878, 4525, 3398, 4523, 2284, 1714, 2280, 2279, - 4541, 4320, 4537, 4539, 1192, 1191, 1751, 3810, 4546, 48, - 746, 3311, 2767, 4292, 4542, 4538, 4540, 4561, 2138, 1038, - 2609, 117, 42, 4569, 133, 116, 2245, 201, 63, 4552, - 4553, 200, 62, 4554, 1712, 1970, 18, 4555, 4556, 131, - 4195, 198, 61, 47, 2221, 46, 1723, 196, 2220, 111, - 743, 110, 109, 108, 130, 195, 60, 232, 231, 234, - 233, 230, 2902, 4582, 2903, 4583, 229, 4584, 4586, 4585, - 1758, 4590, 2238, 228, 4530, 4116, 4511, 1669, 992, 45, - 44, 2226, 202, 43, 118, 64, 1683, 41, 155, 40, - 4601, 2637, 4603, 4604, 3534, 2152, 3828, 3093, 4599, 4593, - 2593, 4597, 1306, 39, 35, 13, 12, 1720, 2059, 4418, - 4607, 36, 23, 22, 1842, 21, 4608, 27, 4609, 4610, - 33, 4343, 32, 148, 4614, 147, 31, 146, 145, 144, - 143, 4617, 4616, 4615, 142, 4620, 141, 140, 30, 20, - 55, 54, 4624, 4634, 4622, 53, 4642, 52, 51, 4641, - 50, 9, 4628, 4629, 4630, 4631, 136, 4394, 4395, 134, - 129, 127, 29, 128, 1306, 125, 4646, 126, 4612, 121, - 120, 119, 114, 112, 92, 91, 4463, 4653, 4652, 155, - 90, 4655, 4656, 105, 104, 4662, 103, 102, 4666, 101, - 100, 4663, 98, 99, 155, 1095, 89, 155, 155, 88, - 87, 86, 85, 122, 107, 4217, 115, 113, 96, 4674, - 3380, 155, 3382, 106, 97, 95, 94, 93, 84, 4642, - 4682, 83, 4641, 4681, 82, 4192, 124, 123, 135, 203, - 1878, 4666, 4683, 65, 2395, 180, 179, 4687, 178, 2059, - 177, 176, 174, 175, 2059, 173, 172, 1667, 1668, 171, - 1670, 1966, 170, 1674, 169, 1678, 1679, 1680, 1963, 168, - 56, 57, 1965, 1962, 1964, 1968, 1969, 58, 59, 191, - 1967, 190, 192, 194, 4134, 197, 193, 199, 188, 186, - 189, 187, 185, 74, 3433, 11, 132, 19, 1728, 1729, - 1730, 1731, 1732, 1733, 1734, 4, 1736, 1737, 1738, 1739, - 1740, 0, 0, 0, 1746, 0, 1748, 1749, 1750, 0, - 0, 3455, 0, 0, 3939, 0, 0, 0, 1416, 0, - 3918, 0, 0, 0, 0, 0, 4197, 4198, 1395, 1394, - 1404, 1405, 1406, 1407, 1397, 1398, 1399, 1400, 1401, 1402, - 1403, 1396, 4193, 4194, 0, 4201, 4200, 4199, 4212, 4213, - 4214, 4202, 4203, 4206, 4208, 4207, 4204, 4205, 4209, 4210, - 4211, 3930, 0, 0, 0, 4215, 0, 0, 0, 0, - 0, 0, 0, 0, 3921, 0, 4216, 0, 0, 0, - 0, 0, 0, 0, 0, 3916, 0, 0, 0, 0, - 3941, 3942, 0, 0, 0, 0, 3917, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1951, 1952, 1953, 1954, - 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1973, 1974, 1975, - 1976, 1977, 1978, 1971, 1972, 0, 0, 0, 0, 0, - 4364, 4365, 0, 0, 0, 0, 3922, 4369, 4370, 4371, - 4372, 4373, 4374, 0, 0, 0, 4378, 4379, 4380, 4381, - 0, 0, 0, 4383, 4384, 0, 4386, 0, 0, 0, - 0, 183, 223, 182, 214, 184, 0, 0, 0, 0, - 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, - 0, 215, 4132, 0, 0, 0, 0, 0, 206, 0, - 0, 0, 216, 0, 0, 2093, 2094, 2095, 0, 0, - 183, 223, 182, 214, 184, 0, 0, 0, 0, 0, - 0, 153, 0, 0, 0, 3647, 0, 0, 0, 0, - 215, 0, 0, 0, 0, 0, 139, 206, 2127, 0, - 0, 216, 0, 2132, 0, 219, 1395, 1394, 1404, 1405, - 1406, 1407, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1396, - 153, 0, 4446, 3940, 0, 2679, 0, 0, 4451, 4452, - 0, 0, 0, 0, 0, 139, 2287, 0, 0, 0, - 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, - 3926, 0, 0, 0, 0, 0, 0, 0, 0, 4472, - 0, 0, 0, 0, 0, 0, 3681, 0, 0, 0, - 0, 0, 3923, 3927, 3925, 3924, 0, 0, 0, 0, - 0, 0, 0, 2196, 2197, 0, 1084, 0, 0, 0, - 0, 0, 0, 0, 162, 163, 0, 164, 165, 0, - 0, 0, 166, 0, 0, 167, 0, 0, 0, 0, - 0, 0, 0, 155, 0, 0, 155, 155, 0, 155, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3933, 3934, 0, 162, 163, 0, 164, 165, 0, 0, - 0, 166, 0, 0, 167, 0, 2334, 0, 0, 0, - 0, 0, 2334, 2334, 2334, 0, 0, 0, 1080, 1081, - 0, 0, 0, 0, 0, 0, 1166, 0, 0, 1126, - 0, 0, 1188, 0, 0, 0, 0, 181, 212, 221, - 213, 75, 137, 0, 155, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, - 0, 211, 205, 204, 0, 0, 0, 0, 76, 0, - 3919, 0, 0, 3932, 0, 0, 181, 212, 221, 213, - 75, 137, 0, 0, 0, 3773, 161, 0, 0, 0, - 0, 0, 0, 3775, 3776, 0, 0, 0, 0, 0, - 211, 205, 204, 0, 0, 0, 0, 76, 0, 0, - 0, 0, 0, 0, 0, 0, 1189, 0, 1409, 0, - 1413, 3784, 0, 3786, 1128, 161, 0, 1127, 1416, 207, - 208, 209, 3796, 0, 0, 0, 1410, 1412, 1408, 0, - 1411, 1395, 1394, 1404, 1405, 1406, 1407, 1397, 1398, 1399, - 1400, 1401, 1402, 1403, 1396, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 207, 208, - 209, 3681, 2482, 0, 0, 0, 1112, 0, 0, 0, - 0, 0, 0, 2339, 0, 0, 1085, 0, 0, 0, - 0, 0, 0, 0, 0, 2349, 0, 0, 0, 217, - 0, 1182, 1177, 1172, 1176, 1180, 0, 0, 3937, 0, - 0, 0, 0, 1087, 0, 0, 0, 1409, 0, 1413, - 149, 0, 0, 0, 210, 0, 150, 0, 0, 1185, - 0, 0, 0, 1175, 0, 1410, 1412, 1408, 217, 1411, - 1395, 1394, 1404, 1405, 1406, 1407, 1397, 1398, 1399, 1400, - 1401, 1402, 1403, 1396, 0, 0, 0, 0, 0, 149, - 0, 0, 0, 210, 0, 150, 0, 0, 0, 0, - 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 151, 0, 0, 1183, 0, 0, 0, 3931, 0, - 1108, 0, 1110, 1107, 68, 3936, 0, 1111, 0, 0, - 0, 0, 0, 3938, 0, 0, 1186, 0, 0, 1061, - 0, 1062, 0, 1187, 0, 0, 0, 0, 0, 0, - 151, 0, 0, 0, 0, 2059, 0, 0, 0, 0, - 0, 0, 0, 68, 0, 0, 0, 1106, 0, 0, - 0, 0, 2059, 0, 0, 3988, 0, 71, 3990, 1079, - 1042, 0, 1173, 0, 0, 0, 1166, 0, 155, 0, - 1086, 1121, 0, 0, 1056, 2576, 1052, 2578, 0, 0, - 0, 0, 3999, 0, 0, 0, 1184, 0, 0, 0, - 0, 0, 1117, 159, 220, 160, 71, 0, 2598, 2599, - 2600, 0, 0, 0, 0, 0, 0, 0, 66, 0, - 0, 0, 0, 0, 2617, 2618, 2619, 2620, 0, 0, - 0, 0, 0, 0, 1174, 0, 0, 0, 1118, 1122, - 0, 0, 159, 220, 160, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1033, 0, 0, 66, 1103, 0, - 1101, 1105, 1125, 0, 0, 0, 1102, 1099, 1098, 0, - 1104, 1089, 1090, 1088, 2247, 1078, 1091, 1092, 1093, 1094, - 1075, 0, 0, 1123, 0, 1124, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1119, 1120, 152, 49, - 0, 0, 0, 0, 0, 67, 0, 0, 0, 5, - 0, 0, 2249, 0, 0, 1181, 0, 2798, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, - 157, 0, 0, 158, 1115, 0, 0, 152, 49, 1058, - 1114, 1051, 0, 0, 67, 0, 0, 0, 1076, 0, - 1055, 1054, 1178, 0, 0, 1179, 4342, 1109, 0, 0, - 0, 0, 0, 0, 1171, 0, 2224, 0, 156, 157, - 1683, 1043, 158, 0, 1381, 1382, 1383, 1380, 0, 0, - 0, 0, 0, 0, 798, 797, 804, 794, 2287, 0, - 0, 1050, 0, 0, 0, 0, 155, 801, 802, 0, - 803, 807, 0, 0, 788, 0, 0, 0, 0, 0, - 1060, 0, 0, 0, 812, 1049, 0, 1720, 0, 1048, - 0, 0, 3457, 0, 0, 1036, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 2334, 0, 0, 0, 0, - 0, 0, 0, 0, 1041, 0, 2240, 0, 0, 1113, - 0, 0, 0, 0, 0, 1082, 1083, 0, 0, 1074, - 816, 0, 0, 818, 1077, 1970, 0, 0, 817, 0, - 0, 1190, 0, 0, 0, 1170, 1395, 1394, 1404, 1405, - 1406, 1407, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1396, - 1039, 0, 0, 0, 798, 797, 804, 794, 0, 3452, - 0, 0, 0, 0, 0, 0, 0, 801, 802, 0, - 803, 807, 0, 0, 788, 0, 0, 0, 0, 798, - 797, 804, 794, 0, 812, 0, 0, 0, 0, 1059, - 0, 0, 801, 802, 0, 803, 807, 0, 0, 788, - 2228, 0, 0, 0, 0, 0, 0, 0, 0, 812, - 0, 2234, 1040, 1395, 1394, 1404, 1405, 1406, 1407, 1397, - 1398, 1399, 1400, 1401, 1402, 1403, 1396, 0, 0, 0, - 816, 2222, 2256, 818, 0, 2223, 2225, 2227, 817, 2229, - 2230, 2231, 2235, 2236, 2237, 2239, 2242, 2243, 2244, 0, - 0, 0, 0, 0, 0, 0, 2232, 2241, 2233, 2487, - 0, 0, 1395, 1394, 1404, 1405, 1406, 1407, 1397, 1398, - 1399, 1400, 1401, 1402, 1403, 1396, 0, 0, 0, 0, - 4354, 0, 0, 0, 0, 0, 0, 0, 0, 789, - 791, 790, 0, 1057, 0, 0, 0, 0, 0, 0, - 0, 796, 0, 0, 0, 0, 0, 0, 0, 155, - 2248, 0, 0, 800, 0, 0, 0, 0, 0, 0, - 815, 0, 155, 0, 0, 0, 0, 793, 0, 0, - 0, 783, 0, 1046, 0, 0, 3077, 3078, 3079, 0, - 0, 1966, 1035, 0, 0, 0, 0, 0, 1963, 0, - 0, 0, 1965, 1962, 1964, 1968, 1969, 0, 0, 0, - 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2245, 0, 0, 0, 1437, 0, - 0, 0, 0, 2247, 0, 0, 0, 0, 2208, 3166, - 0, 2255, 2221, 0, 0, 0, 2220, 0, 0, 789, - 791, 790, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 796, 0, 0, 0, 0, 0, 0, 0, 0, - 2238, 2249, 2217, 800, 789, 791, 790, 0, 0, 2226, - 815, 2250, 2251, 0, 0, 0, 796, 793, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 800, 1034, - 0, 0, 0, 1032, 0, 815, 0, 2216, 0, 0, - 0, 0, 793, 0, 4470, 2287, 2287, 2287, 2287, 2287, - 2287, 0, 0, 0, 0, 2224, 795, 799, 805, 0, - 806, 808, 0, 2287, 809, 810, 811, 0, 0, 0, - 813, 814, 2973, 0, 0, 0, 1951, 1952, 1953, 1954, - 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1973, 1974, 1975, - 1976, 1977, 1978, 1971, 1972, 2461, 1395, 1394, 1404, 1405, - 1406, 1407, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1396, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, - 1394, 1404, 1405, 1406, 1407, 1397, 1398, 1399, 1400, 1401, - 1402, 1403, 1396, 0, 0, 2240, 1395, 1394, 1404, 1405, - 1406, 1407, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1396, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3364, 3365, 4566, 0, 0, 795, 799, 805, 4570, - 806, 808, 0, 155, 809, 810, 811, 0, 155, 0, - 813, 814, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 795, 799, 805, 0, 806, 808, 0, 0, 809, - 810, 811, 0, 0, 0, 813, 814, 155, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2207, 2209, - 2206, 0, 0, 0, 2203, 0, 0, 0, 0, 2228, - 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2234, 0, 0, 0, 0, 0, 0, 0, 2219, 0, - 2202, 0, 0, 0, 0, 0, 0, 0, 4566, 0, - 2222, 2256, 0, 0, 2223, 2225, 2227, 0, 2229, 2230, - 2231, 2235, 2236, 2237, 2239, 2242, 2243, 2244, 819, 820, - 821, 822, 823, 0, 2247, 2232, 2241, 2233, 0, 2208, - 0, 0, 2255, 0, 0, 0, 0, 2211, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4566, 0, 0, 0, 0, 0, - 0, 0, 2249, 2217, 0, 0, 0, 0, 0, 0, - 0, 0, 2250, 2251, 0, 0, 0, 0, 0, 2248, - 792, 0, 0, 2247, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2216, 0, - 0, 0, 0, 0, 0, 792, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 4685, 2224, 0, 0, 0, - 0, 2249, 0, 0, 0, 2204, 2205, 0, 819, 820, - 821, 822, 823, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2245, 0, 0, 0, 0, 0, 0, - 0, 3514, 0, 0, 0, 0, 0, 0, 0, 0, - 1166, 2221, 155, 0, 0, 2220, 0, 0, 0, 155, - 0, 0, 0, 0, 155, 2224, 0, 0, 0, 0, - 0, 2287, 0, 0, 0, 0, 0, 0, 0, 2238, - 0, 0, 0, 0, 0, 0, 2240, 0, 2226, 0, - 155, 0, 0, 0, 0, 0, 3580, 0, 0, 0, - 0, 2253, 2252, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3594, 0, 3595, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 4312, 0, 0, 0, 2240, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2213, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207, - 3191, 2206, 0, 0, 0, 3190, 0, 0, 0, 0, - 2228, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2234, 0, 3683, 0, 0, 2247, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2254, 0, 0, 0, - 0, 2222, 2256, 0, 0, 2223, 2225, 2227, 0, 2229, - 2230, 2231, 2235, 2236, 2237, 2239, 2242, 2243, 2244, 0, - 0, 0, 0, 0, 2249, 0, 2232, 2241, 2233, 2228, - 0, 0, 0, 0, 0, 0, 0, 0, 2211, 0, - 2234, 2334, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2222, 2256, 0, 0, 2223, 2225, 2227, 0, 2229, 2230, - 2231, 2235, 2236, 2237, 2239, 2242, 2243, 2244, 2224, 0, - 2248, 0, 0, 0, 0, 2232, 2241, 2233, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2204, 2205, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2248, - 0, 0, 0, 0, 2245, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, - 0, 0, 2221, 0, 0, 0, 2220, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3767, 0, 0, 0, 0, 0, - 2238, 0, 0, 0, 0, 0, 0, 0, 0, 2226, - 0, 0, 0, 2245, 0, 0, 0, 0, 0, 0, - 0, 0, 2253, 2252, 0, 0, 0, 0, 0, 0, - 0, 2221, 0, 0, 0, 2220, 0, 0, 3683, 0, - 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, - 0, 0, 0, 155, 0, 0, 0, 0, 0, 2238, - 0, 0, 2228, 0, 0, 0, 0, 0, 2226, 0, - 0, 0, 0, 2234, 0, 0, 0, 0, 0, 2213, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 2222, 2256, 0, 0, 2223, 2225, 2227, - 0, 2229, 2230, 2231, 2235, 2236, 2237, 2239, 2242, 2243, - 2244, 0, 0, 2287, 0, 0, 0, 0, 2232, 2241, - 2233, 0, 0, 0, 0, 0, 0, 2254, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 2248, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 2287, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 2245, 0, 0, 0, - 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2221, 0, 0, 0, 2220, 0, - 0, 0, 0, 0, 0, 2334, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 2238, 0, 0, 0, 0, 0, 0, 0, - 0, 2226, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 894, 0, 0, 0, 0, 0, 0, 0, 0, 451, - 0, 0, 590, 624, 613, 698, 578, 0, 0, 0, - 0, 0, 0, 845, 0, 0, 155, 367, 0, 0, - 419, 628, 609, 620, 610, 595, 596, 597, 604, 379, - 598, 599, 600, 570, 601, 571, 602, 603, 885, 627, - 577, 489, 435, 0, 644, 0, 0, 963, 971, 0, - 0, 0, 0, 0, 0, 0, 0, 959, 0, 0, - 0, 0, 837, 0, 0, 874, 940, 939, 861, 871, - 0, 0, 335, 246, 572, 694, 574, 573, 862, 0, - 863, 867, 870, 866, 864, 865, 0, 954, 0, 0, - 0, 0, 0, 0, 829, 841, 0, 846, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 838, 839, 0, 0, 0, 0, 895, - 0, 840, 0, 0, 0, 0, 0, 490, 519, 0, - 532, 0, 404, 405, 890, 868, 872, 0, 0, 4168, - 0, 322, 497, 516, 336, 484, 530, 341, 492, 509, - 331, 450, 481, 0, 0, 324, 514, 491, 432, 323, - 0, 475, 364, 381, 361, 448, 869, 0, 893, 897, - 360, 977, 891, 524, 326, 0, 523, 447, 510, 515, - 433, 426, 0, 325, 512, 431, 425, 410, 371, 978, - 411, 412, 385, 462, 423, 463, 386, 437, 436, 438, + 862, 838, 4681, 864, 4655, 3214, 240, 4673, 1814, 4587, + 4581, 3915, 4029, 1889, 4591, 3677, 2222, 4580, 4592, 3976, + 4357, 3640, 4481, 3534, 847, 2835, 4430, 4538, 4253, 3944, + 3765, 4187, 4335, 1718, 3208, 4295, 3536, 4421, 840, 3766, + 4024, 4356, 1885, 1462, 4458, 3861, 4115, 3763, 893, 3211, + 721, 1163, 1644, 3107, 1303, 4325, 38, 4034, 227, 3, + 4431, 1650, 4433, 3405, 3875, 3649, 1955, 2161, 740, 3869, + 2687, 1942, 751, 3931, 4136, 3187, 1308, 751, 764, 773, + 4125, 3895, 773, 3334, 1892, 3605, 2939, 4096, 225, 3588, + 3823, 3563, 2327, 3592, 1957, 3335, 2345, 154, 2289, 4130, + 3859, 2324, 3303, 3237, 3669, 3034, 836, 3333, 791, 2804, + 3115, 1172, 70, 3651, 3658, 2408, 3897, 70, 3696, 2369, + 3815, 2440, 1939, 3747, 1938, 2842, 3365, 1961, 3570, 3553, + 3725, 3568, 2690, 3657, 770, 2647, 3330, 3143, 786, 2946, + 782, 3566, 3616, 3321, 3564, 2180, 3565, 3561, 1305, 37, + 3158, 2572, 830, 1711, 3516, 1798, 1787, 2571, 2474, 835, + 1615, 2417, 2416, 2409, 2069, 2406, 2920, 1034, 2436, 2374, + 1802, 1807, 2320, 2435, 1803, 2293, 2787, 2805, 1819, 3125, + 3131, 3239, 2782, 751, 1072, 2688, 3219, 2646, 3174, 70, + 1956, 2212, 236, 8, 235, 7, 2840, 1604, 6, 2290, + 1157, 2625, 1227, 2132, 2470, 1883, 2437, 839, 739, 1760, + 1696, 1690, 1727, 2616, 1633, 2415, 721, 1569, 2403, 2153, + 2683, 829, 2412, 2574, 848, 1925, 1949, 2619, 1874, 1326, + 2391, 2127, 1882, 24, 779, 1156, 2179, 1767, 1547, 1695, + 240, 15, 240, 2812, 1217, 1218, 2131, 1629, 1071, 2783, + 720, 751, 1692, 755, 1750, 25, 1197, 1645, 226, 789, + 1542, 1051, 1653, 1120, 788, 997, 26, 218, 1654, 17, + 1962, 1069, 10, 772, 748, 222, 1104, 1057, 1518, 1463, + 2444, 785, 4443, 4321, 34, 1389, 1390, 1391, 1388, 2814, + 3079, 1888, 1389, 1390, 1391, 1388, 1389, 1390, 1391, 1388, + 2093, 1249, 28, 999, 3079, 1000, 3079, 1214, 837, 3912, + 3779, 3628, 3526, 3525, 3428, 3427, 2454, 1169, 1543, 768, + 1309, 4079, 16, 3878, 1310, 3758, 2984, 2926, 1213, 2923, + 1215, 2924, 1544, 2921, 2082, 1774, 70, 1770, 746, 1209, + 14, 1210, 224, 741, 2570, 1537, 1611, 1612, 1613, 1694, + 758, 70, 4408, 70, 1503, 777, 2836, 4063, 3527, 3523, + 1021, 1210, 766, 2585, 2577, 1619, 2089, 1210, 1546, 1171, + 3509, 3506, 3511, 3508, 1309, 4667, 1142, 1670, 5, 2076, + 769, 1533, 1833, 4022, 3401, 1772, 3399, 2379, 4180, 3071, + 3069, 4589, 4588, 3772, 1815, 1389, 1390, 1391, 1388, 4416, + 765, 4260, 1018, 1389, 1390, 1391, 1388, 4254, 4025, 3033, + 3764, 2402, 4435, 2411, 1457, 998, 2944, 8, 767, 7, + 3480, 3551, 2762, 1208, 2398, 2728, 4687, 4429, 1009, 4664, + 1940, 1941, 4268, 3073, 4427, 1267, 1268, 1230, 820, 4307, + 4266, 822, 3850, 3011, 2592, 820, 821, 4494, 822, 1735, + 1554, 1552, 1548, 821, 1551, 3845, 3554, 1067, 1257, 1261, + 1263, 1265, 1270, 2606, 1275, 1271, 1272, 1273, 1274, 2272, + 1022, 1252, 1253, 1254, 1255, 1228, 1229, 1258, 2315, 1231, + 1019, 1233, 1234, 1235, 1236, 1232, 1237, 1238, 1239, 1240, + 1241, 1248, 1250, 1242, 1243, 1244, 1245, 1246, 1247, 1276, + 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1285, 1284, 1286, + 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1260, 1262, 1264, + 1266, 1269, 1167, 1249, 1168, 1173, 4068, 784, 1831, 988, + 3478, 987, 989, 990, 820, 991, 992, 822, 1596, 3955, + 4066, 2452, 821, 3328, 1010, 3934, 2103, 831, 2101, 1830, + 1135, 1133, 2620, 1134, 2832, 1016, 2819, 4309, 1251, 2818, + 1386, 2833, 2820, 3373, 3374, 1997, 183, 223, 182, 214, + 184, 2337, 1561, 3510, 3507, 1666, 3372, 1578, 1667, 2304, + 2305, 1138, 2108, 2109, 1022, 1697, 3946, 1699, 2303, 1610, + 2768, 2767, 1649, 3533, 3644, 2940, 1648, 1651, 1652, 3937, + 1129, 1576, 183, 223, 182, 214, 184, 2173, 1366, 1379, + 3932, 1367, 2720, 2194, 1641, 3957, 3958, 1651, 1652, 3127, + 1891, 3933, 3354, 3642, 2796, 2797, 1019, 1680, 4051, 3128, + 1384, 1166, 1165, 4438, 4552, 4437, 4551, 4438, 1875, 1369, + 219, 1879, 4436, 4550, 4437, 4595, 4596, 4436, 831, 1773, + 1771, 2549, 4564, 4621, 1143, 4659, 4660, 1267, 1268, 1230, + 3767, 3938, 4419, 1219, 4540, 1878, 3406, 4540, 1321, 3074, + 4422, 4423, 4424, 4425, 4543, 4257, 219, 3106, 3126, 1669, + 1257, 1261, 1263, 1265, 1270, 3102, 1275, 1271, 1272, 1273, + 1274, 1895, 3767, 1252, 1253, 1254, 1255, 1228, 1229, 1258, + 1139, 1231, 1020, 1233, 1234, 1235, 1236, 1232, 1237, 1238, + 1239, 1240, 1241, 1248, 1250, 1242, 1243, 1244, 1245, 1246, + 1247, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1285, + 1284, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1260, + 1262, 1264, 1266, 1269, 1017, 3407, 2965, 3408, 1577, 1315, + 751, 3411, 3104, 2456, 4454, 751, 1314, 2171, 1364, 3258, + 3099, 3782, 1141, 2321, 183, 223, 182, 214, 184, 3860, + 2448, 4107, 2311, 3867, 1870, 773, 773, 1340, 3584, 751, + 1251, 2770, 3956, 1063, 2691, 2614, 3959, 752, 1880, 3322, + 4311, 4312, 3441, 210, 4566, 183, 223, 182, 214, 184, + 3112, 3134, 2777, 4050, 3774, 1382, 1383, 3439, 2975, 3942, + 1013, 4052, 1877, 1381, 1354, 783, 770, 770, 770, 3072, + 1365, 1324, 183, 223, 182, 214, 184, 3103, 1536, 3582, + 1840, 3939, 3943, 3941, 3940, 3100, 2453, 4594, 219, 2104, + 2726, 2102, 1220, 1140, 1639, 1894, 1893, 1432, 2092, 1683, + 1313, 4023, 3400, 4442, 4320, 2773, 2774, 3316, 1377, 1378, + 1169, 3785, 3578, 1329, 1332, 4317, 2335, 2336, 1668, 219, + 2772, 70, 70, 70, 4104, 3445, 4070, 3078, 2837, 4064, + 1310, 3590, 1310, 3579, 3580, 1014, 1259, 3081, 1318, 3949, + 3950, 1310, 1137, 2780, 1376, 1314, 219, 738, 3975, 3581, + 2172, 1368, 4384, 3589, 1579, 1901, 1904, 1905, 3646, 1661, + 2270, 2761, 1171, 2764, 3429, 3673, 1902, 3674, 3676, 3675, + 3862, 3426, 1346, 1755, 2763, 2479, 183, 223, 182, 214, + 184, 1876, 1553, 1466, 1333, 2443, 3671, 3672, 2314, 1169, + 3971, 1210, 3670, 1210, 4276, 1310, 4277, 1210, 1210, 1371, + 1664, 1665, 1372, 1210, 1550, 1210, 3959, 2459, 2461, 2462, + 1015, 4446, 4271, 4067, 4298, 4131, 2455, 4080, 3884, 3935, + 3105, 3751, 3948, 3603, 3130, 3617, 4474, 4469, 3101, 4347, + 1374, 4267, 4339, 3175, 2922, 183, 223, 3829, 3827, 1467, + 3576, 1171, 1775, 1023, 775, 4248, 2627, 1334, 774, 1136, + 219, 768, 768, 768, 944, 3326, 1855, 2622, 3964, 1225, + 3517, 4459, 4279, 1539, 1541, 4476, 1545, 998, 3916, 4482, + 823, 824, 825, 826, 827, 1544, 1302, 823, 824, 825, + 826, 827, 1565, 1549, 3641, 153, 1568, 3070, 1651, 1652, + 1343, 1575, 4278, 1544, 766, 766, 766, 2271, 1338, 1339, + 1301, 1516, 1168, 3590, 1521, 3213, 1323, 4310, 3923, 219, + 771, 1628, 769, 769, 769, 1345, 1832, 1651, 1652, 751, + 3679, 1072, 4304, 2706, 1433, 4072, 4073, 4074, 3841, 2686, + 2709, 2798, 765, 765, 765, 4088, 3209, 3210, 1556, 3213, + 2603, 3838, 1320, 3546, 2760, 3980, 771, 1331, 1330, 1370, + 767, 767, 767, 1012, 4453, 1640, 4175, 3953, 1259, 4693, + 1317, 1319, 1322, 1982, 1560, 2738, 823, 824, 825, 826, + 827, 2837, 1626, 2737, 183, 223, 71, 1558, 3590, 3170, + 4037, 4164, 4565, 1336, 4069, 3140, 751, 2708, 1679, 1375, + 1065, 1685, 1066, 2758, 2759, 751, 3840, 2166, 3166, 721, + 721, 1428, 1429, 1430, 1431, 1707, 1647, 3585, 1706, 721, + 721, 1373, 71, 1722, 1722, 2322, 751, 3323, 4348, 1344, + 4313, 4340, 3442, 1643, 1642, 1625, 181, 212, 221, 213, + 1624, 4483, 3133, 2776, 3647, 1478, 1479, 3947, 773, 1751, + 740, 4326, 3650, 3898, 3952, 1306, 1763, 1724, 3164, 1903, + 211, 3500, 3954, 2707, 2729, 1720, 1720, 4108, 2686, 1583, + 3259, 240, 3260, 3261, 4020, 4361, 1600, 1570, 4579, 3904, + 721, 784, 1729, 2791, 2795, 2796, 2797, 2792, 2801, 2793, + 2799, 1225, 2312, 2794, 1871, 2800, 1426, 3137, 3138, 1359, + 1329, 1332, 1361, 2703, 2448, 1618, 3577, 1681, 3167, 4537, + 2693, 2460, 3136, 1627, 1693, 1571, 1572, 1573, 771, 1351, + 1637, 1582, 1584, 1585, 1586, 1587, 3824, 1589, 1656, 1657, + 1362, 1659, 1660, 1595, 3700, 1662, 1522, 3666, 1520, 2971, + 4676, 2824, 3383, 3384, 3671, 3672, 1684, 2766, 2784, 771, + 2724, 3678, 2575, 2445, 2310, 1811, 2287, 1567, 1423, 1422, + 1816, 3147, 3153, 3154, 3155, 3148, 3152, 3149, 3151, 3150, + 1829, 1333, 1635, 1636, 1716, 1717, 771, 1609, 1588, 4272, + 1581, 3990, 70, 4273, 71, 2791, 2795, 2796, 2797, 2792, + 2801, 2793, 2799, 1978, 1555, 2794, 1853, 2800, 3853, 2626, + 1975, 1856, 1603, 1601, 1977, 1974, 1976, 1980, 1981, 1701, + 1703, 1722, 1979, 1722, 1314, 71, 1818, 3715, 1580, 1714, + 1715, 3088, 1825, 1350, 1630, 1634, 1634, 1634, 3287, 3702, + 1671, 1672, 2693, 2696, 4360, 3444, 1594, 770, 1593, 832, + 770, 770, 71, 1655, 2085, 1785, 1658, 1788, 1789, 1355, + 1592, 1630, 1630, 4170, 1864, 3367, 3369, 1782, 1752, 1790, + 1791, 1792, 1793, 1794, 1795, 1591, 2604, 1144, 778, 1800, + 1801, 1705, 4166, 2276, 2274, 1357, 4165, 2692, 2275, 2696, + 1776, 3310, 2694, 1073, 1722, 1064, 4578, 4178, 1360, 1363, + 2471, 1130, 70, 4677, 3667, 70, 70, 1557, 1559, 3256, + 746, 1314, 1959, 1743, 1730, 3816, 1809, 1806, 1890, 70, + 1810, 1356, 1621, 1764, 2962, 1991, 1992, 1607, 2010, 1996, + 3096, 1749, 1028, 2457, 2458, 2596, 1943, 2011, 1564, 1765, + 1570, 2111, 3169, 1805, 1331, 1330, 2695, 4276, 3599, 4277, + 2018, 3905, 2020, 2702, 2021, 2022, 2023, 2700, 1963, 1964, + 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1985, + 1986, 1987, 1988, 1989, 1990, 1983, 1984, 2598, 2597, 1911, + 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, 1920, 1921, + 1922, 1887, 1865, 220, 1866, 1032, 1132, 1936, 1937, 1131, + 1030, 1029, 1358, 2112, 2697, 1314, 2693, 2696, 2084, 2692, + 2686, 2691, 1169, 2689, 2694, 4279, 1035, 2094, 3831, 1868, + 2095, 1960, 1838, 2098, 1906, 1841, 1995, 2110, 751, 751, + 751, 1821, 768, 1562, 1563, 768, 768, 2113, 2115, 2067, + 2116, 2595, 2118, 2119, 2120, 4278, 1994, 740, 1751, 2019, + 2697, 4674, 4675, 2128, 1024, 1722, 2134, 2135, 1804, 2137, + 1685, 751, 2090, 1862, 1171, 3368, 751, 2750, 2695, 1722, + 1075, 1076, 1077, 1072, 1858, 766, 2162, 1861, 766, 766, + 1857, 1886, 2070, 1025, 1881, 1130, 3089, 1211, 1212, 2009, + 4137, 1031, 1216, 769, 1722, 3185, 769, 769, 1028, 2078, + 1685, 2028, 2030, 2029, 3600, 764, 3278, 3279, 1145, 2086, + 1863, 1927, 1884, 765, 4695, 3118, 765, 765, 4247, 3288, + 3290, 3291, 3292, 3289, 3627, 2193, 2442, 2155, 4171, 4172, + 1860, 767, 1685, 1620, 767, 767, 3668, 2202, 2202, 1130, + 1685, 1839, 1685, 1685, 1842, 1843, 751, 751, 1859, 2269, + 3119, 3120, 2798, 2128, 2280, 1850, 4689, 1722, 2284, 2285, + 1620, 1027, 4547, 2300, 2802, 721, 1030, 1029, 2697, 2073, + 2488, 1847, 1848, 2692, 2686, 2691, 3722, 2689, 2694, 721, + 1132, 1722, 2515, 1131, 1304, 2514, 2723, 4683, 2027, 2681, + 2197, 1387, 2837, 1923, 1924, 2949, 2136, 1934, 1935, 2666, + 1389, 1390, 1391, 1388, 2442, 2138, 3717, 2342, 2344, 751, + 2128, 1722, 3186, 2350, 4702, 751, 751, 751, 782, 782, + 2158, 2024, 2025, 4670, 1351, 2360, 3856, 2362, 2363, 2364, + 4634, 2618, 2695, 2370, 1132, 3503, 2368, 1131, 3721, 1620, + 240, 2450, 3784, 240, 240, 3277, 240, 2224, 2442, 2074, + 3186, 4607, 2338, 2068, 2798, 2122, 2177, 2178, 2487, 2564, + 2970, 2278, 1389, 1390, 1391, 1388, 2124, 2125, 2126, 2083, + 1387, 2087, 4684, 2187, 2188, 1852, 2091, 2198, 1387, 2140, + 2141, 2142, 2143, 3683, 1851, 2205, 1351, 2803, 2330, 2331, + 1389, 1390, 1391, 1388, 2199, 1873, 4604, 2010, 2010, 2419, + 3681, 3557, 2123, 4603, 2803, 4597, 2426, 4575, 4635, 1426, + 3515, 2352, 2353, 2354, 3722, 4635, 2316, 2000, 2001, 2002, + 3504, 2159, 4272, 2186, 4530, 2307, 4432, 2309, 2378, 2163, + 2016, 2381, 2382, 2017, 2384, 2191, 4608, 3513, 2328, 2329, + 2162, 2401, 2176, 2803, 1722, 2439, 2133, 1387, 2665, 2323, + 2349, 70, 2036, 2037, 70, 70, 2204, 70, 2182, 4529, + 2149, 2617, 4504, 1630, 2970, 2301, 2181, 3386, 2183, 2184, + 1872, 770, 3501, 2954, 2388, 3075, 2420, 1634, 2206, 2207, + 2066, 4605, 2190, 1304, 2277, 2174, 2201, 2203, 2450, 1634, + 2489, 2366, 4576, 3722, 1169, 2168, 2169, 4477, 4465, 1389, + 1390, 1391, 1388, 2282, 2945, 4406, 2288, 2433, 2441, 1387, + 2306, 2441, 2308, 2302, 2317, 4405, 4376, 2679, 4375, 1675, + 1676, 2569, 1678, 4374, 2563, 1682, 70, 1686, 1687, 1688, + 4188, 4189, 4190, 4194, 4192, 4193, 4195, 4196, 4197, 4191, + 2562, 2347, 2524, 2348, 1387, 2341, 1171, 2489, 2283, 2395, + 1351, 2167, 2355, 2356, 1205, 1206, 1207, 3502, 2414, 1349, + 1736, 1737, 1738, 1739, 1740, 1741, 1742, 2523, 1744, 1745, + 1746, 1747, 1748, 2185, 2375, 1348, 1754, 2557, 1756, 1757, + 1758, 2522, 2450, 4466, 2555, 2432, 2333, 2286, 1204, 2192, + 4407, 1201, 2195, 2196, 4373, 1602, 1169, 1946, 2393, 4351, + 2644, 2489, 1884, 2489, 1389, 1390, 1391, 1388, 2489, 2468, + 2469, 1389, 1390, 1391, 1388, 4350, 1708, 4685, 4403, 2031, + 2032, 2033, 2034, 4323, 2162, 2038, 2039, 2040, 2041, 2043, + 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2051, 2052, 2053, + 1002, 1003, 1004, 1005, 2428, 2430, 1517, 4236, 1171, 1389, + 1390, 1391, 1388, 3912, 4292, 2576, 768, 2578, 3391, 2580, + 2581, 4103, 2558, 2584, 1349, 1389, 1390, 1391, 1388, 2556, + 2477, 2434, 751, 1685, 751, 1685, 3188, 4289, 3985, 2489, + 1389, 1390, 1391, 1388, 2450, 2599, 2447, 1002, 1003, 1004, + 1005, 2547, 830, 3084, 2973, 751, 751, 751, 2491, 766, + 2450, 2615, 2463, 1389, 1390, 1391, 1388, 3475, 2489, 3925, + 2472, 751, 751, 751, 751, 3007, 3008, 769, 3886, 3808, + 3804, 2972, 3001, 3474, 1927, 2465, 3691, 2010, 2010, 3464, + 2548, 2550, 2551, 2552, 2648, 2554, 2651, 765, 2561, 1387, + 2964, 2673, 2653, 2654, 2655, 2481, 2658, 1685, 3362, 1275, + 1271, 1272, 1273, 1274, 3178, 767, 3006, 2510, 3005, 3004, + 3002, 2485, 2644, 2837, 3052, 2429, 3040, 3032, 2493, 2986, + 1733, 2968, 2431, 2373, 2956, 1685, 1198, 1199, 1200, 1203, + 2951, 1202, 2358, 1392, 2088, 2155, 2936, 2934, 2932, 2930, + 2643, 1425, 2715, 2565, 3926, 2531, 1835, 2530, 2513, 2504, + 1435, 2503, 2502, 3887, 3809, 3805, 1007, 2387, 1441, 2466, + 2467, 3692, 1335, 2490, 1387, 865, 875, 2449, 2589, 1299, + 2591, 1844, 1294, 3866, 1169, 866, 1445, 867, 871, 874, + 870, 868, 869, 2803, 2476, 2475, 4234, 1026, 4041, 2952, + 2637, 3632, 2670, 3003, 3983, 2652, 2722, 2332, 2672, 2644, + 2674, 1387, 1387, 1007, 1387, 4470, 2644, 751, 2202, 2957, + 3618, 1404, 3436, 2566, 1631, 2952, 2807, 2807, 2300, 2807, + 4696, 2937, 2935, 2931, 2931, 2644, 1171, 4663, 2564, 4138, + 1387, 2579, 1387, 1387, 1387, 2583, 1387, 1387, 2414, 721, + 721, 1663, 872, 2464, 1780, 1779, 2721, 1314, 2489, 1423, + 1422, 4471, 2450, 1722, 751, 3901, 1845, 1407, 1408, 1409, + 1410, 1411, 1404, 2607, 2095, 1999, 1998, 1999, 1998, 4341, + 2675, 2667, 751, 873, 1710, 4139, 4444, 2685, 1314, 2903, + 740, 3756, 2684, 4398, 3899, 1466, 1616, 1763, 1933, 2300, + 1617, 3619, 2911, 2640, 2913, 2830, 2993, 240, 2641, 2638, + 1712, 3902, 2765, 4322, 1930, 1932, 1929, 4264, 1931, 4206, + 2907, 1713, 4168, 4167, 4040, 1169, 2811, 1405, 1406, 1407, + 1408, 1409, 1410, 1411, 1404, 2921, 2809, 2678, 2813, 2821, + 3900, 2822, 2671, 2659, 4153, 1632, 3392, 3620, 4111, 3877, + 3723, 1467, 3713, 2959, 1616, 3705, 2525, 2526, 1617, 2528, + 2827, 2828, 2966, 1033, 3693, 2439, 2535, 2698, 2699, 4342, + 2704, 3594, 1722, 3319, 1722, 3318, 1722, 1171, 2839, 2662, + 3145, 1314, 2845, 2376, 2668, 2815, 3080, 2669, 1709, 2985, + 2983, 2042, 2955, 2035, 2826, 2916, 1634, 2910, 1412, 1413, + 1414, 1415, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1404, + 2844, 1389, 1390, 1391, 1388, 4343, 2976, 2582, 70, 1722, + 1314, 2423, 3759, 2422, 3014, 2915, 2775, 2421, 2781, 1395, + 1396, 1397, 1398, 1399, 1400, 1401, 1393, 1389, 1390, 1391, + 1388, 3023, 1598, 2816, 1597, 1316, 1722, 3535, 2925, 1701, + 1703, 3538, 3009, 1389, 1390, 1391, 1388, 1389, 1390, 1391, + 1388, 1720, 3757, 1950, 3110, 2482, 2995, 2506, 1389, 1390, + 1391, 1388, 1950, 2660, 2661, 2980, 2831, 2917, 1768, 3024, + 2376, 1391, 1388, 2663, 2664, 2117, 3538, 2834, 1720, 2351, + 1389, 1390, 1391, 1388, 1389, 1390, 1391, 1388, 2904, 1769, + 4549, 2361, 1768, 4637, 4291, 1762, 2909, 4290, 3082, 1388, + 3029, 3030, 2943, 3086, 4183, 3466, 3090, 4584, 4182, 3621, + 4612, 2947, 2948, 751, 751, 751, 3248, 3022, 3246, 3535, + 1389, 1390, 1391, 1388, 3225, 3223, 3018, 2505, 4692, 2996, + 1314, 2998, 4159, 2982, 1389, 1390, 1391, 1388, 1722, 2977, + 2941, 1685, 4519, 4520, 3025, 4491, 4574, 1685, 2280, 2497, + 4573, 3012, 3537, 2991, 1389, 1390, 1391, 1388, 4378, 4379, + 2967, 2969, 4112, 4113, 4522, 3181, 3184, 2974, 3465, 1443, + 2425, 4521, 1389, 1390, 1391, 1388, 3190, 3066, 3451, 3054, + 4518, 3055, 1442, 3057, 4517, 3059, 3060, 1389, 1390, 1391, + 1388, 2987, 2988, 4691, 3200, 1389, 1390, 1391, 1388, 3062, + 3010, 3063, 1834, 4105, 1314, 4516, 3165, 4515, 2014, 4513, + 3000, 4512, 3222, 4058, 1389, 1390, 1391, 1388, 3864, 1314, + 1314, 1314, 2202, 2015, 2845, 1314, 4511, 3232, 3233, 3234, + 3235, 1314, 3242, 4510, 3243, 3244, 4509, 3245, 4508, 3247, + 1389, 1390, 1391, 1388, 3299, 3159, 1389, 1390, 1391, 1388, + 3242, 2908, 2844, 1884, 4055, 3162, 1389, 1390, 1391, 1388, + 4506, 3297, 2807, 4106, 4505, 3067, 4472, 4054, 1389, 1390, + 1391, 1388, 3201, 3295, 70, 3284, 3300, 3176, 3865, 3108, + 4364, 1389, 1390, 1391, 1388, 4354, 3141, 1896, 1897, 1898, + 1899, 1900, 3160, 721, 1389, 1390, 1391, 1388, 3217, 2224, + 2990, 2280, 4344, 4316, 3298, 1314, 2300, 2300, 2300, 2300, + 2300, 2300, 4288, 3217, 3228, 3229, 3191, 4255, 4177, 3231, + 3203, 3296, 4141, 1314, 2300, 3238, 3121, 2807, 3122, 4140, + 3124, 3917, 1947, 3294, 3305, 3283, 1951, 1952, 1953, 1954, + 3192, 3139, 3220, 3370, 1425, 1722, 3220, 1993, 3216, 3197, + 3198, 3168, 3903, 3183, 3863, 8, 2004, 7, 751, 751, + 3846, 3180, 2727, 3227, 3583, 2730, 2731, 2732, 2733, 2734, + 2735, 2736, 3432, 3404, 2739, 2740, 2741, 2742, 2743, 2744, + 2745, 2746, 2747, 2748, 2749, 3202, 2751, 2752, 2753, 2754, + 2755, 3205, 2756, 3218, 4044, 3311, 3403, 3308, 3282, 3336, + 3224, 3281, 3280, 3272, 3230, 4043, 3358, 3266, 2058, 3015, + 2060, 2061, 2062, 2063, 2064, 3265, 3264, 3336, 4042, 2071, + 3144, 1389, 1390, 1391, 1388, 3263, 3189, 3388, 3221, 2133, + 3076, 3262, 1389, 1390, 1391, 1388, 2938, 3371, 240, 3274, + 2823, 3324, 2568, 240, 2397, 1389, 1390, 1391, 1388, 2396, + 2394, 2390, 3968, 2389, 3199, 2339, 2100, 2097, 1836, 1535, + 3035, 3036, 3870, 3876, 3569, 1297, 3041, 4314, 4315, 3314, + 4688, 4686, 4030, 2010, 4661, 2010, 3320, 4627, 3425, 1389, + 1390, 1391, 1388, 4561, 3790, 3431, 4559, 4296, 3387, 4694, + 4535, 1722, 4456, 3317, 3438, 4116, 3355, 4450, 4441, 3359, + 4439, 4426, 4417, 4393, 3361, 3337, 3338, 3339, 3340, 3341, + 3342, 1389, 1390, 1391, 1388, 4392, 4383, 2486, 3375, 3378, + 2170, 4382, 3360, 4368, 1296, 1789, 3393, 4363, 3420, 3505, + 4362, 3397, 4319, 3379, 1704, 1790, 1791, 1792, 1793, 1794, + 1795, 4303, 3476, 4301, 1800, 1801, 2189, 3215, 3193, 70, + 4287, 4256, 4161, 3196, 70, 3470, 1389, 1390, 1391, 1388, + 4503, 4120, 4109, 2070, 3469, 4093, 4092, 4090, 3424, 1389, + 1390, 1391, 1388, 4648, 4085, 1809, 1806, 4083, 4062, 1810, + 4061, 4060, 1389, 1390, 1391, 1388, 4057, 4056, 4032, 3422, + 4028, 1389, 1390, 1391, 1388, 1389, 1390, 1391, 1388, 4026, + 3996, 3993, 3521, 3987, 3304, 3524, 3395, 3858, 3394, 3848, + 3528, 2071, 751, 1685, 3833, 3817, 2071, 2071, 3467, 3440, + 3796, 3540, 3542, 3543, 3545, 3051, 3547, 3548, 3794, 3788, + 3435, 3413, 3773, 3416, 3421, 3418, 3050, 3423, 1314, 2484, + 3409, 4649, 3734, 3711, 1314, 1389, 1390, 1391, 1388, 3710, + 3572, 3574, 1389, 1390, 1391, 1388, 3708, 3707, 3694, 3434, + 3689, 3587, 3447, 1389, 1390, 1391, 1388, 751, 2377, 3448, + 3688, 2380, 3049, 3463, 2383, 3595, 3555, 2385, 3454, 3455, + 3549, 3457, 3602, 3048, 3606, 1314, 3459, 3460, 751, 3539, + 751, 2280, 1314, 1314, 4489, 3456, 3529, 3458, 3522, 1389, + 1390, 1391, 1388, 3520, 2573, 3446, 2300, 2648, 3443, 3631, + 1389, 1390, 1391, 1388, 3430, 3402, 3377, 1389, 1390, 1391, + 1388, 3312, 2407, 3514, 3309, 3306, 3293, 3285, 2715, 3275, + 3273, 3598, 3558, 3047, 2298, 3269, 3268, 3267, 3217, 3111, + 3656, 3097, 3659, 3609, 3659, 3659, 3085, 3129, 3077, 1314, + 3615, 3519, 3591, 3518, 2963, 3623, 2942, 3601, 3531, 2905, + 1389, 1390, 1391, 1388, 944, 943, 4485, 3684, 2600, 2587, + 2586, 3680, 3046, 2400, 2392, 1722, 1722, 3159, 2200, 3217, + 2130, 3639, 1169, 2099, 2096, 2081, 3217, 3217, 2080, 1837, + 1474, 3643, 3645, 3634, 1470, 3575, 3045, 1469, 3162, 1389, + 1390, 1391, 1388, 1300, 3629, 1011, 750, 1982, 3685, 3686, + 4293, 753, 4283, 4282, 4269, 3624, 4265, 1720, 1720, 4091, + 4059, 4038, 751, 1389, 1390, 1391, 1388, 4007, 3597, 3988, + 183, 223, 3608, 3894, 1171, 3893, 3572, 3890, 3855, 3613, + 3614, 3813, 3630, 3217, 3811, 3810, 2478, 3807, 3806, 1685, + 2483, 3626, 2280, 2280, 3655, 3254, 3255, 3044, 2492, 3795, + 3654, 3664, 3043, 183, 223, 3638, 3793, 3777, 2685, 3762, + 3270, 3271, 3042, 2684, 3761, 3746, 3745, 3660, 3661, 3625, + 3637, 3039, 3559, 2157, 1389, 1390, 1391, 1388, 3556, 1389, + 1390, 1391, 1388, 3665, 3038, 3512, 3472, 2501, 3682, 1389, + 1390, 1391, 1388, 3315, 219, 2508, 3461, 1314, 1389, 1390, + 1391, 1388, 3014, 2154, 3453, 3452, 3450, 750, 3690, 3385, + 3760, 1389, 1390, 1391, 1388, 3698, 223, 182, 214, 184, + 3037, 2933, 2929, 2527, 2928, 877, 155, 2156, 2532, 2533, + 2534, 155, 2927, 2537, 2538, 2539, 2540, 2541, 2542, 2543, + 2544, 2545, 2546, 2536, 183, 223, 3695, 1389, 1390, 1391, + 1388, 3481, 3482, 3662, 3704, 751, 3703, 3483, 3484, 3485, + 3486, 2529, 3487, 3488, 3489, 3490, 3491, 3492, 3493, 3494, + 3495, 3496, 3497, 3712, 3709, 753, 2521, 3031, 3718, 3719, + 3730, 3706, 3731, 2520, 3716, 2519, 3019, 183, 223, 219, + 4611, 2845, 183, 223, 3622, 2518, 747, 2516, 2512, 2511, + 2509, 3739, 2500, 155, 1389, 1390, 1391, 1388, 2496, 2495, + 3742, 3743, 3744, 1389, 1390, 1391, 1388, 1978, 219, 2844, + 3013, 2399, 3749, 2059, 1975, 2057, 2056, 2055, 1977, 1974, + 1976, 1980, 1981, 2054, 3819, 2013, 1979, 3419, 3820, 2012, + 2370, 2003, 153, 3770, 1734, 3778, 1732, 1389, 1390, 1391, + 1388, 2992, 3834, 223, 3836, 4528, 183, 223, 4490, 3842, + 1464, 219, 4484, 3797, 4412, 4409, 219, 3781, 3633, 4391, + 3830, 4372, 4365, 3635, 3636, 2560, 1827, 4250, 1389, 1390, + 1391, 1388, 3780, 4249, 4201, 3843, 3786, 1402, 1412, 1413, + 1414, 1415, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1404, + 751, 2280, 1389, 1390, 1391, 1388, 1824, 4181, 3837, 4179, + 3839, 4174, 4501, 2559, 3885, 4152, 4135, 3799, 3825, 3801, + 4008, 3803, 4005, 3892, 3966, 3965, 219, 3854, 2553, 3962, + 1826, 3961, 3924, 3921, 3857, 3919, 3879, 2807, 2300, 3909, + 1389, 1390, 1391, 1388, 3832, 1170, 3828, 3552, 3462, 3818, + 155, 1784, 1799, 3814, 1786, 1389, 1390, 1391, 1388, 3822, + 3698, 3927, 1805, 1808, 1314, 155, 1796, 155, 1781, 1605, + 3347, 3874, 3307, 3656, 3301, 3226, 3172, 1314, 3851, 3171, + 3163, 3847, 3123, 1985, 1986, 1987, 1988, 1989, 1990, 1983, + 1984, 3852, 1314, 1945, 3982, 3053, 3871, 2071, 1722, 2071, + 2950, 2825, 3883, 2757, 3977, 3978, 3979, 2642, 2609, 2608, + 3720, 2567, 3891, 3991, 1928, 219, 2357, 3911, 2071, 2071, + 1389, 1390, 1391, 1388, 2077, 1869, 751, 1828, 2280, 1797, + 1534, 3984, 2300, 1314, 3738, 3960, 3908, 3873, 1519, 1515, + 1720, 1514, 1513, 1512, 1511, 1510, 3907, 1509, 3906, 1508, + 1507, 1506, 1505, 1504, 1503, 1502, 1762, 1501, 3928, 3918, + 1500, 3920, 4014, 1499, 3951, 3914, 1498, 1497, 240, 1496, + 1495, 3970, 1494, 1493, 1492, 1491, 1490, 1489, 1488, 1487, + 1486, 1485, 3972, 1484, 1483, 4000, 3238, 3967, 1482, 3969, + 1481, 1480, 1477, 1476, 3997, 3981, 1475, 1473, 1472, 4013, + 1471, 1468, 1461, 1460, 1458, 3986, 1457, 1456, 1455, 1454, + 1453, 2958, 1452, 2961, 1451, 3992, 3995, 1450, 3994, 3989, + 1449, 1448, 3998, 1447, 1446, 1440, 4003, 3336, 4002, 1439, + 4001, 1438, 1437, 1436, 1353, 3999, 1298, 3726, 3727, 2162, + 4499, 4497, 4075, 3963, 2657, 4036, 4081, 2624, 1341, 4641, + 4639, 4593, 4087, 3729, 3701, 3313, 4021, 3146, 2838, 2636, + 1614, 1352, 3345, 3357, 3352, 3350, 3348, 1314, 4031, 3353, + 3351, 3349, 2994, 3344, 3737, 2997, 4010, 3736, 3735, 70, + 3732, 3356, 3343, 4548, 4428, 4157, 4011, 3016, 3017, 138, + 1314, 1722, 1722, 73, 72, 4121, 3020, 3021, 3606, 3179, + 2953, 1599, 4084, 3593, 4086, 3415, 4071, 69, 2151, 2152, + 2725, 3973, 3026, 3027, 3028, 1314, 3750, 4129, 2146, 2147, + 2148, 4129, 2261, 3652, 4118, 3653, 4065, 3775, 3776, 1777, + 1314, 4146, 1314, 1720, 1943, 3177, 4009, 4123, 4124, 1820, + 4078, 2947, 2948, 4149, 2981, 4151, 3056, 4098, 3058, 1722, + 2594, 3061, 2593, 1896, 2071, 4100, 4117, 4126, 4099, 3250, + 742, 3217, 1817, 4119, 743, 744, 3251, 3252, 3253, 2601, + 751, 4110, 1314, 1314, 1307, 2359, 1314, 1314, 745, 1312, + 2273, 1347, 4369, 4122, 4089, 4134, 4095, 3567, 4133, 3560, + 3204, 1943, 3173, 2677, 4203, 3911, 2420, 2634, 4142, 4145, + 2160, 4235, 4155, 1342, 4205, 3910, 4198, 2121, 4652, 3336, + 4367, 3960, 3687, 3913, 4158, 2778, 2162, 1999, 1998, 4242, + 4162, 1530, 1531, 2771, 1890, 2281, 1890, 1528, 1529, 1526, + 1527, 1524, 1525, 4251, 4252, 1674, 1673, 1380, 4185, 4186, + 3951, 4154, 4199, 4200, 2424, 3748, 3741, 1722, 2602, 2427, + 2165, 4160, 1623, 1622, 1590, 3194, 3195, 1646, 4102, 2979, + 2650, 4618, 4616, 4238, 4567, 4545, 4544, 4101, 2978, 4542, + 4237, 4460, 4413, 4245, 4284, 4285, 4244, 751, 4147, 4027, + 4263, 3798, 3769, 4240, 4275, 3768, 3754, 2404, 4204, 1720, + 2710, 4297, 2680, 4299, 1822, 3753, 3390, 1620, 4643, 4642, + 4622, 4082, 3835, 3821, 3433, 4258, 3092, 4262, 3091, 3083, + 2906, 2498, 1337, 802, 801, 808, 798, 4643, 1311, 4300, + 4270, 4302, 4274, 4642, 4176, 4012, 805, 806, 4097, 807, + 811, 3896, 3412, 792, 1002, 1003, 1004, 1005, 2628, 1304, + 1813, 1304, 4331, 816, 1638, 81, 4336, 2, 4665, 4666, + 1, 3068, 4305, 2075, 1532, 4329, 1006, 1001, 4306, 4016, + 1698, 2817, 2334, 1314, 1726, 2079, 1008, 3363, 3364, 3740, + 4280, 4281, 3366, 2340, 1608, 3098, 4353, 4324, 2446, 4033, + 3325, 2769, 2613, 4359, 3586, 155, 155, 155, 1170, 1606, + 4318, 1074, 4330, 2005, 4327, 4036, 1849, 1328, 1846, 1327, + 4333, 4332, 1325, 4053, 1948, 2071, 2026, 4349, 4345, 879, + 2410, 3302, 1314, 4045, 3276, 4046, 4241, 4651, 4680, 4610, + 4654, 3880, 3881, 3882, 1867, 863, 4536, 3771, 3410, 3888, + 3889, 4418, 4614, 4420, 4261, 4366, 2451, 4077, 1385, 3417, + 1100, 923, 891, 1459, 1722, 1823, 3479, 4404, 3477, 890, + 3868, 3135, 4246, 3382, 4338, 1101, 2386, 1890, 4415, 4259, + 1778, 1783, 2676, 4346, 4480, 4156, 3648, 1424, 3212, 1812, + 4475, 3922, 4049, 4047, 4048, 790, 2313, 4401, 4377, 719, + 1154, 4202, 2635, 4143, 4144, 2656, 1720, 4207, 4371, 1048, + 3849, 2623, 1049, 750, 1041, 3157, 3156, 1907, 1394, 1926, + 3498, 4440, 3499, 1434, 834, 2480, 4434, 3132, 3945, 4445, + 3376, 3396, 80, 3398, 79, 78, 4414, 77, 4452, 1403, + 1402, 1412, 1413, 1414, 1415, 1405, 1406, 1407, 1408, 1409, + 1410, 1411, 1404, 248, 882, 2407, 247, 4447, 4294, 4448, + 2071, 4114, 4531, 4656, 860, 2071, 859, 858, 857, 856, + 2517, 855, 2789, 4457, 2790, 4461, 2788, 2786, 2785, 2295, + 1677, 2294, 793, 795, 794, 3389, 3752, 2365, 4449, 1691, + 2367, 3604, 3241, 3974, 800, 3236, 2213, 4239, 4455, 2211, + 1689, 2705, 2712, 1314, 4479, 3449, 804, 4463, 4464, 2210, + 1728, 4590, 3787, 819, 4039, 4507, 4492, 4212, 4493, 4173, + 797, 3286, 1314, 4496, 4498, 4500, 4502, 4035, 4473, 4478, + 2145, 2701, 3471, 1722, 4524, 4514, 2230, 3257, 4525, 2227, + 4487, 2226, 3249, 4532, 4169, 4163, 1523, 2258, 4334, 4128, + 3929, 3930, 3936, 1256, 2633, 1226, 4533, 1221, 1223, 1224, + 1222, 2999, 3714, 2682, 3562, 3117, 4523, 3116, 3114, 4495, + 3113, 1574, 4451, 4563, 4094, 1720, 2843, 4560, 2841, 1295, + 3728, 3724, 3532, 1540, 1538, 4534, 2418, 3733, 4541, 4539, + 3346, 2405, 3414, 1722, 4553, 4555, 2296, 4336, 4557, 2292, + 2291, 4211, 1196, 1195, 1759, 3826, 4562, 4554, 4556, 4558, + 48, 3327, 2779, 4577, 4308, 2150, 1890, 1042, 2621, 4585, + 117, 1417, 42, 1421, 133, 4568, 4569, 116, 201, 4570, + 63, 200, 62, 18, 131, 1720, 4571, 4572, 198, 1418, + 1420, 1416, 61, 1419, 1403, 1402, 1412, 1413, 1414, 1415, + 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1404, 47, 4598, + 46, 4599, 196, 4600, 4602, 4601, 111, 4606, 110, 799, + 803, 809, 109, 810, 812, 2494, 108, 813, 814, 815, + 130, 195, 60, 817, 818, 4617, 232, 4619, 4620, 231, + 234, 233, 230, 4609, 2918, 2919, 229, 4615, 1314, 4613, + 1766, 228, 4546, 4132, 4434, 4527, 4623, 996, 4624, 45, + 4625, 44, 202, 43, 1731, 118, 64, 4626, 747, 41, + 4630, 40, 2649, 3550, 4359, 2164, 4233, 4633, 4632, 4631, + 4636, 3844, 3109, 2605, 39, 35, 3663, 4640, 13, 4650, + 4638, 12, 4658, 36, 23, 4657, 22, 1854, 4644, 4645, + 4646, 4647, 21, 27, 33, 32, 155, 148, 147, 31, + 1314, 4662, 146, 145, 144, 143, 142, 141, 140, 30, + 20, 55, 4669, 4668, 4410, 4411, 4671, 4672, 54, 4479, + 53, 4678, 52, 51, 4682, 50, 9, 4679, 136, 134, + 4208, 129, 4628, 127, 29, 128, 125, 126, 121, 120, + 119, 114, 112, 92, 91, 4690, 90, 105, 104, 103, + 102, 101, 100, 98, 99, 4658, 4698, 3697, 4657, 4697, + 1099, 89, 88, 87, 86, 85, 122, 4682, 4699, 107, + 115, 113, 96, 4703, 106, 97, 95, 94, 93, 84, + 83, 82, 124, 796, 123, 135, 203, 65, 180, 179, + 178, 155, 177, 176, 1890, 174, 175, 173, 172, 171, + 170, 169, 168, 56, 57, 58, 155, 59, 191, 155, + 155, 190, 192, 194, 197, 193, 199, 188, 186, 189, + 187, 185, 74, 155, 11, 132, 19, 4, 0, 0, + 0, 4213, 4214, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2105, 2106, 2107, 0, 0, 4209, 4210, 0, + 4217, 4216, 4215, 4228, 4229, 4230, 4218, 4219, 4222, 4224, + 4223, 4220, 4221, 4225, 4226, 4227, 0, 0, 0, 0, + 4231, 0, 0, 0, 0, 2139, 0, 0, 0, 0, + 2144, 4232, 0, 0, 4355, 0, 0, 0, 0, 0, + 0, 4380, 4381, 0, 0, 0, 0, 0, 4385, 4386, + 4387, 4388, 4389, 4390, 0, 0, 0, 4394, 4395, 4396, + 4397, 0, 0, 0, 4399, 4400, 3789, 4402, 1249, 0, + 0, 0, 0, 0, 3791, 3792, 0, 0, 0, 0, + 1424, 0, 0, 0, 0, 1065, 0, 1066, 1403, 1402, + 1412, 1413, 1414, 1415, 1405, 1406, 1407, 1408, 1409, 1410, + 1411, 1404, 3800, 0, 3802, 0, 0, 0, 0, 0, + 2208, 2209, 0, 3812, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1046, 0, 4150, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1060, 0, 1056, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 3697, 0, 0, 0, 0, 4148, 0, 0, + 0, 0, 0, 4462, 0, 0, 0, 0, 0, 4467, + 4468, 0, 0, 2346, 0, 0, 0, 0, 0, 2346, + 2346, 2346, 1403, 1402, 1412, 1413, 1414, 1415, 1405, 1406, + 1407, 1408, 1409, 1410, 1411, 1404, 0, 0, 0, 0, + 4488, 0, 1267, 1268, 1230, 0, 0, 0, 0, 0, + 1037, 1403, 1402, 1412, 1413, 1414, 1415, 1405, 1406, 1407, + 1408, 1409, 1410, 1411, 1404, 1257, 1261, 1263, 1265, 1270, + 0, 1275, 1271, 1272, 1273, 1274, 155, 0, 1252, 1253, + 1254, 1255, 1228, 1229, 1258, 0, 1231, 0, 1233, 1234, + 1235, 1236, 1232, 1237, 1238, 1239, 1240, 1241, 1248, 1250, + 1242, 1243, 1244, 1245, 1246, 1247, 1276, 1277, 1278, 1279, + 1280, 1281, 1282, 1283, 1285, 1284, 1286, 1287, 1288, 1289, + 1290, 1291, 1292, 1293, 1260, 1262, 1264, 1266, 1269, 0, + 0, 0, 0, 183, 223, 182, 214, 184, 0, 1062, + 0, 1055, 0, 0, 0, 0, 2071, 0, 0, 0, + 1059, 1058, 0, 215, 0, 0, 0, 1192, 0, 0, + 206, 0, 0, 2071, 216, 1251, 4004, 0, 2299, 4006, + 0, 1047, 183, 223, 182, 214, 184, 0, 0, 0, + 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, + 0, 1054, 215, 4015, 0, 0, 0, 0, 139, 206, + 0, 0, 0, 216, 0, 0, 0, 219, 0, 0, + 1064, 0, 0, 0, 0, 1053, 0, 3473, 0, 1052, + 0, 0, 153, 0, 0, 1040, 0, 0, 0, 0, + 0, 0, 0, 0, 1417, 0, 1421, 139, 0, 0, + 0, 1193, 0, 0, 1045, 155, 219, 0, 155, 155, + 0, 155, 1418, 1420, 1416, 0, 1419, 1403, 1402, 1412, + 1413, 1414, 1415, 1405, 1406, 1407, 1408, 1409, 1410, 1411, + 1404, 1403, 1402, 1412, 1413, 1414, 1415, 1405, 1406, 1407, + 1408, 1409, 1410, 1411, 1404, 0, 0, 0, 0, 0, + 1043, 0, 0, 0, 0, 0, 162, 163, 0, 164, + 165, 0, 1170, 0, 166, 0, 0, 167, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 155, 0, 0, 0, 0, 0, 0, 0, 0, 1063, + 1186, 1181, 1176, 1180, 1184, 162, 163, 0, 164, 165, + 0, 0, 0, 166, 0, 0, 167, 0, 0, 0, + 0, 0, 1044, 0, 0, 0, 0, 0, 1189, 0, + 0, 0, 1179, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2588, 0, 2590, 0, + 0, 0, 0, 181, 212, 221, 213, 75, 137, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, + 2611, 2612, 0, 0, 1424, 0, 0, 211, 205, 204, + 0, 0, 0, 1187, 76, 2629, 2630, 2631, 2632, 0, + 0, 0, 181, 212, 221, 213, 75, 137, 0, 0, + 0, 0, 161, 1061, 0, 1190, 0, 0, 0, 0, + 0, 0, 1191, 0, 0, 0, 211, 205, 204, 0, + 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 3468, 0, 1088, 0, 0, 0, 0, + 0, 161, 0, 1050, 0, 207, 208, 209, 0, 0, + 0, 1177, 1039, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1259, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1188, 0, 0, 0, 0, + 0, 0, 0, 0, 207, 208, 209, 1403, 1402, 1412, + 1413, 1414, 1415, 1405, 1406, 1407, 1408, 1409, 1410, 1411, + 1404, 0, 0, 0, 0, 0, 0, 1084, 1085, 0, + 0, 0, 0, 1178, 0, 217, 0, 0, 1130, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1691, 0, 0, 0, 0, 149, 0, 0, 0, + 210, 0, 150, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 217, 0, 0, 0, 0, 1038, + 0, 0, 0, 1036, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 149, 0, 0, 1728, 210, + 0, 150, 0, 0, 0, 0, 1225, 0, 0, 0, + 0, 4370, 0, 0, 1185, 0, 2346, 151, 802, 801, + 808, 798, 1170, 0, 155, 0, 0, 0, 0, 0, + 68, 805, 806, 1132, 807, 811, 1131, 0, 792, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 816, 0, + 0, 1182, 0, 0, 1183, 0, 151, 0, 0, 0, + 0, 0, 0, 1175, 0, 0, 0, 0, 0, 68, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 71, 0, 0, 0, 0, 0, 1116, + 0, 0, 0, 0, 820, 0, 0, 822, 0, 1089, + 0, 0, 821, 0, 0, 0, 0, 0, 0, 1445, + 0, 2989, 0, 0, 0, 0, 0, 0, 0, 159, + 220, 160, 71, 0, 0, 0, 1091, 0, 0, 0, + 0, 0, 0, 0, 66, 1403, 1402, 1412, 1413, 1414, + 1415, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1404, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 159, 220, + 160, 0, 0, 2810, 802, 801, 808, 798, 0, 0, + 1194, 0, 0, 66, 1174, 0, 0, 805, 806, 0, + 807, 811, 0, 0, 792, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 816, 4486, 0, 0, 0, 0, + 0, 0, 0, 1112, 0, 1114, 1111, 0, 0, 0, + 1115, 0, 0, 0, 152, 49, 0, 0, 0, 0, + 0, 67, 0, 0, 0, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 2299, 0, 0, 0, 0, 0, + 820, 0, 155, 822, 0, 156, 157, 0, 821, 158, + 1110, 0, 0, 152, 49, 0, 0, 3093, 3094, 3095, + 67, 0, 1083, 0, 0, 0, 0, 793, 795, 794, + 0, 0, 0, 1090, 1125, 0, 0, 0, 0, 800, + 0, 0, 0, 0, 156, 157, 0, 0, 158, 0, + 0, 804, 0, 0, 0, 1121, 0, 0, 819, 0, + 0, 0, 0, 0, 0, 797, 0, 0, 0, 787, + 3182, 0, 0, 0, 4582, 0, 0, 0, 0, 0, + 4586, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2499, 1122, 1126, 1403, 1402, 1412, 1413, 1414, 1415, 1405, + 1406, 1407, 1408, 1409, 1410, 1411, 1404, 0, 0, 0, + 0, 1107, 0, 1105, 1109, 1129, 0, 0, 0, 1106, + 1103, 1102, 0, 1108, 1093, 1094, 1092, 2473, 1082, 1095, + 1096, 1097, 1098, 1079, 0, 0, 1127, 0, 1128, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, + 1124, 1403, 1402, 1412, 1413, 1414, 1415, 1405, 1406, 1407, + 1408, 1409, 1410, 1411, 1404, 0, 0, 0, 0, 4582, + 0, 0, 0, 793, 795, 794, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 800, 0, 1119, 0, 0, + 0, 0, 0, 1118, 0, 2259, 0, 804, 0, 0, + 2220, 1080, 0, 2267, 819, 0, 0, 0, 0, 0, + 1113, 797, 0, 0, 799, 803, 809, 0, 810, 812, + 0, 0, 813, 814, 815, 4582, 0, 0, 817, 818, + 0, 0, 0, 2261, 2229, 0, 0, 0, 0, 0, + 0, 0, 0, 2262, 2263, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, + 0, 0, 3380, 3381, 0, 0, 0, 0, 155, 2228, + 1403, 1402, 1412, 1413, 1414, 1415, 1405, 1406, 1407, 1408, + 1409, 1410, 1411, 1404, 0, 0, 4701, 2236, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1117, 0, 0, 0, 0, 0, 1086, 1087, + 0, 0, 1078, 0, 0, 0, 0, 1081, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 799, 803, 809, 0, 810, 812, 0, 2252, 813, 814, + 815, 0, 0, 0, 817, 818, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 796, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2299, 2299, 2299, 2299, 2299, 2299, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2299, + 0, 0, 0, 0, 0, 0, 823, 824, 825, 826, + 827, 0, 0, 0, 2259, 0, 0, 0, 0, 2220, + 0, 0, 2267, 0, 2219, 2221, 2218, 0, 0, 0, + 2215, 0, 0, 0, 0, 2240, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 2246, 0, 0, 0, + 0, 0, 2261, 2229, 2231, 0, 2214, 0, 0, 0, + 0, 0, 2262, 2263, 0, 0, 2234, 2268, 0, 0, + 2235, 2237, 2239, 0, 2241, 2242, 2243, 2247, 2248, 2249, + 2251, 2254, 2255, 2256, 0, 0, 3530, 0, 2228, 0, + 0, 2244, 2253, 2245, 1389, 1390, 1391, 1388, 0, 0, + 0, 0, 0, 2223, 796, 0, 2236, 0, 0, 0, + 0, 0, 0, 155, 0, 0, 0, 0, 155, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3596, 0, 0, 0, 2260, 0, 155, 0, 0, + 0, 0, 823, 824, 825, 826, 827, 0, 0, 0, + 0, 0, 3610, 0, 3611, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1982, 2252, 0, 0, 0, + 0, 2216, 2217, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2259, 0, 0, 0, 2257, + 0, 0, 0, 183, 223, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2233, 0, 0, + 0, 2232, 0, 0, 0, 0, 0, 4127, 0, 0, + 0, 0, 0, 2261, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2250, 0, 0, 0, 0, + 0, 0, 0, 0, 2238, 0, 0, 0, 0, 0, + 0, 0, 0, 2219, 3207, 2218, 0, 2265, 2264, 3206, + 0, 0, 0, 0, 2240, 0, 0, 219, 0, 0, + 0, 0, 0, 0, 0, 2246, 2346, 2236, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2259, + 0, 0, 0, 0, 0, 2234, 2268, 0, 0, 2235, + 2237, 2239, 0, 2241, 2242, 2243, 2247, 2248, 2249, 2251, + 2254, 2255, 2256, 0, 2225, 0, 0, 0, 0, 0, + 2244, 2253, 2245, 0, 0, 0, 0, 2261, 0, 0, + 0, 0, 2223, 0, 0, 0, 0, 0, 0, 0, + 1170, 0, 155, 0, 0, 0, 0, 0, 0, 155, + 0, 0, 0, 0, 155, 0, 0, 2252, 0, 0, + 0, 2299, 2266, 0, 0, 0, 0, 0, 0, 0, + 0, 4358, 0, 0, 2260, 0, 0, 0, 0, 0, + 155, 2236, 0, 0, 0, 1978, 0, 0, 0, 0, + 0, 0, 1975, 0, 0, 0, 1977, 1974, 1976, 1980, + 1981, 0, 0, 0, 1979, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 3783, + 2216, 2217, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2257, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2240, 2233, 0, 0, 0, + 2232, 2252, 0, 0, 0, 0, 2246, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 3699, 2250, 0, 2234, 2268, 0, 0, + 2235, 2237, 2239, 2238, 2241, 2242, 2243, 2247, 2248, 2249, + 2251, 2254, 2255, 2256, 0, 0, 2265, 2264, 0, 0, + 0, 2244, 2253, 2245, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, + 1973, 1985, 1986, 1987, 1988, 1989, 1990, 1983, 1984, 2240, + 0, 0, 0, 2225, 2346, 2260, 0, 0, 0, 0, + 2246, 0, 2259, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2234, 2268, 0, 0, 2235, 2237, 2239, 0, 2241, 2242, + 2243, 2247, 2248, 2249, 2251, 2254, 2255, 2256, 0, 0, + 2261, 2266, 0, 0, 0, 2244, 2253, 2245, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2257, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2233, 0, 0, + 0, 2232, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2236, 0, 0, 0, 0, 2260, + 0, 0, 0, 0, 0, 2250, 0, 0, 0, 0, + 0, 0, 0, 0, 2238, 0, 0, 0, 0, 0, + 2346, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 3699, 0, + 0, 0, 0, 2257, 0, 0, 155, 0, 0, 0, + 4328, 0, 0, 155, 2252, 0, 0, 0, 0, 0, + 0, 2233, 0, 0, 0, 2232, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2250, + 0, 0, 0, 0, 0, 0, 0, 0, 2238, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2299, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2240, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2246, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2234, 2268, 0, 0, 2235, 2237, 2239, + 0, 2241, 2242, 2243, 2247, 2248, 2249, 2251, 2254, 2255, + 2256, 0, 0, 0, 0, 0, 0, 2299, 2244, 2253, + 2245, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 155, 4184, 898, 0, 0, 0, 0, + 0, 0, 0, 0, 455, 0, 0, 594, 628, 617, + 702, 582, 2260, 0, 0, 0, 0, 0, 849, 0, + 0, 0, 367, 0, 0, 423, 632, 613, 624, 614, + 599, 600, 601, 608, 379, 602, 603, 604, 574, 605, + 575, 606, 607, 889, 631, 581, 493, 439, 0, 648, + 0, 0, 967, 975, 0, 0, 0, 0, 0, 0, + 0, 0, 963, 0, 0, 0, 0, 841, 3699, 0, + 878, 944, 943, 865, 875, 0, 2257, 335, 246, 576, + 698, 578, 577, 866, 0, 867, 871, 874, 870, 868, + 869, 0, 958, 0, 2233, 0, 0, 0, 2232, 833, + 845, 4286, 850, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 2250, 0, 0, 0, 155, 0, 842, 843, + 0, 2238, 0, 0, 899, 0, 844, 0, 0, 0, + 0, 0, 494, 523, 0, 536, 0, 404, 405, 894, + 872, 876, 0, 0, 0, 0, 322, 501, 520, 336, + 488, 534, 341, 496, 513, 331, 454, 485, 0, 0, + 324, 518, 495, 436, 323, 0, 479, 364, 381, 361, + 452, 873, 0, 897, 901, 360, 981, 895, 528, 326, + 0, 527, 451, 514, 519, 437, 430, 0, 325, 516, + 435, 429, 410, 371, 982, 411, 412, 413, 414, 415, + 416, 385, 466, 427, 467, 386, 441, 440, 442, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 0, 0, 0, 0, 558, 559, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 691, 892, 0, 695, 0, 530, 0, 0, 965, + 0, 0, 0, 499, 0, 0, 417, 0, 0, 0, + 896, 0, 482, 457, 978, 0, 0, 480, 425, 515, + 468, 521, 502, 529, 474, 469, 316, 503, 363, 438, + 332, 334, 723, 365, 368, 372, 373, 447, 448, 462, + 487, 506, 507, 508, 362, 346, 481, 347, 382, 348, + 317, 354, 352, 355, 489, 356, 319, 463, 512, 0, + 378, 477, 433, 320, 432, 464, 511, 510, 333, 538, + 545, 546, 636, 0, 551, 734, 735, 736, 560, 0, + 470, 329, 328, 0, 0, 0, 358, 465, 342, 344, + 345, 343, 460, 461, 565, 566, 567, 569, 0, 570, + 571, 0, 0, 155, 0, 572, 637, 653, 621, 590, + 553, 645, 587, 591, 592, 399, 400, 401, 656, 2007, + 2006, 2008, 544, 418, 419, 0, 370, 369, 434, 321, + 0, 0, 407, 398, 471, 327, 366, 409, 403, 420, + 421, 422, 376, 311, 312, 729, 962, 453, 658, 693, + 694, 583, 0, 977, 957, 959, 960, 964, 968, 969, + 970, 971, 972, 974, 976, 980, 728, 0, 638, 652, + 732, 651, 725, 459, 0, 486, 649, 596, 0, 642, + 615, 616, 0, 643, 611, 647, 0, 585, 0, 554, + 557, 586, 671, 672, 673, 318, 556, 675, 676, 677, + 678, 679, 680, 681, 674, 979, 619, 595, 622, 535, + 598, 597, 0, 0, 633, 900, 634, 635, 443, 444, + 445, 446, 966, 659, 340, 555, 473, 0, 620, 0, + 0, 0, 0, 0, 0, 0, 0, 625, 626, 623, + 737, 0, 682, 683, 0, 0, 549, 550, 375, 0, + 568, 383, 339, 458, 377, 533, 406, 0, 561, 627, + 562, 475, 476, 685, 690, 686, 687, 689, 709, 450, + 397, 402, 490, 408, 426, 478, 532, 456, 483, 337, + 522, 492, 431, 612, 640, 988, 961, 987, 989, 990, + 986, 991, 992, 973, 854, 0, 907, 908, 984, 983, + 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 667, 666, 665, 664, 663, 662, 661, 660, + 0, 0, 609, 509, 353, 305, 349, 350, 357, 726, + 722, 727, 710, 713, 712, 688, 861, 313, 589, 424, + 472, 374, 654, 655, 0, 708, 951, 916, 917, 918, + 851, 919, 913, 914, 852, 915, 952, 905, 948, 949, + 880, 910, 920, 947, 921, 950, 881, 953, 993, 994, + 927, 911, 275, 995, 924, 954, 946, 945, 922, 906, + 955, 956, 888, 883, 925, 926, 912, 931, 932, 933, + 936, 853, 937, 938, 939, 940, 941, 935, 934, 902, + 903, 904, 928, 929, 909, 500, 884, 885, 886, 887, + 0, 0, 539, 540, 541, 564, 0, 542, 524, 588, + 384, 314, 504, 531, 724, 0, 0, 0, 0, 0, + 0, 0, 639, 650, 684, 0, 696, 697, 699, 701, + 942, 703, 497, 498, 711, 0, 0, 930, 706, 707, + 704, 428, 484, 505, 491, 0, 730, 579, 580, 731, + 692, 315, 0, 846, 183, 223, 898, 0, 0, 0, + 0, 0, 0, 0, 0, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 849, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 889, 631, 581, 493, 439, 0, + 648, 0, 0, 967, 975, 0, 0, 0, 0, 0, + 0, 0, 0, 963, 0, 0, 0, 0, 841, 0, + 0, 878, 944, 943, 865, 875, 0, 0, 335, 246, + 576, 698, 578, 577, 866, 0, 867, 871, 874, 870, + 868, 869, 0, 958, 0, 0, 0, 0, 0, 0, + 833, 845, 0, 850, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, + 843, 0, 0, 0, 0, 899, 0, 844, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 894, 872, 876, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 873, 0, 897, 901, 360, 981, 895, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 982, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 0, 0, 0, 0, 0, 554, 555, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 687, 888, 0, 691, 0, 526, 0, 0, - 961, 0, 0, 155, 495, 0, 4270, 413, 0, 0, - 0, 892, 0, 478, 453, 974, 0, 0, 476, 421, - 511, 464, 517, 498, 525, 470, 465, 316, 499, 363, - 434, 332, 334, 719, 365, 368, 372, 373, 443, 444, - 458, 483, 502, 503, 504, 362, 346, 477, 347, 382, - 348, 317, 354, 352, 355, 485, 356, 319, 459, 508, - 0, 378, 473, 429, 320, 428, 460, 507, 506, 333, - 534, 541, 542, 632, 0, 547, 730, 731, 732, 556, - 0, 466, 329, 328, 0, 0, 0, 358, 461, 342, - 344, 345, 343, 456, 457, 561, 562, 563, 565, 0, - 566, 567, 0, 0, 0, 0, 568, 633, 649, 617, - 586, 549, 641, 583, 587, 588, 399, 400, 401, 652, - 1995, 1994, 1996, 540, 414, 415, 0, 370, 369, 430, - 321, 0, 0, 407, 398, 467, 327, 366, 409, 403, - 416, 417, 418, 376, 311, 312, 725, 958, 449, 654, - 689, 690, 579, 0, 973, 953, 955, 956, 960, 964, - 965, 966, 967, 968, 970, 972, 976, 724, 0, 634, - 648, 728, 647, 721, 455, 0, 482, 645, 592, 0, - 638, 611, 612, 0, 639, 607, 643, 0, 581, 0, - 550, 553, 582, 667, 668, 669, 318, 552, 671, 672, - 673, 674, 675, 676, 677, 670, 975, 615, 591, 618, - 531, 594, 593, 0, 0, 629, 896, 630, 631, 439, - 440, 441, 442, 962, 655, 340, 551, 469, 0, 616, - 0, 0, 0, 0, 0, 0, 0, 0, 621, 622, - 619, 733, 0, 678, 679, 0, 0, 545, 546, 375, - 0, 564, 383, 339, 454, 377, 529, 406, 0, 557, - 623, 558, 471, 472, 681, 686, 682, 683, 685, 705, - 446, 397, 402, 486, 408, 422, 474, 528, 452, 479, - 337, 518, 488, 427, 608, 636, 984, 957, 983, 985, - 986, 982, 987, 988, 969, 850, 0, 903, 904, 980, - 979, 981, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 663, 662, 661, 660, 659, 658, 657, - 656, 0, 0, 605, 505, 353, 305, 349, 350, 357, - 722, 718, 723, 706, 709, 708, 684, 857, 313, 585, - 420, 468, 374, 650, 651, 0, 704, 947, 912, 913, - 914, 847, 915, 909, 910, 848, 911, 948, 901, 944, - 945, 876, 906, 916, 943, 917, 946, 877, 949, 989, - 990, 923, 907, 275, 991, 920, 950, 942, 941, 918, - 902, 951, 952, 884, 879, 921, 922, 908, 927, 928, - 929, 932, 849, 933, 934, 935, 936, 937, 931, 930, - 898, 899, 900, 924, 925, 905, 496, 880, 881, 882, - 883, 0, 0, 535, 536, 537, 560, 0, 538, 520, - 584, 384, 314, 500, 527, 720, 0, 0, 0, 0, - 0, 0, 0, 635, 646, 680, 0, 692, 693, 695, - 697, 938, 699, 493, 494, 707, 0, 0, 926, 702, - 703, 700, 424, 480, 501, 487, 0, 726, 575, 576, - 727, 688, 315, 0, 842, 183, 223, 894, 0, 0, - 0, 0, 0, 0, 0, 0, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 845, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 885, 627, 577, 489, 435, - 0, 644, 0, 0, 963, 971, 0, 0, 0, 0, - 0, 0, 0, 0, 959, 0, 0, 0, 0, 837, - 0, 0, 874, 940, 939, 861, 871, 0, 0, 335, - 246, 572, 694, 574, 573, 862, 0, 863, 867, 870, - 866, 864, 865, 0, 954, 0, 0, 0, 0, 0, - 0, 829, 841, 0, 846, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 838, 839, 0, 0, 0, 0, 895, 0, 840, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 890, 868, 872, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 869, 0, 893, 897, 360, 977, 891, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 978, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 888, 0, 691, 0, 526, 0, 0, 961, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 892, 0, - 478, 453, 974, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 958, 449, 654, 689, 690, 579, - 0, 973, 953, 955, 956, 960, 964, 965, 966, 967, - 968, 970, 972, 976, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 975, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 896, 630, 631, 439, 440, 441, 442, - 962, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 984, 957, 983, 985, 986, 982, 987, - 988, 969, 850, 0, 903, 904, 980, 979, 981, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 857, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 947, 912, 913, 914, 847, 915, - 909, 910, 848, 911, 948, 901, 944, 945, 876, 906, - 916, 943, 917, 946, 877, 949, 989, 990, 923, 907, - 275, 991, 920, 950, 942, 941, 918, 902, 951, 952, - 884, 879, 921, 922, 908, 927, 928, 929, 932, 849, - 933, 934, 935, 936, 937, 931, 930, 898, 899, 900, - 924, 925, 905, 496, 880, 881, 882, 883, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 938, 699, - 493, 494, 707, 0, 0, 926, 702, 703, 700, 424, - 480, 501, 487, 894, 726, 575, 576, 727, 688, 315, - 0, 842, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 845, 0, 0, 0, - 367, 2060, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 885, 627, 577, 489, 435, 0, 644, 0, 0, - 963, 971, 0, 0, 0, 0, 0, 0, 0, 0, - 959, 0, 2313, 0, 0, 837, 0, 0, 874, 940, - 939, 861, 871, 0, 0, 335, 246, 572, 694, 574, - 573, 862, 0, 863, 867, 870, 866, 864, 865, 0, - 954, 0, 0, 0, 0, 0, 0, 829, 841, 0, - 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 838, 839, 0, 0, - 0, 0, 895, 0, 840, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 2314, 868, 872, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 869, - 0, 893, 897, 360, 977, 891, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 978, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 888, 0, 691, 0, - 526, 0, 0, 961, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 892, 0, 478, 453, 974, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 958, 449, 654, 689, 690, 579, 0, 973, 953, 955, - 956, 960, 964, 965, 966, 967, 968, 970, 972, 976, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 975, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 896, - 630, 631, 439, 440, 441, 442, 962, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 984, - 957, 983, 985, 986, 982, 987, 988, 969, 850, 0, - 903, 904, 980, 979, 981, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 857, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 947, 912, 913, 914, 847, 915, 909, 910, 848, 911, - 948, 901, 944, 945, 876, 906, 916, 943, 917, 946, - 877, 949, 989, 990, 923, 907, 275, 991, 920, 950, - 942, 941, 918, 902, 951, 952, 884, 879, 921, 922, - 908, 927, 928, 929, 932, 849, 933, 934, 935, 936, - 937, 931, 930, 898, 899, 900, 924, 925, 905, 496, - 880, 881, 882, 883, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 938, 699, 493, 494, 707, 0, - 0, 926, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 0, 842, 183, 223, - 894, 0, 0, 0, 0, 0, 0, 0, 0, 451, - 0, 0, 590, 624, 613, 698, 578, 0, 0, 0, - 0, 0, 0, 845, 0, 0, 0, 367, 0, 0, - 419, 628, 609, 620, 610, 595, 596, 597, 604, 379, - 598, 599, 600, 570, 601, 571, 602, 603, 1419, 627, - 577, 489, 435, 0, 644, 0, 0, 963, 971, 0, - 0, 0, 0, 0, 0, 0, 0, 959, 0, 0, - 0, 0, 837, 0, 0, 874, 940, 939, 861, 871, - 0, 0, 335, 246, 572, 694, 574, 573, 862, 0, - 863, 867, 870, 866, 864, 865, 0, 954, 0, 0, - 0, 0, 0, 0, 829, 841, 0, 846, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 838, 839, 0, 0, 0, 0, 895, - 0, 840, 0, 0, 0, 0, 0, 490, 519, 0, - 532, 0, 404, 405, 890, 868, 872, 0, 0, 0, - 0, 322, 497, 516, 336, 484, 530, 341, 492, 509, - 331, 450, 481, 0, 0, 324, 514, 491, 432, 323, - 0, 475, 364, 381, 361, 448, 869, 0, 893, 897, - 360, 977, 891, 524, 326, 0, 523, 447, 510, 515, - 433, 426, 0, 325, 512, 431, 425, 410, 371, 978, - 411, 412, 385, 462, 423, 463, 386, 437, 436, 438, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 892, 0, 695, 0, 530, 0, 0, + 965, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 896, 0, 482, 457, 978, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 962, 453, 658, + 693, 694, 583, 0, 977, 957, 959, 960, 964, 968, + 969, 970, 971, 972, 974, 976, 980, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 979, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 900, 634, 635, 443, + 444, 445, 446, 966, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 988, 961, 987, 989, + 990, 986, 991, 992, 973, 854, 0, 907, 908, 984, + 983, 985, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 861, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 951, 916, 917, + 918, 851, 919, 913, 914, 852, 915, 952, 905, 948, + 949, 880, 910, 920, 947, 921, 950, 881, 953, 993, + 994, 927, 911, 275, 995, 924, 954, 946, 945, 922, + 906, 955, 956, 888, 883, 925, 926, 912, 931, 932, + 933, 936, 853, 937, 938, 939, 940, 941, 935, 934, + 902, 903, 904, 928, 929, 909, 500, 884, 885, 886, + 887, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 942, 703, 497, 498, 711, 0, 0, 930, 706, + 707, 704, 428, 484, 505, 491, 898, 730, 579, 580, + 731, 692, 315, 0, 846, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 849, + 0, 0, 0, 367, 2072, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 889, 631, 581, 493, 439, 0, + 648, 0, 0, 967, 975, 0, 0, 0, 0, 0, + 0, 0, 0, 963, 0, 2325, 0, 0, 841, 0, + 0, 878, 944, 943, 865, 875, 0, 0, 335, 246, + 576, 698, 578, 577, 866, 0, 867, 871, 874, 870, + 868, 869, 0, 958, 0, 0, 0, 0, 0, 0, + 833, 845, 0, 850, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, + 843, 0, 0, 0, 0, 899, 0, 844, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 2326, 872, 876, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 873, 0, 897, 901, 360, 981, 895, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 982, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 0, 0, 0, 0, 0, 554, 555, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 687, 888, 0, 691, 0, 526, 0, 0, - 961, 0, 0, 0, 495, 0, 0, 413, 0, 0, - 0, 892, 0, 478, 453, 974, 0, 0, 476, 421, - 511, 464, 517, 498, 525, 470, 465, 316, 499, 363, - 434, 332, 334, 719, 365, 368, 372, 373, 443, 444, - 458, 483, 502, 503, 504, 362, 346, 477, 347, 382, - 348, 317, 354, 352, 355, 485, 356, 319, 459, 508, - 0, 378, 473, 429, 320, 428, 460, 507, 506, 333, - 534, 541, 542, 632, 0, 547, 730, 731, 732, 556, - 0, 466, 329, 328, 0, 0, 0, 358, 461, 342, - 344, 345, 343, 456, 457, 561, 562, 563, 565, 0, - 566, 567, 0, 0, 0, 0, 568, 633, 649, 617, - 586, 549, 641, 583, 587, 588, 399, 400, 401, 652, - 0, 0, 0, 540, 414, 415, 0, 370, 369, 430, - 321, 0, 0, 407, 398, 467, 327, 366, 409, 403, - 416, 417, 418, 376, 311, 312, 725, 958, 449, 654, - 689, 690, 579, 0, 973, 953, 955, 956, 960, 964, - 965, 966, 967, 968, 970, 972, 976, 724, 0, 634, - 648, 728, 647, 721, 455, 0, 482, 645, 592, 0, - 638, 611, 612, 0, 639, 607, 643, 0, 581, 0, - 550, 553, 582, 667, 668, 669, 318, 552, 671, 672, - 673, 674, 675, 676, 677, 670, 975, 615, 591, 618, - 531, 594, 593, 0, 0, 629, 896, 630, 631, 439, - 440, 441, 442, 962, 655, 340, 551, 469, 0, 616, - 0, 0, 0, 0, 0, 0, 0, 0, 621, 622, - 619, 733, 0, 678, 679, 0, 0, 545, 546, 375, - 0, 564, 383, 339, 454, 377, 529, 406, 0, 557, - 623, 558, 471, 472, 681, 686, 682, 683, 685, 705, - 446, 397, 402, 486, 408, 422, 474, 528, 452, 479, - 337, 518, 488, 427, 608, 636, 984, 957, 983, 985, - 986, 982, 987, 988, 969, 850, 0, 903, 904, 980, - 979, 981, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 663, 662, 661, 660, 659, 658, 657, - 656, 0, 0, 605, 505, 353, 305, 349, 350, 357, - 722, 718, 723, 706, 709, 708, 684, 857, 313, 585, - 420, 468, 374, 650, 651, 0, 704, 947, 912, 913, - 914, 847, 915, 909, 910, 848, 911, 948, 901, 944, - 945, 876, 906, 916, 943, 917, 946, 877, 949, 989, - 990, 923, 907, 275, 991, 920, 950, 942, 941, 918, - 902, 951, 952, 884, 879, 921, 922, 908, 927, 928, - 929, 932, 849, 933, 934, 935, 936, 937, 931, 930, - 898, 899, 900, 924, 925, 905, 496, 880, 881, 882, - 883, 0, 0, 535, 536, 537, 560, 0, 538, 520, - 584, 384, 314, 500, 527, 720, 0, 0, 0, 0, - 0, 0, 0, 635, 646, 680, 0, 692, 693, 695, - 697, 938, 699, 493, 494, 707, 0, 0, 926, 702, - 703, 700, 424, 480, 501, 487, 894, 726, 575, 576, - 727, 688, 315, 0, 842, 451, 0, 0, 590, 624, - 613, 698, 578, 0, 0, 0, 0, 0, 0, 845, - 0, 0, 0, 367, 4684, 0, 419, 628, 609, 620, - 610, 595, 596, 597, 604, 379, 598, 599, 600, 570, - 601, 571, 602, 603, 885, 627, 577, 489, 435, 0, - 644, 0, 0, 963, 971, 0, 0, 0, 0, 0, - 0, 0, 0, 959, 0, 0, 0, 0, 837, 0, - 0, 874, 940, 939, 861, 871, 0, 0, 335, 246, - 572, 694, 574, 573, 862, 0, 863, 867, 870, 866, - 864, 865, 0, 954, 0, 0, 0, 0, 0, 0, - 829, 841, 0, 846, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, - 839, 0, 0, 0, 0, 895, 0, 840, 0, 0, - 0, 0, 0, 490, 519, 0, 532, 0, 404, 405, - 890, 868, 872, 0, 0, 0, 0, 322, 497, 516, - 336, 484, 530, 341, 492, 509, 331, 450, 481, 0, - 0, 324, 514, 491, 432, 323, 0, 475, 364, 381, - 361, 448, 869, 0, 893, 897, 360, 977, 891, 524, - 326, 0, 523, 447, 510, 515, 433, 426, 0, 325, - 512, 431, 425, 410, 371, 978, 411, 412, 385, 462, - 423, 463, 386, 437, 436, 438, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, - 0, 554, 555, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 687, 888, - 0, 691, 0, 526, 0, 0, 961, 0, 0, 0, - 495, 0, 0, 413, 0, 0, 0, 892, 0, 478, - 453, 974, 0, 0, 476, 421, 511, 464, 517, 498, - 525, 470, 465, 316, 499, 363, 434, 332, 334, 719, - 365, 368, 372, 373, 443, 444, 458, 483, 502, 503, - 504, 362, 346, 477, 347, 382, 348, 317, 354, 352, - 355, 485, 356, 319, 459, 508, 0, 378, 473, 429, - 320, 428, 460, 507, 506, 333, 534, 541, 542, 632, - 0, 547, 730, 731, 732, 556, 0, 466, 329, 328, - 0, 0, 0, 358, 461, 342, 344, 345, 343, 456, - 457, 561, 562, 563, 565, 0, 566, 567, 0, 0, - 0, 0, 568, 633, 649, 617, 586, 549, 641, 583, - 587, 588, 399, 400, 401, 652, 0, 0, 0, 540, - 414, 415, 0, 370, 369, 430, 321, 0, 0, 407, - 398, 467, 327, 366, 409, 403, 416, 417, 418, 376, - 311, 312, 725, 958, 449, 654, 689, 690, 579, 0, - 973, 953, 955, 956, 960, 964, 965, 966, 967, 968, - 970, 972, 976, 724, 0, 634, 648, 728, 647, 721, - 455, 0, 482, 645, 592, 0, 638, 611, 612, 0, - 639, 607, 643, 0, 581, 0, 550, 553, 582, 667, - 668, 669, 318, 552, 671, 672, 673, 674, 675, 676, - 677, 670, 975, 615, 591, 618, 531, 594, 593, 0, - 0, 629, 896, 630, 631, 439, 440, 441, 442, 962, - 655, 340, 551, 469, 0, 616, 0, 0, 0, 0, - 0, 0, 0, 0, 621, 622, 619, 733, 0, 678, - 679, 0, 0, 545, 546, 375, 0, 564, 383, 339, - 454, 377, 529, 406, 0, 557, 623, 558, 471, 472, - 681, 686, 682, 683, 685, 705, 446, 397, 402, 486, - 408, 422, 474, 528, 452, 479, 337, 518, 488, 427, - 608, 636, 984, 957, 983, 985, 986, 982, 987, 988, - 969, 850, 0, 903, 904, 980, 979, 981, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, - 662, 661, 660, 659, 658, 657, 656, 0, 0, 605, - 505, 353, 305, 349, 350, 357, 722, 718, 723, 706, - 709, 708, 684, 857, 313, 585, 420, 468, 374, 650, - 651, 0, 704, 947, 912, 913, 914, 847, 915, 909, - 910, 848, 911, 948, 901, 944, 945, 876, 906, 916, - 943, 917, 946, 877, 949, 989, 990, 923, 907, 275, - 991, 920, 950, 942, 941, 918, 902, 951, 952, 884, - 879, 921, 922, 908, 927, 928, 929, 932, 849, 933, - 934, 935, 936, 937, 931, 930, 898, 899, 900, 924, - 925, 905, 496, 880, 881, 882, 883, 0, 0, 535, - 536, 537, 560, 0, 538, 520, 584, 384, 314, 500, - 527, 720, 0, 0, 0, 0, 0, 0, 0, 635, - 646, 680, 0, 692, 693, 695, 697, 938, 699, 493, - 494, 707, 0, 0, 926, 702, 703, 700, 424, 480, - 501, 487, 894, 726, 575, 576, 727, 688, 315, 0, - 842, 451, 0, 0, 590, 624, 613, 698, 578, 0, - 0, 0, 0, 0, 0, 845, 0, 0, 0, 367, - 0, 0, 419, 628, 609, 620, 610, 595, 596, 597, - 604, 379, 598, 599, 600, 570, 601, 571, 602, 603, - 885, 627, 577, 489, 435, 0, 644, 0, 0, 963, - 971, 0, 0, 0, 0, 0, 0, 0, 0, 959, - 0, 0, 0, 0, 837, 0, 0, 874, 940, 939, - 861, 871, 0, 0, 335, 246, 572, 694, 574, 573, - 862, 0, 863, 867, 870, 866, 864, 865, 0, 954, - 0, 0, 0, 0, 0, 0, 829, 841, 0, 846, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 838, 839, 0, 0, 0, - 0, 895, 0, 840, 0, 0, 0, 0, 0, 490, - 519, 0, 532, 0, 404, 405, 890, 868, 872, 0, - 0, 0, 0, 322, 497, 516, 336, 484, 530, 341, - 492, 509, 331, 450, 481, 0, 0, 324, 514, 491, - 432, 323, 0, 475, 364, 381, 361, 448, 869, 0, - 893, 897, 360, 977, 891, 524, 326, 0, 523, 447, - 510, 515, 433, 426, 0, 325, 512, 431, 425, 410, - 371, 978, 411, 412, 385, 462, 423, 463, 386, 437, - 436, 438, 387, 388, 389, 390, 391, 392, 393, 394, - 395, 396, 0, 0, 0, 0, 0, 554, 555, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 687, 888, 0, 691, 0, 526, - 0, 0, 961, 0, 0, 0, 495, 0, 0, 413, - 0, 0, 0, 892, 0, 478, 453, 974, 4567, 0, - 476, 421, 511, 464, 517, 498, 525, 470, 465, 316, - 499, 363, 434, 332, 334, 719, 365, 368, 372, 373, - 443, 444, 458, 483, 502, 503, 504, 362, 346, 477, - 347, 382, 348, 317, 354, 352, 355, 485, 356, 319, - 459, 508, 0, 378, 473, 429, 320, 428, 460, 507, - 506, 333, 534, 541, 542, 632, 0, 547, 730, 731, - 732, 556, 0, 466, 329, 328, 0, 0, 0, 358, - 461, 342, 344, 345, 343, 456, 457, 561, 562, 563, - 565, 0, 566, 567, 0, 0, 0, 0, 568, 633, - 649, 617, 586, 549, 641, 583, 587, 588, 399, 400, - 401, 652, 0, 0, 0, 540, 414, 415, 0, 370, - 369, 430, 321, 0, 0, 407, 398, 467, 327, 366, - 409, 403, 416, 417, 418, 376, 311, 312, 725, 958, - 449, 654, 689, 690, 579, 0, 973, 953, 955, 956, - 960, 964, 965, 966, 967, 968, 970, 972, 976, 724, - 0, 634, 648, 728, 647, 721, 455, 0, 482, 645, - 592, 0, 638, 611, 612, 0, 639, 607, 643, 0, - 581, 0, 550, 553, 582, 667, 668, 669, 318, 552, - 671, 672, 673, 674, 675, 676, 677, 670, 975, 615, - 591, 618, 531, 594, 593, 0, 0, 629, 896, 630, - 631, 439, 440, 441, 442, 962, 655, 340, 551, 469, - 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, - 621, 622, 619, 733, 0, 678, 679, 0, 0, 545, - 546, 375, 0, 564, 383, 339, 454, 377, 529, 406, - 0, 557, 623, 558, 471, 472, 681, 686, 682, 683, - 685, 705, 446, 397, 402, 486, 408, 422, 474, 528, - 452, 479, 337, 518, 488, 427, 608, 636, 984, 957, - 983, 985, 986, 982, 987, 988, 969, 850, 0, 903, - 904, 980, 979, 981, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 663, 662, 661, 660, 659, - 658, 657, 656, 0, 0, 605, 505, 353, 305, 349, - 350, 357, 722, 718, 723, 706, 709, 708, 684, 857, - 313, 585, 420, 468, 374, 650, 651, 0, 704, 947, - 912, 913, 914, 847, 915, 909, 910, 848, 911, 948, - 901, 944, 945, 876, 906, 916, 943, 917, 946, 877, - 949, 989, 990, 923, 907, 275, 991, 920, 950, 942, - 941, 918, 902, 951, 952, 884, 879, 921, 922, 908, - 927, 928, 929, 932, 849, 933, 934, 935, 936, 937, - 931, 930, 898, 899, 900, 924, 925, 905, 496, 880, - 881, 882, 883, 0, 0, 535, 536, 537, 560, 0, - 538, 520, 584, 384, 314, 500, 527, 720, 0, 0, - 0, 0, 0, 0, 0, 635, 646, 680, 0, 692, - 693, 695, 697, 938, 699, 493, 494, 707, 0, 0, - 926, 702, 703, 700, 424, 480, 501, 487, 894, 726, - 575, 576, 727, 688, 315, 0, 842, 451, 0, 0, - 590, 624, 613, 698, 578, 0, 0, 0, 0, 0, - 0, 845, 0, 0, 0, 367, 2060, 0, 419, 628, - 609, 620, 610, 595, 596, 597, 604, 379, 598, 599, - 600, 570, 601, 571, 602, 603, 885, 627, 577, 489, - 435, 0, 644, 0, 0, 963, 971, 0, 0, 0, - 0, 0, 0, 0, 0, 959, 0, 0, 0, 0, - 837, 0, 0, 874, 940, 939, 861, 871, 0, 0, - 335, 246, 572, 694, 574, 573, 862, 0, 863, 867, - 870, 866, 864, 865, 0, 954, 0, 0, 0, 0, - 0, 0, 829, 841, 0, 846, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 838, 839, 0, 0, 0, 0, 895, 0, 840, - 0, 0, 0, 0, 0, 490, 519, 0, 532, 0, - 404, 405, 890, 868, 872, 0, 0, 0, 0, 322, - 497, 516, 336, 484, 530, 341, 492, 509, 331, 450, - 481, 0, 0, 324, 514, 491, 432, 323, 0, 475, - 364, 381, 361, 448, 869, 0, 893, 897, 360, 977, - 891, 524, 326, 0, 523, 447, 510, 515, 433, 426, - 0, 325, 512, 431, 425, 410, 371, 978, 411, 412, - 385, 462, 423, 463, 386, 437, 436, 438, 387, 388, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 892, 0, 695, 0, 530, 0, 0, + 965, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 896, 0, 482, 457, 978, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 962, 453, 658, + 693, 694, 583, 0, 977, 957, 959, 960, 964, 968, + 969, 970, 971, 972, 974, 976, 980, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 979, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 900, 634, 635, 443, + 444, 445, 446, 966, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 988, 961, 987, 989, + 990, 986, 991, 992, 973, 854, 0, 907, 908, 984, + 983, 985, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 861, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 951, 916, 917, + 918, 851, 919, 913, 914, 852, 915, 952, 905, 948, + 949, 880, 910, 920, 947, 921, 950, 881, 953, 993, + 994, 927, 911, 275, 995, 924, 954, 946, 945, 922, + 906, 955, 956, 888, 883, 925, 926, 912, 931, 932, + 933, 936, 853, 937, 938, 939, 940, 941, 935, 934, + 902, 903, 904, 928, 929, 909, 500, 884, 885, 886, + 887, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 942, 703, 497, 498, 711, 0, 0, 930, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 0, 846, 183, 223, 898, 0, 0, + 0, 0, 0, 0, 0, 0, 455, 0, 0, 594, + 628, 617, 702, 582, 0, 0, 0, 0, 0, 0, + 849, 0, 0, 0, 367, 0, 0, 423, 632, 613, + 624, 614, 599, 600, 601, 608, 379, 602, 603, 604, + 574, 605, 575, 606, 607, 1427, 631, 581, 493, 439, + 0, 648, 0, 0, 967, 975, 0, 0, 0, 0, + 0, 0, 0, 0, 963, 0, 0, 0, 0, 841, + 0, 0, 878, 944, 943, 865, 875, 0, 0, 335, + 246, 576, 698, 578, 577, 866, 0, 867, 871, 874, + 870, 868, 869, 0, 958, 0, 0, 0, 0, 0, + 0, 833, 845, 0, 850, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 842, 843, 0, 0, 0, 0, 899, 0, 844, 0, + 0, 0, 0, 0, 494, 523, 0, 536, 0, 404, + 405, 894, 872, 876, 0, 0, 0, 0, 322, 501, + 520, 336, 488, 534, 341, 496, 513, 331, 454, 485, + 0, 0, 324, 518, 495, 436, 323, 0, 479, 364, + 381, 361, 452, 873, 0, 897, 901, 360, 981, 895, + 528, 326, 0, 527, 451, 514, 519, 437, 430, 0, + 325, 516, 435, 429, 410, 371, 982, 411, 412, 413, + 414, 415, 416, 385, 466, 427, 467, 386, 441, 440, + 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 558, 559, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 691, 892, 0, 695, 0, 530, 0, + 0, 965, 0, 0, 0, 499, 0, 0, 417, 0, + 0, 0, 896, 0, 482, 457, 978, 0, 0, 480, + 425, 515, 468, 521, 502, 529, 474, 469, 316, 503, + 363, 438, 332, 334, 723, 365, 368, 372, 373, 447, + 448, 462, 487, 506, 507, 508, 362, 346, 481, 347, + 382, 348, 317, 354, 352, 355, 489, 356, 319, 463, + 512, 0, 378, 477, 433, 320, 432, 464, 511, 510, + 333, 538, 545, 546, 636, 0, 551, 734, 735, 736, + 560, 0, 470, 329, 328, 0, 0, 0, 358, 465, + 342, 344, 345, 343, 460, 461, 565, 566, 567, 569, + 0, 570, 571, 0, 0, 0, 0, 572, 637, 653, + 621, 590, 553, 645, 587, 591, 592, 399, 400, 401, + 656, 0, 0, 0, 544, 418, 419, 0, 370, 369, + 434, 321, 0, 0, 407, 398, 471, 327, 366, 409, + 403, 420, 421, 422, 376, 311, 312, 729, 962, 453, + 658, 693, 694, 583, 0, 977, 957, 959, 960, 964, + 968, 969, 970, 971, 972, 974, 976, 980, 728, 0, + 638, 652, 732, 651, 725, 459, 0, 486, 649, 596, + 0, 642, 615, 616, 0, 643, 611, 647, 0, 585, + 0, 554, 557, 586, 671, 672, 673, 318, 556, 675, + 676, 677, 678, 679, 680, 681, 674, 979, 619, 595, + 622, 535, 598, 597, 0, 0, 633, 900, 634, 635, + 443, 444, 445, 446, 966, 659, 340, 555, 473, 0, + 620, 0, 0, 0, 0, 0, 0, 0, 0, 625, + 626, 623, 737, 0, 682, 683, 0, 0, 549, 550, + 375, 0, 568, 383, 339, 458, 377, 533, 406, 0, + 561, 627, 562, 475, 476, 685, 690, 686, 687, 689, + 709, 450, 397, 402, 490, 408, 426, 478, 532, 456, + 483, 337, 522, 492, 431, 612, 640, 988, 961, 987, + 989, 990, 986, 991, 992, 973, 854, 0, 907, 908, + 984, 983, 985, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 667, 666, 665, 664, 663, 662, + 661, 660, 0, 0, 609, 509, 353, 305, 349, 350, + 357, 726, 722, 727, 710, 713, 712, 688, 861, 313, + 589, 424, 472, 374, 654, 655, 0, 708, 951, 916, + 917, 918, 851, 919, 913, 914, 852, 915, 952, 905, + 948, 949, 880, 910, 920, 947, 921, 950, 881, 953, + 993, 994, 927, 911, 275, 995, 924, 954, 946, 945, + 922, 906, 955, 956, 888, 883, 925, 926, 912, 931, + 932, 933, 936, 853, 937, 938, 939, 940, 941, 935, + 934, 902, 903, 904, 928, 929, 909, 500, 884, 885, + 886, 887, 0, 0, 539, 540, 541, 564, 0, 542, + 524, 588, 384, 314, 504, 531, 724, 0, 0, 0, + 0, 0, 0, 0, 639, 650, 684, 0, 696, 697, + 699, 701, 942, 703, 497, 498, 711, 0, 0, 930, + 706, 707, 704, 428, 484, 505, 491, 898, 730, 579, + 580, 731, 692, 315, 0, 846, 455, 0, 0, 594, + 628, 617, 702, 582, 0, 0, 0, 0, 0, 0, + 849, 0, 0, 0, 367, 4700, 0, 423, 632, 613, + 624, 614, 599, 600, 601, 608, 379, 602, 603, 604, + 574, 605, 575, 606, 607, 889, 631, 581, 493, 439, + 0, 648, 0, 0, 967, 975, 0, 0, 0, 0, + 0, 0, 0, 0, 963, 0, 0, 0, 0, 841, + 0, 0, 878, 944, 943, 865, 875, 0, 0, 335, + 246, 576, 698, 578, 577, 866, 0, 867, 871, 874, + 870, 868, 869, 0, 958, 0, 0, 0, 0, 0, + 0, 833, 845, 0, 850, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 842, 843, 0, 0, 0, 0, 899, 0, 844, 0, + 0, 0, 0, 0, 494, 523, 0, 536, 0, 404, + 405, 894, 872, 876, 0, 0, 0, 0, 322, 501, + 520, 336, 488, 534, 341, 496, 513, 331, 454, 485, + 0, 0, 324, 518, 495, 436, 323, 0, 479, 364, + 381, 361, 452, 873, 0, 897, 901, 360, 981, 895, + 528, 326, 0, 527, 451, 514, 519, 437, 430, 0, + 325, 516, 435, 429, 410, 371, 982, 411, 412, 413, + 414, 415, 416, 385, 466, 427, 467, 386, 441, 440, + 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 558, 559, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 691, 892, 0, 695, 0, 530, 0, + 0, 965, 0, 0, 0, 499, 0, 0, 417, 0, + 0, 0, 896, 0, 482, 457, 978, 0, 0, 480, + 425, 515, 468, 521, 502, 529, 474, 469, 316, 503, + 363, 438, 332, 334, 723, 365, 368, 372, 373, 447, + 448, 462, 487, 506, 507, 508, 362, 346, 481, 347, + 382, 348, 317, 354, 352, 355, 489, 356, 319, 463, + 512, 0, 378, 477, 433, 320, 432, 464, 511, 510, + 333, 538, 545, 546, 636, 0, 551, 734, 735, 736, + 560, 0, 470, 329, 328, 0, 0, 0, 358, 465, + 342, 344, 345, 343, 460, 461, 565, 566, 567, 569, + 0, 570, 571, 0, 0, 0, 0, 572, 637, 653, + 621, 590, 553, 645, 587, 591, 592, 399, 400, 401, + 656, 0, 0, 0, 544, 418, 419, 0, 370, 369, + 434, 321, 0, 0, 407, 398, 471, 327, 366, 409, + 403, 420, 421, 422, 376, 311, 312, 729, 962, 453, + 658, 693, 694, 583, 0, 977, 957, 959, 960, 964, + 968, 969, 970, 971, 972, 974, 976, 980, 728, 0, + 638, 652, 732, 651, 725, 459, 0, 486, 649, 596, + 0, 642, 615, 616, 0, 643, 611, 647, 0, 585, + 0, 554, 557, 586, 671, 672, 673, 318, 556, 675, + 676, 677, 678, 679, 680, 681, 674, 979, 619, 595, + 622, 535, 598, 597, 0, 0, 633, 900, 634, 635, + 443, 444, 445, 446, 966, 659, 340, 555, 473, 0, + 620, 0, 0, 0, 0, 0, 0, 0, 0, 625, + 626, 623, 737, 0, 682, 683, 0, 0, 549, 550, + 375, 0, 568, 383, 339, 458, 377, 533, 406, 0, + 561, 627, 562, 475, 476, 685, 690, 686, 687, 689, + 709, 450, 397, 402, 490, 408, 426, 478, 532, 456, + 483, 337, 522, 492, 431, 612, 640, 988, 961, 987, + 989, 990, 986, 991, 992, 973, 854, 0, 907, 908, + 984, 983, 985, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 667, 666, 665, 664, 663, 662, + 661, 660, 0, 0, 609, 509, 353, 305, 349, 350, + 357, 726, 722, 727, 710, 713, 712, 688, 861, 313, + 589, 424, 472, 374, 654, 655, 0, 708, 951, 916, + 917, 918, 851, 919, 913, 914, 852, 915, 952, 905, + 948, 949, 880, 910, 920, 947, 921, 950, 881, 953, + 993, 994, 927, 911, 275, 995, 924, 954, 946, 945, + 922, 906, 955, 956, 888, 883, 925, 926, 912, 931, + 932, 933, 936, 853, 937, 938, 939, 940, 941, 935, + 934, 902, 903, 904, 928, 929, 909, 500, 884, 885, + 886, 887, 0, 0, 539, 540, 541, 564, 0, 542, + 524, 588, 384, 314, 504, 531, 724, 0, 0, 0, + 0, 0, 0, 0, 639, 650, 684, 0, 696, 697, + 699, 701, 942, 703, 497, 498, 711, 0, 0, 930, + 706, 707, 704, 428, 484, 505, 491, 898, 730, 579, + 580, 731, 692, 315, 0, 846, 455, 0, 0, 594, + 628, 617, 702, 582, 0, 0, 0, 0, 0, 0, + 849, 0, 0, 0, 367, 0, 0, 423, 632, 613, + 624, 614, 599, 600, 601, 608, 379, 602, 603, 604, + 574, 605, 575, 606, 607, 889, 631, 581, 493, 439, + 0, 648, 0, 0, 967, 975, 0, 0, 0, 0, + 0, 0, 0, 0, 963, 0, 0, 0, 0, 841, + 0, 0, 878, 944, 943, 865, 875, 0, 0, 335, + 246, 576, 698, 578, 577, 866, 0, 867, 871, 874, + 870, 868, 869, 0, 958, 0, 0, 0, 0, 0, + 0, 833, 845, 0, 850, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 842, 843, 0, 0, 0, 0, 899, 0, 844, 0, + 0, 0, 0, 0, 494, 523, 0, 536, 0, 404, + 405, 894, 872, 876, 0, 0, 0, 0, 322, 501, + 520, 336, 488, 534, 341, 496, 513, 331, 454, 485, + 0, 0, 324, 518, 495, 436, 323, 0, 479, 364, + 381, 361, 452, 873, 0, 897, 901, 360, 981, 895, + 528, 326, 0, 527, 451, 514, 519, 437, 430, 0, + 325, 516, 435, 429, 410, 371, 982, 411, 412, 413, + 414, 415, 416, 385, 466, 427, 467, 386, 441, 440, + 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 558, 559, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 691, 892, 0, 695, 0, 530, 0, + 0, 965, 0, 0, 0, 499, 0, 0, 417, 0, + 0, 0, 896, 0, 482, 457, 978, 4583, 0, 480, + 425, 515, 468, 521, 502, 529, 474, 469, 316, 503, + 363, 438, 332, 334, 723, 365, 368, 372, 373, 447, + 448, 462, 487, 506, 507, 508, 362, 346, 481, 347, + 382, 348, 317, 354, 352, 355, 489, 356, 319, 463, + 512, 0, 378, 477, 433, 320, 432, 464, 511, 510, + 333, 538, 545, 546, 636, 0, 551, 734, 735, 736, + 560, 0, 470, 329, 328, 0, 0, 0, 358, 465, + 342, 344, 345, 343, 460, 461, 565, 566, 567, 569, + 0, 570, 571, 0, 0, 0, 0, 572, 637, 653, + 621, 590, 553, 645, 587, 591, 592, 399, 400, 401, + 656, 0, 0, 0, 544, 418, 419, 0, 370, 369, + 434, 321, 0, 0, 407, 398, 471, 327, 366, 409, + 403, 420, 421, 422, 376, 311, 312, 729, 962, 453, + 658, 693, 694, 583, 0, 977, 957, 959, 960, 964, + 968, 969, 970, 971, 972, 974, 976, 980, 728, 0, + 638, 652, 732, 651, 725, 459, 0, 486, 649, 596, + 0, 642, 615, 616, 0, 643, 611, 647, 0, 585, + 0, 554, 557, 586, 671, 672, 673, 318, 556, 675, + 676, 677, 678, 679, 680, 681, 674, 979, 619, 595, + 622, 535, 598, 597, 0, 0, 633, 900, 634, 635, + 443, 444, 445, 446, 966, 659, 340, 555, 473, 0, + 620, 0, 0, 0, 0, 0, 0, 0, 0, 625, + 626, 623, 737, 0, 682, 683, 0, 0, 549, 550, + 375, 0, 568, 383, 339, 458, 377, 533, 406, 0, + 561, 627, 562, 475, 476, 685, 690, 686, 687, 689, + 709, 450, 397, 402, 490, 408, 426, 478, 532, 456, + 483, 337, 522, 492, 431, 612, 640, 988, 961, 987, + 989, 990, 986, 991, 992, 973, 854, 0, 907, 908, + 984, 983, 985, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 667, 666, 665, 664, 663, 662, + 661, 660, 0, 0, 609, 509, 353, 305, 349, 350, + 357, 726, 722, 727, 710, 713, 712, 688, 861, 313, + 589, 424, 472, 374, 654, 655, 0, 708, 951, 916, + 917, 918, 851, 919, 913, 914, 852, 915, 952, 905, + 948, 949, 880, 910, 920, 947, 921, 950, 881, 953, + 993, 994, 927, 911, 275, 995, 924, 954, 946, 945, + 922, 906, 955, 956, 888, 883, 925, 926, 912, 931, + 932, 933, 936, 853, 937, 938, 939, 940, 941, 935, + 934, 902, 903, 904, 928, 929, 909, 500, 884, 885, + 886, 887, 0, 0, 539, 540, 541, 564, 0, 542, + 524, 588, 384, 314, 504, 531, 724, 0, 0, 0, + 0, 0, 0, 0, 639, 650, 684, 0, 696, 697, + 699, 701, 942, 703, 497, 498, 711, 0, 0, 930, + 706, 707, 704, 428, 484, 505, 491, 898, 730, 579, + 580, 731, 692, 315, 0, 846, 455, 0, 0, 594, + 628, 617, 702, 582, 0, 0, 0, 0, 0, 0, + 849, 0, 0, 0, 367, 2072, 0, 423, 632, 613, + 624, 614, 599, 600, 601, 608, 379, 602, 603, 604, + 574, 605, 575, 606, 607, 889, 631, 581, 493, 439, + 0, 648, 0, 0, 967, 975, 0, 0, 0, 0, + 0, 0, 0, 0, 963, 0, 0, 0, 0, 841, + 0, 0, 878, 944, 943, 865, 875, 0, 0, 335, + 246, 576, 698, 578, 577, 866, 0, 867, 871, 874, + 870, 868, 869, 0, 958, 0, 0, 0, 0, 0, + 0, 833, 845, 0, 850, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 842, 843, 0, 0, 0, 0, 899, 0, 844, 0, + 0, 0, 0, 0, 494, 523, 0, 536, 0, 404, + 405, 894, 872, 876, 0, 0, 0, 0, 322, 501, + 520, 336, 488, 534, 341, 496, 513, 331, 454, 485, + 0, 0, 324, 518, 495, 436, 323, 0, 479, 364, + 381, 361, 452, 873, 0, 897, 901, 360, 981, 895, + 528, 326, 0, 527, 451, 514, 519, 437, 430, 0, + 325, 516, 435, 429, 410, 371, 982, 411, 412, 413, + 414, 415, 416, 385, 466, 427, 467, 386, 441, 440, + 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 558, 559, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 691, 892, 0, 695, 0, 530, 0, + 0, 965, 0, 0, 0, 499, 0, 0, 417, 0, + 0, 0, 896, 0, 482, 457, 978, 0, 0, 480, + 425, 515, 468, 521, 502, 529, 474, 469, 316, 503, + 363, 438, 332, 334, 723, 365, 368, 372, 373, 447, + 448, 462, 487, 506, 507, 508, 362, 346, 481, 347, + 382, 348, 317, 354, 352, 355, 489, 356, 319, 463, + 512, 0, 378, 477, 433, 320, 432, 464, 511, 510, + 333, 538, 545, 546, 636, 0, 551, 734, 735, 736, + 560, 0, 470, 329, 328, 0, 0, 0, 358, 465, + 342, 344, 345, 343, 460, 461, 565, 566, 567, 569, + 0, 570, 571, 0, 0, 0, 0, 572, 637, 653, + 621, 590, 553, 645, 587, 591, 592, 399, 400, 401, + 656, 0, 0, 0, 544, 418, 419, 0, 370, 369, + 434, 321, 0, 0, 407, 398, 471, 327, 366, 409, + 403, 420, 421, 422, 376, 311, 312, 729, 962, 453, + 658, 693, 694, 583, 0, 977, 957, 959, 960, 964, + 968, 969, 970, 971, 972, 974, 976, 980, 728, 0, + 638, 652, 732, 651, 725, 459, 0, 486, 649, 596, + 0, 642, 615, 616, 0, 643, 611, 647, 0, 585, + 0, 554, 557, 586, 671, 672, 673, 318, 556, 675, + 676, 677, 678, 679, 680, 681, 674, 979, 619, 595, + 622, 535, 598, 597, 0, 0, 633, 900, 634, 635, + 443, 444, 445, 446, 966, 659, 340, 555, 473, 0, + 620, 0, 0, 0, 0, 0, 0, 0, 0, 625, + 626, 623, 737, 0, 682, 683, 0, 0, 549, 550, + 375, 0, 568, 383, 339, 458, 377, 533, 406, 0, + 561, 627, 562, 475, 476, 685, 690, 686, 687, 689, + 709, 450, 397, 402, 490, 408, 426, 478, 532, 456, + 483, 337, 522, 492, 431, 612, 640, 988, 961, 987, + 989, 990, 986, 991, 992, 973, 854, 0, 907, 908, + 984, 983, 985, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 667, 666, 665, 664, 663, 662, + 661, 660, 0, 0, 609, 509, 353, 305, 349, 350, + 357, 726, 722, 727, 710, 713, 712, 688, 861, 313, + 589, 424, 472, 374, 654, 655, 0, 708, 951, 916, + 917, 918, 851, 919, 913, 914, 852, 915, 952, 905, + 948, 949, 880, 910, 920, 947, 921, 950, 881, 953, + 993, 994, 927, 911, 275, 995, 924, 954, 946, 945, + 922, 906, 955, 956, 888, 883, 925, 926, 912, 931, + 932, 933, 936, 853, 937, 938, 939, 940, 941, 935, + 934, 902, 903, 904, 928, 929, 909, 500, 884, 885, + 886, 887, 0, 0, 539, 540, 541, 564, 0, 542, + 524, 588, 384, 314, 504, 531, 724, 0, 0, 0, + 0, 0, 0, 0, 639, 650, 684, 0, 696, 697, + 699, 701, 942, 703, 497, 498, 711, 0, 0, 930, + 706, 707, 704, 428, 484, 505, 491, 898, 730, 579, + 580, 731, 692, 315, 0, 846, 455, 0, 0, 594, + 628, 617, 702, 582, 0, 0, 0, 0, 0, 0, + 849, 0, 0, 0, 367, 0, 0, 423, 632, 613, + 624, 614, 599, 600, 601, 608, 379, 602, 603, 604, + 574, 605, 575, 606, 607, 889, 631, 581, 493, 439, + 0, 648, 0, 0, 967, 975, 0, 0, 0, 0, + 0, 0, 0, 0, 963, 0, 0, 0, 0, 841, + 0, 0, 878, 944, 943, 865, 875, 0, 0, 335, + 246, 576, 698, 578, 577, 866, 0, 867, 871, 874, + 870, 868, 869, 0, 958, 0, 0, 0, 0, 0, + 0, 833, 845, 0, 850, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 842, 843, 1761, 0, 0, 0, 899, 0, 844, 0, + 0, 0, 0, 0, 494, 523, 0, 536, 0, 404, + 405, 894, 872, 876, 0, 0, 0, 0, 322, 501, + 520, 336, 488, 534, 341, 496, 513, 331, 454, 485, + 0, 0, 324, 518, 495, 436, 323, 0, 479, 364, + 381, 361, 452, 873, 0, 897, 901, 360, 981, 895, + 528, 326, 0, 527, 451, 514, 519, 437, 430, 0, + 325, 516, 435, 429, 410, 371, 982, 411, 412, 413, + 414, 415, 416, 385, 466, 427, 467, 386, 441, 440, + 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 558, 559, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 691, 892, 0, 695, 0, 530, 0, + 0, 965, 0, 0, 0, 499, 0, 0, 417, 0, + 0, 0, 896, 0, 482, 457, 978, 0, 0, 480, + 425, 515, 468, 521, 502, 529, 474, 469, 316, 503, + 363, 438, 332, 334, 723, 365, 368, 372, 373, 447, + 448, 462, 487, 506, 507, 508, 362, 346, 481, 347, + 382, 348, 317, 354, 352, 355, 489, 356, 319, 463, + 512, 0, 378, 477, 433, 320, 432, 464, 511, 510, + 333, 538, 545, 546, 636, 0, 551, 734, 735, 736, + 560, 0, 470, 329, 328, 0, 0, 0, 358, 465, + 342, 344, 345, 343, 460, 461, 565, 566, 567, 569, + 0, 570, 571, 0, 0, 0, 0, 572, 637, 653, + 621, 590, 553, 645, 587, 591, 592, 399, 400, 401, + 656, 0, 0, 0, 544, 418, 419, 0, 370, 369, + 434, 321, 0, 0, 407, 398, 471, 327, 366, 409, + 403, 420, 421, 422, 376, 311, 312, 729, 962, 453, + 658, 693, 694, 583, 0, 977, 957, 959, 960, 964, + 968, 969, 970, 971, 972, 974, 976, 980, 728, 0, + 638, 652, 732, 651, 725, 459, 0, 486, 649, 596, + 0, 642, 615, 616, 0, 643, 611, 647, 0, 585, + 0, 554, 557, 586, 671, 672, 673, 318, 556, 675, + 676, 677, 678, 679, 680, 681, 674, 979, 619, 595, + 622, 535, 598, 597, 0, 0, 633, 900, 634, 635, + 443, 444, 445, 446, 966, 659, 340, 555, 473, 0, + 620, 0, 0, 0, 0, 0, 0, 0, 0, 625, + 626, 623, 737, 0, 682, 683, 0, 0, 549, 550, + 375, 0, 568, 383, 339, 458, 377, 533, 406, 0, + 561, 627, 562, 475, 476, 685, 690, 686, 687, 689, + 709, 450, 397, 402, 490, 408, 426, 478, 532, 456, + 483, 337, 522, 492, 431, 612, 640, 988, 961, 987, + 989, 990, 986, 991, 992, 973, 854, 0, 907, 908, + 984, 983, 985, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 667, 666, 665, 664, 663, 662, + 661, 660, 0, 0, 609, 509, 353, 305, 349, 350, + 357, 726, 722, 727, 710, 713, 712, 688, 861, 313, + 589, 424, 472, 374, 654, 655, 0, 708, 951, 916, + 917, 918, 851, 919, 913, 914, 852, 915, 952, 905, + 948, 949, 880, 910, 920, 947, 921, 950, 881, 953, + 993, 994, 927, 911, 275, 995, 924, 954, 946, 945, + 922, 906, 955, 956, 888, 883, 925, 926, 912, 931, + 932, 933, 936, 853, 937, 938, 939, 940, 941, 935, + 934, 902, 903, 904, 928, 929, 909, 500, 884, 885, + 886, 887, 0, 0, 539, 540, 541, 564, 0, 542, + 524, 588, 384, 314, 504, 531, 724, 0, 0, 0, + 0, 0, 0, 0, 639, 650, 684, 0, 696, 697, + 699, 701, 942, 703, 497, 498, 711, 0, 0, 930, + 706, 707, 704, 428, 484, 505, 491, 0, 730, 579, + 580, 731, 692, 315, 898, 846, 0, 2507, 0, 0, + 0, 0, 0, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 849, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 889, 631, 581, 493, 439, 0, 648, 0, + 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, + 0, 963, 0, 0, 0, 0, 841, 0, 0, 878, + 944, 943, 865, 875, 0, 0, 335, 246, 576, 698, + 578, 577, 866, 0, 867, 871, 874, 870, 868, 869, + 0, 958, 0, 0, 0, 0, 0, 0, 833, 845, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 842, 843, 0, + 0, 0, 0, 899, 0, 844, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 894, 872, + 876, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 873, 0, 897, 901, 360, 981, 895, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 982, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, - 0, 0, 0, 554, 555, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 687, 888, 0, 691, 0, 526, 0, 0, 961, 0, - 0, 0, 495, 0, 0, 413, 0, 0, 0, 892, - 0, 478, 453, 974, 0, 0, 476, 421, 511, 464, - 517, 498, 525, 470, 465, 316, 499, 363, 434, 332, - 334, 719, 365, 368, 372, 373, 443, 444, 458, 483, - 502, 503, 504, 362, 346, 477, 347, 382, 348, 317, - 354, 352, 355, 485, 356, 319, 459, 508, 0, 378, - 473, 429, 320, 428, 460, 507, 506, 333, 534, 541, - 542, 632, 0, 547, 730, 731, 732, 556, 0, 466, - 329, 328, 0, 0, 0, 358, 461, 342, 344, 345, - 343, 456, 457, 561, 562, 563, 565, 0, 566, 567, - 0, 0, 0, 0, 568, 633, 649, 617, 586, 549, - 641, 583, 587, 588, 399, 400, 401, 652, 0, 0, - 0, 540, 414, 415, 0, 370, 369, 430, 321, 0, - 0, 407, 398, 467, 327, 366, 409, 403, 416, 417, - 418, 376, 311, 312, 725, 958, 449, 654, 689, 690, - 579, 0, 973, 953, 955, 956, 960, 964, 965, 966, - 967, 968, 970, 972, 976, 724, 0, 634, 648, 728, - 647, 721, 455, 0, 482, 645, 592, 0, 638, 611, - 612, 0, 639, 607, 643, 0, 581, 0, 550, 553, - 582, 667, 668, 669, 318, 552, 671, 672, 673, 674, - 675, 676, 677, 670, 975, 615, 591, 618, 531, 594, - 593, 0, 0, 629, 896, 630, 631, 439, 440, 441, - 442, 962, 655, 340, 551, 469, 0, 616, 0, 0, - 0, 0, 0, 0, 0, 0, 621, 622, 619, 733, - 0, 678, 679, 0, 0, 545, 546, 375, 0, 564, - 383, 339, 454, 377, 529, 406, 0, 557, 623, 558, - 471, 472, 681, 686, 682, 683, 685, 705, 446, 397, - 402, 486, 408, 422, 474, 528, 452, 479, 337, 518, - 488, 427, 608, 636, 984, 957, 983, 985, 986, 982, - 987, 988, 969, 850, 0, 903, 904, 980, 979, 981, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 663, 662, 661, 660, 659, 658, 657, 656, 0, - 0, 605, 505, 353, 305, 349, 350, 357, 722, 718, - 723, 706, 709, 708, 684, 857, 313, 585, 420, 468, - 374, 650, 651, 0, 704, 947, 912, 913, 914, 847, - 915, 909, 910, 848, 911, 948, 901, 944, 945, 876, - 906, 916, 943, 917, 946, 877, 949, 989, 990, 923, - 907, 275, 991, 920, 950, 942, 941, 918, 902, 951, - 952, 884, 879, 921, 922, 908, 927, 928, 929, 932, - 849, 933, 934, 935, 936, 937, 931, 930, 898, 899, - 900, 924, 925, 905, 496, 880, 881, 882, 883, 0, - 0, 535, 536, 537, 560, 0, 538, 520, 584, 384, - 314, 500, 527, 720, 0, 0, 0, 0, 0, 0, - 0, 635, 646, 680, 0, 692, 693, 695, 697, 938, - 699, 493, 494, 707, 0, 0, 926, 702, 703, 700, - 424, 480, 501, 487, 894, 726, 575, 576, 727, 688, - 315, 0, 842, 451, 0, 0, 590, 624, 613, 698, - 578, 0, 0, 0, 0, 0, 0, 845, 0, 0, - 0, 367, 0, 0, 419, 628, 609, 620, 610, 595, - 596, 597, 604, 379, 598, 599, 600, 570, 601, 571, - 602, 603, 885, 627, 577, 489, 435, 0, 644, 0, - 0, 963, 971, 0, 0, 0, 0, 0, 0, 0, - 0, 959, 0, 0, 0, 0, 837, 0, 0, 874, - 940, 939, 861, 871, 0, 0, 335, 246, 572, 694, - 574, 573, 862, 0, 863, 867, 870, 866, 864, 865, - 0, 954, 0, 0, 0, 0, 0, 0, 829, 841, - 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 838, 839, 1753, - 0, 0, 0, 895, 0, 840, 0, 0, 0, 0, - 0, 490, 519, 0, 532, 0, 404, 405, 890, 868, - 872, 0, 0, 0, 0, 322, 497, 516, 336, 484, - 530, 341, 492, 509, 331, 450, 481, 0, 0, 324, - 514, 491, 432, 323, 0, 475, 364, 381, 361, 448, - 869, 0, 893, 897, 360, 977, 891, 524, 326, 0, - 523, 447, 510, 515, 433, 426, 0, 325, 512, 431, - 425, 410, 371, 978, 411, 412, 385, 462, 423, 463, - 386, 437, 436, 438, 387, 388, 389, 390, 391, 392, - 393, 394, 395, 396, 0, 0, 0, 0, 0, 554, - 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 687, 888, 0, 691, - 0, 526, 0, 0, 961, 0, 0, 0, 495, 0, - 0, 413, 0, 0, 0, 892, 0, 478, 453, 974, - 0, 0, 476, 421, 511, 464, 517, 498, 525, 470, - 465, 316, 499, 363, 434, 332, 334, 719, 365, 368, - 372, 373, 443, 444, 458, 483, 502, 503, 504, 362, - 346, 477, 347, 382, 348, 317, 354, 352, 355, 485, - 356, 319, 459, 508, 0, 378, 473, 429, 320, 428, - 460, 507, 506, 333, 534, 541, 542, 632, 0, 547, - 730, 731, 732, 556, 0, 466, 329, 328, 0, 0, - 0, 358, 461, 342, 344, 345, 343, 456, 457, 561, - 562, 563, 565, 0, 566, 567, 0, 0, 0, 0, - 568, 633, 649, 617, 586, 549, 641, 583, 587, 588, - 399, 400, 401, 652, 0, 0, 0, 540, 414, 415, - 0, 370, 369, 430, 321, 0, 0, 407, 398, 467, - 327, 366, 409, 403, 416, 417, 418, 376, 311, 312, - 725, 958, 449, 654, 689, 690, 579, 0, 973, 953, - 955, 956, 960, 964, 965, 966, 967, 968, 970, 972, - 976, 724, 0, 634, 648, 728, 647, 721, 455, 0, - 482, 645, 592, 0, 638, 611, 612, 0, 639, 607, - 643, 0, 581, 0, 550, 553, 582, 667, 668, 669, - 318, 552, 671, 672, 673, 674, 675, 676, 677, 670, - 975, 615, 591, 618, 531, 594, 593, 0, 0, 629, - 896, 630, 631, 439, 440, 441, 442, 962, 655, 340, - 551, 469, 0, 616, 0, 0, 0, 0, 0, 0, - 0, 0, 621, 622, 619, 733, 0, 678, 679, 0, - 0, 545, 546, 375, 0, 564, 383, 339, 454, 377, - 529, 406, 0, 557, 623, 558, 471, 472, 681, 686, - 682, 683, 685, 705, 446, 397, 402, 486, 408, 422, - 474, 528, 452, 479, 337, 518, 488, 427, 608, 636, - 984, 957, 983, 985, 986, 982, 987, 988, 969, 850, - 0, 903, 904, 980, 979, 981, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 663, 662, 661, - 660, 659, 658, 657, 656, 0, 0, 605, 505, 353, - 305, 349, 350, 357, 722, 718, 723, 706, 709, 708, - 684, 857, 313, 585, 420, 468, 374, 650, 651, 0, - 704, 947, 912, 913, 914, 847, 915, 909, 910, 848, - 911, 948, 901, 944, 945, 876, 906, 916, 943, 917, - 946, 877, 949, 989, 990, 923, 907, 275, 991, 920, - 950, 942, 941, 918, 902, 951, 952, 884, 879, 921, - 922, 908, 927, 928, 929, 932, 849, 933, 934, 935, - 936, 937, 931, 930, 898, 899, 900, 924, 925, 905, - 496, 880, 881, 882, 883, 0, 0, 535, 536, 537, - 560, 0, 538, 520, 584, 384, 314, 500, 527, 720, - 0, 0, 0, 0, 0, 0, 0, 635, 646, 680, - 0, 692, 693, 695, 697, 938, 699, 493, 494, 707, - 0, 0, 926, 702, 703, 700, 424, 480, 501, 487, - 0, 726, 575, 576, 727, 688, 315, 894, 842, 0, - 2495, 0, 0, 0, 0, 0, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 845, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 885, 627, 577, 489, 435, - 0, 644, 0, 0, 963, 971, 0, 0, 0, 0, - 0, 0, 0, 0, 959, 0, 0, 0, 0, 837, - 0, 0, 874, 940, 939, 861, 871, 0, 0, 335, - 246, 572, 694, 574, 573, 862, 0, 863, 867, 870, - 866, 864, 865, 0, 954, 0, 0, 0, 0, 0, - 0, 829, 841, 0, 846, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 838, 839, 0, 0, 0, 0, 895, 0, 840, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 890, 868, 872, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 869, 0, 893, 897, 360, 977, 891, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 978, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 888, 0, 691, 0, 526, 0, 0, 961, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 892, 0, - 478, 453, 974, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 958, 449, 654, 689, 690, 579, - 0, 973, 953, 955, 956, 960, 964, 965, 966, 967, - 968, 970, 972, 976, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 975, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 896, 630, 631, 439, 440, 441, 442, - 962, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 984, 957, 983, 985, 986, 982, 987, - 988, 969, 850, 0, 903, 904, 980, 979, 981, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 857, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 947, 912, 913, 914, 847, 915, - 909, 910, 848, 911, 948, 901, 944, 945, 876, 906, - 916, 943, 917, 946, 877, 949, 989, 990, 923, 907, - 275, 991, 920, 950, 942, 941, 918, 902, 951, 952, - 884, 879, 921, 922, 908, 927, 928, 929, 932, 849, - 933, 934, 935, 936, 937, 931, 930, 898, 899, 900, - 924, 925, 905, 496, 880, 881, 882, 883, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 938, 699, - 493, 494, 707, 0, 0, 926, 702, 703, 700, 424, - 480, 501, 487, 894, 726, 575, 576, 727, 688, 315, - 0, 842, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 845, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 885, 627, 577, 489, 435, 0, 644, 0, 0, - 963, 971, 0, 0, 0, 0, 0, 0, 0, 0, - 959, 0, 0, 0, 0, 837, 0, 0, 874, 940, - 939, 861, 871, 0, 0, 335, 246, 572, 694, 574, - 573, 862, 0, 863, 867, 870, 866, 864, 865, 0, - 954, 0, 0, 0, 0, 0, 0, 829, 841, 0, - 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 838, 839, 2053, 0, - 0, 0, 895, 0, 840, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 890, 868, 872, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 869, - 0, 893, 897, 360, 977, 891, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 978, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 888, 0, 691, 0, - 526, 0, 0, 961, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 892, 0, 478, 453, 974, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 958, 449, 654, 689, 690, 579, 0, 973, 953, 955, - 956, 960, 964, 965, 966, 967, 968, 970, 972, 976, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 975, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 896, - 630, 631, 439, 440, 441, 442, 962, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 984, - 957, 983, 985, 986, 982, 987, 988, 969, 850, 0, - 903, 904, 980, 979, 981, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 857, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 947, 912, 913, 914, 847, 915, 909, 910, 848, 911, - 948, 901, 944, 945, 876, 906, 916, 943, 917, 946, - 877, 949, 989, 990, 923, 907, 275, 991, 920, 950, - 942, 941, 918, 902, 951, 952, 884, 879, 921, 922, - 908, 927, 928, 929, 932, 849, 933, 934, 935, 936, - 937, 931, 930, 898, 899, 900, 924, 925, 905, 496, - 880, 881, 882, 883, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 938, 699, 493, 494, 707, 0, - 0, 926, 702, 703, 700, 424, 480, 501, 487, 894, - 726, 575, 576, 727, 688, 315, 0, 842, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 845, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 885, 627, 577, - 489, 435, 0, 644, 0, 0, 963, 971, 0, 0, - 0, 0, 0, 0, 0, 0, 959, 0, 0, 0, - 0, 837, 0, 0, 874, 940, 939, 861, 871, 0, - 0, 335, 246, 572, 694, 574, 573, 862, 0, 863, - 867, 870, 866, 864, 865, 0, 954, 0, 0, 0, - 0, 0, 0, 829, 841, 0, 846, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 838, 839, 0, 0, 0, 0, 895, 0, - 840, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 890, 868, 872, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 869, 0, 893, 897, 360, - 977, 891, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 978, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 888, 0, 691, 0, 526, 0, 0, 961, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 892, 0, 478, 453, 974, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 958, 449, 654, 689, - 690, 579, 0, 973, 953, 955, 956, 960, 964, 965, - 966, 967, 968, 970, 972, 976, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 975, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 896, 630, 631, 439, 440, - 441, 442, 962, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 984, 957, 983, 985, 986, - 982, 987, 988, 969, 850, 0, 903, 904, 980, 979, - 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 857, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 947, 912, 913, 914, - 847, 915, 909, 910, 848, 911, 948, 901, 944, 945, - 876, 906, 916, 943, 917, 946, 877, 949, 989, 990, - 923, 907, 275, 991, 920, 950, 942, 941, 918, 902, - 951, 952, 884, 879, 921, 922, 908, 927, 928, 929, - 932, 849, 933, 934, 935, 936, 937, 931, 930, 898, - 899, 900, 924, 925, 905, 496, 880, 881, 882, 883, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 938, 699, 493, 494, 707, 0, 0, 926, 702, 703, - 700, 424, 480, 501, 487, 894, 726, 575, 576, 727, - 688, 315, 0, 842, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 845, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 885, 627, 577, 489, 435, 0, 644, - 0, 0, 963, 971, 0, 0, 0, 0, 0, 0, - 0, 0, 959, 0, 0, 0, 0, 837, 0, 0, - 874, 940, 939, 861, 871, 0, 0, 335, 246, 572, - 694, 574, 573, 862, 0, 863, 867, 870, 866, 864, - 865, 0, 954, 0, 0, 0, 0, 0, 0, 829, - 841, 0, 846, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 838, 839, - 0, 0, 0, 0, 895, 0, 840, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 890, - 868, 872, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 869, 0, 893, 897, 360, 977, 891, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 978, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 888, 0, - 691, 0, 526, 0, 0, 961, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 892, 0, 478, 453, - 974, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 958, 449, 654, 689, 690, 579, 0, 973, - 953, 955, 956, 960, 964, 965, 966, 967, 968, 970, - 972, 976, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 975, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 896, 630, 631, 439, 440, 441, 442, 962, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 984, 957, 983, 985, 986, 982, 987, 988, 969, - 850, 0, 903, 904, 980, 979, 981, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 857, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 947, 912, 913, 914, 847, 915, 909, 910, - 848, 911, 948, 901, 944, 945, 876, 906, 916, 943, - 917, 946, 877, 949, 989, 990, 923, 907, 275, 991, - 920, 950, 942, 941, 918, 902, 951, 952, 884, 879, - 921, 922, 908, 927, 928, 929, 932, 849, 933, 934, - 935, 936, 937, 931, 930, 898, 899, 900, 924, 925, - 905, 496, 880, 881, 882, 883, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 938, 699, 493, 494, - 707, 0, 0, 4001, 702, 4002, 4003, 424, 480, 501, - 487, 894, 726, 575, 576, 727, 688, 315, 0, 842, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 845, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 885, - 627, 577, 489, 435, 0, 644, 0, 0, 963, 971, - 0, 0, 0, 0, 0, 0, 0, 0, 959, 0, - 0, 0, 0, 837, 0, 0, 874, 940, 939, 861, - 871, 0, 0, 335, 246, 572, 694, 574, 573, 3048, - 0, 3049, 867, 870, 866, 864, 865, 0, 954, 0, - 0, 0, 0, 0, 0, 829, 841, 0, 846, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 838, 839, 0, 0, 0, 0, - 895, 0, 840, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 890, 868, 872, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 869, 0, 893, - 897, 360, 977, 891, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 978, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 888, 0, 691, 0, 526, 0, - 0, 961, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 892, 0, 478, 453, 974, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 958, 449, - 654, 689, 690, 579, 0, 973, 953, 955, 956, 960, - 964, 965, 966, 967, 968, 970, 972, 976, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 975, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 896, 630, 631, - 439, 440, 441, 442, 962, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 984, 957, 983, - 985, 986, 982, 987, 988, 969, 850, 0, 903, 904, - 980, 979, 981, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 857, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 947, 912, - 913, 914, 847, 915, 909, 910, 848, 911, 948, 901, - 944, 945, 876, 906, 916, 943, 917, 946, 877, 949, - 989, 990, 923, 907, 275, 991, 920, 950, 942, 941, - 918, 902, 951, 952, 884, 879, 921, 922, 908, 927, - 928, 929, 932, 849, 933, 934, 935, 936, 937, 931, - 930, 898, 899, 900, 924, 925, 905, 496, 880, 881, - 882, 883, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 938, 699, 493, 494, 707, 0, 0, 926, - 702, 703, 700, 424, 480, 501, 487, 894, 726, 575, - 576, 727, 688, 315, 0, 842, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 1896, 0, 0, 0, - 845, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 885, 627, 577, 489, 435, - 0, 644, 0, 0, 963, 971, 0, 0, 0, 0, - 0, 0, 0, 0, 959, 0, 0, 0, 0, 837, - 0, 0, 874, 940, 939, 861, 871, 0, 0, 335, - 246, 572, 694, 574, 573, 862, 0, 863, 867, 870, - 866, 864, 865, 0, 954, 0, 0, 0, 0, 0, - 0, 0, 841, 0, 846, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 838, 839, 0, 0, 0, 0, 895, 0, 840, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 890, 868, 872, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 869, 0, 893, 897, 360, 977, 891, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 978, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 888, 0, 691, 0, 526, 0, 0, 961, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 892, 0, - 478, 453, 974, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 1897, 1898, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 958, 449, 654, 689, 690, 579, - 0, 973, 953, 955, 956, 960, 964, 965, 966, 967, - 968, 970, 972, 976, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 975, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 896, 630, 631, 439, 440, 441, 442, - 962, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 984, 957, 983, 985, 986, 982, 987, - 988, 969, 850, 0, 903, 904, 980, 979, 981, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 857, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 947, 912, 913, 914, 847, 915, - 909, 910, 848, 911, 948, 901, 944, 945, 876, 906, - 916, 943, 917, 946, 877, 949, 989, 990, 923, 907, - 275, 991, 920, 950, 942, 941, 918, 902, 951, 952, - 884, 879, 921, 922, 908, 927, 928, 929, 932, 849, - 933, 934, 935, 936, 937, 931, 930, 898, 899, 900, - 924, 925, 905, 496, 880, 881, 882, 883, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 938, 699, - 493, 494, 707, 0, 0, 926, 702, 703, 700, 424, - 480, 501, 487, 894, 726, 575, 576, 727, 688, 315, - 0, 842, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 845, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 885, 627, 577, 489, 435, 0, 644, 0, 0, - 963, 971, 0, 0, 0, 0, 0, 0, 0, 0, - 959, 0, 0, 0, 0, 1436, 0, 0, 874, 940, - 939, 861, 871, 0, 0, 335, 246, 572, 694, 574, - 573, 862, 0, 863, 867, 870, 866, 864, 865, 0, - 954, 0, 0, 0, 0, 0, 0, 829, 841, 0, - 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 838, 839, 0, 0, - 0, 0, 895, 0, 840, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 890, 868, 872, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 869, - 0, 893, 897, 360, 977, 891, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 978, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 888, 0, 691, 0, - 526, 0, 0, 961, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 892, 0, 478, 453, 974, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 958, 449, 654, 689, 690, 579, 0, 973, 953, 955, - 956, 960, 964, 965, 966, 967, 968, 970, 972, 976, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 975, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 896, - 630, 631, 439, 440, 441, 442, 962, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 984, - 957, 983, 985, 986, 982, 987, 988, 969, 850, 0, - 903, 904, 980, 979, 981, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 857, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 947, 912, 913, 914, 847, 915, 909, 910, 848, 911, - 948, 901, 944, 945, 876, 906, 916, 943, 917, 946, - 877, 949, 989, 990, 923, 907, 275, 991, 920, 950, - 942, 941, 918, 902, 951, 952, 884, 879, 921, 922, - 908, 927, 928, 929, 932, 849, 933, 934, 935, 936, - 937, 931, 930, 898, 899, 900, 924, 925, 905, 496, - 880, 881, 882, 883, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 938, 699, 493, 494, 707, 0, - 0, 926, 702, 703, 700, 424, 480, 501, 487, 894, - 726, 575, 576, 727, 688, 315, 0, 842, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 845, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 885, 627, 577, - 489, 435, 0, 644, 0, 0, 963, 971, 0, 0, - 0, 0, 0, 0, 0, 0, 959, 0, 0, 0, - 0, 837, 0, 0, 874, 940, 939, 861, 871, 0, - 0, 335, 246, 572, 694, 574, 573, 862, 0, 863, - 867, 870, 866, 864, 865, 0, 954, 0, 0, 0, - 0, 0, 0, 0, 841, 0, 846, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 838, 839, 0, 0, 0, 0, 895, 0, - 840, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 890, 868, 872, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 869, 0, 893, 897, 360, - 977, 891, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 978, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 892, 0, 695, 0, 530, 0, 0, 965, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 896, + 0, 482, 457, 978, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 962, 453, 658, 693, 694, + 583, 0, 977, 957, 959, 960, 964, 968, 969, 970, + 971, 972, 974, 976, 980, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 979, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 900, 634, 635, 443, 444, 445, + 446, 966, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 988, 961, 987, 989, 990, 986, + 991, 992, 973, 854, 0, 907, 908, 984, 983, 985, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 861, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 951, 916, 917, 918, 851, + 919, 913, 914, 852, 915, 952, 905, 948, 949, 880, + 910, 920, 947, 921, 950, 881, 953, 993, 994, 927, + 911, 275, 995, 924, 954, 946, 945, 922, 906, 955, + 956, 888, 883, 925, 926, 912, 931, 932, 933, 936, + 853, 937, 938, 939, 940, 941, 935, 934, 902, 903, + 904, 928, 929, 909, 500, 884, 885, 886, 887, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 942, + 703, 497, 498, 711, 0, 0, 930, 706, 707, 704, + 428, 484, 505, 491, 898, 730, 579, 580, 731, 692, + 315, 0, 846, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 849, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 889, 631, 581, 493, 439, 0, 648, 0, + 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, + 0, 963, 0, 0, 0, 0, 841, 0, 0, 878, + 944, 943, 865, 875, 0, 0, 335, 246, 576, 698, + 578, 577, 866, 0, 867, 871, 874, 870, 868, 869, + 0, 958, 0, 0, 0, 0, 0, 0, 833, 845, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 842, 843, 2065, + 0, 0, 0, 899, 0, 844, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 894, 872, + 876, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 873, 0, 897, 901, 360, 981, 895, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 982, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 892, 0, 695, 0, 530, 0, 0, 965, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 896, + 0, 482, 457, 978, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 962, 453, 658, 693, 694, + 583, 0, 977, 957, 959, 960, 964, 968, 969, 970, + 971, 972, 974, 976, 980, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 979, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 900, 634, 635, 443, 444, 445, + 446, 966, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 988, 961, 987, 989, 990, 986, + 991, 992, 973, 854, 0, 907, 908, 984, 983, 985, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 861, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 951, 916, 917, 918, 851, + 919, 913, 914, 852, 915, 952, 905, 948, 949, 880, + 910, 920, 947, 921, 950, 881, 953, 993, 994, 927, + 911, 275, 995, 924, 954, 946, 945, 922, 906, 955, + 956, 888, 883, 925, 926, 912, 931, 932, 933, 936, + 853, 937, 938, 939, 940, 941, 935, 934, 902, 903, + 904, 928, 929, 909, 500, 884, 885, 886, 887, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 942, + 703, 497, 498, 711, 0, 0, 930, 706, 707, 704, + 428, 484, 505, 491, 898, 730, 579, 580, 731, 692, + 315, 0, 846, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 849, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 889, 631, 581, 493, 439, 0, 648, 0, + 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, + 0, 963, 0, 0, 0, 0, 841, 0, 0, 878, + 944, 943, 865, 875, 0, 0, 335, 246, 576, 698, + 578, 577, 866, 0, 867, 871, 874, 870, 868, 869, + 0, 958, 0, 0, 0, 0, 0, 0, 833, 845, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 842, 843, 0, + 0, 0, 0, 899, 0, 844, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 894, 872, + 876, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 873, 0, 897, 901, 360, 981, 895, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 982, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 892, 0, 695, 0, 530, 0, 0, 965, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 896, + 0, 482, 457, 978, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 962, 453, 658, 693, 694, + 583, 0, 977, 957, 959, 960, 964, 968, 969, 970, + 971, 972, 974, 976, 980, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 979, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 900, 634, 635, 443, 444, 445, + 446, 966, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 988, 961, 987, 989, 990, 986, + 991, 992, 973, 854, 0, 907, 908, 984, 983, 985, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 861, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 951, 916, 917, 918, 851, + 919, 913, 914, 852, 915, 952, 905, 948, 949, 880, + 910, 920, 947, 921, 950, 881, 953, 993, 994, 927, + 911, 275, 995, 924, 954, 946, 945, 922, 906, 955, + 956, 888, 883, 925, 926, 912, 931, 932, 933, 936, + 853, 937, 938, 939, 940, 941, 935, 934, 902, 903, + 904, 928, 929, 909, 500, 884, 885, 886, 887, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 942, + 703, 497, 498, 711, 0, 0, 930, 706, 707, 704, + 428, 484, 505, 491, 898, 730, 579, 580, 731, 692, + 315, 0, 846, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 849, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 889, 631, 581, 493, 439, 0, 648, 0, + 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, + 0, 963, 0, 0, 0, 0, 841, 0, 0, 878, + 944, 943, 865, 875, 0, 0, 335, 246, 576, 698, + 578, 577, 866, 0, 867, 871, 874, 870, 868, 869, + 0, 958, 0, 0, 0, 0, 0, 0, 833, 845, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 842, 843, 0, + 0, 0, 0, 899, 0, 844, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 894, 872, + 876, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 873, 0, 897, 901, 360, 981, 895, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 982, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 892, 0, 695, 0, 530, 0, 0, 965, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 896, + 0, 482, 457, 978, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 962, 453, 658, 693, 694, + 583, 0, 977, 957, 959, 960, 964, 968, 969, 970, + 971, 972, 974, 976, 980, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 979, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 900, 634, 635, 443, 444, 445, + 446, 966, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 988, 961, 987, 989, 990, 986, + 991, 992, 973, 854, 0, 907, 908, 984, 983, 985, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 861, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 951, 916, 917, 918, 851, + 919, 913, 914, 852, 915, 952, 905, 948, 949, 880, + 910, 920, 947, 921, 950, 881, 953, 993, 994, 927, + 911, 275, 995, 924, 954, 946, 945, 922, 906, 955, + 956, 888, 883, 925, 926, 912, 931, 932, 933, 936, + 853, 937, 938, 939, 940, 941, 935, 934, 902, 903, + 904, 928, 929, 909, 500, 884, 885, 886, 887, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 942, + 703, 497, 498, 711, 0, 0, 4017, 706, 4018, 4019, + 428, 484, 505, 491, 898, 730, 579, 580, 731, 692, + 315, 0, 846, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 849, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 889, 631, 581, 493, 439, 0, 648, 0, + 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, + 0, 963, 0, 0, 0, 0, 841, 0, 0, 878, + 944, 943, 865, 875, 0, 0, 335, 246, 576, 698, + 578, 577, 3064, 0, 3065, 871, 874, 870, 868, 869, + 0, 958, 0, 0, 0, 0, 0, 0, 833, 845, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 842, 843, 0, + 0, 0, 0, 899, 0, 844, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 894, 872, + 876, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 873, 0, 897, 901, 360, 981, 895, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 982, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 892, 0, 695, 0, 530, 0, 0, 965, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 896, + 0, 482, 457, 978, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 962, 453, 658, 693, 694, + 583, 0, 977, 957, 959, 960, 964, 968, 969, 970, + 971, 972, 974, 976, 980, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 979, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 900, 634, 635, 443, 444, 445, + 446, 966, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 988, 961, 987, 989, 990, 986, + 991, 992, 973, 854, 0, 907, 908, 984, 983, 985, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 861, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 951, 916, 917, 918, 851, + 919, 913, 914, 852, 915, 952, 905, 948, 949, 880, + 910, 920, 947, 921, 950, 881, 953, 993, 994, 927, + 911, 275, 995, 924, 954, 946, 945, 922, 906, 955, + 956, 888, 883, 925, 926, 912, 931, 932, 933, 936, + 853, 937, 938, 939, 940, 941, 935, 934, 902, 903, + 904, 928, 929, 909, 500, 884, 885, 886, 887, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 942, + 703, 497, 498, 711, 0, 0, 930, 706, 707, 704, + 428, 484, 505, 491, 898, 730, 579, 580, 731, 692, + 315, 0, 846, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 1908, 0, 0, 0, 849, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 889, 631, 581, 493, 439, 0, 648, 0, + 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, + 0, 963, 0, 0, 0, 0, 841, 0, 0, 878, + 944, 943, 865, 875, 0, 0, 335, 246, 576, 698, + 578, 577, 866, 0, 867, 871, 874, 870, 868, 869, + 0, 958, 0, 0, 0, 0, 0, 0, 0, 845, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 842, 843, 0, + 0, 0, 0, 899, 0, 844, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 894, 872, + 876, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 873, 0, 897, 901, 360, 981, 895, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 982, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 892, 0, 695, 0, 530, 0, 0, 965, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 896, + 0, 482, 457, 978, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 1909, + 1910, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 962, 453, 658, 693, 694, + 583, 0, 977, 957, 959, 960, 964, 968, 969, 970, + 971, 972, 974, 976, 980, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 979, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 900, 634, 635, 443, 444, 445, + 446, 966, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 988, 961, 987, 989, 990, 986, + 991, 992, 973, 854, 0, 907, 908, 984, 983, 985, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 861, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 951, 916, 917, 918, 851, + 919, 913, 914, 852, 915, 952, 905, 948, 949, 880, + 910, 920, 947, 921, 950, 881, 953, 993, 994, 927, + 911, 275, 995, 924, 954, 946, 945, 922, 906, 955, + 956, 888, 883, 925, 926, 912, 931, 932, 933, 936, + 853, 937, 938, 939, 940, 941, 935, 934, 902, 903, + 904, 928, 929, 909, 500, 884, 885, 886, 887, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 942, + 703, 497, 498, 711, 0, 0, 930, 706, 707, 704, + 428, 484, 505, 491, 898, 730, 579, 580, 731, 692, + 315, 0, 846, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 849, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 889, 631, 581, 493, 439, 0, 648, 0, + 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, + 0, 963, 0, 0, 0, 0, 1444, 0, 0, 878, + 944, 943, 865, 875, 0, 0, 335, 246, 576, 698, + 578, 577, 866, 0, 867, 871, 874, 870, 868, 869, + 0, 958, 0, 0, 0, 0, 0, 0, 833, 845, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 842, 843, 0, + 0, 0, 0, 899, 0, 844, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 894, 872, + 876, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 873, 0, 897, 901, 360, 981, 895, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 982, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 892, 0, 695, 0, 530, 0, 0, 965, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 896, + 0, 482, 457, 978, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 962, 453, 658, 693, 694, + 583, 0, 977, 957, 959, 960, 964, 968, 969, 970, + 971, 972, 974, 976, 980, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 979, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 900, 634, 635, 443, 444, 445, + 446, 966, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 988, 961, 987, 989, 990, 986, + 991, 992, 973, 854, 0, 907, 908, 984, 983, 985, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 861, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 951, 916, 917, 918, 851, + 919, 913, 914, 852, 915, 952, 905, 948, 949, 880, + 910, 920, 947, 921, 950, 881, 953, 993, 994, 927, + 911, 275, 995, 924, 954, 946, 945, 922, 906, 955, + 956, 888, 883, 925, 926, 912, 931, 932, 933, 936, + 853, 937, 938, 939, 940, 941, 935, 934, 902, 903, + 904, 928, 929, 909, 500, 884, 885, 886, 887, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 942, + 703, 497, 498, 711, 0, 0, 930, 706, 707, 704, + 428, 484, 505, 491, 898, 730, 579, 580, 731, 692, + 315, 0, 846, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 849, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 889, 631, 581, 493, 439, 0, 648, 0, + 0, 967, 975, 0, 0, 0, 0, 0, 0, 0, + 0, 963, 0, 0, 0, 0, 841, 0, 0, 878, + 944, 943, 865, 875, 0, 0, 335, 246, 576, 698, + 578, 577, 866, 0, 867, 871, 874, 870, 868, 869, + 0, 958, 0, 0, 0, 0, 0, 0, 0, 845, + 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 842, 843, 0, + 0, 0, 0, 899, 0, 844, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 894, 872, + 876, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 873, 0, 897, 901, 360, 981, 895, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 982, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 892, 0, 695, 0, 530, 0, 0, 965, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 896, + 0, 482, 457, 978, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 962, 453, 658, 693, 694, + 583, 0, 977, 957, 959, 960, 964, 968, 969, 970, + 971, 972, 974, 976, 980, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 979, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 900, 634, 635, 443, 444, 445, + 446, 966, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 988, 961, 987, 989, 990, 986, + 991, 992, 973, 854, 0, 907, 908, 984, 983, 985, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 861, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 951, 916, 917, 918, 851, + 919, 913, 914, 852, 915, 952, 905, 948, 949, 880, + 910, 920, 947, 921, 950, 881, 953, 993, 994, 927, + 911, 275, 995, 924, 954, 946, 945, 922, 906, 955, + 956, 888, 883, 925, 926, 912, 931, 932, 933, 936, + 853, 937, 938, 939, 940, 941, 935, 934, 902, 903, + 904, 928, 929, 909, 500, 884, 885, 886, 887, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 942, + 703, 497, 498, 711, 0, 0, 930, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 0, 846, 183, 223, 182, 214, 184, 0, 0, + 0, 0, 0, 0, 455, 0, 0, 594, 628, 617, + 702, 582, 0, 215, 0, 0, 0, 0, 0, 0, + 206, 0, 367, 0, 216, 423, 632, 613, 624, 614, + 599, 600, 601, 608, 379, 602, 603, 604, 574, 605, + 575, 606, 607, 153, 631, 581, 493, 439, 0, 648, + 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, + 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, + 245, 0, 0, 0, 0, 0, 0, 335, 246, 576, + 698, 578, 577, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 494, 523, 0, 536, 0, 404, 405, 0, + 0, 0, 0, 0, 0, 0, 322, 501, 520, 336, + 488, 534, 341, 496, 513, 331, 454, 485, 0, 0, + 324, 518, 495, 436, 323, 0, 479, 364, 381, 361, + 452, 0, 0, 517, 547, 360, 537, 0, 528, 326, + 0, 527, 451, 514, 519, 437, 430, 0, 325, 516, + 435, 429, 410, 371, 563, 411, 412, 413, 414, 415, + 416, 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 888, 0, 691, 0, 526, 0, 0, 961, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 892, 0, 478, 453, 974, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 958, 449, 654, 689, - 690, 579, 0, 973, 953, 955, 956, 960, 964, 965, - 966, 967, 968, 970, 972, 976, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 975, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 896, 630, 631, 439, 440, - 441, 442, 962, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 984, 957, 983, 985, 986, - 982, 987, 988, 969, 850, 0, 903, 904, 980, 979, - 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 857, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 947, 912, 913, 914, - 847, 915, 909, 910, 848, 911, 948, 901, 944, 945, - 876, 906, 916, 943, 917, 946, 877, 949, 989, 990, - 923, 907, 275, 991, 920, 950, 942, 941, 918, 902, - 951, 952, 884, 879, 921, 922, 908, 927, 928, 929, - 932, 849, 933, 934, 935, 936, 937, 931, 930, 898, - 899, 900, 924, 925, 905, 496, 880, 881, 882, 883, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 938, 699, 493, 494, 707, 0, 0, 926, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 0, 842, 183, 223, 182, 214, 184, 0, - 0, 0, 0, 0, 0, 451, 0, 0, 590, 624, - 613, 698, 578, 0, 215, 0, 0, 0, 0, 0, - 0, 206, 0, 367, 0, 216, 419, 628, 609, 620, - 610, 595, 596, 597, 604, 379, 598, 599, 600, 570, - 601, 571, 602, 603, 153, 627, 577, 489, 435, 0, - 644, 0, 0, 0, 0, 0, 0, 0, 0, 139, - 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, - 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, - 572, 694, 574, 573, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 558, 559, 0, 0, 0, 0, + 0, 0, 0, 181, 212, 221, 213, 75, 137, 0, + 0, 691, 0, 0, 695, 0, 530, 0, 0, 238, + 0, 0, 0, 499, 0, 0, 417, 211, 205, 204, + 548, 0, 482, 457, 250, 0, 0, 480, 425, 515, + 468, 521, 502, 529, 474, 469, 316, 503, 363, 438, + 332, 334, 258, 365, 368, 372, 373, 447, 448, 462, + 487, 506, 507, 508, 362, 346, 481, 347, 382, 348, + 317, 354, 352, 355, 489, 356, 319, 463, 512, 0, + 378, 477, 433, 320, 432, 464, 511, 510, 333, 538, + 545, 546, 636, 0, 551, 668, 669, 670, 560, 0, + 470, 329, 328, 0, 0, 0, 358, 465, 342, 344, + 345, 343, 460, 461, 565, 566, 567, 569, 0, 570, + 571, 0, 0, 0, 0, 572, 637, 653, 621, 590, + 553, 645, 587, 591, 592, 399, 400, 401, 656, 0, + 0, 0, 544, 418, 419, 0, 370, 369, 434, 321, + 0, 0, 407, 398, 471, 327, 366, 409, 403, 420, + 421, 422, 376, 311, 312, 525, 359, 453, 658, 693, + 694, 583, 0, 646, 584, 593, 351, 618, 630, 629, + 449, 543, 241, 641, 644, 573, 251, 0, 638, 652, + 610, 651, 252, 459, 0, 486, 649, 596, 0, 642, + 615, 616, 0, 643, 611, 647, 0, 585, 0, 554, + 557, 586, 671, 672, 673, 318, 556, 675, 676, 677, + 678, 679, 680, 681, 674, 526, 619, 595, 622, 535, + 598, 597, 0, 0, 633, 552, 634, 635, 443, 444, + 445, 446, 380, 659, 340, 555, 473, 151, 620, 0, + 0, 0, 0, 0, 0, 0, 0, 625, 626, 623, + 249, 0, 682, 683, 0, 0, 549, 550, 375, 0, + 568, 383, 339, 458, 377, 533, 406, 0, 561, 627, + 562, 475, 476, 685, 690, 686, 687, 689, 709, 450, + 397, 402, 490, 408, 426, 478, 532, 456, 483, 337, + 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, + 0, 0, 0, 71, 0, 0, 298, 299, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 667, 666, 665, 664, 663, 662, 661, 660, + 0, 0, 609, 509, 353, 305, 349, 350, 357, 256, + 330, 257, 710, 713, 712, 688, 0, 313, 589, 424, + 472, 374, 654, 655, 66, 708, 259, 260, 261, 262, + 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, + 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, + 657, 274, 275, 284, 285, 286, 287, 288, 289, 290, + 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, + 0, 307, 714, 715, 716, 717, 718, 0, 0, 308, + 309, 310, 0, 0, 300, 500, 301, 302, 303, 304, + 0, 0, 539, 540, 541, 564, 0, 542, 524, 588, + 384, 314, 504, 531, 253, 49, 239, 242, 244, 243, + 0, 67, 639, 650, 684, 5, 696, 697, 699, 701, + 700, 703, 497, 498, 711, 0, 0, 705, 706, 707, + 704, 428, 484, 505, 491, 156, 254, 579, 580, 255, + 692, 315, 183, 223, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 153, 631, 581, 493, 439, 0, 648, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 219, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 2693, 2696, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 2697, 530, 0, 0, 0, 2692, + 0, 2691, 499, 2689, 2694, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 2695, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1465, 0, 0, 245, 0, 0, + 865, 875, 0, 0, 335, 246, 576, 698, 578, 577, + 866, 0, 867, 871, 874, 870, 868, 869, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 490, 519, 0, 532, 0, 404, 405, - 0, 0, 0, 0, 0, 0, 0, 322, 497, 516, - 336, 484, 530, 341, 492, 509, 331, 450, 481, 0, - 0, 324, 514, 491, 432, 323, 0, 475, 364, 381, - 361, 448, 0, 0, 513, 543, 360, 533, 0, 524, - 326, 0, 523, 447, 510, 515, 433, 426, 0, 325, - 512, 431, 425, 410, 371, 559, 411, 412, 385, 462, - 423, 463, 386, 437, 436, 438, 387, 388, 389, 390, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 872, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 873, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, - 0, 554, 555, 0, 0, 0, 0, 0, 0, 0, - 181, 212, 221, 213, 75, 137, 0, 0, 687, 0, - 0, 691, 0, 526, 0, 0, 238, 0, 0, 0, - 495, 0, 0, 413, 211, 205, 204, 544, 0, 478, - 453, 250, 0, 0, 476, 421, 511, 464, 517, 498, - 525, 470, 465, 316, 499, 363, 434, 332, 334, 258, - 365, 368, 372, 373, 443, 444, 458, 483, 502, 503, - 504, 362, 346, 477, 347, 382, 348, 317, 354, 352, - 355, 485, 356, 319, 459, 508, 0, 378, 473, 429, - 320, 428, 460, 507, 506, 333, 534, 541, 542, 632, - 0, 547, 664, 665, 666, 556, 0, 466, 329, 328, - 0, 0, 0, 358, 461, 342, 344, 345, 343, 456, - 457, 561, 562, 563, 565, 0, 566, 567, 0, 0, - 0, 0, 568, 633, 649, 617, 586, 549, 641, 583, - 587, 588, 399, 400, 401, 652, 0, 0, 0, 540, - 414, 415, 0, 370, 369, 430, 321, 0, 0, 407, - 398, 467, 327, 366, 409, 403, 416, 417, 418, 376, - 311, 312, 521, 359, 449, 654, 689, 690, 579, 0, - 642, 580, 589, 351, 614, 626, 625, 445, 539, 241, - 637, 640, 569, 251, 0, 634, 648, 606, 647, 252, - 455, 0, 482, 645, 592, 0, 638, 611, 612, 0, - 639, 607, 643, 0, 581, 0, 550, 553, 582, 667, - 668, 669, 318, 552, 671, 672, 673, 674, 675, 676, - 677, 670, 522, 615, 591, 618, 531, 594, 593, 0, - 0, 629, 548, 630, 631, 439, 440, 441, 442, 380, - 655, 340, 551, 469, 151, 616, 0, 0, 0, 0, - 0, 0, 0, 0, 621, 622, 619, 249, 0, 678, - 679, 0, 0, 545, 546, 375, 0, 564, 383, 339, - 454, 377, 529, 406, 0, 557, 623, 558, 471, 472, - 681, 686, 682, 683, 685, 705, 446, 397, 402, 486, - 408, 422, 474, 528, 452, 479, 337, 518, 488, 427, - 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, - 71, 0, 0, 298, 299, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, - 662, 661, 660, 659, 658, 657, 656, 0, 0, 605, - 505, 353, 305, 349, 350, 357, 256, 330, 257, 706, - 709, 708, 684, 0, 313, 585, 420, 468, 374, 650, - 651, 66, 704, 259, 260, 261, 262, 263, 264, 265, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, - 277, 278, 279, 280, 281, 282, 283, 653, 274, 275, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, - 294, 295, 296, 297, 0, 0, 0, 0, 307, 710, - 711, 712, 713, 714, 0, 0, 308, 309, 310, 0, - 0, 300, 496, 301, 302, 303, 304, 0, 0, 535, - 536, 537, 560, 0, 538, 520, 584, 384, 314, 500, - 527, 253, 49, 239, 242, 244, 243, 0, 67, 635, - 646, 680, 5, 692, 693, 695, 697, 696, 699, 493, - 494, 707, 0, 0, 701, 702, 703, 700, 424, 480, - 501, 487, 156, 254, 575, 576, 255, 688, 315, 183, - 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 183, + 223, 182, 214, 184, 0, 0, 0, 0, 0, 0, + 455, 756, 0, 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 153, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 219, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 2681, - 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 2685, 526, 0, - 0, 0, 2680, 0, 2679, 495, 2677, 2682, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 2683, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, + 0, 423, 632, 613, 624, 614, 599, 600, 601, 608, + 379, 602, 603, 604, 574, 605, 575, 606, 607, 0, + 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 763, 0, 0, 0, 0, + 0, 0, 0, 762, 0, 0, 245, 0, 0, 0, + 0, 0, 0, 335, 246, 576, 698, 578, 577, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1457, 0, 0, - 245, 0, 0, 861, 871, 0, 0, 335, 246, 572, - 694, 574, 573, 862, 0, 863, 867, 870, 866, 864, - 865, 0, 338, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 868, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 869, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 494, 523, + 0, 536, 0, 404, 405, 0, 0, 0, 0, 0, + 0, 0, 322, 501, 520, 336, 488, 534, 341, 496, + 513, 331, 454, 485, 0, 0, 324, 518, 495, 436, + 323, 0, 479, 364, 381, 361, 452, 0, 0, 517, + 547, 360, 537, 0, 528, 326, 0, 527, 451, 514, + 519, 437, 430, 0, 325, 516, 435, 429, 410, 371, + 563, 411, 412, 413, 414, 415, 416, 385, 466, 427, + 467, 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 558, 559, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 760, 761, 0, 691, 0, 0, + 695, 0, 530, 0, 0, 0, 0, 0, 0, 499, + 0, 0, 417, 0, 0, 0, 548, 0, 482, 457, + 733, 0, 0, 480, 425, 515, 468, 521, 502, 529, + 474, 469, 316, 503, 363, 438, 332, 334, 723, 365, + 368, 372, 373, 447, 448, 462, 487, 506, 507, 508, + 362, 346, 481, 347, 382, 348, 317, 354, 352, 355, + 489, 356, 319, 463, 512, 0, 378, 477, 433, 320, + 432, 464, 511, 510, 333, 538, 545, 546, 636, 0, + 551, 734, 735, 736, 560, 0, 470, 329, 328, 0, + 0, 0, 358, 465, 342, 344, 345, 343, 460, 461, + 565, 566, 567, 569, 0, 570, 571, 0, 0, 0, + 0, 572, 637, 653, 621, 590, 553, 645, 587, 591, + 592, 399, 400, 401, 656, 0, 0, 0, 544, 418, + 419, 0, 370, 369, 434, 321, 0, 0, 407, 398, + 471, 327, 366, 409, 403, 420, 421, 422, 376, 311, + 312, 729, 359, 453, 658, 693, 694, 583, 0, 646, + 584, 593, 351, 618, 630, 629, 449, 543, 0, 641, + 644, 573, 728, 0, 638, 652, 732, 651, 725, 459, + 0, 486, 649, 596, 0, 642, 615, 616, 0, 643, + 611, 647, 0, 585, 0, 554, 557, 586, 671, 672, + 673, 318, 556, 675, 676, 677, 678, 679, 680, 681, + 674, 526, 619, 595, 622, 535, 598, 597, 0, 0, + 633, 552, 634, 635, 443, 444, 445, 446, 757, 759, + 340, 555, 473, 771, 620, 0, 0, 0, 0, 0, + 0, 0, 0, 625, 626, 623, 737, 0, 682, 683, + 0, 0, 549, 550, 375, 0, 568, 383, 339, 458, + 377, 533, 406, 0, 561, 627, 562, 475, 476, 685, + 690, 686, 687, 689, 709, 450, 397, 402, 490, 408, + 426, 478, 532, 456, 483, 337, 522, 492, 431, 612, + 640, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, + 0, 0, 0, 0, 0, 0, 0, 0, 667, 666, + 665, 664, 663, 662, 661, 660, 0, 0, 609, 509, + 353, 305, 349, 350, 357, 726, 722, 727, 710, 713, + 712, 688, 0, 313, 589, 424, 472, 374, 654, 655, + 0, 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, + 278, 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 183, 223, - 182, 214, 184, 0, 0, 0, 0, 0, 0, 451, - 752, 0, 590, 624, 613, 698, 578, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, - 419, 628, 609, 620, 610, 595, 596, 597, 604, 379, - 598, 599, 600, 570, 601, 571, 602, 603, 0, 627, - 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, - 0, 0, 758, 0, 0, 245, 0, 0, 0, 0, - 0, 0, 335, 246, 572, 694, 574, 573, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, + 295, 296, 297, 0, 0, 0, 0, 307, 714, 715, + 716, 717, 718, 0, 0, 308, 309, 310, 0, 0, + 300, 500, 301, 302, 303, 304, 0, 0, 539, 540, + 541, 564, 0, 542, 524, 588, 384, 314, 504, 531, + 724, 0, 0, 0, 0, 0, 0, 0, 639, 650, + 684, 0, 696, 697, 699, 701, 700, 703, 497, 498, + 711, 0, 0, 705, 706, 707, 704, 428, 484, 505, + 491, 0, 730, 579, 580, 731, 692, 315, 455, 0, + 0, 594, 628, 617, 702, 582, 0, 1249, 0, 0, + 0, 0, 0, 0, 0, 0, 367, 0, 0, 423, + 632, 613, 624, 614, 599, 600, 601, 608, 379, 602, + 603, 604, 574, 605, 575, 606, 607, 0, 631, 581, + 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, + 0, 335, 246, 576, 698, 578, 577, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 490, 519, 0, - 532, 0, 404, 405, 0, 0, 0, 0, 0, 0, - 0, 322, 497, 516, 336, 484, 530, 341, 492, 509, - 331, 450, 481, 0, 0, 324, 514, 491, 432, 323, - 0, 475, 364, 381, 361, 448, 0, 0, 513, 543, - 360, 533, 0, 524, 326, 0, 523, 447, 510, 515, - 433, 426, 0, 325, 512, 431, 425, 410, 371, 559, - 411, 412, 385, 462, 423, 463, 386, 437, 436, 438, - 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 0, 0, 0, 0, 0, 554, 555, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 756, - 757, 0, 687, 0, 0, 691, 0, 526, 0, 0, - 0, 0, 0, 0, 495, 0, 0, 413, 0, 0, - 0, 544, 0, 478, 453, 729, 0, 0, 476, 421, - 511, 464, 517, 498, 525, 470, 465, 316, 499, 363, - 434, 332, 334, 719, 365, 368, 372, 373, 443, 444, - 458, 483, 502, 503, 504, 362, 346, 477, 347, 382, - 348, 317, 354, 352, 355, 485, 356, 319, 459, 508, - 0, 378, 473, 429, 320, 428, 460, 507, 506, 333, - 534, 541, 542, 632, 0, 547, 730, 731, 732, 556, - 0, 466, 329, 328, 0, 0, 0, 358, 461, 342, - 344, 345, 343, 456, 457, 561, 562, 563, 565, 0, - 566, 567, 0, 0, 0, 0, 568, 633, 649, 617, - 586, 549, 641, 583, 587, 588, 399, 400, 401, 652, - 0, 0, 0, 540, 414, 415, 0, 370, 369, 430, - 321, 0, 0, 407, 398, 467, 327, 366, 409, 403, - 416, 417, 418, 376, 311, 312, 725, 359, 449, 654, - 689, 690, 579, 0, 642, 580, 589, 351, 614, 626, - 625, 445, 539, 0, 637, 640, 569, 724, 0, 634, - 648, 728, 647, 721, 455, 0, 482, 645, 592, 0, - 638, 611, 612, 0, 639, 607, 643, 0, 581, 0, - 550, 553, 582, 667, 668, 669, 318, 552, 671, 672, - 673, 674, 675, 676, 677, 670, 522, 615, 591, 618, - 531, 594, 593, 0, 0, 629, 548, 630, 631, 439, - 440, 441, 442, 753, 755, 340, 551, 469, 767, 616, - 0, 0, 0, 0, 0, 0, 0, 0, 621, 622, - 619, 733, 0, 678, 679, 0, 0, 545, 546, 375, - 0, 564, 383, 339, 454, 377, 529, 406, 0, 557, - 623, 558, 471, 472, 681, 686, 682, 683, 685, 705, - 446, 397, 402, 486, 408, 422, 474, 528, 452, 479, - 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, - 0, 0, 0, 0, 71, 0, 0, 298, 299, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 663, 662, 661, 660, 659, 658, 657, - 656, 0, 0, 605, 505, 353, 305, 349, 350, 357, - 722, 718, 723, 706, 709, 708, 684, 0, 313, 585, - 420, 468, 374, 650, 651, 0, 704, 259, 260, 261, - 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, - 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, - 283, 653, 274, 275, 284, 285, 286, 287, 288, 289, - 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, - 0, 0, 307, 710, 711, 712, 713, 714, 0, 0, - 308, 309, 310, 0, 0, 300, 496, 301, 302, 303, - 304, 0, 0, 535, 536, 537, 560, 0, 538, 520, - 584, 384, 314, 500, 527, 720, 0, 0, 0, 0, - 0, 0, 0, 635, 646, 680, 0, 692, 693, 695, - 697, 696, 699, 493, 494, 707, 0, 0, 701, 702, - 703, 700, 424, 480, 501, 487, 0, 726, 575, 576, - 727, 688, 315, 451, 0, 0, 590, 624, 613, 698, - 578, 0, 1241, 0, 0, 0, 0, 0, 0, 0, - 0, 367, 0, 0, 419, 628, 609, 620, 610, 595, - 596, 597, 604, 379, 598, 599, 600, 570, 601, 571, - 602, 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, - 0, 0, 0, 0, 0, 0, 335, 246, 572, 694, - 574, 573, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 494, 523, 0, 536, + 0, 2877, 2878, 1230, 0, 0, 0, 0, 0, 0, + 322, 501, 520, 336, 488, 534, 341, 496, 513, 331, + 454, 485, 0, 0, 2871, 2874, 2875, 2876, 2879, 0, + 2884, 2880, 2881, 2882, 2883, 0, 0, 2867, 2868, 2869, + 2870, 1228, 2847, 2872, 0, 2848, 451, 2849, 2850, 2851, + 2852, 1232, 2853, 2854, 2855, 2856, 2857, 2864, 2865, 2858, + 2859, 2860, 2861, 2862, 2863, 2885, 2886, 2887, 2888, 2889, + 2890, 2891, 2892, 2894, 2893, 2895, 2896, 2897, 2898, 2899, + 2900, 2901, 2902, 1260, 1262, 1264, 1266, 1269, 558, 559, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 691, 0, 0, 695, 0, + 530, 0, 0, 0, 0, 0, 0, 499, 0, 0, + 417, 0, 0, 0, 2866, 0, 482, 457, 733, 0, + 0, 480, 425, 515, 468, 521, 502, 529, 474, 469, + 316, 503, 363, 438, 332, 334, 723, 365, 368, 372, + 373, 447, 448, 462, 487, 506, 507, 508, 362, 346, + 481, 347, 382, 348, 317, 354, 352, 355, 489, 356, + 319, 463, 512, 0, 378, 477, 433, 320, 432, 464, + 511, 510, 333, 538, 545, 546, 636, 0, 551, 734, + 735, 736, 560, 0, 470, 329, 328, 0, 0, 0, + 358, 465, 342, 344, 345, 343, 460, 461, 565, 566, + 567, 569, 0, 570, 571, 0, 0, 0, 0, 572, + 637, 653, 621, 590, 553, 645, 587, 591, 592, 399, + 400, 401, 656, 0, 0, 0, 544, 418, 419, 0, + 370, 369, 434, 321, 0, 0, 407, 398, 471, 327, + 366, 409, 403, 420, 421, 422, 376, 311, 312, 729, + 359, 453, 658, 693, 694, 583, 0, 646, 584, 593, + 351, 618, 630, 629, 449, 543, 0, 641, 644, 573, + 728, 0, 638, 652, 732, 651, 725, 459, 0, 486, + 649, 596, 0, 642, 615, 616, 0, 643, 611, 647, + 0, 585, 0, 554, 557, 586, 671, 672, 673, 318, + 556, 675, 676, 677, 678, 679, 680, 681, 674, 526, + 619, 595, 622, 535, 598, 597, 0, 0, 633, 552, + 634, 635, 443, 444, 445, 446, 380, 659, 340, 555, + 473, 0, 620, 0, 0, 0, 0, 0, 0, 0, + 0, 625, 626, 623, 737, 0, 682, 683, 0, 0, + 549, 550, 375, 0, 568, 383, 339, 458, 377, 533, + 406, 0, 561, 627, 562, 475, 476, 685, 690, 686, + 687, 689, 709, 450, 397, 402, 490, 408, 426, 478, + 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 667, 666, 665, 664, + 663, 662, 661, 660, 0, 0, 609, 509, 353, 305, + 349, 350, 357, 726, 722, 727, 710, 713, 712, 688, + 0, 313, 2873, 424, 472, 374, 654, 655, 0, 708, + 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, + 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, + 280, 281, 282, 283, 657, 274, 275, 284, 285, 286, + 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, + 297, 0, 0, 0, 0, 307, 714, 715, 716, 717, + 718, 0, 0, 308, 309, 310, 0, 0, 300, 500, + 301, 302, 303, 304, 0, 0, 539, 540, 541, 564, + 0, 542, 524, 588, 384, 314, 504, 531, 724, 0, + 0, 0, 0, 0, 0, 0, 639, 650, 684, 0, + 696, 697, 699, 701, 700, 703, 497, 498, 711, 0, + 0, 705, 706, 707, 704, 428, 484, 505, 491, 0, + 730, 579, 580, 731, 692, 2846, 455, 0, 0, 594, + 628, 617, 702, 582, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 367, 0, 0, 423, 632, 613, + 624, 614, 599, 600, 601, 608, 379, 602, 603, 604, + 574, 605, 575, 606, 607, 0, 631, 581, 493, 439, + 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, + 246, 576, 698, 578, 577, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 338, 2693, 2696, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 494, 523, 0, 536, 0, 404, + 405, 0, 0, 0, 0, 0, 0, 0, 322, 501, + 520, 336, 488, 534, 341, 496, 513, 331, 454, 485, + 0, 0, 324, 518, 495, 436, 323, 0, 479, 364, + 381, 361, 452, 0, 0, 517, 547, 360, 537, 0, + 528, 326, 0, 527, 451, 514, 519, 437, 430, 0, + 325, 516, 435, 429, 410, 371, 563, 411, 412, 413, + 414, 415, 416, 385, 466, 427, 467, 386, 441, 440, + 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 558, 559, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 691, 0, 0, 695, 2697, 530, 0, + 0, 0, 2692, 0, 2691, 499, 2689, 2694, 417, 0, + 0, 0, 548, 0, 482, 457, 733, 0, 0, 480, + 425, 515, 468, 521, 502, 529, 474, 469, 316, 503, + 363, 438, 332, 334, 723, 365, 368, 372, 373, 447, + 448, 462, 487, 506, 507, 508, 362, 346, 481, 347, + 382, 348, 317, 354, 352, 355, 489, 356, 319, 463, + 512, 2695, 378, 477, 433, 320, 432, 464, 511, 510, + 333, 538, 545, 546, 636, 0, 551, 734, 735, 736, + 560, 0, 470, 329, 328, 0, 0, 0, 358, 465, + 342, 344, 345, 343, 460, 461, 565, 566, 567, 569, + 0, 570, 571, 0, 0, 0, 0, 572, 637, 653, + 621, 590, 553, 645, 587, 591, 592, 399, 400, 401, + 656, 0, 0, 0, 544, 418, 419, 0, 370, 369, + 434, 321, 0, 0, 407, 398, 471, 327, 366, 409, + 403, 420, 421, 422, 376, 311, 312, 729, 359, 453, + 658, 693, 694, 583, 0, 646, 584, 593, 351, 618, + 630, 629, 449, 543, 0, 641, 644, 573, 728, 0, + 638, 652, 732, 651, 725, 459, 0, 486, 649, 596, + 0, 642, 615, 616, 0, 643, 611, 647, 0, 585, + 0, 554, 557, 586, 671, 672, 673, 318, 556, 675, + 676, 677, 678, 679, 680, 681, 674, 526, 619, 595, + 622, 535, 598, 597, 0, 0, 633, 552, 634, 635, + 443, 444, 445, 446, 380, 659, 340, 555, 473, 0, + 620, 0, 0, 0, 0, 0, 0, 0, 0, 625, + 626, 623, 737, 0, 682, 683, 0, 0, 549, 550, + 375, 0, 568, 383, 339, 458, 377, 533, 406, 0, + 561, 627, 562, 475, 476, 685, 690, 686, 687, 689, + 709, 450, 397, 402, 490, 408, 426, 478, 532, 456, + 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 667, 666, 665, 664, 663, 662, + 661, 660, 0, 0, 609, 509, 353, 305, 349, 350, + 357, 726, 722, 727, 710, 713, 712, 688, 0, 313, + 589, 424, 472, 374, 654, 655, 0, 708, 259, 260, + 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, + 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, + 282, 283, 657, 274, 275, 284, 285, 286, 287, 288, + 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, + 0, 0, 0, 307, 714, 715, 716, 717, 718, 0, + 0, 308, 309, 310, 0, 0, 300, 500, 301, 302, + 303, 304, 0, 0, 539, 540, 541, 564, 0, 542, + 524, 588, 384, 314, 504, 531, 724, 0, 0, 0, + 0, 0, 0, 0, 639, 650, 684, 0, 696, 697, + 699, 701, 700, 703, 497, 498, 711, 0, 0, 705, + 706, 707, 704, 428, 484, 505, 491, 0, 730, 579, + 580, 731, 692, 315, 455, 0, 0, 594, 628, 617, + 702, 582, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 367, 0, 0, 423, 632, 613, 624, 614, + 599, 600, 601, 608, 379, 602, 603, 604, 574, 605, + 575, 606, 607, 0, 631, 581, 493, 439, 0, 648, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 0, 0, 0, 0, 335, 246, 576, + 698, 578, 577, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 338, 0, 2714, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 494, 523, 0, 536, 0, 404, 405, 0, + 0, 0, 0, 0, 0, 0, 322, 501, 520, 336, + 488, 534, 341, 496, 513, 331, 454, 485, 0, 0, + 324, 518, 495, 436, 323, 0, 479, 364, 381, 361, + 452, 0, 0, 517, 547, 360, 537, 0, 528, 326, + 0, 527, 451, 514, 519, 437, 430, 0, 325, 516, + 435, 429, 410, 371, 563, 411, 412, 413, 414, 415, + 416, 385, 466, 427, 467, 386, 441, 440, 442, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, + 0, 0, 0, 0, 558, 559, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 691, 0, 0, 695, 2713, 530, 0, 0, 0, + 2719, 2716, 2718, 499, 0, 2717, 417, 0, 0, 0, + 548, 0, 482, 457, 733, 0, 2711, 480, 425, 515, + 468, 521, 502, 529, 474, 469, 316, 503, 363, 438, + 332, 334, 723, 365, 368, 372, 373, 447, 448, 462, + 487, 506, 507, 508, 362, 346, 481, 347, 382, 348, + 317, 354, 352, 355, 489, 356, 319, 463, 512, 0, + 378, 477, 433, 320, 432, 464, 511, 510, 333, 538, + 545, 546, 636, 0, 551, 734, 735, 736, 560, 0, + 470, 329, 328, 0, 0, 0, 358, 465, 342, 344, + 345, 343, 460, 461, 565, 566, 567, 569, 0, 570, + 571, 0, 0, 0, 0, 572, 637, 653, 621, 590, + 553, 645, 587, 591, 592, 399, 400, 401, 656, 0, + 0, 0, 544, 418, 419, 0, 370, 369, 434, 321, + 0, 0, 407, 398, 471, 327, 366, 409, 403, 420, + 421, 422, 376, 311, 312, 729, 359, 453, 658, 693, + 694, 583, 0, 646, 584, 593, 351, 618, 630, 629, + 449, 543, 0, 641, 644, 573, 728, 0, 638, 652, + 732, 651, 725, 459, 0, 486, 649, 596, 0, 642, + 615, 616, 0, 643, 611, 647, 0, 585, 0, 554, + 557, 586, 671, 672, 673, 318, 556, 675, 676, 677, + 678, 679, 680, 681, 674, 526, 619, 595, 622, 535, + 598, 597, 0, 0, 633, 552, 634, 635, 443, 444, + 445, 446, 380, 659, 340, 555, 473, 0, 620, 0, + 0, 0, 0, 0, 0, 0, 0, 625, 626, 623, + 737, 0, 682, 683, 0, 0, 549, 550, 375, 0, + 568, 383, 339, 458, 377, 533, 406, 0, 561, 627, + 562, 475, 476, 685, 690, 686, 687, 689, 709, 450, + 397, 402, 490, 408, 426, 478, 532, 456, 483, 337, + 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 490, 519, 0, 532, 0, 2861, 2862, 1226, 0, - 0, 0, 0, 0, 0, 322, 497, 516, 336, 484, - 530, 341, 492, 509, 331, 450, 481, 0, 0, 2855, - 2858, 2859, 2860, 2863, 0, 2868, 2864, 2865, 2866, 2867, - 0, 0, 2851, 2852, 2853, 2854, 1224, 2835, 2856, 0, - 2836, 447, 2837, 2838, 2839, 2840, 1228, 2841, 2842, 2843, - 2844, 2845, 2848, 2849, 2846, 2847, 2869, 2870, 2871, 2872, - 2873, 2874, 2875, 2876, 2878, 2877, 2879, 2880, 2881, 2882, - 2883, 2884, 2885, 2886, 1252, 1254, 1256, 1258, 1261, 554, - 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 687, 0, 0, 691, - 0, 526, 0, 0, 0, 0, 0, 0, 495, 0, - 0, 413, 0, 0, 0, 2850, 0, 478, 453, 729, - 0, 0, 476, 421, 511, 464, 517, 498, 525, 470, - 465, 316, 499, 363, 434, 332, 334, 719, 365, 368, - 372, 373, 443, 444, 458, 483, 502, 503, 504, 362, - 346, 477, 347, 382, 348, 317, 354, 352, 355, 485, - 356, 319, 459, 508, 0, 378, 473, 429, 320, 428, - 460, 507, 506, 333, 534, 541, 542, 632, 0, 547, - 730, 731, 732, 556, 0, 466, 329, 328, 0, 0, - 0, 358, 461, 342, 344, 345, 343, 456, 457, 561, - 562, 563, 565, 0, 566, 567, 0, 0, 0, 0, - 568, 633, 649, 617, 586, 549, 641, 583, 587, 588, - 399, 400, 401, 652, 0, 0, 0, 540, 414, 415, - 0, 370, 369, 430, 321, 0, 0, 407, 398, 467, - 327, 366, 409, 403, 416, 417, 418, 376, 311, 312, - 725, 359, 449, 654, 689, 690, 579, 0, 642, 580, - 589, 351, 614, 626, 625, 445, 539, 0, 637, 640, - 569, 724, 0, 634, 648, 728, 647, 721, 455, 0, - 482, 645, 592, 0, 638, 611, 612, 0, 639, 607, - 643, 0, 581, 0, 550, 553, 582, 667, 668, 669, - 318, 552, 671, 672, 673, 674, 675, 676, 677, 670, - 522, 615, 591, 618, 531, 594, 593, 0, 0, 629, - 548, 630, 631, 439, 440, 441, 442, 380, 655, 340, - 551, 469, 0, 616, 0, 0, 0, 0, 0, 0, - 0, 0, 621, 622, 619, 733, 0, 678, 679, 0, - 0, 545, 546, 375, 0, 564, 383, 339, 454, 377, - 529, 406, 0, 557, 623, 558, 471, 472, 681, 686, - 682, 683, 685, 705, 446, 397, 402, 486, 408, 422, - 474, 528, 452, 479, 337, 518, 488, 427, 608, 636, + 0, 0, 667, 666, 665, 664, 663, 662, 661, 660, + 0, 0, 609, 509, 353, 305, 349, 350, 357, 726, + 722, 727, 710, 713, 712, 688, 0, 313, 589, 424, + 472, 374, 654, 655, 0, 708, 259, 260, 261, 262, + 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, + 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, + 657, 274, 275, 284, 285, 286, 287, 288, 289, 290, + 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, + 0, 307, 714, 715, 716, 717, 718, 0, 0, 308, + 309, 310, 0, 0, 300, 500, 301, 302, 303, 304, + 0, 0, 539, 540, 541, 564, 0, 542, 524, 588, + 384, 314, 504, 531, 724, 0, 0, 0, 0, 0, + 0, 0, 639, 650, 684, 0, 696, 697, 699, 701, + 700, 703, 497, 498, 711, 0, 0, 705, 706, 707, + 704, 428, 484, 505, 491, 0, 730, 579, 580, 731, + 692, 315, 455, 0, 0, 594, 628, 617, 702, 582, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 367, 0, 0, 423, 632, 613, 624, 614, 599, 600, + 601, 608, 379, 602, 603, 604, 574, 605, 575, 606, + 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 663, 662, 661, - 660, 659, 658, 657, 656, 0, 0, 605, 505, 353, - 305, 349, 350, 357, 722, 718, 723, 706, 709, 708, - 684, 0, 313, 2857, 420, 468, 374, 650, 651, 0, - 704, 259, 260, 261, 262, 263, 264, 265, 266, 306, - 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, - 279, 280, 281, 282, 283, 653, 274, 275, 284, 285, - 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, - 296, 297, 0, 0, 0, 0, 307, 710, 711, 712, - 713, 714, 0, 0, 308, 309, 310, 0, 0, 300, - 496, 301, 302, 303, 304, 0, 0, 535, 536, 537, - 560, 0, 538, 520, 584, 384, 314, 500, 527, 720, - 0, 0, 0, 0, 0, 0, 0, 635, 646, 680, - 0, 692, 693, 695, 697, 696, 699, 493, 494, 707, - 0, 0, 701, 702, 703, 700, 424, 480, 501, 487, - 0, 726, 575, 576, 727, 688, 2834, 451, 0, 0, - 590, 624, 613, 698, 578, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 367, 0, 0, 419, 628, - 609, 620, 610, 595, 596, 597, 604, 379, 598, 599, - 600, 570, 601, 571, 602, 603, 0, 627, 577, 489, - 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, + 0, 0, 0, 0, 0, 335, 246, 576, 698, 578, + 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 338, 0, 2714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, - 335, 246, 572, 694, 574, 573, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 338, 2681, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 494, 523, 0, 536, 0, 404, 405, 0, 0, 0, + 0, 0, 0, 0, 322, 501, 520, 336, 488, 534, + 341, 496, 513, 331, 454, 485, 0, 0, 324, 518, + 495, 436, 323, 0, 479, 364, 381, 361, 452, 0, + 0, 517, 547, 360, 537, 0, 528, 326, 0, 527, + 451, 514, 519, 437, 430, 0, 325, 516, 435, 429, + 410, 371, 563, 411, 412, 413, 414, 415, 416, 385, + 466, 427, 467, 386, 441, 440, 442, 387, 388, 389, + 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, + 0, 0, 558, 559, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, + 0, 0, 695, 2713, 530, 0, 0, 0, 2719, 2716, + 2718, 499, 0, 2717, 417, 0, 0, 0, 548, 0, + 482, 457, 733, 0, 0, 480, 425, 515, 468, 521, + 502, 529, 474, 469, 316, 503, 363, 438, 332, 334, + 723, 365, 368, 372, 373, 447, 448, 462, 487, 506, + 507, 508, 362, 346, 481, 347, 382, 348, 317, 354, + 352, 355, 489, 356, 319, 463, 512, 0, 378, 477, + 433, 320, 432, 464, 511, 510, 333, 538, 545, 546, + 636, 0, 551, 734, 735, 736, 560, 0, 470, 329, + 328, 0, 0, 0, 358, 465, 342, 344, 345, 343, + 460, 461, 565, 566, 567, 569, 0, 570, 571, 0, + 0, 0, 0, 572, 637, 653, 621, 590, 553, 645, + 587, 591, 592, 399, 400, 401, 656, 0, 0, 0, + 544, 418, 419, 0, 370, 369, 434, 321, 0, 0, + 407, 398, 471, 327, 366, 409, 403, 420, 421, 422, + 376, 311, 312, 729, 359, 453, 658, 693, 694, 583, + 0, 646, 584, 593, 351, 618, 630, 629, 449, 543, + 0, 641, 644, 573, 728, 0, 638, 652, 732, 651, + 725, 459, 0, 486, 649, 596, 0, 642, 615, 616, + 0, 643, 611, 647, 0, 585, 0, 554, 557, 586, + 671, 672, 673, 318, 556, 675, 676, 677, 678, 679, + 680, 681, 674, 526, 619, 595, 622, 535, 598, 597, + 0, 0, 633, 552, 634, 635, 443, 444, 445, 446, + 380, 659, 340, 555, 473, 0, 620, 0, 0, 0, + 0, 0, 0, 0, 0, 625, 626, 623, 737, 0, + 682, 683, 0, 0, 549, 550, 375, 0, 568, 383, + 339, 458, 377, 533, 406, 0, 561, 627, 562, 475, + 476, 685, 690, 686, 687, 689, 709, 450, 397, 402, + 490, 408, 426, 478, 532, 456, 483, 337, 522, 492, + 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 490, 519, 0, 532, 0, - 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, - 497, 516, 336, 484, 530, 341, 492, 509, 331, 450, - 481, 0, 0, 324, 514, 491, 432, 323, 0, 475, - 364, 381, 361, 448, 0, 0, 513, 543, 360, 533, - 0, 524, 326, 0, 523, 447, 510, 515, 433, 426, - 0, 325, 512, 431, 425, 410, 371, 559, 411, 412, - 385, 462, 423, 463, 386, 437, 436, 438, 387, 388, - 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, - 0, 0, 0, 554, 555, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 687, 0, 0, 691, 2685, 526, 0, 0, 0, 2680, - 0, 2679, 495, 2677, 2682, 413, 0, 0, 0, 544, - 0, 478, 453, 729, 0, 0, 476, 421, 511, 464, - 517, 498, 525, 470, 465, 316, 499, 363, 434, 332, - 334, 719, 365, 368, 372, 373, 443, 444, 458, 483, - 502, 503, 504, 362, 346, 477, 347, 382, 348, 317, - 354, 352, 355, 485, 356, 319, 459, 508, 2683, 378, - 473, 429, 320, 428, 460, 507, 506, 333, 534, 541, - 542, 632, 0, 547, 730, 731, 732, 556, 0, 466, - 329, 328, 0, 0, 0, 358, 461, 342, 344, 345, - 343, 456, 457, 561, 562, 563, 565, 0, 566, 567, - 0, 0, 0, 0, 568, 633, 649, 617, 586, 549, - 641, 583, 587, 588, 399, 400, 401, 652, 0, 0, - 0, 540, 414, 415, 0, 370, 369, 430, 321, 0, - 0, 407, 398, 467, 327, 366, 409, 403, 416, 417, - 418, 376, 311, 312, 725, 359, 449, 654, 689, 690, - 579, 0, 642, 580, 589, 351, 614, 626, 625, 445, - 539, 0, 637, 640, 569, 724, 0, 634, 648, 728, - 647, 721, 455, 0, 482, 645, 592, 0, 638, 611, - 612, 0, 639, 607, 643, 0, 581, 0, 550, 553, - 582, 667, 668, 669, 318, 552, 671, 672, 673, 674, - 675, 676, 677, 670, 522, 615, 591, 618, 531, 594, - 593, 0, 0, 629, 548, 630, 631, 439, 440, 441, - 442, 380, 655, 340, 551, 469, 0, 616, 0, 0, - 0, 0, 0, 0, 0, 0, 621, 622, 619, 733, - 0, 678, 679, 0, 0, 545, 546, 375, 0, 564, - 383, 339, 454, 377, 529, 406, 0, 557, 623, 558, - 471, 472, 681, 686, 682, 683, 685, 705, 446, 397, - 402, 486, 408, 422, 474, 528, 452, 479, 337, 518, - 488, 427, 608, 636, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, + 667, 666, 665, 664, 663, 662, 661, 660, 0, 0, + 609, 509, 353, 305, 349, 350, 357, 726, 722, 727, + 710, 713, 712, 688, 0, 313, 589, 424, 472, 374, + 654, 655, 0, 708, 259, 260, 261, 262, 263, 264, + 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, + 276, 277, 278, 279, 280, 281, 282, 283, 657, 274, + 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, + 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, + 714, 715, 716, 717, 718, 0, 0, 308, 309, 310, + 0, 0, 300, 500, 301, 302, 303, 304, 0, 0, + 539, 540, 541, 564, 0, 542, 524, 588, 384, 314, + 504, 531, 724, 0, 0, 0, 0, 0, 0, 0, + 639, 650, 684, 0, 696, 697, 699, 701, 700, 703, + 497, 498, 711, 0, 0, 705, 706, 707, 704, 428, + 484, 505, 491, 0, 730, 579, 580, 731, 692, 315, + 455, 0, 0, 594, 628, 617, 702, 582, 0, 0, + 0, 0, 0, 2371, 0, 0, 0, 0, 367, 0, + 0, 423, 632, 613, 624, 614, 599, 600, 601, 608, + 379, 602, 603, 604, 574, 605, 575, 606, 607, 0, + 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 245, 0, 0, 2372, + 0, 0, 0, 335, 246, 576, 698, 578, 577, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, + 0, 1389, 1390, 1391, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 663, 662, 661, 660, 659, 658, 657, 656, 0, - 0, 605, 505, 353, 305, 349, 350, 357, 722, 718, - 723, 706, 709, 708, 684, 0, 313, 585, 420, 468, - 374, 650, 651, 0, 704, 259, 260, 261, 262, 263, - 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, - 273, 276, 277, 278, 279, 280, 281, 282, 283, 653, - 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, - 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, - 307, 710, 711, 712, 713, 714, 0, 0, 308, 309, - 310, 0, 0, 300, 496, 301, 302, 303, 304, 0, - 0, 535, 536, 537, 560, 0, 538, 520, 584, 384, - 314, 500, 527, 720, 0, 0, 0, 0, 0, 0, - 0, 635, 646, 680, 0, 692, 693, 695, 697, 696, - 699, 493, 494, 707, 0, 0, 701, 702, 703, 700, - 424, 480, 501, 487, 0, 726, 575, 576, 727, 688, - 315, 451, 0, 0, 590, 624, 613, 698, 578, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, - 0, 0, 419, 628, 609, 620, 610, 595, 596, 597, - 604, 379, 598, 599, 600, 570, 601, 571, 602, 603, - 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, - 0, 0, 0, 0, 335, 246, 572, 694, 574, 573, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, - 0, 2702, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, - 519, 0, 532, 0, 404, 405, 0, 0, 0, 0, - 0, 0, 0, 322, 497, 516, 336, 484, 530, 341, - 492, 509, 331, 450, 481, 0, 0, 324, 514, 491, - 432, 323, 0, 475, 364, 381, 361, 448, 0, 0, - 513, 543, 360, 533, 0, 524, 326, 0, 523, 447, - 510, 515, 433, 426, 0, 325, 512, 431, 425, 410, - 371, 559, 411, 412, 385, 462, 423, 463, 386, 437, - 436, 438, 387, 388, 389, 390, 391, 392, 393, 394, - 395, 396, 0, 0, 0, 0, 0, 554, 555, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 687, 0, 0, 691, 2701, 526, - 0, 0, 0, 2707, 2704, 2706, 495, 0, 2705, 413, - 0, 0, 0, 544, 0, 478, 453, 729, 0, 2699, - 476, 421, 511, 464, 517, 498, 525, 470, 465, 316, - 499, 363, 434, 332, 334, 719, 365, 368, 372, 373, - 443, 444, 458, 483, 502, 503, 504, 362, 346, 477, - 347, 382, 348, 317, 354, 352, 355, 485, 356, 319, - 459, 508, 0, 378, 473, 429, 320, 428, 460, 507, - 506, 333, 534, 541, 542, 632, 0, 547, 730, 731, - 732, 556, 0, 466, 329, 328, 0, 0, 0, 358, - 461, 342, 344, 345, 343, 456, 457, 561, 562, 563, - 565, 0, 566, 567, 0, 0, 0, 0, 568, 633, - 649, 617, 586, 549, 641, 583, 587, 588, 399, 400, - 401, 652, 0, 0, 0, 540, 414, 415, 0, 370, - 369, 430, 321, 0, 0, 407, 398, 467, 327, 366, - 409, 403, 416, 417, 418, 376, 311, 312, 725, 359, - 449, 654, 689, 690, 579, 0, 642, 580, 589, 351, - 614, 626, 625, 445, 539, 0, 637, 640, 569, 724, - 0, 634, 648, 728, 647, 721, 455, 0, 482, 645, - 592, 0, 638, 611, 612, 0, 639, 607, 643, 0, - 581, 0, 550, 553, 582, 667, 668, 669, 318, 552, - 671, 672, 673, 674, 675, 676, 677, 670, 522, 615, - 591, 618, 531, 594, 593, 0, 0, 629, 548, 630, - 631, 439, 440, 441, 442, 380, 655, 340, 551, 469, - 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, - 621, 622, 619, 733, 0, 678, 679, 0, 0, 545, - 546, 375, 0, 564, 383, 339, 454, 377, 529, 406, - 0, 557, 623, 558, 471, 472, 681, 686, 682, 683, - 685, 705, 446, 397, 402, 486, 408, 422, 474, 528, - 452, 479, 337, 518, 488, 427, 608, 636, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, - 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 663, 662, 661, 660, 659, - 658, 657, 656, 0, 0, 605, 505, 353, 305, 349, - 350, 357, 722, 718, 723, 706, 709, 708, 684, 0, - 313, 585, 420, 468, 374, 650, 651, 0, 704, 259, - 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, - 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, - 281, 282, 283, 653, 274, 275, 284, 285, 286, 287, - 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, - 0, 0, 0, 0, 307, 710, 711, 712, 713, 714, - 0, 0, 308, 309, 310, 0, 0, 300, 496, 301, - 302, 303, 304, 0, 0, 535, 536, 537, 560, 0, - 538, 520, 584, 384, 314, 500, 527, 720, 0, 0, - 0, 0, 0, 0, 0, 635, 646, 680, 0, 692, - 693, 695, 697, 696, 699, 493, 494, 707, 0, 0, - 701, 702, 703, 700, 424, 480, 501, 487, 0, 726, - 575, 576, 727, 688, 315, 451, 0, 0, 590, 624, - 613, 698, 578, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 367, 0, 0, 419, 628, 609, 620, - 610, 595, 596, 597, 604, 379, 598, 599, 600, 570, - 601, 571, 602, 603, 0, 627, 577, 489, 435, 0, - 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, - 572, 694, 574, 573, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 338, 0, 2702, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 494, 523, + 0, 536, 0, 404, 405, 0, 0, 0, 0, 0, + 0, 0, 322, 501, 520, 336, 488, 534, 341, 496, + 513, 331, 454, 485, 0, 0, 324, 518, 495, 436, + 323, 0, 479, 364, 381, 361, 452, 0, 0, 517, + 547, 360, 537, 0, 528, 326, 0, 527, 451, 514, + 519, 437, 430, 0, 325, 516, 435, 429, 410, 371, + 563, 411, 412, 413, 414, 415, 416, 385, 466, 427, + 467, 386, 441, 440, 442, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, + 558, 559, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, + 695, 0, 530, 0, 0, 0, 0, 0, 0, 499, + 0, 0, 417, 0, 0, 0, 548, 0, 482, 457, + 733, 0, 0, 480, 425, 515, 468, 521, 502, 529, + 474, 469, 316, 503, 363, 438, 332, 334, 723, 365, + 368, 372, 373, 447, 448, 462, 487, 506, 507, 508, + 362, 346, 481, 347, 382, 348, 317, 354, 352, 355, + 489, 356, 319, 463, 512, 0, 378, 477, 433, 320, + 432, 464, 511, 510, 333, 538, 545, 546, 636, 0, + 551, 734, 735, 736, 560, 0, 470, 329, 328, 0, + 0, 0, 358, 465, 342, 344, 345, 343, 460, 461, + 565, 566, 567, 569, 0, 570, 571, 0, 0, 0, + 0, 572, 637, 653, 621, 590, 553, 645, 587, 591, + 592, 399, 400, 401, 656, 0, 0, 0, 544, 418, + 419, 0, 370, 369, 434, 321, 0, 0, 407, 398, + 471, 327, 366, 409, 403, 420, 421, 422, 376, 311, + 312, 729, 359, 453, 658, 693, 694, 583, 0, 646, + 584, 593, 351, 618, 630, 629, 449, 543, 0, 641, + 644, 573, 728, 0, 638, 652, 732, 651, 725, 459, + 0, 486, 649, 596, 0, 642, 615, 616, 0, 643, + 611, 647, 0, 585, 0, 554, 557, 586, 671, 672, + 673, 318, 556, 675, 676, 677, 678, 679, 680, 681, + 674, 526, 619, 595, 622, 535, 598, 597, 0, 0, + 633, 552, 634, 635, 443, 444, 445, 446, 380, 659, + 340, 555, 473, 0, 620, 0, 0, 0, 0, 0, + 0, 0, 0, 625, 626, 623, 737, 0, 682, 683, + 0, 0, 549, 550, 375, 0, 568, 383, 339, 458, + 377, 533, 406, 0, 561, 627, 562, 475, 476, 685, + 690, 686, 687, 689, 709, 450, 397, 402, 490, 408, + 426, 478, 532, 456, 483, 337, 522, 492, 431, 612, + 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 667, 666, + 665, 664, 663, 662, 661, 660, 0, 0, 609, 509, + 353, 305, 349, 350, 357, 726, 722, 727, 710, 713, + 712, 688, 0, 313, 589, 424, 472, 374, 654, 655, + 0, 708, 259, 260, 261, 262, 263, 264, 265, 266, + 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, + 278, 279, 280, 281, 282, 283, 657, 274, 275, 284, + 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, + 295, 296, 297, 0, 0, 0, 0, 307, 714, 715, + 716, 717, 718, 0, 0, 308, 309, 310, 0, 0, + 300, 500, 301, 302, 303, 304, 0, 0, 539, 540, + 541, 564, 0, 542, 524, 588, 384, 314, 504, 531, + 724, 0, 0, 0, 0, 0, 0, 0, 639, 650, + 684, 0, 696, 697, 699, 701, 700, 703, 497, 498, + 711, 0, 0, 705, 706, 707, 704, 428, 484, 505, + 491, 0, 730, 579, 580, 731, 692, 315, 183, 223, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 153, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 219, 2639, 0, 245, 0, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 490, 519, 0, 532, 0, 404, 405, - 0, 0, 0, 0, 0, 0, 0, 322, 497, 516, - 336, 484, 530, 341, 492, 509, 331, 450, 481, 0, - 0, 324, 514, 491, 432, 323, 0, 475, 364, 381, - 361, 448, 0, 0, 513, 543, 360, 533, 0, 524, - 326, 0, 523, 447, 510, 515, 433, 426, 0, 325, - 512, 431, 425, 410, 371, 559, 411, 412, 385, 462, - 423, 463, 386, 437, 436, 438, 387, 388, 389, 390, - 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, - 0, 554, 555, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, - 0, 691, 2701, 526, 0, 0, 0, 2707, 2704, 2706, - 495, 0, 2705, 413, 0, 0, 0, 544, 0, 478, - 453, 729, 0, 0, 476, 421, 511, 464, 517, 498, - 525, 470, 465, 316, 499, 363, 434, 332, 334, 719, - 365, 368, 372, 373, 443, 444, 458, 483, 502, 503, - 504, 362, 346, 477, 347, 382, 348, 317, 354, 352, - 355, 485, 356, 319, 459, 508, 0, 378, 473, 429, - 320, 428, 460, 507, 506, 333, 534, 541, 542, 632, - 0, 547, 730, 731, 732, 556, 0, 466, 329, 328, - 0, 0, 0, 358, 461, 342, 344, 345, 343, 456, - 457, 561, 562, 563, 565, 0, 566, 567, 0, 0, - 0, 0, 568, 633, 649, 617, 586, 549, 641, 583, - 587, 588, 399, 400, 401, 652, 0, 0, 0, 540, - 414, 415, 0, 370, 369, 430, 321, 0, 0, 407, - 398, 467, 327, 366, 409, 403, 416, 417, 418, 376, - 311, 312, 725, 359, 449, 654, 689, 690, 579, 0, - 642, 580, 589, 351, 614, 626, 625, 445, 539, 0, - 637, 640, 569, 724, 0, 634, 648, 728, 647, 721, - 455, 0, 482, 645, 592, 0, 638, 611, 612, 0, - 639, 607, 643, 0, 581, 0, 550, 553, 582, 667, - 668, 669, 318, 552, 671, 672, 673, 674, 675, 676, - 677, 670, 522, 615, 591, 618, 531, 594, 593, 0, - 0, 629, 548, 630, 631, 439, 440, 441, 442, 380, - 655, 340, 551, 469, 0, 616, 0, 0, 0, 0, - 0, 0, 0, 0, 621, 622, 619, 733, 0, 678, - 679, 0, 0, 545, 546, 375, 0, 564, 383, 339, - 454, 377, 529, 406, 0, 557, 623, 558, 471, 472, - 681, 686, 682, 683, 685, 705, 446, 397, 402, 486, - 408, 422, 474, 528, 452, 479, 337, 518, 488, 427, - 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, - 662, 661, 660, 659, 658, 657, 656, 0, 0, 605, - 505, 353, 305, 349, 350, 357, 722, 718, 723, 706, - 709, 708, 684, 0, 313, 585, 420, 468, 374, 650, - 651, 0, 704, 259, 260, 261, 262, 263, 264, 265, - 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, - 277, 278, 279, 280, 281, 282, 283, 653, 274, 275, - 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, - 294, 295, 296, 297, 0, 0, 0, 0, 307, 710, - 711, 712, 713, 714, 0, 0, 308, 309, 310, 0, - 0, 300, 496, 301, 302, 303, 304, 0, 0, 535, - 536, 537, 560, 0, 538, 520, 584, 384, 314, 500, - 527, 720, 0, 0, 0, 0, 0, 0, 0, 635, - 646, 680, 0, 692, 693, 695, 697, 696, 699, 493, - 494, 707, 0, 0, 701, 702, 703, 700, 424, 480, - 501, 487, 0, 726, 575, 576, 727, 688, 315, 451, - 0, 0, 590, 624, 613, 698, 578, 0, 0, 0, - 0, 0, 2359, 0, 0, 0, 0, 367, 0, 0, - 419, 628, 609, 620, 610, 595, 596, 597, 604, 379, - 598, 599, 600, 570, 601, 571, 602, 603, 0, 627, - 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 245, 0, 0, 2360, 0, - 0, 0, 335, 246, 572, 694, 574, 573, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, - 1381, 1382, 1383, 1380, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 490, 519, 0, - 532, 0, 404, 405, 0, 0, 0, 0, 0, 0, - 0, 322, 497, 516, 336, 484, 530, 341, 492, 509, - 331, 450, 481, 0, 0, 324, 514, 491, 432, 323, - 0, 475, 364, 381, 361, 448, 0, 0, 513, 543, - 360, 533, 0, 524, 326, 0, 523, 447, 510, 515, - 433, 426, 0, 325, 512, 431, 425, 410, 371, 559, - 411, 412, 385, 462, 423, 463, 386, 437, 436, 438, - 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 0, 0, 0, 0, 0, 554, 555, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 687, 0, 0, 691, 0, 526, 0, 0, - 0, 0, 0, 0, 495, 0, 0, 413, 0, 0, - 0, 544, 0, 478, 453, 729, 0, 0, 476, 421, - 511, 464, 517, 498, 525, 470, 465, 316, 499, 363, - 434, 332, 334, 719, 365, 368, 372, 373, 443, 444, - 458, 483, 502, 503, 504, 362, 346, 477, 347, 382, - 348, 317, 354, 352, 355, 485, 356, 319, 459, 508, - 0, 378, 473, 429, 320, 428, 460, 507, 506, 333, - 534, 541, 542, 632, 0, 547, 730, 731, 732, 556, - 0, 466, 329, 328, 0, 0, 0, 358, 461, 342, - 344, 345, 343, 456, 457, 561, 562, 563, 565, 0, - 566, 567, 0, 0, 0, 0, 568, 633, 649, 617, - 586, 549, 641, 583, 587, 588, 399, 400, 401, 652, - 0, 0, 0, 540, 414, 415, 0, 370, 369, 430, - 321, 0, 0, 407, 398, 467, 327, 366, 409, 403, - 416, 417, 418, 376, 311, 312, 725, 359, 449, 654, - 689, 690, 579, 0, 642, 580, 589, 351, 614, 626, - 625, 445, 539, 0, 637, 640, 569, 724, 0, 634, - 648, 728, 647, 721, 455, 0, 482, 645, 592, 0, - 638, 611, 612, 0, 639, 607, 643, 0, 581, 0, - 550, 553, 582, 667, 668, 669, 318, 552, 671, 672, - 673, 674, 675, 676, 677, 670, 522, 615, 591, 618, - 531, 594, 593, 0, 0, 629, 548, 630, 631, 439, - 440, 441, 442, 380, 655, 340, 551, 469, 0, 616, - 0, 0, 0, 0, 0, 0, 0, 0, 621, 622, - 619, 733, 0, 678, 679, 0, 0, 545, 546, 375, - 0, 564, 383, 339, 454, 377, 529, 406, 0, 557, - 623, 558, 471, 472, 681, 686, 682, 683, 685, 705, - 446, 397, 402, 486, 408, 422, 474, 528, 452, 479, - 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 183, 223, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, + 0, 594, 628, 617, 702, 582, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 367, 0, 0, 423, + 632, 613, 624, 614, 599, 600, 601, 608, 379, 602, + 603, 604, 574, 605, 575, 606, 607, 153, 631, 581, + 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 219, 2413, 0, 245, 0, 0, 0, 0, 0, + 0, 335, 246, 576, 698, 578, 577, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 663, 662, 661, 660, 659, 658, 657, - 656, 0, 0, 605, 505, 353, 305, 349, 350, 357, - 722, 718, 723, 706, 709, 708, 684, 0, 313, 585, - 420, 468, 374, 650, 651, 0, 704, 259, 260, 261, - 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, - 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, - 283, 653, 274, 275, 284, 285, 286, 287, 288, 289, - 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, - 0, 0, 307, 710, 711, 712, 713, 714, 0, 0, - 308, 309, 310, 0, 0, 300, 496, 301, 302, 303, - 304, 0, 0, 535, 536, 537, 560, 0, 538, 520, - 584, 384, 314, 500, 527, 720, 0, 0, 0, 0, - 0, 0, 0, 635, 646, 680, 0, 692, 693, 695, - 697, 696, 699, 493, 494, 707, 0, 0, 701, 702, - 703, 700, 424, 480, 501, 487, 0, 726, 575, 576, - 727, 688, 315, 183, 223, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 153, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 219, 2627, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 494, 523, 0, 536, + 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, + 322, 501, 520, 336, 488, 534, 341, 496, 513, 331, + 454, 485, 0, 0, 324, 518, 495, 436, 323, 0, + 479, 364, 381, 361, 452, 0, 0, 517, 547, 360, + 537, 0, 528, 326, 0, 527, 451, 514, 519, 437, + 430, 0, 325, 516, 435, 429, 410, 371, 563, 411, + 412, 413, 414, 415, 416, 385, 466, 427, 467, 386, + 441, 440, 442, 387, 388, 389, 390, 391, 392, 393, + 394, 395, 396, 0, 0, 0, 0, 0, 558, 559, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 691, 0, 0, 695, 0, + 530, 0, 0, 0, 0, 0, 0, 499, 0, 0, + 417, 0, 0, 0, 548, 0, 482, 457, 733, 0, + 0, 480, 425, 515, 468, 521, 502, 529, 474, 469, + 316, 503, 363, 438, 332, 334, 723, 365, 368, 372, + 373, 447, 448, 462, 487, 506, 507, 508, 362, 346, + 481, 347, 382, 348, 317, 354, 352, 355, 489, 356, + 319, 463, 512, 0, 378, 477, 433, 320, 432, 464, + 511, 510, 333, 538, 545, 546, 636, 0, 551, 734, + 735, 736, 560, 0, 470, 329, 328, 0, 0, 0, + 358, 465, 342, 344, 345, 343, 460, 461, 565, 566, + 567, 569, 0, 570, 571, 0, 0, 0, 0, 572, + 637, 653, 621, 590, 553, 645, 587, 591, 592, 399, + 400, 401, 656, 0, 0, 0, 544, 418, 419, 0, + 370, 369, 434, 321, 0, 0, 407, 398, 471, 327, + 366, 409, 403, 420, 421, 422, 376, 311, 312, 729, + 359, 453, 658, 693, 694, 583, 0, 646, 584, 593, + 351, 618, 630, 629, 449, 543, 0, 641, 644, 573, + 728, 0, 638, 652, 732, 651, 725, 459, 0, 486, + 649, 596, 0, 642, 615, 616, 0, 643, 611, 647, + 0, 585, 0, 554, 557, 586, 671, 672, 673, 318, + 556, 675, 676, 677, 678, 679, 680, 681, 674, 526, + 619, 595, 622, 535, 598, 597, 0, 0, 633, 552, + 634, 635, 443, 444, 445, 446, 380, 659, 340, 555, + 473, 0, 620, 0, 0, 0, 0, 0, 0, 0, + 0, 625, 626, 623, 737, 0, 682, 683, 0, 0, + 549, 550, 375, 0, 568, 383, 339, 458, 377, 533, + 406, 0, 561, 627, 562, 475, 476, 685, 690, 686, + 687, 689, 709, 450, 397, 402, 490, 408, 426, 478, + 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 183, 223, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, - 0, 0, 590, 624, 613, 698, 578, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, - 419, 628, 609, 620, 610, 595, 596, 597, 604, 379, - 598, 599, 600, 570, 601, 571, 602, 603, 153, 627, - 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, + 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 667, 666, 665, 664, + 663, 662, 661, 660, 0, 0, 609, 509, 353, 305, + 349, 350, 357, 726, 722, 727, 710, 713, 712, 688, + 0, 313, 589, 424, 472, 374, 654, 655, 0, 708, + 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, + 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, + 280, 281, 282, 283, 657, 274, 275, 284, 285, 286, + 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, + 297, 0, 0, 0, 0, 307, 714, 715, 716, 717, + 718, 0, 0, 308, 309, 310, 0, 0, 300, 500, + 301, 302, 303, 304, 0, 0, 539, 540, 541, 564, + 0, 542, 524, 588, 384, 314, 504, 531, 724, 0, + 0, 0, 0, 0, 0, 0, 639, 650, 684, 0, + 696, 697, 699, 701, 700, 703, 497, 498, 711, 0, + 0, 705, 706, 707, 704, 428, 484, 505, 491, 0, + 730, 579, 580, 731, 692, 315, 455, 0, 0, 594, + 628, 617, 702, 582, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 367, 1153, 0, 423, 632, 613, + 624, 614, 599, 600, 601, 608, 379, 602, 603, 604, + 574, 605, 575, 606, 607, 0, 631, 581, 493, 439, + 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 245, 1160, 1161, 0, 0, 0, 0, 335, + 246, 576, 698, 578, 577, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1164, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 494, 523, 0, 536, 0, 404, + 405, 0, 0, 0, 0, 0, 0, 0, 322, 501, + 1147, 336, 488, 534, 341, 496, 513, 331, 454, 485, + 0, 0, 324, 518, 495, 436, 323, 0, 479, 364, + 381, 361, 452, 0, 0, 517, 547, 360, 537, 1132, + 528, 326, 1131, 527, 451, 514, 519, 437, 430, 0, + 325, 516, 435, 429, 410, 371, 563, 411, 412, 413, + 414, 415, 416, 385, 466, 427, 467, 386, 441, 440, + 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, + 396, 0, 0, 0, 0, 0, 558, 559, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 691, 0, 0, 695, 0, 530, 0, + 0, 0, 0, 0, 0, 499, 0, 0, 417, 0, + 0, 0, 548, 0, 482, 457, 733, 0, 0, 480, + 425, 515, 468, 521, 502, 529, 1151, 469, 316, 503, + 363, 438, 332, 334, 723, 365, 368, 372, 373, 447, + 448, 462, 487, 506, 507, 508, 362, 346, 481, 347, + 382, 348, 317, 354, 352, 355, 489, 356, 319, 463, + 512, 0, 378, 477, 433, 320, 432, 464, 511, 510, + 333, 538, 545, 546, 636, 0, 551, 734, 735, 736, + 560, 0, 470, 329, 328, 0, 0, 0, 358, 465, + 342, 344, 345, 343, 460, 461, 565, 566, 567, 569, + 0, 570, 571, 0, 0, 0, 0, 572, 637, 653, + 621, 590, 553, 645, 587, 591, 592, 399, 400, 401, + 656, 0, 0, 0, 544, 418, 419, 0, 370, 369, + 434, 321, 0, 0, 407, 398, 471, 327, 366, 409, + 403, 420, 421, 422, 376, 311, 312, 729, 359, 453, + 658, 693, 694, 583, 0, 646, 584, 593, 351, 618, + 630, 629, 449, 543, 0, 641, 644, 573, 728, 0, + 638, 652, 732, 651, 725, 459, 0, 486, 649, 596, + 0, 642, 615, 616, 0, 643, 611, 647, 0, 585, + 0, 554, 557, 586, 671, 672, 673, 318, 556, 675, + 676, 677, 678, 679, 680, 1152, 674, 526, 619, 595, + 622, 535, 598, 597, 0, 0, 633, 1155, 634, 635, + 443, 444, 445, 446, 380, 659, 1150, 555, 473, 0, + 620, 0, 0, 0, 0, 0, 0, 0, 0, 625, + 626, 623, 737, 0, 682, 683, 0, 0, 549, 550, + 375, 0, 568, 383, 339, 458, 377, 533, 406, 0, + 561, 627, 562, 475, 476, 685, 690, 686, 687, 689, + 709, 1162, 1148, 1158, 1149, 408, 426, 478, 532, 456, + 483, 337, 522, 492, 1159, 612, 640, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 219, 2401, 0, 245, 0, 0, 0, 0, - 0, 0, 335, 246, 572, 694, 574, 573, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, + 0, 0, 0, 0, 667, 666, 665, 664, 663, 662, + 661, 660, 0, 0, 609, 509, 353, 305, 349, 350, + 357, 726, 722, 727, 710, 713, 712, 688, 0, 313, + 589, 424, 472, 374, 654, 655, 0, 708, 259, 260, + 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, + 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, + 282, 283, 657, 274, 275, 284, 285, 286, 287, 288, + 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, + 0, 0, 0, 307, 714, 715, 716, 717, 718, 0, + 0, 308, 309, 310, 0, 0, 300, 500, 301, 302, + 303, 304, 0, 0, 539, 540, 541, 564, 0, 542, + 524, 588, 384, 314, 504, 531, 724, 0, 0, 0, + 0, 0, 0, 0, 639, 650, 684, 0, 696, 697, + 699, 701, 700, 703, 497, 498, 711, 0, 0, 705, + 706, 707, 704, 1146, 484, 505, 491, 0, 730, 579, + 580, 731, 692, 315, 183, 223, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 153, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 2297, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 490, 519, 0, - 532, 0, 404, 405, 0, 0, 0, 0, 0, 0, - 0, 322, 497, 516, 336, 484, 530, 341, 492, 509, - 331, 450, 481, 0, 0, 324, 514, 491, 432, 323, - 0, 475, 364, 381, 361, 448, 0, 0, 513, 543, - 360, 533, 0, 524, 326, 0, 523, 447, 510, 515, - 433, 426, 0, 325, 512, 431, 425, 410, 371, 559, - 411, 412, 385, 462, 423, 463, 386, 437, 436, 438, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, - 0, 0, 0, 0, 0, 554, 555, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 687, 0, 0, 691, 0, 526, 0, 0, - 0, 0, 0, 0, 495, 0, 0, 413, 0, 0, - 0, 544, 0, 478, 453, 729, 0, 0, 476, 421, - 511, 464, 517, 498, 525, 470, 465, 316, 499, 363, - 434, 332, 334, 719, 365, 368, 372, 373, 443, 444, - 458, 483, 502, 503, 504, 362, 346, 477, 347, 382, - 348, 317, 354, 352, 355, 485, 356, 319, 459, 508, - 0, 378, 473, 429, 320, 428, 460, 507, 506, 333, - 534, 541, 542, 632, 0, 547, 730, 731, 732, 556, - 0, 466, 329, 328, 0, 0, 0, 358, 461, 342, - 344, 345, 343, 456, 457, 561, 562, 563, 565, 0, - 566, 567, 0, 0, 0, 0, 568, 633, 649, 617, - 586, 549, 641, 583, 587, 588, 399, 400, 401, 652, - 0, 0, 0, 540, 414, 415, 0, 370, 369, 430, - 321, 0, 0, 407, 398, 467, 327, 366, 409, 403, - 416, 417, 418, 376, 311, 312, 725, 359, 449, 654, - 689, 690, 579, 0, 642, 580, 589, 351, 614, 626, - 625, 445, 539, 0, 637, 640, 569, 724, 0, 634, - 648, 728, 647, 721, 455, 0, 482, 645, 592, 0, - 638, 611, 612, 0, 639, 607, 643, 0, 581, 0, - 550, 553, 582, 667, 668, 669, 318, 552, 671, 672, - 673, 674, 675, 676, 677, 670, 522, 615, 591, 618, - 531, 594, 593, 0, 0, 629, 548, 630, 631, 439, - 440, 441, 442, 380, 655, 340, 551, 469, 0, 616, - 0, 0, 0, 0, 0, 0, 0, 0, 621, 622, - 619, 733, 0, 678, 679, 0, 0, 545, 546, 375, - 0, 564, 383, 339, 454, 377, 529, 406, 0, 557, - 623, 558, 471, 472, 681, 686, 682, 683, 685, 705, - 446, 397, 402, 486, 408, 422, 474, 528, 452, 479, - 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 663, 662, 661, 660, 659, 658, 657, - 656, 0, 0, 605, 505, 353, 305, 349, 350, 357, - 722, 718, 723, 706, 709, 708, 684, 0, 313, 585, - 420, 468, 374, 650, 651, 0, 704, 259, 260, 261, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, - 283, 653, 274, 275, 284, 285, 286, 287, 288, 289, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, - 0, 0, 307, 710, 711, 712, 713, 714, 0, 0, - 308, 309, 310, 0, 0, 300, 496, 301, 302, 303, - 304, 0, 0, 535, 536, 537, 560, 0, 538, 520, - 584, 384, 314, 500, 527, 720, 0, 0, 0, 0, - 0, 0, 0, 635, 646, 680, 0, 692, 693, 695, - 697, 696, 699, 493, 494, 707, 0, 0, 701, 702, - 703, 700, 424, 480, 501, 487, 0, 726, 575, 576, - 727, 688, 315, 451, 0, 0, 590, 624, 613, 698, - 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 367, 1149, 0, 419, 628, 609, 620, 610, 595, - 596, 597, 604, 379, 598, 599, 600, 570, 601, 571, - 602, 603, 0, 627, 577, 489, 435, 0, 644, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, - 1156, 1157, 0, 0, 0, 0, 335, 246, 572, 694, - 574, 573, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 490, 519, 0, 532, 0, 404, 405, 0, 0, - 0, 0, 0, 0, 0, 322, 497, 1143, 336, 484, - 530, 341, 492, 509, 331, 450, 481, 0, 0, 324, - 514, 491, 432, 323, 0, 475, 364, 381, 361, 448, - 0, 0, 513, 543, 360, 533, 1128, 524, 326, 1127, - 523, 447, 510, 515, 433, 426, 0, 325, 512, 431, - 425, 410, 371, 559, 411, 412, 385, 462, 423, 463, - 386, 437, 436, 438, 387, 388, 389, 390, 391, 392, - 393, 394, 395, 396, 0, 0, 0, 0, 0, 554, - 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 687, 0, 0, 691, - 0, 526, 0, 0, 0, 0, 0, 0, 495, 0, - 0, 413, 0, 0, 0, 544, 0, 478, 453, 729, - 0, 0, 476, 421, 511, 464, 517, 498, 525, 1147, - 465, 316, 499, 363, 434, 332, 334, 719, 365, 368, - 372, 373, 443, 444, 458, 483, 502, 503, 504, 362, - 346, 477, 347, 382, 348, 317, 354, 352, 355, 485, - 356, 319, 459, 508, 0, 378, 473, 429, 320, 428, - 460, 507, 506, 333, 534, 541, 542, 632, 0, 547, - 730, 731, 732, 556, 0, 466, 329, 328, 0, 0, - 0, 358, 461, 342, 344, 345, 343, 456, 457, 561, - 562, 563, 565, 0, 566, 567, 0, 0, 0, 0, - 568, 633, 649, 617, 586, 549, 641, 583, 587, 588, - 399, 400, 401, 652, 0, 0, 0, 540, 414, 415, - 0, 370, 369, 430, 321, 0, 0, 407, 398, 467, - 327, 366, 409, 403, 416, 417, 418, 376, 311, 312, - 725, 359, 449, 654, 689, 690, 579, 0, 642, 580, - 589, 351, 614, 626, 625, 445, 539, 0, 637, 640, - 569, 724, 0, 634, 648, 728, 647, 721, 455, 0, - 482, 645, 592, 0, 638, 611, 612, 0, 639, 607, - 643, 0, 581, 0, 550, 553, 582, 667, 668, 669, - 318, 552, 671, 672, 673, 674, 675, 676, 1148, 670, - 522, 615, 591, 618, 531, 594, 593, 0, 0, 629, - 1151, 630, 631, 439, 440, 441, 442, 380, 655, 1146, - 551, 469, 0, 616, 0, 0, 0, 0, 0, 0, - 0, 0, 621, 622, 619, 733, 0, 678, 679, 0, - 0, 545, 546, 375, 0, 564, 383, 339, 454, 377, - 529, 406, 0, 557, 623, 558, 471, 472, 681, 686, - 682, 683, 685, 705, 1158, 1144, 1154, 1145, 408, 422, - 474, 528, 452, 479, 337, 518, 488, 1155, 608, 636, + 1160, 1161, 0, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 663, 662, 661, - 660, 659, 658, 657, 656, 0, 0, 605, 505, 353, - 305, 349, 350, 357, 722, 718, 723, 706, 709, 708, - 684, 0, 313, 585, 420, 468, 374, 650, 651, 0, - 704, 259, 260, 261, 262, 263, 264, 265, 266, 306, - 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, - 279, 280, 281, 282, 283, 653, 274, 275, 284, 285, - 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, - 296, 297, 0, 0, 0, 0, 307, 710, 711, 712, - 713, 714, 0, 0, 308, 309, 310, 0, 0, 300, - 496, 301, 302, 303, 304, 0, 0, 535, 536, 537, - 560, 0, 538, 520, 584, 384, 314, 500, 527, 720, - 0, 0, 0, 0, 0, 0, 0, 635, 646, 680, - 0, 692, 693, 695, 697, 696, 699, 493, 494, 707, - 0, 0, 701, 702, 703, 700, 1142, 480, 501, 487, - 0, 726, 575, 576, 727, 688, 315, 183, 223, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 153, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2285, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 1132, 528, 326, 1131, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 1162, 2318, + 1158, 2319, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 1159, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 3329, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 1156, - 1157, 0, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 1128, 524, 326, 1127, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 1158, 2306, 1154, 2307, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 1155, 608, 636, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 3313, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 3332, 0, 0, 0, 0, 3331, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 1725, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 1723, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3316, 0, 0, 0, 0, 3315, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 1717, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 1715, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 1721, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 1719, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 1723, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 1721, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 4653, + 0, 245, 944, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 1713, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 1711, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 1715, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 1723, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 1713, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4637, 0, 245, 940, 0, 0, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 1721, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 1723, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 1715, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 1713, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 1715, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 1932, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 2794, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 2796, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 1944, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 2806, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 2808, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 2359, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 2360, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 2371, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 2372, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 3555, 3557, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 3571, 3573, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 2817, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 1715, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 745, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 2829, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 1723, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 1064, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 749, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 940, 0, 0, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 1068, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 4613, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 944, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 4321, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 4629, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 4510, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 4337, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1946, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4336, 0, 245, 0, 0, 0, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 4526, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1958, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 4227, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 3591, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 4352, 0, 245, 0, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 4060, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2285, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 4243, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 3607, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3616, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 3856, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 4076, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2297, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3739, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 3596, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 3525, 0, 0, 0, - 0, 0, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 3632, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 3872, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 1715, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 2796, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, + 0, 3755, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 3612, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 3224, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 3541, 0, 0, 0, 0, 0, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 3145, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3126, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, + 0, 3437, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 3071, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 1723, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 2808, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2426, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 2944, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 3240, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2898, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 2896, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 2633, 0, 0, 0, 0, 0, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 3161, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 2117, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3142, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 3087, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 1540, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 2331, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 2267, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 2960, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 1715, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 2163, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2914, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 2912, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 2645, 0, 0, 0, 0, 0, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 2129, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 1548, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 2343, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 2279, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 1723, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 2175, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 1753, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 749, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 754, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 520, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 1070, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 513, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 723, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 3544, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 1745, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 745, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 2114, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 474, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 681, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 1702, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 688, 0, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 0, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 455, 0, 0, 594, 628, 617, 702, + 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 367, 0, 0, 423, 632, 613, 624, 614, 599, + 600, 601, 608, 379, 602, 603, 604, 574, 605, 575, + 606, 607, 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 750, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, + 0, 0, 0, 0, 0, 0, 335, 246, 576, 698, + 578, 577, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 494, 523, 0, 536, 0, 404, 405, 0, 0, + 0, 0, 0, 0, 0, 322, 501, 1700, 336, 488, + 534, 341, 496, 513, 331, 454, 485, 0, 0, 324, + 518, 495, 436, 323, 0, 479, 364, 381, 361, 452, + 0, 0, 517, 547, 360, 537, 0, 528, 326, 0, + 527, 451, 514, 519, 437, 430, 0, 325, 516, 435, + 429, 410, 371, 563, 411, 412, 413, 414, 415, 416, + 385, 466, 427, 467, 386, 441, 440, 442, 387, 388, + 389, 390, 391, 392, 393, 394, 395, 396, 0, 0, + 0, 0, 0, 558, 559, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 691, 0, 0, 695, 0, 530, 0, 0, 0, 0, + 0, 0, 499, 0, 0, 417, 0, 0, 0, 548, + 0, 482, 457, 733, 0, 0, 480, 425, 515, 468, + 521, 502, 529, 474, 469, 316, 503, 363, 438, 332, + 334, 723, 365, 368, 372, 373, 447, 448, 462, 487, + 506, 507, 508, 362, 346, 481, 347, 382, 348, 317, + 354, 352, 355, 489, 356, 319, 463, 512, 0, 378, + 477, 433, 320, 432, 464, 511, 510, 333, 538, 545, + 546, 636, 0, 551, 734, 735, 736, 560, 0, 470, + 329, 328, 0, 0, 0, 358, 465, 342, 344, 345, + 343, 460, 461, 565, 566, 567, 569, 0, 570, 571, + 0, 0, 0, 0, 572, 637, 653, 621, 590, 553, + 645, 587, 591, 592, 399, 400, 401, 656, 0, 0, + 0, 544, 418, 419, 0, 370, 369, 434, 321, 0, + 0, 407, 398, 471, 327, 366, 409, 403, 420, 421, + 422, 376, 311, 312, 729, 359, 453, 658, 693, 694, + 583, 0, 646, 584, 593, 351, 618, 630, 629, 449, + 543, 0, 641, 644, 573, 728, 0, 638, 652, 732, + 651, 725, 459, 0, 486, 649, 596, 0, 642, 615, + 616, 0, 643, 611, 647, 0, 585, 0, 554, 557, + 586, 671, 672, 673, 318, 556, 675, 676, 677, 678, + 679, 680, 681, 674, 526, 619, 595, 622, 535, 598, + 597, 0, 0, 633, 552, 634, 635, 443, 444, 445, + 446, 380, 659, 340, 555, 473, 0, 620, 0, 0, + 0, 0, 0, 0, 0, 0, 625, 626, 623, 737, + 0, 682, 683, 0, 0, 549, 550, 375, 0, 568, + 383, 339, 458, 377, 533, 406, 0, 561, 627, 562, + 475, 476, 685, 690, 686, 687, 689, 709, 450, 397, + 402, 490, 408, 426, 478, 532, 456, 483, 337, 522, + 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 516, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, + 0, 667, 666, 665, 664, 663, 662, 661, 660, 0, + 0, 609, 509, 353, 305, 349, 350, 357, 726, 722, + 727, 710, 713, 712, 688, 0, 313, 589, 424, 472, + 374, 654, 655, 0, 708, 259, 260, 261, 262, 263, + 264, 265, 266, 306, 267, 268, 269, 270, 271, 272, + 273, 276, 277, 278, 279, 280, 281, 282, 283, 657, + 274, 275, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 294, 295, 296, 297, 0, 0, 0, 0, + 307, 714, 715, 716, 717, 718, 0, 0, 308, 309, + 310, 0, 0, 300, 500, 301, 302, 303, 304, 0, + 0, 539, 540, 541, 564, 0, 542, 524, 588, 384, + 314, 504, 531, 724, 0, 0, 0, 0, 0, 0, + 0, 639, 650, 684, 0, 696, 697, 699, 701, 700, + 703, 497, 498, 711, 0, 0, 705, 706, 707, 704, + 428, 484, 505, 491, 0, 730, 579, 580, 731, 692, + 315, 455, 0, 0, 594, 628, 617, 702, 582, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, + 0, 0, 423, 632, 613, 624, 614, 599, 600, 601, + 608, 379, 602, 603, 604, 574, 605, 575, 606, 607, + 0, 631, 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 1066, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, + 0, 0, 0, 0, 335, 246, 576, 698, 578, 577, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 509, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, + 523, 0, 536, 0, 404, 405, 0, 0, 0, 0, + 0, 0, 0, 322, 501, 520, 336, 488, 534, 341, + 496, 1566, 331, 454, 485, 0, 0, 324, 518, 495, + 436, 323, 0, 479, 364, 381, 361, 452, 0, 0, + 517, 547, 360, 537, 0, 528, 326, 0, 527, 451, + 514, 519, 437, 430, 0, 325, 516, 435, 429, 410, + 371, 563, 411, 412, 413, 414, 415, 416, 385, 466, + 427, 467, 386, 441, 440, 442, 387, 388, 389, 390, + 391, 392, 393, 394, 395, 396, 0, 0, 0, 0, + 0, 558, 559, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, + 0, 695, 0, 530, 0, 0, 0, 0, 0, 0, + 499, 0, 0, 417, 0, 0, 0, 548, 0, 482, + 457, 733, 0, 0, 480, 425, 515, 468, 521, 502, + 529, 474, 469, 316, 503, 363, 438, 332, 334, 723, + 365, 368, 372, 373, 447, 448, 462, 487, 506, 507, + 508, 362, 346, 481, 347, 382, 348, 317, 354, 352, + 355, 489, 356, 319, 463, 512, 0, 378, 477, 433, + 320, 432, 464, 511, 510, 333, 538, 545, 546, 636, + 0, 551, 734, 735, 736, 560, 0, 470, 329, 328, + 0, 0, 0, 358, 465, 342, 344, 345, 343, 460, + 461, 565, 566, 567, 569, 0, 570, 571, 0, 0, + 0, 0, 572, 637, 653, 621, 590, 553, 645, 587, + 591, 592, 399, 400, 401, 656, 0, 0, 0, 544, + 418, 419, 0, 370, 369, 434, 321, 0, 0, 407, + 398, 471, 327, 366, 409, 403, 420, 421, 422, 376, + 311, 312, 729, 359, 453, 658, 693, 694, 583, 0, + 646, 584, 593, 351, 618, 630, 629, 449, 543, 0, + 641, 644, 573, 728, 0, 638, 652, 732, 651, 725, + 459, 0, 486, 649, 596, 0, 642, 615, 616, 0, + 643, 611, 647, 0, 585, 0, 554, 557, 586, 671, + 672, 673, 318, 556, 675, 676, 677, 678, 679, 680, + 681, 674, 526, 619, 595, 622, 535, 598, 597, 0, + 0, 633, 552, 634, 635, 443, 444, 445, 446, 380, + 659, 340, 555, 473, 0, 620, 0, 0, 0, 0, + 0, 0, 0, 0, 625, 626, 623, 737, 0, 682, + 683, 0, 0, 549, 550, 375, 0, 568, 383, 339, + 458, 377, 533, 406, 0, 561, 627, 562, 475, 476, + 685, 690, 686, 687, 689, 709, 450, 397, 402, 490, + 408, 426, 478, 532, 456, 483, 337, 522, 492, 431, + 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, + 666, 665, 664, 663, 662, 661, 660, 0, 0, 609, + 509, 353, 305, 349, 350, 357, 726, 722, 727, 710, + 713, 712, 688, 0, 313, 589, 424, 472, 374, 654, + 655, 0, 708, 259, 260, 261, 262, 263, 264, 265, + 266, 306, 267, 268, 269, 270, 271, 272, 273, 276, + 277, 278, 279, 280, 281, 282, 283, 657, 274, 275, + 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 0, 0, 0, 0, 307, 714, + 715, 716, 717, 718, 0, 0, 308, 309, 310, 0, + 0, 300, 500, 301, 302, 303, 304, 0, 0, 539, + 540, 541, 564, 0, 542, 524, 588, 384, 314, 504, + 531, 724, 0, 0, 0, 0, 0, 0, 0, 639, + 650, 684, 0, 696, 697, 699, 701, 700, 703, 497, + 498, 711, 0, 0, 705, 706, 707, 704, 428, 484, + 505, 491, 0, 730, 579, 580, 731, 692, 315, 455, + 0, 0, 594, 628, 617, 702, 582, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, + 423, 632, 613, 624, 614, 599, 600, 601, 608, 379, + 602, 603, 604, 574, 605, 575, 606, 607, 0, 631, + 581, 493, 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, + 0, 0, 335, 246, 576, 698, 578, 577, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 719, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 3528, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, + 0, 0, 0, 0, 0, 0, 0, 494, 523, 0, + 536, 0, 404, 405, 0, 0, 0, 0, 0, 0, + 0, 322, 501, 520, 336, 488, 534, 341, 496, 513, + 331, 454, 485, 0, 0, 324, 518, 495, 436, 323, + 0, 479, 364, 381, 361, 452, 0, 0, 517, 547, + 360, 537, 0, 528, 326, 0, 527, 451, 514, 519, + 437, 430, 0, 325, 516, 435, 429, 410, 371, 563, + 411, 412, 413, 414, 415, 416, 385, 466, 427, 467, + 386, 441, 440, 442, 387, 388, 389, 390, 391, 392, + 393, 394, 395, 396, 0, 0, 0, 0, 0, 558, + 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 691, 0, 0, 695, + 0, 530, 0, 0, 0, 0, 0, 0, 499, 0, + 0, 417, 0, 0, 0, 548, 0, 482, 457, 733, + 0, 0, 480, 425, 515, 468, 521, 502, 529, 474, + 469, 316, 503, 363, 438, 332, 334, 828, 365, 368, + 372, 373, 447, 448, 462, 487, 506, 507, 508, 362, + 346, 481, 347, 382, 348, 317, 354, 352, 355, 489, + 356, 319, 463, 512, 0, 378, 477, 433, 320, 432, + 464, 511, 510, 333, 538, 545, 546, 636, 0, 551, + 734, 735, 736, 560, 0, 470, 329, 328, 0, 0, + 0, 358, 465, 342, 344, 345, 343, 460, 461, 565, + 566, 567, 569, 0, 570, 571, 0, 0, 0, 0, + 572, 637, 653, 621, 590, 553, 645, 587, 591, 592, + 399, 400, 401, 656, 0, 0, 0, 544, 418, 419, + 0, 370, 369, 434, 321, 0, 0, 407, 398, 471, + 327, 366, 409, 403, 420, 421, 422, 376, 311, 312, + 729, 359, 453, 658, 693, 694, 583, 0, 646, 584, + 593, 351, 618, 630, 629, 449, 543, 0, 641, 644, + 573, 728, 0, 638, 652, 732, 651, 725, 459, 0, + 486, 649, 596, 0, 642, 615, 616, 0, 643, 611, + 647, 0, 585, 0, 554, 557, 586, 671, 672, 673, + 318, 556, 675, 676, 677, 678, 679, 680, 681, 674, + 526, 619, 595, 622, 535, 598, 597, 0, 0, 633, + 552, 634, 635, 443, 444, 445, 446, 380, 659, 340, + 555, 473, 0, 620, 0, 0, 0, 0, 0, 0, + 0, 0, 625, 626, 623, 737, 0, 682, 683, 0, + 0, 549, 550, 375, 0, 568, 383, 339, 458, 377, + 533, 406, 0, 561, 627, 562, 475, 476, 685, 690, + 686, 687, 689, 709, 450, 397, 402, 490, 408, 426, + 478, 532, 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 667, 666, 665, + 664, 663, 662, 661, 660, 0, 0, 609, 509, 353, + 305, 349, 350, 357, 726, 722, 727, 710, 713, 712, + 688, 0, 313, 589, 424, 472, 374, 654, 655, 0, + 708, 259, 260, 261, 262, 263, 264, 265, 266, 306, + 267, 268, 269, 270, 271, 272, 273, 276, 277, 278, + 279, 280, 281, 282, 283, 657, 274, 275, 284, 285, + 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 0, 0, 0, 0, 307, 714, 715, 716, + 717, 718, 0, 0, 308, 309, 310, 0, 0, 300, + 500, 301, 302, 303, 304, 0, 0, 539, 540, 541, + 564, 0, 542, 524, 588, 384, 314, 504, 531, 724, + 0, 0, 0, 0, 0, 0, 0, 639, 650, 684, + 0, 696, 697, 699, 701, 700, 703, 497, 498, 711, + 0, 0, 705, 706, 707, 704, 428, 484, 505, 491, + 0, 730, 579, 580, 731, 692, 315, 455, 0, 0, + 594, 628, 617, 702, 582, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 367, 0, 0, 423, 632, + 613, 624, 614, 599, 600, 601, 608, 379, 602, 603, + 604, 574, 605, 575, 606, 607, 0, 631, 581, 493, + 439, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, + 335, 246, 576, 698, 578, 577, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 2102, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 470, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 677, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 494, 523, 0, 536, 0, + 404, 405, 0, 0, 0, 0, 0, 0, 0, 322, + 501, 520, 336, 488, 534, 341, 496, 513, 331, 454, + 485, 0, 0, 324, 518, 495, 436, 323, 0, 479, + 364, 381, 361, 452, 0, 0, 517, 547, 360, 537, + 0, 528, 326, 0, 527, 451, 514, 519, 437, 430, + 0, 325, 516, 435, 429, 410, 371, 563, 411, 412, + 413, 414, 415, 416, 385, 466, 427, 467, 386, 441, + 440, 442, 387, 388, 389, 390, 391, 392, 393, 394, + 395, 396, 0, 0, 0, 0, 0, 558, 559, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 691, 0, 0, 695, 0, 530, + 0, 0, 0, 0, 0, 0, 499, 0, 0, 417, + 0, 0, 0, 548, 0, 482, 457, 733, 0, 0, + 480, 425, 515, 468, 521, 502, 529, 780, 469, 316, + 503, 363, 438, 332, 334, 723, 365, 368, 372, 373, + 447, 448, 462, 487, 506, 507, 508, 362, 346, 481, + 347, 382, 348, 317, 354, 352, 355, 489, 356, 319, + 463, 512, 0, 378, 477, 433, 320, 432, 464, 511, + 510, 333, 538, 545, 546, 636, 0, 551, 734, 735, + 736, 560, 0, 470, 329, 328, 0, 0, 0, 358, + 465, 342, 344, 345, 343, 460, 461, 565, 566, 567, + 569, 0, 570, 571, 0, 0, 0, 0, 572, 637, + 653, 621, 590, 553, 645, 587, 591, 592, 399, 400, + 401, 656, 0, 0, 0, 544, 418, 419, 0, 370, + 369, 434, 321, 0, 0, 407, 398, 471, 327, 366, + 409, 403, 420, 421, 422, 376, 311, 312, 729, 359, + 453, 658, 693, 694, 583, 0, 646, 584, 593, 351, + 618, 630, 629, 449, 543, 0, 641, 644, 573, 728, + 0, 638, 652, 732, 651, 725, 459, 0, 486, 649, + 596, 0, 642, 615, 616, 0, 643, 611, 647, 0, + 585, 0, 554, 557, 586, 671, 672, 673, 318, 556, + 675, 676, 677, 678, 679, 680, 781, 674, 526, 619, + 595, 622, 535, 598, 597, 0, 0, 633, 552, 634, + 635, 443, 444, 445, 446, 380, 659, 340, 555, 473, + 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, + 625, 626, 623, 737, 0, 682, 683, 0, 0, 549, + 550, 375, 0, 568, 383, 339, 458, 377, 533, 406, + 0, 561, 627, 562, 475, 476, 685, 690, 686, 687, + 689, 709, 450, 397, 402, 490, 408, 426, 478, 532, + 456, 483, 337, 522, 492, 431, 612, 640, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, + 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 667, 666, 665, 664, 663, + 662, 661, 660, 0, 0, 609, 509, 353, 305, 349, + 350, 357, 726, 722, 727, 710, 713, 712, 688, 0, + 313, 589, 424, 472, 374, 654, 655, 0, 708, 259, + 260, 261, 262, 263, 264, 265, 266, 306, 267, 268, + 269, 270, 271, 272, 273, 276, 277, 278, 279, 280, + 281, 282, 283, 657, 274, 275, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, + 0, 0, 0, 0, 307, 714, 715, 716, 717, 718, + 0, 0, 308, 309, 310, 0, 0, 300, 500, 301, + 302, 303, 304, 0, 0, 539, 540, 541, 564, 0, + 542, 524, 588, 384, 314, 504, 531, 724, 0, 0, + 0, 0, 0, 0, 0, 639, 650, 684, 0, 696, + 697, 699, 701, 700, 703, 497, 498, 711, 0, 0, + 705, 706, 707, 704, 428, 484, 505, 491, 0, 730, + 579, 580, 731, 692, 315, 455, 0, 0, 594, 628, + 617, 702, 582, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 367, 0, 0, 423, 632, 613, 624, + 614, 599, 600, 601, 608, 379, 602, 603, 604, 574, + 605, 575, 606, 607, 0, 631, 581, 493, 439, 0, + 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 245, 0, 0, 0, 0, 0, 0, 335, 246, + 576, 698, 578, 577, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 1694, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 684, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, 451, 0, - 0, 590, 624, 613, 698, 578, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 367, 0, 0, 419, - 628, 609, 620, 610, 595, 596, 597, 604, 379, 598, - 599, 600, 570, 601, 571, 602, 603, 0, 627, 577, - 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, - 0, 335, 246, 572, 694, 574, 573, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 494, 523, 0, 536, 0, 404, 405, + 0, 0, 0, 0, 0, 0, 0, 322, 501, 520, + 336, 488, 534, 341, 496, 513, 331, 454, 485, 0, + 0, 324, 518, 495, 436, 323, 0, 479, 364, 381, + 361, 452, 0, 0, 517, 547, 360, 537, 0, 528, + 326, 0, 527, 451, 514, 519, 437, 430, 0, 325, + 516, 435, 429, 410, 371, 563, 411, 412, 413, 414, + 415, 416, 385, 466, 427, 467, 386, 441, 440, 442, + 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, + 0, 0, 0, 0, 0, 558, 559, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 691, 0, 0, 695, 0, 530, 0, 0, + 0, 0, 0, 0, 499, 0, 0, 417, 0, 0, + 0, 548, 0, 482, 457, 733, 0, 0, 480, 425, + 515, 468, 521, 502, 529, 474, 469, 316, 503, 363, + 438, 332, 334, 723, 365, 368, 372, 373, 447, 448, + 462, 487, 506, 507, 508, 362, 346, 481, 347, 382, + 348, 317, 354, 352, 355, 489, 356, 319, 463, 512, + 0, 378, 477, 433, 320, 432, 464, 511, 510, 333, + 538, 545, 546, 636, 0, 551, 734, 735, 736, 560, + 0, 470, 329, 328, 0, 0, 0, 358, 465, 342, + 344, 345, 343, 460, 461, 565, 566, 567, 569, 0, + 570, 571, 0, 0, 0, 0, 572, 637, 653, 621, + 590, 553, 645, 587, 591, 592, 399, 400, 401, 656, + 0, 0, 0, 544, 418, 419, 0, 370, 369, 434, + 321, 0, 0, 407, 398, 471, 327, 366, 409, 403, + 420, 421, 422, 376, 311, 312, 729, 359, 453, 658, + 693, 694, 583, 0, 646, 584, 593, 351, 618, 630, + 629, 449, 543, 0, 641, 644, 573, 728, 0, 638, + 652, 732, 651, 725, 459, 0, 486, 649, 596, 0, + 642, 615, 616, 0, 643, 611, 647, 0, 585, 0, + 554, 557, 586, 671, 672, 673, 318, 556, 675, 676, + 677, 678, 679, 680, 681, 674, 526, 619, 595, 622, + 535, 598, 597, 0, 0, 633, 552, 634, 635, 443, + 444, 445, 446, 380, 659, 340, 555, 473, 0, 620, + 0, 0, 0, 0, 0, 0, 0, 0, 625, 626, + 623, 737, 0, 682, 683, 0, 0, 549, 550, 375, + 0, 568, 383, 339, 458, 377, 533, 406, 0, 561, + 627, 562, 475, 476, 685, 690, 686, 687, 689, 709, + 450, 397, 402, 490, 408, 426, 478, 532, 456, 483, + 337, 522, 492, 431, 612, 640, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 298, 299, 2259, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 667, 666, 665, 664, 663, 662, 661, + 660, 0, 0, 609, 509, 353, 305, 349, 350, 357, + 726, 722, 727, 710, 713, 712, 776, 2261, 313, 589, + 424, 472, 374, 654, 655, 0, 708, 259, 260, 261, + 262, 263, 264, 265, 266, 306, 267, 268, 269, 270, + 271, 272, 273, 276, 277, 278, 279, 280, 281, 282, + 283, 657, 274, 275, 284, 285, 286, 287, 288, 289, + 290, 291, 292, 293, 294, 295, 296, 297, 0, 0, + 0, 2236, 307, 714, 715, 716, 717, 718, 0, 0, + 308, 309, 310, 0, 0, 300, 500, 301, 302, 303, + 304, 0, 0, 539, 540, 541, 564, 0, 542, 524, + 588, 384, 314, 504, 531, 724, 0, 0, 0, 0, + 0, 0, 0, 639, 650, 684, 0, 696, 697, 699, + 701, 700, 703, 497, 498, 711, 0, 0, 705, 706, + 707, 704, 428, 484, 505, 491, 0, 730, 579, 580, + 731, 692, 315, 0, 0, 0, 0, 0, 0, 0, + 0, 2252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 490, 519, 0, 532, - 0, 404, 405, 0, 0, 0, 0, 0, 0, 0, - 322, 497, 1692, 336, 484, 530, 341, 492, 509, 331, - 450, 481, 0, 0, 324, 514, 491, 432, 323, 0, - 475, 364, 381, 361, 448, 0, 0, 513, 543, 360, - 533, 0, 524, 326, 0, 523, 447, 510, 515, 433, - 426, 0, 325, 512, 431, 425, 410, 371, 559, 411, - 412, 385, 462, 423, 463, 386, 437, 436, 438, 387, - 388, 389, 390, 391, 392, 393, 394, 395, 396, 0, - 0, 0, 0, 0, 554, 555, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 687, 0, 0, 691, 0, 526, 0, 0, 0, - 0, 0, 0, 495, 0, 0, 413, 0, 0, 0, - 544, 0, 478, 453, 729, 0, 0, 476, 421, 511, - 464, 517, 498, 525, 470, 465, 316, 499, 363, 434, - 332, 334, 719, 365, 368, 372, 373, 443, 444, 458, - 483, 502, 503, 504, 362, 346, 477, 347, 382, 348, - 317, 354, 352, 355, 485, 356, 319, 459, 508, 0, - 378, 473, 429, 320, 428, 460, 507, 506, 333, 534, - 541, 542, 632, 0, 547, 730, 731, 732, 556, 0, - 466, 329, 328, 0, 0, 0, 358, 461, 342, 344, - 345, 343, 456, 457, 561, 562, 563, 565, 0, 566, - 567, 0, 0, 0, 0, 568, 633, 649, 617, 586, - 549, 641, 583, 587, 588, 399, 400, 401, 652, 0, - 0, 0, 540, 414, 415, 0, 370, 369, 430, 321, - 0, 0, 407, 398, 467, 327, 366, 409, 403, 416, - 417, 418, 376, 311, 312, 725, 359, 449, 654, 689, - 690, 579, 0, 642, 580, 589, 351, 614, 626, 625, - 445, 539, 0, 637, 640, 569, 724, 0, 634, 648, - 728, 647, 721, 455, 0, 482, 645, 592, 0, 638, - 611, 612, 0, 639, 607, 643, 0, 581, 0, 550, - 553, 582, 667, 668, 669, 318, 552, 671, 672, 673, - 674, 675, 676, 677, 670, 522, 615, 591, 618, 531, - 594, 593, 0, 0, 629, 548, 630, 631, 439, 440, - 441, 442, 380, 655, 340, 551, 469, 0, 616, 0, - 0, 0, 0, 0, 0, 0, 0, 621, 622, 619, - 733, 0, 678, 679, 0, 0, 545, 546, 375, 0, - 564, 383, 339, 454, 377, 529, 406, 0, 557, 623, - 558, 471, 472, 681, 686, 682, 683, 685, 705, 446, - 397, 402, 486, 408, 422, 474, 528, 452, 479, 337, - 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 663, 662, 661, 660, 659, 658, 657, 656, - 0, 0, 605, 505, 353, 305, 349, 350, 357, 722, - 718, 723, 706, 709, 708, 684, 0, 313, 585, 420, - 468, 374, 650, 651, 0, 704, 259, 260, 261, 262, - 263, 264, 265, 266, 306, 267, 268, 269, 270, 271, - 272, 273, 276, 277, 278, 279, 280, 281, 282, 283, - 653, 274, 275, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 0, 0, 0, - 0, 307, 710, 711, 712, 713, 714, 0, 0, 308, - 309, 310, 0, 0, 300, 496, 301, 302, 303, 304, - 0, 0, 535, 536, 537, 560, 0, 538, 520, 584, - 384, 314, 500, 527, 720, 0, 0, 0, 0, 0, - 0, 0, 635, 646, 680, 0, 692, 693, 695, 697, - 696, 699, 493, 494, 707, 0, 0, 701, 702, 703, - 700, 424, 480, 501, 487, 0, 726, 575, 576, 727, - 688, 315, 451, 0, 0, 590, 624, 613, 698, 578, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 367, 0, 0, 419, 628, 609, 620, 610, 595, 596, - 597, 604, 379, 598, 599, 600, 570, 601, 571, 602, - 603, 0, 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, - 0, 0, 0, 0, 0, 335, 246, 572, 694, 574, - 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 490, 519, 0, 532, 0, 404, 405, 0, 0, 0, - 0, 0, 0, 0, 322, 497, 516, 336, 484, 530, - 341, 492, 1558, 331, 450, 481, 0, 0, 324, 514, - 491, 432, 323, 0, 475, 364, 381, 361, 448, 0, - 0, 513, 543, 360, 533, 0, 524, 326, 0, 523, - 447, 510, 515, 433, 426, 0, 325, 512, 431, 425, - 410, 371, 559, 411, 412, 385, 462, 423, 463, 386, - 437, 436, 438, 387, 388, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 0, 0, 0, 554, 555, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 687, 0, 0, 691, 0, - 526, 0, 0, 0, 0, 0, 0, 495, 0, 0, - 413, 0, 0, 0, 544, 0, 478, 453, 729, 0, - 0, 476, 421, 511, 464, 517, 498, 525, 470, 465, - 316, 499, 363, 434, 332, 334, 719, 365, 368, 372, - 373, 443, 444, 458, 483, 502, 503, 504, 362, 346, - 477, 347, 382, 348, 317, 354, 352, 355, 485, 356, - 319, 459, 508, 0, 378, 473, 429, 320, 428, 460, - 507, 506, 333, 534, 541, 542, 632, 0, 547, 730, - 731, 732, 556, 0, 466, 329, 328, 0, 0, 0, - 358, 461, 342, 344, 345, 343, 456, 457, 561, 562, - 563, 565, 0, 566, 567, 0, 0, 0, 0, 568, - 633, 649, 617, 586, 549, 641, 583, 587, 588, 399, - 400, 401, 652, 0, 0, 0, 540, 414, 415, 0, - 370, 369, 430, 321, 0, 0, 407, 398, 467, 327, - 366, 409, 403, 416, 417, 418, 376, 311, 312, 725, - 359, 449, 654, 689, 690, 579, 0, 642, 580, 589, - 351, 614, 626, 625, 445, 539, 0, 637, 640, 569, - 724, 0, 634, 648, 728, 647, 721, 455, 0, 482, - 645, 592, 0, 638, 611, 612, 0, 639, 607, 643, - 0, 581, 0, 550, 553, 582, 667, 668, 669, 318, - 552, 671, 672, 673, 674, 675, 676, 677, 670, 522, - 615, 591, 618, 531, 594, 593, 0, 0, 629, 548, - 630, 631, 439, 440, 441, 442, 380, 655, 340, 551, - 469, 0, 616, 0, 0, 0, 0, 0, 0, 0, - 0, 621, 622, 619, 733, 0, 678, 679, 0, 0, - 545, 546, 375, 0, 564, 383, 339, 454, 377, 529, - 406, 0, 557, 623, 558, 471, 472, 681, 686, 682, - 683, 685, 705, 446, 397, 402, 486, 408, 422, 474, - 528, 452, 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 663, 662, 661, 660, - 659, 658, 657, 656, 0, 0, 605, 505, 353, 305, - 349, 350, 357, 722, 718, 723, 706, 709, 708, 684, - 0, 313, 585, 420, 468, 374, 650, 651, 0, 704, - 259, 260, 261, 262, 263, 264, 265, 266, 306, 267, - 268, 269, 270, 271, 272, 273, 276, 277, 278, 279, - 280, 281, 282, 283, 653, 274, 275, 284, 285, 286, - 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, - 297, 0, 0, 0, 0, 307, 710, 711, 712, 713, - 714, 0, 0, 308, 309, 310, 0, 0, 300, 496, - 301, 302, 303, 304, 0, 0, 535, 536, 537, 560, - 0, 538, 520, 584, 384, 314, 500, 527, 720, 0, - 0, 0, 0, 0, 0, 0, 635, 646, 680, 0, - 692, 693, 695, 697, 696, 699, 493, 494, 707, 0, - 0, 701, 702, 703, 700, 424, 480, 501, 487, 0, - 726, 575, 576, 727, 688, 315, 451, 0, 0, 590, - 624, 613, 698, 578, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 367, 0, 0, 419, 628, 609, - 620, 610, 595, 596, 597, 604, 379, 598, 599, 600, - 570, 601, 571, 602, 603, 0, 627, 577, 489, 435, - 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 245, 0, 0, 0, 0, 0, 0, 335, - 246, 572, 694, 574, 573, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2234, 2268, 0, 0, 2235, 2237, 2239, 0, 2241, 2242, + 2243, 2247, 2248, 2249, 2251, 2254, 2255, 2256, 0, 0, + 0, 0, 0, 0, 0, 2244, 2253, 2245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 490, 519, 0, 532, 0, 404, - 405, 0, 0, 0, 0, 0, 0, 0, 322, 497, - 516, 336, 484, 530, 341, 492, 509, 331, 450, 481, - 0, 0, 324, 514, 491, 432, 323, 0, 475, 364, - 381, 361, 448, 0, 0, 513, 543, 360, 533, 0, - 524, 326, 0, 523, 447, 510, 515, 433, 426, 0, - 325, 512, 431, 425, 410, 371, 559, 411, 412, 385, - 462, 423, 463, 386, 437, 436, 438, 387, 388, 389, - 390, 391, 392, 393, 394, 395, 396, 0, 0, 0, - 0, 0, 554, 555, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, - 0, 0, 691, 0, 526, 0, 0, 0, 0, 0, - 0, 495, 0, 0, 413, 0, 0, 0, 544, 0, - 478, 453, 729, 0, 0, 476, 421, 511, 464, 517, - 498, 525, 470, 465, 316, 499, 363, 434, 332, 334, - 824, 365, 368, 372, 373, 443, 444, 458, 483, 502, - 503, 504, 362, 346, 477, 347, 382, 348, 317, 354, - 352, 355, 485, 356, 319, 459, 508, 0, 378, 473, - 429, 320, 428, 460, 507, 506, 333, 534, 541, 542, - 632, 0, 547, 730, 731, 732, 556, 0, 466, 329, - 328, 0, 0, 0, 358, 461, 342, 344, 345, 343, - 456, 457, 561, 562, 563, 565, 0, 566, 567, 0, - 0, 0, 0, 568, 633, 649, 617, 586, 549, 641, - 583, 587, 588, 399, 400, 401, 652, 0, 0, 0, - 540, 414, 415, 0, 370, 369, 430, 321, 0, 0, - 407, 398, 467, 327, 366, 409, 403, 416, 417, 418, - 376, 311, 312, 725, 359, 449, 654, 689, 690, 579, - 0, 642, 580, 589, 351, 614, 626, 625, 445, 539, - 0, 637, 640, 569, 724, 0, 634, 648, 728, 647, - 721, 455, 0, 482, 645, 592, 0, 638, 611, 612, - 0, 639, 607, 643, 0, 581, 0, 550, 553, 582, - 667, 668, 669, 318, 552, 671, 672, 673, 674, 675, - 676, 677, 670, 522, 615, 591, 618, 531, 594, 593, - 0, 0, 629, 548, 630, 631, 439, 440, 441, 442, - 380, 655, 340, 551, 469, 0, 616, 0, 0, 0, - 0, 0, 0, 0, 0, 621, 622, 619, 733, 0, - 678, 679, 0, 0, 545, 546, 375, 0, 564, 383, - 339, 454, 377, 529, 406, 0, 557, 623, 558, 471, - 472, 681, 686, 682, 683, 685, 705, 446, 397, 402, - 486, 408, 422, 474, 528, 452, 479, 337, 518, 488, - 427, 608, 636, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 663, 662, 661, 660, 659, 658, 657, 656, 0, 0, - 605, 505, 353, 305, 349, 350, 357, 722, 718, 723, - 706, 709, 708, 684, 0, 313, 585, 420, 468, 374, - 650, 651, 0, 704, 259, 260, 261, 262, 263, 264, - 265, 266, 306, 267, 268, 269, 270, 271, 272, 273, - 276, 277, 278, 279, 280, 281, 282, 283, 653, 274, - 275, 284, 285, 286, 287, 288, 289, 290, 291, 292, - 293, 294, 295, 296, 297, 0, 0, 0, 0, 307, - 710, 711, 712, 713, 714, 0, 0, 308, 309, 310, - 0, 0, 300, 496, 301, 302, 303, 304, 0, 0, - 535, 536, 537, 560, 0, 538, 520, 584, 384, 314, - 500, 527, 720, 0, 0, 0, 0, 0, 0, 0, - 635, 646, 680, 0, 692, 693, 695, 697, 696, 699, - 493, 494, 707, 0, 0, 701, 702, 703, 700, 424, - 480, 501, 487, 0, 726, 575, 576, 727, 688, 315, - 451, 0, 0, 590, 624, 613, 698, 578, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, - 0, 419, 628, 609, 620, 610, 595, 596, 597, 604, - 379, 598, 599, 600, 570, 601, 571, 602, 603, 0, - 627, 577, 489, 435, 0, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, - 0, 0, 0, 335, 246, 572, 694, 574, 573, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, - 0, 532, 0, 404, 405, 0, 0, 0, 0, 0, - 0, 0, 322, 497, 516, 336, 484, 530, 341, 492, - 509, 331, 450, 481, 0, 0, 324, 514, 491, 432, - 323, 0, 475, 364, 381, 361, 448, 0, 0, 513, - 543, 360, 533, 0, 524, 326, 0, 523, 447, 510, - 515, 433, 426, 0, 325, 512, 431, 425, 410, 371, - 559, 411, 412, 385, 462, 423, 463, 386, 437, 436, - 438, 387, 388, 389, 390, 391, 392, 393, 394, 395, - 396, 0, 0, 0, 0, 0, 554, 555, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 687, 0, 0, 691, 0, 526, 0, - 0, 0, 0, 0, 0, 495, 0, 0, 413, 0, - 0, 0, 544, 0, 478, 453, 729, 0, 0, 476, - 421, 511, 464, 517, 498, 525, 776, 465, 316, 499, - 363, 434, 332, 334, 719, 365, 368, 372, 373, 443, - 444, 458, 483, 502, 503, 504, 362, 346, 477, 347, - 382, 348, 317, 354, 352, 355, 485, 356, 319, 459, - 508, 0, 378, 473, 429, 320, 428, 460, 507, 506, - 333, 534, 541, 542, 632, 0, 547, 730, 731, 732, - 556, 0, 466, 329, 328, 0, 0, 0, 358, 461, - 342, 344, 345, 343, 456, 457, 561, 562, 563, 565, - 0, 566, 567, 0, 0, 0, 0, 568, 633, 649, - 617, 586, 549, 641, 583, 587, 588, 399, 400, 401, - 652, 0, 0, 0, 540, 414, 415, 0, 370, 369, - 430, 321, 0, 0, 407, 398, 467, 327, 366, 409, - 403, 416, 417, 418, 376, 311, 312, 725, 359, 449, - 654, 689, 690, 579, 0, 642, 580, 589, 351, 614, - 626, 625, 445, 539, 0, 637, 640, 569, 724, 0, - 634, 648, 728, 647, 721, 455, 0, 482, 645, 592, - 0, 638, 611, 612, 0, 639, 607, 643, 0, 581, - 0, 550, 553, 582, 667, 668, 669, 318, 552, 671, - 672, 673, 674, 675, 676, 777, 670, 522, 615, 591, - 618, 531, 594, 593, 0, 0, 629, 548, 630, 631, - 439, 440, 441, 442, 380, 655, 340, 551, 469, 0, - 616, 0, 0, 0, 0, 0, 0, 0, 0, 621, - 622, 619, 733, 0, 678, 679, 0, 0, 545, 546, - 375, 0, 564, 383, 339, 454, 377, 529, 406, 0, - 557, 623, 558, 471, 472, 681, 686, 682, 683, 685, - 705, 446, 397, 402, 486, 408, 422, 474, 528, 452, - 479, 337, 518, 488, 427, 608, 636, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 663, 662, 661, 660, 659, 658, - 657, 656, 0, 0, 605, 505, 353, 305, 349, 350, - 357, 722, 718, 723, 706, 709, 708, 684, 0, 313, - 585, 420, 468, 374, 650, 651, 0, 704, 259, 260, - 261, 262, 263, 264, 265, 266, 306, 267, 268, 269, - 270, 271, 272, 273, 276, 277, 278, 279, 280, 281, - 282, 283, 653, 274, 275, 284, 285, 286, 287, 288, - 289, 290, 291, 292, 293, 294, 295, 296, 297, 0, - 0, 0, 0, 307, 710, 711, 712, 713, 714, 0, - 0, 308, 309, 310, 0, 0, 300, 496, 301, 302, - 303, 304, 0, 0, 535, 536, 537, 560, 0, 538, - 520, 584, 384, 314, 500, 527, 720, 0, 0, 0, - 0, 0, 0, 0, 635, 646, 680, 0, 692, 693, - 695, 697, 696, 699, 493, 494, 707, 0, 0, 701, - 702, 703, 700, 424, 480, 501, 487, 0, 726, 575, - 576, 727, 688, 315, 451, 0, 0, 590, 624, 613, - 698, 578, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 367, 0, 0, 419, 628, 609, 620, 610, - 595, 596, 597, 604, 379, 598, 599, 600, 570, 601, - 571, 602, 603, 0, 627, 577, 489, 435, 0, 644, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 0, 0, 0, 0, 0, 0, 335, 246, 572, - 694, 574, 573, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2233, 0, 0, 0, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 490, 519, 0, 532, 0, 404, 405, 0, - 0, 0, 0, 0, 0, 0, 322, 497, 516, 336, - 484, 530, 341, 492, 509, 331, 450, 481, 0, 0, - 324, 514, 491, 432, 323, 0, 475, 364, 381, 361, - 448, 0, 0, 513, 543, 360, 533, 0, 524, 326, - 0, 523, 447, 510, 515, 433, 426, 0, 325, 512, - 431, 425, 410, 371, 559, 411, 412, 385, 462, 423, - 463, 386, 437, 436, 438, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 0, 0, 0, 0, 0, - 554, 555, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 687, 0, 0, - 691, 0, 526, 0, 0, 0, 0, 0, 0, 495, - 0, 0, 413, 0, 0, 0, 544, 0, 478, 453, - 729, 0, 0, 476, 421, 511, 464, 517, 498, 525, - 470, 465, 316, 499, 363, 434, 332, 334, 719, 365, - 368, 372, 373, 443, 444, 458, 483, 502, 503, 504, - 362, 346, 477, 347, 382, 348, 317, 354, 352, 355, - 485, 356, 319, 459, 508, 0, 378, 473, 429, 320, - 428, 460, 507, 506, 333, 534, 541, 542, 632, 0, - 547, 730, 731, 732, 556, 0, 466, 329, 328, 0, - 0, 0, 358, 461, 342, 344, 345, 343, 456, 457, - 561, 562, 563, 565, 0, 566, 567, 0, 0, 0, - 0, 568, 633, 649, 617, 586, 549, 641, 583, 587, - 588, 399, 400, 401, 652, 0, 0, 0, 540, 414, - 415, 0, 370, 369, 430, 321, 0, 0, 407, 398, - 467, 327, 366, 409, 403, 416, 417, 418, 376, 311, - 312, 725, 359, 449, 654, 689, 690, 579, 0, 642, - 580, 589, 351, 614, 626, 625, 445, 539, 0, 637, - 640, 569, 724, 0, 634, 648, 728, 647, 721, 455, - 0, 482, 645, 592, 0, 638, 611, 612, 0, 639, - 607, 643, 0, 581, 0, 550, 553, 582, 667, 668, - 669, 318, 552, 671, 672, 673, 674, 675, 676, 677, - 670, 522, 615, 591, 618, 531, 594, 593, 0, 0, - 629, 548, 630, 631, 439, 440, 441, 442, 380, 655, - 340, 551, 469, 0, 616, 0, 0, 0, 0, 0, - 0, 0, 0, 621, 622, 619, 733, 0, 678, 679, - 0, 0, 545, 546, 375, 0, 564, 383, 339, 454, - 377, 529, 406, 0, 557, 623, 558, 471, 472, 681, - 686, 682, 683, 685, 705, 446, 397, 402, 486, 408, - 422, 474, 528, 452, 479, 337, 518, 488, 427, 608, - 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 298, 299, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 663, 662, - 661, 660, 659, 658, 657, 656, 0, 0, 605, 505, - 353, 305, 349, 350, 357, 722, 718, 723, 706, 709, - 708, 772, 0, 313, 585, 420, 468, 374, 650, 651, - 0, 704, 259, 260, 261, 262, 263, 264, 265, 266, - 306, 267, 268, 269, 270, 271, 272, 273, 276, 277, - 278, 279, 280, 281, 282, 283, 653, 274, 275, 284, - 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, - 295, 296, 297, 0, 0, 0, 0, 307, 710, 711, - 712, 713, 714, 0, 0, 308, 309, 310, 0, 0, - 300, 496, 301, 302, 303, 304, 0, 0, 535, 536, - 537, 560, 0, 538, 520, 584, 384, 314, 500, 527, - 720, 0, 0, 0, 0, 0, 0, 0, 635, 646, - 680, 0, 692, 693, 695, 697, 696, 699, 493, 494, - 707, 0, 0, 701, 702, 703, 700, 424, 480, 501, - 487, 0, 726, 575, 576, 727, 688, 315, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 2250, + 0, 0, 0, 0, 0, 0, 0, 0, 2238, } var yyPact = [...]int{ - 4897, -1000, -1000, -1000, -399, 18660, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 5059, -1000, -1000, -1000, -401, 18719, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 61567, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 547, 61567, -396, -1000, - 3363, 59425, -1000, -1000, -1000, 436, 60139, 20824, 61567, 723, - 722, 67279, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 61866, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 473, 61866, -398, -1000, + 3391, 59712, -1000, -1000, -1000, 325, 60430, 20895, 61866, 642, + 638, 67610, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1138, -1000, 66565, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 989, - 5669, 65851, 14353, -272, -1000, 2205, -70, 3084, 635, 21, - 19, 712, 1375, 1397, 1538, 1273, 61567, 1341, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 5370, 35851, 60853, 1277, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1134, -1000, 66892, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 941, + 5553, 66174, 14388, -276, -1000, 2064, -77, 3191, 536, 17, + -25, 630, 1402, 1434, 1509, 1343, 61866, 1361, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 5036, 371, 1126, 1277, 26558, 224, 220, 2205, 3447, - -125, 5052, -1000, 1968, 4936, 212, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 14353, 14353, 18660, - -443, 18660, 14353, 61567, 61567, -1000, -1000, -1000, -1000, -396, - 60139, 989, 5669, 14353, 3084, 635, 21, 19, 712, -1000, + -1000, 4806, 36006, 61148, 1339, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 5375, 276, 1133, 1339, 26661, 105, 104, 2064, 3468, + -84, 4997, -1000, 1949, 5098, 212, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 14388, 14388, 18719, + -439, 18719, 14388, 61866, 61866, -1000, -1000, -1000, -1000, -398, + 60430, 941, 5553, 14388, 3191, 536, 17, -25, 630, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -125, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -84, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, @@ -8897,9 +8986,9 @@ var yyPact = [...]int{ -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 220, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, 104, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, @@ -8916,477 +9005,481 @@ var yyPact = [...]int{ -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 360, -1000, 1965, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 2805, 3714, 1961, 3083, -1000, -1000, - -1000, -1000, 2205, 4097, 930, 61567, -1000, 145, 4064, -1000, - 61567, 61567, 331, 2328, -1000, 781, 766, 751, 1219, 480, - 1960, -1000, -1000, -1000, -1000, -1000, -1000, 859, 4063, -1000, - 61567, 61567, 61567, 3724, 61567, -1000, 330, 907, -1000, 5779, - 3874, 1787, 1114, 3738, -1000, -1000, 3712, -1000, 487, 365, - 380, 893, 546, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 431, -1000, 3941, -1000, -1000, 475, -1000, -1000, 462, -1000, - -1000, -1000, 179, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -21, -1000, -1000, 1476, 2920, 14353, - 2506, -1000, 5217, 2131, -1000, -1000, -1000, 9334, 17933, 17933, - 17933, 17933, 61567, -1000, -1000, 3530, 14353, 3709, 3697, 3696, - 3686, -1000, -1000, -1000, -1000, -1000, -1000, 3684, 1955, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 2466, -1000, - -1000, -1000, 17217, -1000, 3683, 3681, 3669, 3666, 3665, 3664, - 3659, 3653, 3651, 3650, 3647, 3646, 3643, 3640, 3639, 3369, - 20099, 3638, 3082, 3081, 3637, 3634, 3633, 3078, 3631, 3626, - 3625, 3369, 3369, 3623, 3615, 3611, 3609, 3607, 3606, 3604, - 3603, 3602, 3600, 3593, 3589, 3588, 3586, 3585, 3583, 3582, - 3581, 3579, 3577, 3576, 3575, 3574, 3568, 3565, 3564, 3562, - 3561, 3560, 3557, 3556, 3555, 3554, 3553, 3552, 3551, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 489, + -1000, 2079, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 2910, 3738, + 2076, 3189, -1000, -1000, -1000, -1000, 2064, 4118, 898, 61866, + -1000, 147, 4089, -1000, 61866, 61866, 244, 2385, -1000, 808, + 588, 552, 971, 342, 2069, -1000, -1000, -1000, -1000, -1000, + -1000, 820, 4083, -1000, 61866, 61866, 61866, 3752, 61866, -1000, + 354, 864, -1000, 5699, 3932, 1920, 1085, 3766, -1000, -1000, + 3736, -1000, 365, 965, 334, 675, 470, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 332, -1000, 3997, -1000, -1000, 359, + -1000, -1000, 348, -1000, -1000, -1000, 103, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -3, -1000, + -1000, 1547, 2632, 14388, 2369, -1000, 5054, 2169, -1000, -1000, + -1000, 9341, 17988, 17988, 17988, 17988, 61866, -1000, -1000, 3617, + 14388, 3735, 3734, 3733, 3731, -1000, -1000, -1000, -1000, -1000, + -1000, 3727, 2065, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 2555, -1000, -1000, -1000, 17268, -1000, 3726, 3725, + 3723, 3722, 3719, 3716, 3714, 3712, 3711, 3710, 3709, 3708, + 3706, 3705, 3704, 3462, 20166, 3703, 3183, 3180, 3702, 3700, + 3699, 3176, 3698, 3695, 3694, 3462, 3462, 3693, 3692, 3690, + 3686, 3685, 3683, 3682, 3681, 3680, 3679, 3678, 3677, 3676, + 3675, 3674, 3672, 3671, 3669, 3668, 3665, 3662, 3659, 3657, + 3656, 3655, 3654, 3653, 3652, 3651, 3649, 3647, 3646, 3645, + 3644, 3643, 3641, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 1683, -1000, 3546, 4096, 3425, -1000, 3922, 3917, - 3913, 3907, -330, 3544, 2705, -1000, -1000, 92, 61567, 61567, - 298, 61567, -352, 414, 631, -158, -159, 617, -160, 1358, - -1000, 536, -1000, -1000, 1419, -1000, 1315, 65137, 1064, -1000, - -1000, 61567, 967, 967, 967, 967, 61567, 342, 1089, 1248, - 967, 967, 967, 967, 1055, 967, 3965, 1120, 1115, 1104, - 1082, 967, -43, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 2326, 2319, 3794, 930, 59425, 1778, 61567, -1000, 3491, 1247, - -1000, -1000, -1000, -1000, 414, -1000, 86, -385, 3737, 2178, - 2178, 4041, 4041, 3963, 3962, 913, 911, 891, 2178, 773, - -1000, 2275, 2275, 2275, 2275, 2178, 609, 905, 3972, 3972, - 140, 2275, 139, 2178, 2178, 139, 2178, 2178, 604, -1000, - 2264, 629, 266, -339, -1000, -1000, -1000, -1000, 2275, 2275, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3933, 3932, 989, - 989, 61567, 989, 61567, 352, 206, 61567, 989, 989, 989, - 61567, 999, -384, 99, 64423, 63709, 2618, 330, 902, 898, - 1785, 2283, -1000, 2229, 61567, 61567, 2229, 2229, 30139, 29425, - -1000, 61567, -1000, 4096, 3425, 3360, 1799, 3359, 3425, -162, - 989, 989, 989, 989, 989, 989, 989, 424, 989, 989, - 989, 989, 989, 61567, 61567, 58711, 989, 612, 989, 989, - 989, 12198, 1968, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 18660, 2567, 2550, 209, - -58, -373, 272, -1000, -1000, 61567, 3838, 2037, -1000, -1000, - -1000, 3489, 3471, -1000, 3481, 3481, 3481, 3481, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3481, 3481, - 3488, 3541, -1000, -1000, 3480, 3480, 3480, 3471, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 1902, -1000, 3640, 4120, + 3528, -1000, 3986, 3984, 3982, 3976, -333, 3632, 2835, -1000, + -1000, 90, 61866, 61866, 299, 61866, -355, 417, 575, -162, + -165, 553, -166, 1070, -1000, 558, -1000, -1000, 1377, -1000, + 1280, 65456, 1019, -1000, -1000, 61866, 937, 937, 937, 937, + 61866, 300, 1081, 1190, 937, 937, 937, 937, 1042, 937, + 4018, 1131, 1116, 1104, 1102, 937, -36, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 2384, 2382, 3829, 898, 59712, 1851, + 61866, -1000, 3571, 1267, -1000, -1000, -1000, -1000, 417, -1000, + 40, -387, 3765, 2228, 2228, 4067, 4067, 4017, 4016, 877, + 872, 819, 2228, 718, -1000, 2275, 2275, 2275, 2275, 2228, + 568, 870, 4023, 4023, 65, 2275, 70, 2228, 2228, 70, + 2228, 2228, 525, -1000, 2286, 571, 301, -341, -1000, -1000, + -1000, -1000, 2275, 2275, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 3992, 3991, 941, 941, 61866, 941, 61866, 347, 247, + 61866, 941, 941, 941, 61866, 975, -386, 31, 64738, 64020, + 2923, 354, 853, 850, 1872, 2328, -1000, 2250, 61866, 61866, + 2250, 2250, 30262, 29544, -1000, 61866, -1000, 4120, 3528, 3447, + 2111, 3445, 3528, -167, 941, 941, 941, 941, 941, 941, + 941, 320, 941, 941, 941, 941, 941, 61866, 61866, 58994, + 941, 544, 941, 941, 941, 12221, 1949, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 1364, 3482, 3484, 3484, 3482, -1000, -1000, + 18719, 2448, 2444, 210, -58, -376, 287, -1000, -1000, 61866, + 3875, 2161, -1000, -1000, -1000, 3570, 3553, -1000, 3556, 3556, + 3556, 3556, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 3556, 3556, 3556, 3556, 3556, 3556, 3568, 3631, + -1000, -1000, 3554, 3554, 3554, 3553, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 61567, 4087, -1000, -1000, - 14353, 61567, 3857, 4096, 3849, 3972, 4032, 3348, 3540, -1000, - -1000, 61567, 336, 2470, -1000, -1000, 1952, 2702, 3073, -1000, - 480, -1000, 740, 480, -1000, 660, 660, 2216, -1000, 1534, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 61567, -21, 1970, - -1000, -1000, -1000, 3035, 3538, -1000, 809, 1743, 1742, -1000, - 359, 5804, 47995, 330, 47995, 61567, -1000, -1000, -1000, -1000, - -1000, -1000, 148, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 1385, 3564, 3565, 3565, 3564, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, 471, -1000, 14353, - 14353, 14353, 14353, 14353, -1000, 861, 16501, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 17933, 17933, 17933, 17933, 17933, 17933, - 17933, 17933, 17933, 17933, 17933, 17933, 17933, 17933, 3522, 2358, - 17933, 17933, 17933, 17933, 269, 32281, 1799, 3661, 1781, 324, - 2131, 2131, 2131, 2131, 14353, -1000, 2352, 2920, 14353, 14353, - 14353, 14353, 39421, 61567, -1000, -1000, 9334, 5558, 14353, 14353, - 4278, 17933, 14353, 3903, 14353, 14353, 14353, 3358, 7164, 61567, - 14353, -1000, 3346, 3344, -1000, -1000, 2486, 14353, -1000, -1000, - 14353, -1000, -1000, 14353, 17933, 14353, -1000, 14353, 14353, 14353, - -1000, -1000, 686, 686, 1276, 3903, 3903, 3903, 2309, 14353, - 14353, 3903, 3903, 3903, 2292, 3903, 3903, 3903, 3903, 3903, - 3903, 3903, 3903, 3903, 3903, 3903, 3342, 3337, 3336, 3328, - 14353, 3322, 14353, 14353, 14353, 14353, 14353, 13637, 3972, -272, - -1000, 11482, 3849, 3972, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -332, 3537, 61567, 3072, 3071, -408, - -411, 1347, -411, 1948, -1000, -354, 1356, 278, 61567, -1000, - -1000, 61567, 3070, 2701, 61567, 3068, 2700, 275, 267, 61567, - 61567, 61567, 69, 1365, 1278, 1335, -1000, -1000, 61567, 62995, - -1000, 61567, 2392, 61567, 61567, 61567, 3899, -1000, 61567, 61567, - 967, 967, 967, -1000, 55855, 3066, 47995, 61567, 61567, 330, - 61567, 61567, 61567, 967, 967, 967, 967, 61567, -1000, 3821, - 47995, 3798, 3210, 930, 61567, 1778, 3898, 61567, 999, -1000, - -1000, -1000, 3958, -1000, -1000, -1000, 890, 4041, 17933, 17933, - -1000, -1000, 14353, -1000, 273, 57997, 2275, 2178, 2178, -1000, - -1000, 61567, -1000, -1000, -1000, 2275, 61567, 2275, 2275, 4041, - 2275, -1000, -1000, -1000, 2178, 2178, -1000, -1000, 14353, -1000, - -1000, 2275, 2275, -1000, -1000, 4041, 61567, 147, 4041, 4041, - 129, -1000, -1000, 61567, -1000, 2178, 3065, -1000, 61567, 61567, - 967, 61567, -1000, 61567, 61567, -1000, -1000, 61567, 61567, 6017, - 61567, 425, 3872, 1291, 55855, 57283, 3931, -1000, 47995, 61567, - 61567, 1766, -1000, 1058, 42991, -1000, 61567, 1724, -1000, 22, - -1000, 59, 99, 2229, 99, 2229, 1057, -1000, 799, 798, - 27997, 761, 47995, 8607, -1000, -1000, 2229, 2229, 8607, 8607, - 1987, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1765, -1000, - 306, 3972, -1000, -1000, -1000, -1000, -1000, 2695, 56569, 61567, - 61567, 55855, 47995, 330, 61567, 989, 61567, 61567, 61567, 61567, - 61567, -1000, 3536, 1939, -1000, 3862, 61567, 989, 61567, 61567, - 61567, 1733, -1000, -1000, 24394, 1938, -1000, -1000, 2346, -1000, - 14353, 18660, -307, 14353, 18660, 18660, 14353, 18660, -1000, 14353, - 1829, -1000, -1000, 491, -1000, -1000, 2694, -1000, 2692, -1000, - -1000, -1000, -1000, -1000, 3064, 3064, -1000, 2690, -1000, -1000, - -1000, -1000, 3482, 2689, -1000, -1000, 2686, -1000, -1000, -1000, - -1000, -197, 3321, 1476, -1000, 3059, 3972, -1000, -278, 4019, - 14353, -1000, -273, -1000, 25844, 61567, 61567, -419, 2318, 2316, - 2314, 3949, 989, 61567, -1000, 3957, -1000, -1000, 480, -1000, - -1000, -1000, 660, 585, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 1935, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -133, -140, 1758, -1000, 61567, -1000, -1000, - 359, 47995, 52279, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 1744, -1000, -1000, 187, -1000, 1042, 389, 2214, -1000, -1000, - 247, 222, 341, 1211, 2920, -1000, 2384, 2384, 2399, -1000, - 887, -1000, -1000, -1000, -1000, 3530, -1000, -1000, -1000, 3351, - 2402, -1000, 2183, 2183, 2013, 2013, 2013, 2013, 2013, 2339, - 2339, 2131, 2131, -1000, -1000, -1000, 9334, 3522, 17933, 17933, - 17933, 17933, 1071, 1071, 6053, 6036, -1000, -1000, 1998, 1998, - -1000, -1000, -1000, -1000, 14353, 177, 2342, -1000, 14353, 3256, - 2089, 3188, 1883, 2211, -1000, 3471, 14353, 1932, 5118, -1000, + -1000, -1000, -1000, -1000, 61866, 4116, -1000, -1000, 14388, 61866, + 3910, 4120, 3887, 4023, 4061, 3532, 3629, -1000, -1000, 61866, + 344, 2582, -1000, -1000, 2053, 2834, 3175, -1000, 342, -1000, + 750, 342, -1000, 781, 781, 2162, -1000, 1591, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, 61866, -3, 912, -1000, -1000, + -1000, 3152, 3627, -1000, 739, 1815, 1806, -1000, 364, 4088, + 48218, 354, 48218, 61866, -1000, -1000, -1000, -1000, -1000, -1000, + 93, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, 336, -1000, 14388, 14388, 14388, + 14388, 14388, -1000, 796, 16548, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 17988, 17988, 17988, 17988, 17988, 17988, 17988, 17988, + 17988, 17988, 17988, 17988, 17988, 17988, 3616, 2246, 17988, 17988, + 17988, 17988, 267, 32416, 2111, 3594, 1853, 326, 2169, 2169, + 2169, 2169, 14388, -1000, 2418, 2632, 14388, 14388, 14388, 14388, + 39596, 61866, -1000, -1000, 9341, 6198, 14388, 14388, 906, 17988, + 14388, 3972, 14388, 14388, 14388, 3442, 7159, 61866, 14388, -1000, + 3440, 3436, -1000, -1000, 2599, 14388, -1000, -1000, 14388, -1000, + -1000, 14388, 17988, 14388, -1000, 14388, 14388, 14388, -1000, -1000, + 3080, 3080, 1427, 3972, 3972, 3972, 2312, 14388, 14388, 3972, + 3972, 3972, 2310, 3972, 3972, 3972, 3972, 3972, 3972, 3972, + 3972, 3972, 3972, 3972, 3434, 3428, 3427, 3426, 14388, 3424, + 14388, 14388, 14388, 14388, 14388, 13668, 4023, -276, -1000, 11501, + 3887, 4023, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -336, 3626, 61866, 3174, 3171, -410, -412, 1365, + -412, 2041, -1000, -357, 1408, 281, 61866, -1000, -1000, 61866, + 3170, 2833, 61866, 3169, 2832, 274, 272, 61866, 61866, 61866, + 25, 1374, 1284, 1347, -1000, -1000, 61866, 63302, -1000, 61866, + 2435, 61866, 61866, 61866, 3959, -1000, 61866, 61866, 937, 937, + 937, -1000, 56122, 3166, 48218, 61866, 61866, 354, 61866, 61866, + 61866, 937, 937, 937, 937, 61866, -1000, 3852, 48218, 3839, + 3329, 898, 61866, 1851, 3952, 61866, 975, -1000, -1000, -1000, + 4014, -1000, -1000, -1000, 842, 4067, 17988, 17988, -1000, -1000, + 14388, -1000, 333, 58276, 2275, 2228, 2228, -1000, -1000, 61866, + -1000, -1000, -1000, 2275, 61866, 2275, 2275, 4067, 2275, -1000, + -1000, -1000, 2228, 2228, -1000, -1000, 14388, -1000, -1000, 2275, + 2275, -1000, -1000, 4067, 61866, 86, 4067, 4067, 85, -1000, + -1000, 61866, -1000, 2228, 3164, -1000, 61866, 61866, 937, 61866, + -1000, 61866, 61866, -1000, -1000, 61866, 61866, 5979, 61866, 435, + 3931, 1147, 56122, 57558, 3981, -1000, 48218, 61866, 61866, 1843, + -1000, 1018, 43186, -1000, 61866, 1769, -1000, 33, -1000, 22, + 31, 2250, 31, 2250, 1016, -1000, 737, 443, 28108, 669, + 48218, 8610, -1000, -1000, 2250, 2250, 8610, 8610, 2117, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 1842, -1000, 310, 4023, + -1000, -1000, -1000, -1000, -1000, 2831, 56840, 61866, 61866, 56122, + 48218, 354, 61866, 941, 61866, 61866, 61866, 61866, 61866, -1000, + 3618, 2039, -1000, 3926, 61866, 941, 61866, 61866, 61866, 1747, + -1000, -1000, 24485, 2030, -1000, -1000, 2424, -1000, 14388, 18719, + -320, 14388, 18719, 18719, 14388, 18719, -1000, 14388, 2071, -1000, + -1000, 4814, -1000, -1000, 2829, -1000, 2827, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 3160, 3160, -1000, 2826, + -1000, -1000, -1000, -1000, 3564, 2825, -1000, -1000, 2820, -1000, + -1000, -1000, -1000, -202, 3422, 1547, -1000, 3159, 4023, -1000, + -281, 4054, 14388, -1000, -278, -1000, 25943, 61866, 61866, -420, + 2367, 2363, 2361, 4007, 941, 61866, -1000, 4013, -1000, -1000, + 342, -1000, -1000, -1000, 781, 594, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 2029, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -129, -139, 1841, -1000, 61866, + -1000, -1000, 364, 48218, 52526, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 1764, -1000, -1000, 188, -1000, 1015, 283, 2158, + -1000, -1000, 262, 222, 250, 1208, 2632, -1000, 2433, 2433, + 2460, -1000, 848, -1000, -1000, -1000, -1000, 3617, -1000, -1000, + -1000, 2323, 3433, -1000, 2186, 2186, 2135, 2135, 2135, 2135, + 2135, 2248, 2248, 2169, 2169, -1000, -1000, -1000, 9341, 3616, + 17988, 17988, 17988, 17988, 1138, 1138, 5917, 5798, -1000, -1000, + 2096, 2096, -1000, -1000, -1000, -1000, 14388, 180, 2409, -1000, + 14388, 3081, 2017, 2959, 1614, 2154, -1000, 3553, 14388, 2025, + 4371, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3319, - 3314, 2993, 4061, 5759, 3308, 14353, -1000, -1000, 2208, 2204, - 2199, -1000, 2631, 12921, -1000, -1000, -1000, 3307, 1931, 3305, - -1000, -1000, -1000, 3297, 2194, 1607, 3294, 1900, 3281, 3280, - 3279, 3277, 1756, 1754, 1745, -1000, -1000, -1000, -1000, 14353, - 14353, 14353, 14353, 3276, 2193, 2182, 14353, 14353, 14353, 14353, - 3259, 14353, 14353, 14353, 14353, 14353, 14353, 14353, 14353, 14353, - 14353, 61567, 276, 276, 276, 276, 3648, 276, 2139, 2100, - 3644, 3627, 2079, 1731, 1729, -1000, -1000, 2180, -1000, 2920, - -1000, -1000, 4019, -1000, 3521, 2685, 1727, -1000, -1000, -393, - 2981, 1030, 61567, -355, 61567, 1030, 61567, 61567, 2312, 1030, - 61567, -360, 3053, -1000, -1000, -1000, 3050, -1000, -1000, 61567, - 61567, 61567, 61567, -167, 3855, 3854, -1000, -1000, 1348, 1295, - 1328, -1000, 61567, -1000, 3049, 3861, 3955, 1092, -151, 61567, - 3519, 3518, 61567, 61567, 61567, 433, -1000, -1000, 61567, 1661, - -1000, 389, -30, 732, 1530, 3719, 1053, 4085, 61567, 61567, - 61567, 61567, 3896, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 3736, -273, -1000, 25119, 61567, 3210, -1000, 3517, 2179, - -1000, 55141, 3977, 61567, 330, -1000, 2131, 2131, 2920, 61567, - 61567, 61567, 3663, 61567, 61567, 4041, 4041, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 2275, 4041, 4041, 1953, 2178, 2275, - -1000, -1000, 2275, -419, -1000, 2275, -1000, -1000, -1000, -419, - 1923, -419, 61567, -1000, -1000, -1000, 3895, 3491, 1726, -1000, - -1000, -1000, 4023, 1906, 950, 950, 1201, 817, 4020, 22966, - -1000, 2187, 1424, 1029, 3827, 483, -1000, 2187, -193, 933, - 2187, 2187, 2187, 2187, 2187, 2187, 2187, 851, 841, 2187, - 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, 2187, - 1392, 2187, 2187, 2187, 2187, 2187, -1000, 2187, 3515, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 883, 806, -1000, -1000, - 302, 330, 1027, 105, 96, 426, 3929, 520, -1000, 511, - 1661, 815, 3927, 544, 61567, 61567, 1099, 1674, -1000, -1000, - -1000, -1000, -1000, 32995, 32995, 27283, 32995, -1000, 210, 2229, - 99, 78, -1000, -1000, 1724, 8607, 1724, 8607, 2683, -1000, - -1000, 1023, -1000, -1000, 1530, -1000, 61567, 61567, -1000, -1000, - 3514, 2308, -1000, -1000, 20099, -1000, 8607, 8607, -1000, -1000, - 35137, 61567, -1000, -24, -1000, -11, 4019, -1000, -368, -1000, - -1000, 61567, -1000, 1498, -1000, -1000, 1721, 1530, 3734, 61567, - 1498, 1498, 1498, -1000, -1000, 21538, 61567, 61567, -1000, 3048, - -1000, 4059, -368, 4041, 12198, -1000, 42991, -1000, -1000, 54421, - -1000, 53707, 2338, -1000, 18660, 2495, 202, -1000, 268, -376, - 201, 2390, 200, 2920, -1000, -1000, 3254, 3251, 3246, 2177, - -1000, 2157, 3245, -1000, 2152, 2151, 2681, -1000, 136, 4019, - 3039, 3849, -247, 1719, -1000, 2615, 1503, -1000, 3512, -1000, - 2150, 3790, -1000, 1711, -1000, 2307, 2145, -1000, -1000, 14353, - 52993, 14353, 1244, 3038, 1922, 326, -1000, -1000, -1000, 61567, - 3035, 2136, 52279, 1590, -1000, 1022, 1914, 1913, -1000, 47995, - 463, 47995, -1000, 47995, -1000, -1000, 3988, -1000, 61567, 3853, - -1000, -1000, -1000, 2981, 2301, -414, 61567, -1000, -1000, -1000, - -1000, -1000, 2115, -1000, 1071, 1071, 6053, 6013, -1000, 17933, - -1000, 17933, -1000, -1000, -1000, -1000, 3566, -1000, 2336, -1000, - 14353, 2469, 269, 14353, 269, 1973, 31567, 39421, -169, 3846, - 3542, 61567, 14353, -1000, -1000, 14353, 14353, 17933, -1000, 3503, - -1000, -1000, -1000, -1000, 14353, 14353, 2835, -1000, 61567, -1000, - -1000, -1000, -1000, 31567, -1000, 17933, -1000, -1000, -1000, -1000, - 14353, 14353, 14353, 1647, 1647, 3498, 2105, 276, 276, 276, - 3445, 3436, 3421, 2093, 276, 3371, 3352, 3326, 3303, 3288, - 3272, 3208, 3203, 3146, 3111, 2087, -1000, 3511, -1000, -1000, - -1000, 276, -1000, 276, 14353, 276, 14353, 276, 276, 14353, - 2485, 15785, 11482, -1000, 3849, 311, 1718, 2680, 3032, 133, - -1000, 2299, -1000, 541, -1000, 61567, 4058, -1000, 1907, 3030, - 51565, -1000, 1338, 61567, -1000, -1000, 4057, 4056, -1000, -1000, - 61567, 61567, 61567, -1000, -1000, -1000, 1270, -1000, 3025, -1000, - 500, 274, 2578, 2351, 3024, 446, 1491, 21538, 3491, 3510, - 3491, 279, 2187, 679, 834, 47995, 868, -1000, 50851, 2443, - 2298, 3733, 1557, 3837, 61567, 50137, 3508, 1385, 3506, 3502, - 3891, 693, 491, -1000, 3840, 1503, 2070, 3788, 1711, -1000, - 4936, -1000, 61567, 61567, 1637, -1000, 1895, -1000, 2678, -1000, - -1000, -1000, -1000, 61567, -1000, 330, -1000, 2178, -1000, -1000, - 4041, -1000, -1000, 14353, 14353, 4041, 2178, 2178, -1000, 2275, - -1000, 61567, -1000, -419, 693, 491, 3888, 6338, 788, 2926, - -1000, 61567, -1000, -1000, -1000, 1067, -1000, 1274, 967, 61567, - 2417, 1274, 2415, 3501, -1000, -1000, 61567, 61567, 61567, 61567, - -1000, -1000, 61567, -1000, 61567, 61567, 61567, 61567, 61567, 49423, - -1000, 61567, 61567, -1000, 61567, 2414, 61567, 2413, 3841, -1000, - 2187, 2187, 1232, -1000, -1000, 826, -1000, 49423, 2667, 2666, - 2664, 2663, 3023, 3019, 3013, 2187, 2187, 2662, 3012, 48709, - 3011, 1525, 2660, 2659, 2658, 2575, 3010, 1658, -1000, 3009, - 2555, 2528, 2525, 61567, 3497, 2922, -1000, -1000, 2578, 3006, - 3495, 2653, 3001, 1164, 330, 3000, 3732, 279, 2187, 518, - 61567, 2289, 2286, 834, 782, 782, 731, -32, 28711, -1000, - -1000, -1000, 61567, 42991, 42991, 42991, 42991, 42991, 42991, -1000, - 3772, 3753, 3493, -1000, 3760, 3758, 3756, 688, 3771, 3016, - 61567, 42991, 3491, -1000, 48709, -1000, -1000, -1000, 1799, 2069, - 1436, 1226, 14353, 8607, -1000, -1000, 73, -8, -1000, -1000, - -1000, -1000, 47995, 2995, 761, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 3849, -1000, -1000, 61567, 61567, 1009, 3244, 1708, - -1000, -1000, -1000, 491, 3489, 3481, 3481, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 3481, 3481, 3488, -1000, - -1000, 3480, 3480, 3480, 3471, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 1364, 3482, 3484, 3484, 3482, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 3410, 3409, 2610, 4082, 5740, 3403, 14388, -1000, -1000, + 2143, 2142, 2140, -1000, 2518, 12948, -1000, -1000, -1000, 3401, + 2014, 3400, -1000, -1000, -1000, 3399, 2139, 1626, 3398, 4166, + 3396, 3386, 3384, 3377, 1837, 1823, 1798, -1000, -1000, -1000, + -1000, 14388, 14388, 14388, 14388, 3362, 2138, 2136, 14388, 14388, + 14388, 14388, 3344, 14388, 14388, 14388, 14388, 14388, 14388, 14388, + 14388, 14388, 14388, 61866, 129, 129, 129, 129, 3529, 129, + 1925, 1918, 3514, 3476, 1994, 1796, 1780, -1000, -1000, 2134, + -1000, 2632, -1000, -1000, 4054, -1000, 3613, 2818, 1777, -1000, + -1000, -395, 3090, 1014, 61866, -359, 61866, 1014, 61866, 61866, + 2357, 1014, 61866, -360, 3156, -1000, -1000, -1000, 3155, -1000, + -1000, 61866, 61866, 61866, 61866, -174, 3900, 3898, -1000, -1000, + 1387, 1277, 1320, -1000, 61866, -1000, 3154, 3920, 4012, 1064, + -150, 61866, 3611, 3610, 61866, 61866, 61866, 312, -1000, -1000, + 61866, 1717, -1000, 283, -15, 652, 1580, 3751, 977, 4114, + 61866, 61866, 61866, 61866, 3949, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 3764, -278, -1000, 25214, 61866, 3329, -1000, + 3609, 2131, -1000, 55404, 4026, 61866, 354, -1000, 2169, 2169, + 2632, 61866, 61866, 61866, 3748, 61866, 61866, 4067, 4067, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 2275, 4067, 4067, 1704, + 2228, 2275, -1000, -1000, 2275, -420, -1000, 2275, -1000, -1000, + -1000, -420, 1998, -420, 61866, -1000, -1000, -1000, 3945, 3571, + 1773, -1000, -1000, -1000, 4059, 1422, 926, 926, 1205, 797, + 4057, 23049, -1000, 2196, 1556, 1012, 3854, 390, -1000, 2196, + -199, 907, 2196, 2196, 2196, 2196, 2196, 2196, 2196, 809, + 801, 2196, 2196, 2196, 2196, 2196, 2196, 2196, 2196, 2196, + 2196, 2196, 1418, 2196, 2196, 2196, 2196, 2196, -1000, 2196, + 3605, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 834, 752, + -1000, -1000, 298, 354, 1009, 49, 48, 308, 3979, 440, + -1000, 425, 1717, 757, 3971, 469, 61866, 61866, 1254, 1660, + -1000, -1000, -1000, -1000, -1000, 33134, 33134, 27390, 33134, -1000, + 201, 2250, 31, 0, -1000, -1000, 1769, 8610, 1769, 8610, + 2816, -1000, -1000, 1003, -1000, -1000, 1580, -1000, 61866, 61866, + -1000, -1000, 3603, 2334, -1000, -1000, 20166, -1000, 8610, 8610, + -1000, -1000, 35288, 61866, -1000, -9, -1000, 4, 4054, -1000, + -369, -1000, -1000, 61866, -1000, 1548, -1000, -1000, 1767, 1580, + 3763, 61866, 1548, 1548, 1548, -1000, -1000, 21613, 61866, 61866, + -1000, 3145, -1000, 4081, -369, 4067, 12221, -1000, 43186, -1000, + -1000, 54680, -1000, 53962, 2368, -1000, 18719, 2422, 205, -1000, + 279, -382, 204, 2391, 200, 2632, -1000, -1000, 3333, 3325, + 3323, 2130, -1000, 2129, 3322, -1000, 2128, 2127, 2812, -1000, + 63, 4054, 3142, 3887, -252, 1760, -1000, 2551, 1551, -1000, + 3602, -1000, 2121, 3826, -1000, 1729, -1000, 2332, 2115, -1000, + -1000, 14388, 53244, 14388, 1249, 3140, 1997, 241, -1000, -1000, + -1000, 61866, 3152, 2112, 52526, 1616, -1000, 1001, 1978, 1951, + -1000, 48218, 355, 48218, -1000, 48218, -1000, -1000, 4035, -1000, + 61866, 3892, -1000, -1000, -1000, 3090, 2330, -418, 61866, -1000, + -1000, -1000, -1000, -1000, 2110, -1000, 1138, 1138, 5917, 5542, + -1000, 17988, -1000, 17988, -1000, -1000, -1000, -1000, 3452, -1000, + 2249, -1000, 14388, 2411, 267, 14388, 267, 1982, 31698, 39596, + -175, 3891, 3421, 61866, 14388, -1000, -1000, 14388, 14388, 17988, + -1000, 3387, -1000, -1000, -1000, -1000, 14388, 14388, 2578, -1000, + 61866, -1000, -1000, -1000, -1000, 31698, -1000, 17988, -1000, -1000, + -1000, -1000, 14388, 14388, 14388, 1703, 1703, 3378, 2108, 129, + 129, 129, 3321, 3285, 3272, 2107, 129, 3263, 3253, 3248, + 3187, 3163, 3124, 3074, 3063, 3027, 3016, 2105, -1000, 3597, + -1000, -1000, -1000, 129, -1000, 129, 14388, 129, 14388, 129, + 129, 14388, 2575, 15828, 11501, -1000, 3887, 324, 1731, 2806, + 3134, 132, -1000, 2326, -1000, 463, -1000, 61866, 4080, -1000, + 1950, 3132, 51808, -1000, 1342, 61866, -1000, -1000, 4079, 4077, + -1000, -1000, 61866, 61866, 61866, -1000, -1000, -1000, 1272, -1000, + 3127, -1000, 411, 403, 2655, 2410, 3125, 339, 1505, 21613, + 3571, 3584, 3571, 144, 2196, 596, 756, 48218, 830, -1000, + 51090, 2799, 2320, 3762, 911, 3868, 61866, 50372, 3582, 1110, + 3581, 3578, 3944, 608, 4814, -1000, 3882, 1551, 2095, 3825, + 1729, -1000, 5098, -1000, 61866, 61866, 1606, -1000, 1933, -1000, + 2802, -1000, -1000, -1000, -1000, 61866, -1000, 354, -1000, 2228, + -1000, -1000, 4067, -1000, -1000, 14388, 14388, 4067, 2228, 2228, + -1000, 2275, -1000, 61866, -1000, -420, 608, 4814, 3942, 6228, + 744, 2939, -1000, 61866, -1000, -1000, -1000, 1136, -1000, 1304, + 937, 61866, 2490, 1304, 2489, 3577, -1000, -1000, 61866, 61866, + 61866, 61866, -1000, -1000, 61866, -1000, 61866, 61866, 61866, 61866, + 61866, 49654, -1000, 61866, 61866, -1000, 61866, 2483, 61866, 2481, + 3905, -1000, 2196, 2196, 1228, -1000, -1000, 715, -1000, 49654, + 2801, 2792, 2791, 2783, 3123, 3122, 3121, 2196, 2196, 2779, + 3116, 48936, 3115, 1588, 2778, 2777, 2774, 2711, 3113, 1324, + -1000, 3112, 2709, 2697, 2680, 61866, 3576, 2990, -1000, -1000, + 2655, 3111, 3574, 2773, 3110, 1154, 354, 3107, 3760, 144, + 2196, 427, 61866, 2315, 2313, 756, 691, 691, 650, -24, + 28826, -1000, -1000, -1000, 61866, 43186, 43186, 43186, 43186, 43186, + 43186, -1000, 3801, 3781, 3572, -1000, 3785, 3784, 3783, 551, + 3800, 3771, 61866, 43186, 3571, -1000, 48936, -1000, -1000, -1000, + 2111, 2089, 1152, 1316, 14388, 8610, -1000, -1000, 20, 3, + -1000, -1000, -1000, -1000, 48218, 3102, 669, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 3887, -1000, -1000, 61866, 61866, 947, + 3310, 1723, -1000, -1000, -1000, 4814, 3570, 3556, 3556, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3556, 3556, + 3556, 3556, 3556, 3556, 3568, -1000, -1000, 3554, 3554, 3554, + 3553, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 1385, 3564, 3565, 3565, 3564, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 61567, -1000, 4039, -1000, 1691, -1000, -1000, 1893, -1000, 2356, - -401, 18660, 2345, 2277, -1000, 14353, 18660, 14353, -308, 496, - -311, -1000, -1000, -1000, -1000, 2990, -1000, -1000, -1000, 2652, - -1000, 2647, -1000, 296, 312, 3849, 332, -1000, 4082, 14353, - 3824, -1000, -1000, -273, 11482, 3386, 61567, -273, 61567, 11482, - -1000, 61567, 173, -426, -427, 169, 2986, -1000, 61567, 2642, - -1000, -1000, -1000, 4054, 47995, 330, 2086, 47281, -1000, 473, - -1000, 1678, 814, 2985, -1000, 1080, 132, 2984, 2981, -1000, - -1000, -1000, -1000, 17933, 2131, -1000, -1000, -1000, 2920, 14353, - 3239, 2845, 3225, 3221, -1000, 3481, 3481, -1000, 3471, 3480, - 3471, 1998, 1998, 3212, -1000, 3470, -1000, 3846, -1000, 2055, - 2605, 3080, 5710, -1000, 3004, 2942, 14353, -1000, 3204, 5633, - 1981, 1966, 2928, -51, -230, 276, 276, -1000, -1000, -1000, - -1000, 276, 276, 276, 276, -1000, 276, 276, 276, 276, - 276, 276, 276, 276, 276, 276, 276, 931, -1000, -1000, - 1667, -1000, 1572, -1000, -1000, 2910, -105, -345, -145, -347, - -1000, -1000, 3200, 1662, -1000, -1000, -1000, -1000, -1000, 4278, - 1608, 748, 748, 2981, 2979, 61567, 2977, -363, 61567, -1000, - -429, -430, -364, 61567, 2976, 61567, 61567, 119, 2350, 2439, - -1000, 2973, -1000, -1000, 46567, 61567, 61567, 62281, 805, 61567, - 61567, 2971, -1000, -199, 3462, -153, 2970, 3199, 1601, -1000, - -1000, 61567, -1000, -1000, -1000, 3196, 3887, 22252, 3883, 2712, - -1000, -1000, -1000, 34423, 61567, 782, -1000, -1000, -1000, 901, - 485, 2641, 759, -1000, 61567, 685, 540, 3804, 2284, 2969, - 61567, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 3837, -1000, 1412, -419, 61567, 662, 41563, 19385, -1000, - 3235, 61567, -1000, 61567, 45847, 22252, 22252, 3235, 683, 2295, - -1000, 2406, 3223, -273, 3194, -1000, 930, 1560, 137, 42991, - 61567, -1000, 43705, -1000, -1000, 1530, 4041, -1000, 2920, 2920, - -419, 4041, 4041, 2178, -1000, -1000, 683, -1000, 3235, -1000, - 1086, 23680, 762, 574, 549, -1000, 912, -1000, -1000, 927, - 3820, 491, -1000, 61567, -1000, 61567, -1000, 61567, 61567, 967, - 14353, 3820, 61567, 1021, -1000, 1380, 676, 601, 997, 997, - 1599, -1000, 3846, -1000, -1000, 1596, -1000, -1000, -1000, -1000, - 61567, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 31567, 31567, - 3920, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 2968, 2963, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 61866, -1000, 4065, -1000, + 1699, -1000, -1000, 1915, -1000, 2293, -402, 18719, 2327, 2277, + -1000, 14388, 18719, 14388, -321, 409, -324, -1000, -1000, -1000, + -1000, 3101, -1000, -1000, -1000, 2772, -1000, 2749, -1000, 150, + 234, 3887, 248, -1000, 4108, 14388, 3848, -1000, -1000, -278, + 11501, 3463, 61866, -278, 61866, 11501, -1000, 61866, 176, -430, + -431, 169, 3100, -1000, 61866, 2748, -1000, -1000, -1000, 4075, + 48218, 354, 2141, 47500, -1000, 353, -1000, 1720, 698, 3094, + -1000, 1101, 130, 3091, 3090, -1000, -1000, -1000, -1000, 17988, + 2169, -1000, -1000, -1000, 2632, 14388, 3307, 2620, 3306, 3305, + -1000, 3556, 3556, -1000, 3553, 3554, 3553, 2096, 2096, 3297, + -1000, 3550, -1000, 3891, -1000, 2070, 2559, 3009, 5314, -1000, + 2955, 2946, 14388, -1000, 3287, 5068, 1979, 1963, 2933, -46, + -233, 129, 129, -1000, -1000, -1000, -1000, 129, 129, 129, + 129, -1000, 129, 129, 129, 129, 129, 129, 129, 129, + 129, 129, 129, 904, -1000, -1000, 1813, -1000, 1676, -1000, + -1000, 2920, -146, -347, -147, -348, -1000, -1000, 3286, 1693, + -1000, -1000, -1000, -1000, -1000, 906, 1666, 660, 660, 3090, + 3089, 61866, 3084, -364, 61866, -1000, -432, -433, -365, 61866, + 3082, 61866, 61866, 60, 2393, 2510, -1000, 3075, -1000, -1000, + 46782, 61866, 61866, 62584, 751, 61866, 61866, 3066, -1000, -205, + 3549, -158, 3062, 3279, 1657, -1000, -1000, 61866, -1000, -1000, + -1000, 3273, 3941, 22331, 3939, 2846, -1000, -1000, -1000, 34570, + 61866, 691, -1000, -1000, -1000, 818, 406, 2740, 674, -1000, + 61866, 629, 457, 3838, 2311, 3061, 61866, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3868, -1000, 1318, + -420, 61866, 595, 41750, 19448, -1000, 3296, 61866, -1000, 61866, + 46058, 22331, 22331, 3296, 598, 2267, -1000, 2474, 3420, -278, + 3270, -1000, 898, 1568, 138, 43186, 61866, -1000, 43904, -1000, + -1000, 1580, 4067, -1000, 2632, 2632, -420, 4067, 4067, 2228, + -1000, -1000, 598, -1000, 3296, -1000, 1258, 23767, 688, 535, + 506, -1000, 793, -1000, -1000, 895, 3865, 4814, -1000, 61866, + -1000, 61866, -1000, 61866, 61866, 937, 14388, 3865, 61866, 999, + -1000, 1400, 548, 521, 950, 950, 1656, -1000, 3891, -1000, + -1000, 1639, -1000, -1000, -1000, -1000, 61866, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 31698, 31698, 3968, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + 3056, 3046, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 61567, 2046, -1000, 2280, 2962, - -153, 7891, -1000, -1000, 1017, -1000, 3730, 1078, 2712, 34423, - 2279, 2229, 2961, 2959, 782, -1000, 2958, 2957, -1000, 2443, - 2274, 1072, 61567, -1000, 1526, 61567, 61567, -1000, 1663, -1000, - 2271, 3723, 3729, 3723, -1000, 3723, -1000, -1000, -1000, -1000, - 3768, 2956, -1000, 3767, -1000, 3766, -1000, 3762, -1000, -1000, - -1000, -1000, 1635, -1000, -1000, -1000, -1000, -1000, 1226, -1000, - 3953, 1274, 1274, 1274, 3193, -1000, -1000, -1000, -1000, 1590, - 3192, -1000, -1000, 3951, -1000, -1000, -1000, -1000, -1000, -1000, - 21538, 3831, 661, 4034, 4018, 45133, -1000, -401, 2335, -1000, - 2408, 197, 2334, 61567, -1000, -1000, -1000, 3189, 3186, -280, - 316, 4017, 4016, 3951, -291, 2955, 469, -1000, -1000, 3813, - -1000, 3182, 1542, -273, -1000, -1000, 1503, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -432, -1000, -1000, 330, -1000, 1649, - -1000, -1000, -1000, -1000, -1000, -1000, 350, -1000, 61567, -1000, - 1539, 127, -1000, 2920, -1000, 269, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, 2954, -1000, -1000, -1000, - 14353, -1000, -1000, -1000, -1000, 2905, -1000, -1000, 14353, 14353, - -1000, 3179, 2948, 3178, 2947, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 4096, -1000, 4015, 276, 14353, 276, 14353, 276, - 2035, 3167, 3163, 2028, 3158, 3157, -1000, 14353, 3154, 4278, - 1241, 2940, 1241, -1000, -1000, -1000, -1000, 61567, -1000, -1000, - -1000, 61567, 4051, 33709, 998, -419, 699, 3461, -1000, 704, - 2350, 1345, 3459, 2938, -1000, 61567, 4050, 61567, 2578, 797, - 2578, 849, 61567, -368, -155, 2633, 7891, -1000, 2925, -1000, - -172, 1491, 491, 1088, 3235, 3152, 1533, -1000, -1000, -1000, - -1000, 3235, -1000, 2924, 374, -1000, -1000, -1000, 606, -1000, - 2632, -1000, -1000, 2518, 1791, 401, -1000, -1000, -1000, -1000, - -1000, -1000, 2573, 61567, 44419, 2573, 2637, 2265, -420, -1000, - 3458, -1000, 2187, 2187, 2187, 998, 643, 61567, 2018, -1000, - 2187, 2187, 3145, -1000, -1000, 998, 61567, 3132, 3131, 4081, - 935, 2237, 2225, -1000, 2625, 1256, -273, -1000, 1503, -1000, - 32995, 42991, 43705, 1614, -1000, 1892, -1000, -1000, -1000, -1000, - -1000, 4041, 935, -1000, 764, 2623, 17933, 3457, 17933, 3455, - 771, 3454, 2012, -1000, 61567, -1000, -1000, 61567, 4726, 3444, - -1000, 3443, 3629, 741, 3442, 3441, 61567, 2888, -1000, 3820, - 61567, 938, 3823, -1000, 560, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 820, -1000, 61567, -1000, 61567, -1000, 1979, - -1000, 31567, -1000, -1000, 1920, -1000, 2922, 2919, -1000, -1000, - 3129, 2920, -1000, 2205, 330, 1069, 61567, -1000, 374, 2917, - 8607, -1000, -1000, -1000, -1000, -1000, 3804, 2909, 2573, 61567, - -1000, 61567, 1526, 1526, 4096, 42991, 61567, 11482, -1000, -1000, - 14353, 3439, -1000, 14353, -1000, -1000, -1000, 3127, -1000, -1000, - -1000, -1000, -1000, -1000, 3435, 3819, -1000, -1000, -1000, -1000, - -1000, -1000, 4069, -1000, 2141, 61567, -1000, 14353, 15069, -1000, - 960, 18660, -312, 494, -1000, -1000, -1000, -283, 2906, -1000, - -1000, 3997, 2904, 2731, -1000, 136, 2903, -1000, 14353, -1000, - -1000, -1000, 1503, -1000, 1530, -1000, -1000, 1506, 858, -1000, - 3121, 2282, -1000, 2867, -1000, 2770, 2763, 276, -1000, 276, - -1000, 335, 14353, -1000, 2756, -1000, 2732, -1000, -1000, 2901, - -1000, -1000, -1000, 2899, -1000, -1000, 2719, -1000, 3116, -1000, - 2896, -1000, -1000, 2894, 2893, -365, -1000, -1000, 532, 998, - -1000, 508, 61567, 733, -1000, 42277, 7891, -421, 641, 61567, - 4047, 2886, 2578, 2885, 2578, 61567, 793, -1000, 3881, 2884, - -1000, 3113, -1000, 2880, 2863, -1000, -1000, 491, 4080, 4081, - 22252, 4080, -1000, -1000, 3983, -1000, 1788, 526, -1000, -1000, - 2496, 838, -1000, -1000, 2854, 789, -1000, 1526, -1000, -1000, - 2262, 2432, 2774, 39421, 31567, 32281, 2853, -1000, 61567, -1000, - -1000, 41563, 2141, 2141, 4030, -1000, 640, 471, 6620, -1000, - 3434, 1402, 2202, -1000, 2622, -1000, 2621, -1000, 61567, -1000, - 1503, 4041, 1614, 135, -1000, -1000, 2040, -1000, 1402, 2926, - 3996, -1000, 4843, 61567, 4635, 61567, 3431, 2261, 17933, -1000, - 927, 3787, -1000, -1000, 4726, -1000, -1000, 2427, 17933, -1000, - -1000, 2851, 32281, 1177, 2259, 2253, 1151, 3428, -1000, 818, - 4068, 2612, -1000, -1000, -1000, 1229, 3406, -1000, -299, 3405, - 2405, 2404, -1000, 61567, -1000, 39421, 39421, 1409, 1409, 39421, - 39421, 3404, 997, -1000, -1000, 17933, -1000, -1000, -1000, 2244, - 4392, 4392, 4392, 4392, -1000, -1000, -1000, 2187, 1846, -1000, - -1000, -1000, -1000, -1000, 61567, 1890, -1000, -1000, -1000, 2637, - -1000, -1000, 1498, -1000, 3972, 1614, -1000, -1000, 2920, 61567, - 2920, -1000, 40849, -1000, 3995, 3994, -1000, -1000, -1000, 2920, - 1571, 264, 3403, 3397, -1000, -401, 61567, 61567, -285, 2611, - -1000, 2846, 307, -1000, -1000, 296, -1000, 1476, -287, 129, - 31567, 2242, -1000, 3106, 364, -181, -1000, -1000, -1000, -1000, - -1000, 3104, -1000, 1043, -1000, -1000, -1000, 1476, 276, 276, - 3101, 3100, -1000, -1000, -1000, -1000, -1000, 61567, 61567, -1000, - 61567, 2842, 2610, -1000, -1000, 1905, -1000, -1000, -1000, 2397, - 2396, 1902, 3099, 2767, 61567, 639, 61567, -368, 2822, -368, - 2821, 787, 2578, -334, -1000, -1000, -1000, -1000, -175, -1000, - -1000, 499, -1000, -1000, -1000, 792, 2749, 2607, -1000, -1000, - 524, -1000, -1000, -1000, 2573, 2812, -1000, -1000, 126, -1000, - 2241, 1886, -1000, -1000, -1000, 606, -1000, -1000, -1000, 925, - -1000, 3235, 6397, -1000, 1424, 61567, -1000, 1506, 925, 37993, - 889, 2161, -1000, 2606, -1000, -1000, 1467, 4096, -1000, 863, - -1000, 769, -1000, 1865, -1000, 1864, 40135, 2602, 3879, -1000, - 5568, 1085, -1000, -1000, 6053, -1000, -1000, -1000, -1000, -1000, - -1000, 2810, 2809, -1000, -1000, -1000, -1000, -1000, 2597, 3395, - 52, -1000, 3911, 2807, 3876, 14353, -1000, -1000, 3394, 1840, - 1834, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 1833, 1822, 39421, -1000, -1000, 6053, 4392, - 2428, -1000, 2187, 2187, 2804, 2802, 595, -1000, -1000, 2187, - 2187, 2187, 2187, 2187, 2187, 3393, 2800, 2799, 2187, 2187, - 2187, 2187, -1000, -1000, 2239, 2187, 2187, 31567, 2187, 1868, - 61567, -1000, -1000, -1000, 1815, 1790, -1000, -1000, -1000, -1000, - -1000, -378, 3388, 14353, 14353, -1000, -1000, -1000, 3387, -1000, - -1000, 3993, -280, -289, 2796, 295, 391, -1000, 2784, -1000, - -176, 3781, -188, -1000, -1000, 784, -276, 255, 234, 230, - -1000, -1000, -1000, 14353, -1000, -1000, -1000, -1000, 2782, -1000, - -1000, -1000, -1000, -1000, 61567, 2776, -1000, -1000, 108, -1000, - 2203, -1000, 61567, 634, -1000, -368, -1000, -368, 2578, 2775, - -1000, 61567, 794, -1000, -1000, -1000, -1000, 343, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, 2774, 2771, -1000, -1000, 752, - 3992, -1000, 6620, -1000, 2187, 606, -1000, 752, 1762, -1000, - 2187, 2187, -1000, 690, -1000, 2123, -1000, 2577, -1000, 3972, - -1000, 687, -1000, 742, -1000, -1000, -1000, 1761, -1000, -1000, - -1000, 5568, 765, -1000, 917, 3378, -1000, -1000, 3093, 14353, - 3369, 2187, 3092, 3368, 2713, -165, 39421, 3500, 3423, 3373, - 2778, 1725, -1000, -1000, 2572, 2569, -1000, -1000, 61567, 2566, - 2561, 2557, 2556, 2548, 2542, 61567, -1000, -1000, 2524, 2522, - 2520, 2519, 2424, 2513, 2503, -1000, 31567, 61567, -1000, -1000, - -1000, 38707, -1000, 3367, 1687, 1682, 61567, 2731, -283, -1000, - 2768, -1000, 994, 294, 391, -1000, 3991, 297, 3990, 3989, - 1463, 3779, -1000, -1000, 2394, -1000, 270, 257, 244, -1000, - -1000, -1000, -1000, -1000, 2440, 2440, -368, 2767, 2766, -1000, - 61567, -1000, -1000, 2761, -368, 750, -1000, 467, -1000, -1000, - -1000, 4392, -1000, 3986, 788, -1000, 31567, -1000, -1000, -1000, - 37993, 2141, 2141, -1000, -1000, 2498, -1000, -1000, -1000, -1000, - 2492, -1000, -1000, -1000, 1655, -1000, 61567, 1134, 10766, -1000, - 2699, -1000, 61567, -1000, 14353, -302, 3727, -1000, 345, 1653, - 4392, 1409, 4392, 1409, 4392, 1409, 4392, 1409, 456, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1595, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, 1589, 14353, -1000, -1000, 1549, -1000, -1000, -285, -1000, - 3365, 2489, 316, 304, 3981, -1000, 2731, 3980, 2731, 2731, - -1000, 271, 4078, 784, -1000, -1000, -1000, -1000, 2350, -1000, - 2350, -1000, -1000, -1000, -1000, -368, -1000, 2752, -1000, -1000, - -1000, 37279, 762, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 765, 6620, -1000, 10766, 1541, -1000, 2920, -1000, 997, -1000, - 2624, -1000, -1000, -1000, -1000, 3726, 2897, 4045, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 3224, - 3091, -1000, 61567, -1000, 3906, 30853, 280, -1000, -1000, -1000, - 2739, -1000, 2731, -1000, -1000, 2184, -183, -1000, -1000, -1000, - -1000, -343, -1000, 61567, 764, -1000, 6620, 1537, -1000, 10766, - -1000, -302, -1000, 4067, -1000, 4046, 1168, 1168, 4392, 4392, - 4392, 4392, 14353, -1000, -1000, -1000, 61567, -1000, 1535, -1000, - -1000, -1000, 1825, -1000, -1000, -1000, -1000, 2726, -190, -1000, - -1000, 2718, 1523, 2926, -1000, -1000, -1000, -1000, -1000, -1000, - 2536, 825, -1000, 2630, 1455, -1000, 2171, -1000, 36565, 61567, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 61567, - 10050, -1000, 1800, -1000, -1000, 2920, 61567, -1000, + -1000, 61866, 2067, -1000, 2304, 3044, -158, 7890, -1000, -1000, + 996, -1000, 3759, 1095, 2846, 34570, 2295, 2250, 3043, 3042, + 691, -1000, 3035, 3029, -1000, 2799, 2292, 1083, 61866, -1000, + 1562, 61866, 61866, -1000, 1749, -1000, 2290, 3742, 3758, 3742, + -1000, 3742, -1000, -1000, -1000, -1000, 3799, 3028, -1000, 3797, + -1000, 3796, -1000, 3793, -1000, -1000, -1000, -1000, 1670, -1000, + -1000, -1000, -1000, -1000, 1316, -1000, 4010, 1304, 1304, 1304, + 3267, -1000, -1000, -1000, -1000, 1616, 3266, -1000, -1000, 4009, + -1000, -1000, -1000, -1000, -1000, -1000, 21613, 3862, 593, 4063, + 4053, 45340, -1000, -402, 2234, -1000, 2407, 198, 2365, 61866, + -1000, -1000, -1000, 3265, 3260, -283, 181, 4052, 4049, 4009, + -305, 3018, 350, -1000, -1000, 3869, -1000, 3258, 1615, -278, + -1000, -1000, 1551, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -435, -1000, -1000, 354, -1000, 1642, -1000, -1000, -1000, -1000, + -1000, -1000, 265, -1000, 61866, -1000, 1598, 116, -1000, 2632, + -1000, 267, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 3015, -1000, -1000, -1000, 14388, -1000, -1000, -1000, + -1000, 2875, -1000, -1000, 14388, 14388, -1000, 3257, 3014, 3250, + 3006, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 4120, -1000, + 4048, 129, 14388, 129, 14388, 129, 2061, 3239, 3238, 2060, + 3236, 3235, -1000, 14388, 3232, 906, 1240, 3001, 1240, -1000, + -1000, -1000, -1000, 61866, -1000, -1000, -1000, 61866, 4074, 33852, + 988, -420, 615, 3548, -1000, 618, 2393, 1364, 3546, 3000, + -1000, 61866, 4073, 61866, 2655, 749, 2655, 804, 61866, -369, + -160, 2736, 7890, -1000, 2995, -1000, -178, 1505, 4814, 1066, + 3296, 3229, 1582, -1000, -1000, -1000, -1000, 3296, -1000, 2993, + 282, -1000, -1000, -1000, 537, -1000, 2730, -1000, -1000, 2654, + 2081, 294, -1000, -1000, -1000, -1000, -1000, -1000, 2844, 61866, + 44622, 2844, 2845, 2289, -421, -1000, 3538, -1000, 2196, 2196, + 2196, 988, 590, 61866, 2059, -1000, 2196, 2196, 3228, -1000, + -1000, 988, 61866, 3226, 3224, 4107, 897, 2260, 2231, -1000, + 2728, 1202, -278, -1000, 1551, -1000, 33134, 43186, 43904, 1643, + -1000, 1910, -1000, -1000, -1000, -1000, -1000, 4067, 897, -1000, + 671, 2707, 17988, 3537, 17988, 3535, 714, 3534, 2050, -1000, + 61866, -1000, -1000, 61866, 501, 3533, -1000, 3531, 3747, 658, + 3527, 3526, 61866, 2843, -1000, 3865, 61866, 896, 3857, -1000, + 486, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 754, + -1000, 61866, -1000, 61866, -1000, 2114, -1000, 31698, -1000, -1000, + 2019, -1000, 2990, 2989, -1000, -1000, 3220, 2632, -1000, 2064, + 354, 1047, 61866, -1000, 282, 2987, 8610, -1000, -1000, -1000, + -1000, -1000, 3838, 2986, 2844, 61866, -1000, 61866, 1562, 1562, + 4120, 43186, 61866, 11501, -1000, -1000, 14388, 3524, -1000, 14388, + -1000, -1000, -1000, 3218, -1000, -1000, -1000, -1000, -1000, -1000, + 3522, 3855, -1000, -1000, -1000, -1000, -1000, -1000, 4098, -1000, + 2151, 61866, -1000, 14388, 15108, -1000, 934, 18719, -325, 408, + -1000, -1000, -1000, -286, 2985, -1000, -1000, 4046, 2976, 2858, + -1000, 63, 2974, -1000, 14388, -1000, -1000, -1000, 1551, -1000, + 1580, -1000, -1000, 1502, 817, -1000, 3212, 2254, -1000, 2809, + -1000, 2796, 2785, 129, -1000, 129, -1000, 286, 14388, -1000, + 2658, -1000, 2645, -1000, -1000, 2973, -1000, -1000, -1000, 2972, + -1000, -1000, 2604, -1000, 3211, -1000, 2967, -1000, -1000, 2966, + 2964, -366, -1000, -1000, 455, 988, -1000, 502, 61866, 705, + -1000, 42468, 7890, -423, 589, 61866, 4072, 2963, 2655, 2960, + 2655, 61866, 743, -1000, 3936, 2953, -1000, 3210, -1000, 2952, + 2951, -1000, -1000, 4814, 4104, 4107, 22331, 4104, -1000, -1000, + 4034, -1000, 1919, 447, -1000, -1000, 2639, 727, -1000, -1000, + 2948, 704, -1000, 1562, -1000, -1000, 2288, 2532, 2881, 39596, + 31698, 32416, 2947, -1000, 61866, -1000, -1000, 41750, 2151, 2151, + 6419, -1000, 587, 336, 68183, -1000, 3518, 1442, 2205, -1000, + 2705, -1000, 2698, -1000, 61866, -1000, 1551, 4067, 1643, 136, + -1000, -1000, 2120, -1000, 1442, 2939, 4045, -1000, 4848, 61866, + 4819, 61866, 3517, 2284, 17988, -1000, 895, 3811, -1000, -1000, + 501, -1000, -1000, 2503, 17988, -1000, -1000, 2938, 32416, 1087, + 2263, 2262, 1349, 3513, -1000, 766, 4097, 2694, -1000, -1000, + -1000, 1216, 3511, -1000, -314, 3509, 2473, 2469, -1000, 61866, + -1000, 39596, 39596, 1570, 1570, 39596, 39596, 3486, 950, -1000, + -1000, 17988, -1000, -1000, -1000, 2259, 4373, 4373, 4373, 4373, + -1000, -1000, -1000, 2196, 2106, -1000, -1000, -1000, -1000, -1000, + 61866, 1904, -1000, -1000, -1000, 2845, -1000, -1000, 1548, -1000, + 4023, 1643, -1000, -1000, 2632, 61866, 2632, -1000, 41032, -1000, + 4043, 4040, -1000, -1000, -1000, 2632, 1634, 266, 3485, 3479, + -1000, -402, 61866, 61866, -288, 2693, -1000, 2937, 162, -1000, + -1000, 150, -1000, 1547, -295, 85, 31698, 2257, -1000, 3207, + 362, -189, -1000, -1000, -1000, -1000, -1000, 3205, -1000, 840, + -1000, -1000, -1000, 1547, 129, 129, 3204, 3203, -1000, -1000, + -1000, -1000, -1000, 61866, 61866, -1000, 61866, 2936, 2688, -1000, + -1000, 2018, -1000, -1000, -1000, 2458, 2455, 1995, 3201, 2873, + 61866, 586, 61866, -369, 2929, -369, 2927, 730, 2655, -338, + -1000, -1000, -1000, -1000, -181, -1000, -1000, 494, -1000, -1000, + -1000, 696, 2853, 2679, -1000, -1000, 438, -1000, -1000, -1000, + 2844, 2918, -1000, -1000, 109, -1000, 2253, 1964, -1000, -1000, + -1000, 537, -1000, -1000, -1000, 894, -1000, 3296, 6806, -1000, + 1556, 61866, -1000, 1502, 894, 38160, 795, 2335, -1000, 2678, + -1000, -1000, 1532, 4120, -1000, 792, -1000, 710, -1000, 1956, + -1000, 1940, 40314, 2661, 4725, -1000, 6523, 1075, -1000, -1000, + 5917, -1000, -1000, -1000, -1000, -1000, -1000, 2916, 2913, -1000, + -1000, -1000, -1000, -1000, 2656, 3474, -37, -1000, 3966, 2909, + 3934, 14388, -1000, -1000, 3473, 1935, 1864, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1859, + 1857, 39596, -1000, -1000, 5917, 4373, 2528, -1000, 2196, 2196, + 2907, 2902, 515, -1000, -1000, 2196, 2196, 2196, 2196, 2196, + 2196, 3471, 2901, 2889, 2196, 2196, 2196, 2196, -1000, -1000, + 2233, 2196, 2196, 31698, 2196, 1875, 61866, -1000, -1000, -1000, + 1856, 1846, -1000, -1000, -1000, -1000, -1000, -379, 3467, 14388, + 14388, -1000, -1000, -1000, 3466, -1000, -1000, 4039, -283, -298, + 2888, 146, 159, -1000, 2887, -1000, -186, 3806, -195, -1000, + -1000, 1373, -279, 124, 121, 114, -1000, -1000, -1000, 14388, + -1000, -1000, -1000, -1000, 2886, -1000, -1000, -1000, -1000, -1000, + 61866, 2884, -1000, -1000, 108, -1000, 2226, -1000, 61866, 583, + -1000, -369, -1000, -369, 2655, 2883, -1000, 61866, 764, -1000, + -1000, -1000, -1000, 258, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, 2881, 2878, -1000, -1000, 662, 4038, -1000, 68183, -1000, + 2196, 537, -1000, 662, 1839, -1000, 2196, 2196, -1000, 601, + -1000, 2181, -1000, 2642, -1000, 4023, -1000, 600, -1000, 667, + -1000, -1000, -1000, 1838, -1000, -1000, -1000, 6523, 672, -1000, + 881, 3464, -1000, -1000, 3157, 14388, 3462, 2196, 3085, 3460, + 2536, -170, 39596, 3745, 3744, 3516, 2954, 1803, -1000, -1000, + 2640, 2636, -1000, -1000, 61866, 2614, 2612, 2609, 2602, 2587, + 2585, 61866, -1000, -1000, 2583, 2581, 2560, 2556, 2512, 2547, + 2540, -1000, 31698, 61866, -1000, -1000, -1000, 38878, -1000, 3457, + 1800, 1765, 61866, 2858, -286, -1000, 2876, -1000, 970, 149, + 159, -1000, 4036, 161, 4033, 4032, 1518, 3805, -1000, -1000, + 2451, -1000, 119, 112, 110, -1000, -1000, -1000, -1000, -1000, + 2475, 2475, -369, 2873, 2872, -1000, 61866, -1000, -1000, 2869, + -369, 614, -1000, 340, -1000, -1000, -1000, 4373, -1000, 4031, + 744, -1000, 31698, -1000, -1000, -1000, 38160, 2151, 2151, -1000, + -1000, 2526, -1000, -1000, -1000, -1000, 2522, -1000, -1000, -1000, + 1748, -1000, 61866, 1130, 10781, -1000, 2498, -1000, 61866, -1000, + 14388, -312, 3756, -1000, 307, 1746, 4373, 1570, 4373, 1570, + 4373, 1570, 4373, 1570, 331, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 1744, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, -1000, 1737, 14388, -1000, + -1000, 1692, -1000, -1000, -288, -1000, 3392, 2486, 181, 152, + 4029, -1000, 2858, 4028, 2858, 2858, -1000, 133, 4076, 1373, + -1000, -1000, -1000, -1000, 2393, -1000, 2393, -1000, -1000, -1000, + -1000, -369, -1000, 2863, -1000, -1000, -1000, 37442, 688, -1000, + -1000, -1000, -1000, -1000, -1000, -1000, 672, 68183, -1000, 10781, + 1671, -1000, 2632, -1000, 950, -1000, 2484, -1000, -1000, -1000, + -1000, 3755, 3754, 4071, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, 2965, 3032, -1000, 61866, -1000, + 3964, 30980, 139, -1000, -1000, -1000, 2860, -1000, 2858, -1000, + -1000, 2177, -192, -1000, -1000, -1000, -1000, -344, -1000, 61866, + 671, -1000, 68183, 1664, -1000, 10781, -1000, -312, -1000, 4090, + -1000, 4095, 1246, 1246, 4373, 4373, 4373, 4373, 14388, -1000, + -1000, -1000, 61866, -1000, 1628, -1000, -1000, -1000, 1874, -1000, + -1000, -1000, -1000, 2857, -196, -1000, -1000, 2856, 1597, 2939, + -1000, -1000, -1000, -1000, -1000, -1000, 2564, 772, -1000, 2880, + 1470, -1000, 2170, -1000, 36724, 61866, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, -1000, 61866, 10061, -1000, 1571, -1000, + -1000, 2632, 61866, -1000, } var yyPgo = [...]int{ - 0, 193, 61, 259, 197, 4745, 118, 274, 323, 3853, - 283, 267, 264, 4737, 4736, 4735, 3846, 3845, 4733, 4732, - 4731, 4730, 4729, 4728, 4727, 4726, 4725, 4723, 4722, 4721, - 4719, 4718, 4717, 4711, 4710, 4709, 4704, 4702, 4699, 4696, - 4695, 4693, 4692, 4691, 4690, 4688, 4686, 4685, 4683, 4679, - 4678, 4677, 4676, 262, 4674, 4671, 4668, 4667, 4666, 4665, - 4664, 4663, 4658, 4657, 4656, 4654, 4653, 4652, 4651, 4650, - 4649, 4646, 4645, 4643, 4642, 4640, 4639, 4637, 4636, 4634, - 4633, 4630, 4625, 4624, 4623, 4622, 4621, 4620, 4619, 4617, - 4615, 4613, 273, 4612, 3844, 4611, 4610, 4609, 4606, 4601, - 4600, 4598, 4597, 4595, 4591, 4590, 4589, 382, 4588, 4587, - 4586, 4584, 4580, 4579, 4578, 4577, 4576, 4575, 4573, 4572, - 4570, 357, 4567, 4565, 4564, 4563, 230, 4562, 335, 4561, - 191, 150, 4556, 4555, 4554, 4553, 4550, 4547, 117, 139, - 4546, 4545, 4544, 4541, 4539, 4537, 4535, 4534, 4533, 4532, - 4530, 4529, 4528, 4526, 249, 173, 81, 4525, 58, 4524, - 256, 222, 4523, 231, 4520, 169, 4516, 164, 4514, 4512, - 4511, 4510, 4509, 4508, 4507, 4506, 4505, 4504, 4503, 4502, - 4501, 4499, 4497, 4495, 4493, 4492, 4491, 4489, 4486, 4482, - 4481, 4478, 4477, 4475, 4474, 4472, 4471, 4470, 60, 4469, - 270, 4468, 86, 4463, 196, 4462, 85, 4461, 4459, 112, - 27, 40, 4457, 56, 93, 266, 3331, 271, 4456, 208, - 4455, 4454, 261, 189, 4449, 4448, 272, 4446, 194, 239, - 175, 100, 141, 4444, 151, 4441, 275, 53, 52, 253, - 218, 159, 4440, 4439, 67, 182, 142, 4437, 221, 111, - 4436, 4435, 4434, 128, 4433, 4432, 123, 4430, 252, 199, - 4429, 124, 4428, 4427, 4426, 23, 4425, 4423, 220, 211, - 4422, 4421, 115, 4419, 4418, 88, 146, 4417, 89, 145, - 187, 138, 4416, 3395, 144, 101, 4415, 147, 130, 4414, - 94, 4413, 4410, 4407, 4404, 201, 4403, 4402, 158, 4401, - 70, 4400, 4399, 4398, 83, 4396, 87, 4395, 33, 4394, - 69, 4391, 4389, 4386, 4384, 4383, 4381, 4375, 4374, 4373, - 4372, 4371, 4370, 39, 4368, 4367, 4366, 4365, 7, 15, - 18, 4364, 29, 4361, 190, 4356, 4355, 181, 4354, 212, - 4353, 4336, 106, 102, 4335, 103, 4332, 177, 4331, 9, - 32, 84, 4330, 4329, 4326, 241, 4325, 4324, 4323, 305, - 4321, 4320, 4319, 178, 4318, 4317, 4316, 540, 4315, 4314, - 4312, 4310, 4308, 4305, 66, 4303, 1, 229, 30, 4300, - 153, 160, 4298, 46, 36, 4297, 57, 171, 232, 148, - 120, 4296, 4295, 4294, 687, 219, 113, 37, 0, 119, - 236, 170, 4293, 4291, 4290, 280, 4289, 250, 235, 255, - 302, 281, 284, 4287, 4286, 71, 4285, 179, 35, 64, - 156, 91, 25, 203, 4284, 1754, 11, 202, 4282, 223, - 4280, 8, 17, 373, 163, 4277, 4276, 42, 276, 4274, - 4273, 4269, 149, 4268, 4264, 206, 90, 4263, 4262, 4261, - 4260, 4259, 48, 4258, 200, 19, 4257, 126, 4256, 257, - 122, 278, 161, 205, 195, 176, 237, 248, 97, 75, - 4254, 2157, 166, 121, 16, 4253, 10, 233, 4252, 215, - 136, 4248, 137, 4247, 260, 277, 228, 4246, 207, 14, - 54, 44, 34, 55, 12, 322, 105, 4245, 4244, 26, - 63, 4243, 59, 4242, 21, 4239, 4238, 51, 47, 4237, - 76, 5, 4236, 4235, 20, 22, 4233, 43, 224, 188, - 143, 107, 74, 4232, 4231, 172, 154, 4230, 155, 174, - 165, 4229, 45, 4226, 4224, 4223, 4222, 3567, 263, 4220, - 4219, 4217, 4216, 4215, 4214, 4213, 4212, 214, 4211, 116, - 49, 4209, 4207, 4206, 4205, 95, 157, 4204, 4202, 4201, - 4200, 38, 92, 4199, 13, 4198, 28, 24, 41, 4197, - 65, 4196, 4195, 4194, 3, 204, 4193, 4192, 4, 4191, - 4189, 2, 4188, 4185, 133, 4184, 104, 31, 185, 125, - 4183, 4182, 98, 217, 162, 4180, 4179, 127, 254, 4178, - 226, 4177, 108, 247, 268, 4174, 227, 4171, 4167, 4165, - 4163, 4162, 1428, 4161, 4160, 246, 80, 96, 4159, 234, - 132, 4158, 4157, 110, 180, 135, 168, 73, 99, 4154, - 129, 225, 4152, 213, 4149, 243, 4148, 4147, 4143, 4142, - 131, 4140, 4139, 4138, 4136, 209, 4131, 4130, 210, 238, - 4129, 4127, 304, 4126, 4125, 4123, 4121, 4119, 4117, 4115, - 4114, 4113, 4111, 258, 325, 4110, + 0, 198, 58, 258, 194, 4757, 88, 272, 340, 3897, + 322, 269, 266, 4756, 4755, 4754, 3884, 3883, 4752, 4751, + 4750, 4749, 4748, 4747, 4746, 4745, 4744, 4743, 4742, 4741, + 4738, 4737, 4735, 4734, 4733, 4732, 4731, 4730, 4729, 4728, + 4727, 4726, 4725, 4723, 4722, 4720, 4719, 4718, 4717, 4716, + 4715, 4714, 4712, 255, 4711, 4710, 4709, 4708, 4707, 4706, + 4705, 4704, 4702, 4701, 4700, 4699, 4696, 4695, 4694, 4693, + 4692, 4691, 4690, 4684, 4683, 4682, 4681, 4680, 4679, 4678, + 4677, 4676, 4674, 4673, 4672, 4671, 4670, 4669, 4668, 4667, + 4666, 4665, 302, 4664, 3879, 4663, 4661, 4659, 4658, 4656, + 4655, 4653, 4652, 4650, 4648, 4641, 4640, 350, 4639, 4638, + 4637, 4636, 4635, 4634, 4633, 4632, 4629, 4628, 4627, 4625, + 4624, 284, 4623, 4622, 4617, 4616, 233, 4614, 241, 4613, + 192, 149, 4611, 4608, 4605, 4604, 4603, 4602, 118, 129, + 4601, 4595, 4593, 4592, 4591, 4589, 4586, 4585, 4583, 4582, + 4581, 4579, 4577, 4575, 265, 178, 84, 4573, 57, 4572, + 260, 223, 4571, 237, 4570, 169, 4566, 166, 4565, 4564, + 4562, 4561, 4560, 4559, 4556, 4552, 4551, 4550, 4546, 4542, + 4538, 4536, 4532, 4530, 4528, 4512, 4508, 4504, 4503, 4502, + 4501, 4500, 4498, 4497, 4494, 4492, 4490, 4488, 69, 4487, + 277, 4485, 87, 4484, 201, 4482, 89, 4481, 4480, 90, + 23, 36, 4475, 56, 97, 267, 3415, 275, 4474, 209, + 4473, 4472, 256, 187, 4470, 4469, 274, 4466, 199, 243, + 175, 98, 139, 4462, 165, 4461, 278, 54, 52, 257, + 218, 161, 4460, 4457, 67, 197, 148, 4456, 222, 136, + 4454, 4453, 4452, 130, 4451, 4450, 123, 4449, 254, 196, + 4448, 125, 4446, 4444, 4443, 25, 4442, 4441, 221, 208, + 4440, 4438, 110, 4437, 4435, 76, 147, 4434, 91, 146, + 185, 141, 4433, 3214, 135, 96, 4432, 144, 124, 4431, + 122, 4430, 4429, 4428, 4427, 202, 4425, 4424, 158, 4423, + 73, 4422, 4421, 4420, 80, 4419, 81, 4418, 33, 4417, + 71, 4415, 4414, 4412, 4411, 4409, 4407, 4406, 4401, 4400, + 4397, 4391, 4389, 40, 4388, 4386, 4384, 4382, 7, 14, + 18, 4381, 29, 4379, 191, 4372, 4371, 182, 4370, 211, + 4369, 4366, 113, 103, 4365, 104, 4363, 181, 4362, 16, + 32, 85, 4361, 4360, 4357, 365, 4356, 4355, 4351, 305, + 4349, 4348, 4347, 176, 4346, 4344, 4342, 565, 4341, 4339, + 4338, 4337, 4336, 4334, 94, 4333, 1, 230, 28, 4332, + 151, 157, 4331, 46, 35, 4328, 51, 145, 236, 154, + 120, 4326, 4324, 4323, 612, 220, 109, 42, 0, 119, + 234, 177, 4307, 4305, 4304, 273, 4302, 253, 214, 249, + 160, 276, 262, 4300, 4298, 70, 4297, 180, 38, 62, + 159, 106, 24, 308, 4295, 1379, 10, 204, 4294, 225, + 4293, 8, 17, 394, 163, 4292, 4290, 43, 279, 4289, + 4288, 4287, 150, 4286, 4285, 217, 99, 4284, 4282, 4281, + 4280, 4279, 66, 4278, 190, 19, 4277, 138, 4275, 259, + 108, 291, 168, 205, 203, 173, 232, 246, 95, 83, + 4272, 2267, 172, 121, 15, 4271, 9, 235, 4270, 200, + 153, 4269, 101, 4266, 264, 281, 228, 4265, 206, 13, + 55, 44, 34, 49, 11, 409, 105, 4264, 4263, 26, + 60, 4262, 61, 4261, 21, 4260, 4259, 53, 45, 4258, + 65, 5, 4256, 4255, 20, 22, 4254, 41, 224, 188, + 142, 116, 74, 4253, 4252, 170, 156, 4251, 155, 174, + 171, 4250, 47, 4249, 4248, 4246, 4245, 815, 261, 4244, + 4243, 4242, 4241, 4240, 4239, 4238, 4236, 219, 4235, 115, + 48, 4233, 4232, 4231, 4230, 92, 164, 4229, 4228, 4226, + 4224, 37, 86, 4223, 12, 4222, 30, 27, 39, 4221, + 63, 4218, 4217, 4216, 3, 207, 4215, 4214, 4, 4210, + 4209, 2, 4208, 4207, 133, 4206, 114, 31, 186, 132, + 4204, 4201, 102, 215, 162, 4200, 4199, 127, 270, 4196, + 226, 4194, 111, 248, 271, 4192, 229, 4189, 4188, 4187, + 4186, 4183, 1423, 4181, 4179, 252, 75, 107, 4174, 231, + 128, 4172, 4171, 100, 179, 131, 143, 64, 93, 4170, + 137, 227, 4168, 213, 4165, 238, 4164, 4163, 4162, 4159, + 126, 4158, 4157, 4156, 4155, 212, 4154, 4152, 210, 239, + 4151, 4150, 303, 4147, 4146, 4144, 4143, 4141, 4140, 4139, + 4138, 4137, 4135, 247, 268, 4134, } -//line mysql_sql.y:14497 +//line mysql_sql.y:14553 type yySymType struct { union interface{} id int @@ -10664,12 +10757,12 @@ var yyR1 = [...]int{ 296, 296, 296, 296, 296, 296, 295, 295, 295, 295, 295, 295, 293, 293, 293, 293, 293, 291, 291, 291, 291, 291, 291, 291, 291, 291, 291, 291, 291, 291, - 291, 291, 291, 291, 291, 291, 129, 130, 130, 292, + 291, 291, 291, 291, 291, 291, 291, 291, 291, 291, + 129, 130, 130, 292, 299, 299, 299, 299, 299, 299, 299, 299, 299, 299, 299, 299, 299, 299, 299, 299, - 299, 299, 299, 299, 299, 299, 299, 299, 377, 377, - 525, 525, 528, 528, 526, 526, 527, 529, 529, 529, - 530, 530, 530, 531, 531, 531, 535, 535, 386, 386, - 386, 394, 394, 393, 393, 393, 393, 393, 393, 393, + 299, 299, 377, 377, 525, 525, 528, 528, 526, 526, + 527, 529, 529, 529, 530, 530, 530, 531, 531, 531, + 535, 535, 386, 386, 386, 394, 394, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, @@ -10710,13 +10803,14 @@ var yyR1 = [...]int{ 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - 393, 393, 393, 393, 393, 393, 393, 392, 392, 392, - 392, 392, 392, 392, 392, 392, 391, 391, 391, 391, + 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, + 393, 393, 393, 393, 393, 392, 392, 392, 392, 392, + 392, 392, 392, 392, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, 391, - 391, 391, 391, 391, 391, 391, 391, 391, + 391, 391, 391, 391, 391, 391, } var yyR2 = [...]int{ @@ -10923,12 +11017,13 @@ var yyR2 = [...]int{ 1, 1, 1, 1, 1, 1, 2, 3, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 2, 2, 4, 4, 1, 2, 3, 5, 1, + 1, 2, 2, 2, 2, 2, 2, 4, 4, 1, + 2, 3, 5, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 3, 0, 1, 0, 3, 0, 3, + 3, 0, 3, 5, 0, 3, 5, 0, 1, 1, + 0, 1, 1, 2, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, - 0, 1, 0, 3, 0, 3, 3, 0, 3, 5, - 0, 3, 5, 0, 1, 1, 0, 1, 1, 2, - 2, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -10975,479 +11070,481 @@ var yyR2 = [...]int{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, } var yyChk = [...]int{ - -1000, -658, -661, -2, -5, 712, -1, -4, -130, -99, + -1000, -658, -661, -2, -5, 716, -1, -4, -130, -99, -7, -15, -132, -133, -8, -128, -10, -11, -188, -13, -106, -123, -125, -127, -126, -53, -12, -122, -92, -93, -108, -116, -119, -120, -121, -134, -129, -131, -213, -135, - -144, -145, -195, -148, -150, -151, -183, -184, -208, 702, + -144, -145, -195, -148, -150, -151, -183, -184, -208, 706, -100, -101, -102, -103, -104, -105, -34, -33, -32, -31, - -175, -185, -189, -191, -146, -48, 621, 708, 517, -9, - -602, 570, -16, -17, -18, 264, 291, -402, -403, -404, + -175, -185, -189, -191, -146, -48, 625, 712, 521, -9, + -602, 574, -16, -17, -18, 268, 295, -402, -403, -404, -406, -662, -54, -55, -56, -67, -68, -69, -70, -71, -81, -82, -83, -57, -58, -59, -62, -60, -74, -73, -75, -76, -77, -78, -79, -80, -61, -65, -178, -179, -180, -181, -84, -63, -85, -64, -193, -196, -147, -86, -87, -88, -66, -51, -52, -90, -89, -95, -91, -96, - -177, -187, -14, -194, -97, -50, -98, 265, -94, 79, - -109, -110, -111, -112, -113, -114, -115, -117, -118, 443, - 449, 504, 701, 64, -214, -216, 732, 733, 736, 606, - 608, 309, 177, 178, 180, 181, 185, 188, -35, -36, + -177, -187, -14, -194, -97, -50, -98, 269, -94, 79, + -109, -110, -111, -112, -113, -114, -115, -117, -118, 447, + 453, 508, 705, 64, -214, -216, 736, 737, 740, 610, + 612, 313, 177, 178, 180, 181, 185, 188, -35, -36, -37, -38, -39, -40, -42, -41, -43, -44, -45, -46, - -47, 260, 16, 14, 18, -19, -22, -20, -23, -21, + -47, 264, 16, 14, 18, -19, -22, -20, -23, -21, -29, -30, -28, -25, -27, -176, -182, -26, -186, -24, - -190, -192, -149, -49, 286, 285, 41, 352, 353, 354, - 447, 284, 261, 263, 17, 34, 45, 422, -215, 88, - 607, 262, -217, 15, 739, -6, -3, -2, -162, -166, - -170, -173, -174, -171, -172, -4, -130, 123, 276, 703, - -398, 439, 704, 706, 705, 91, 99, -391, -393, 517, - 291, 443, 449, 701, 733, 736, 606, 608, 309, 623, - 624, 625, 626, 627, 628, 629, 630, 632, 633, 634, - 635, 636, 637, 638, 648, 649, 639, 640, 641, 642, - 643, 644, 645, 646, 650, 651, 652, 653, 654, 655, - 656, 657, 658, 659, 660, 661, 662, 663, 573, 574, - 681, 683, 684, 685, 686, 602, 631, 668, 676, 677, - 678, 420, 421, 614, 698, 738, 303, 327, 472, 333, - 340, 406, 177, 195, 191, 219, 210, 412, 359, 358, - 607, 186, 307, 345, 308, 98, 180, 556, 113, 529, - 501, 183, 365, 368, 366, 367, 322, 324, 326, 603, - 604, 433, 329, 601, 328, 330, 332, 605, 363, 423, - 206, 200, 321, 305, 198, 310, 413, 43, 311, 404, - 403, 224, 312, 313, 618, 525, 419, 531, 337, 55, - 499, 199, 325, 528, 697, 228, 232, 236, 237, 238, - 239, 240, 241, 242, 243, 244, 245, 547, 410, 392, - 393, 394, 548, 415, 168, 169, 533, 409, 550, 414, - 223, 226, 227, 283, 400, 401, 416, 417, 418, 46, - 616, 295, 551, 230, 728, 222, 217, 559, 341, 339, - 405, 221, 194, 216, 306, 68, 234, 233, 235, 495, - 496, 497, 498, 314, 315, 437, 546, 213, 201, 424, - 187, 25, 554, 290, 530, 450, 369, 370, 316, 334, - 342, 364, 229, 231, 297, 302, 357, 411, 617, 503, - 301, 538, 539, 338, 552, 197, 294, 323, 289, 555, - 729, 188, 452, 317, 181, 331, 549, 731, 558, 67, - 163, 193, 184, 719, 720, 280, 682, 178, 299, 304, - 699, 730, 318, 319, 320, 600, 344, 343, 335, 185, - 214, 296, 220, 204, 192, 215, 179, 298, 557, 164, - 695, 422, 482, 212, 209, 300, 273, 700, 553, 532, - 182, 486, 166, 207, 346, 689, 690, 691, 694, 438, - 399, 347, 348, 205, 287, 523, 524, 351, 492, 387, - 466, 502, 473, 467, 251, 252, 355, 535, 537, 225, - 692, 371, 372, 373, 527, 374, 376, 377, 382, 442, - 59, 61, 100, 103, 102, 734, 735, 66, 32, 428, - 431, 464, 468, 389, 696, 615, 386, 390, 391, 432, - 28, 484, 454, 488, 487, 51, 52, 53, 56, 57, - 58, 60, 62, 63, 54, 599, 447, 461, 560, 48, - 50, 457, 458, 30, 434, 483, 505, 385, 485, 516, - 49, 514, 515, 536, 29, 436, 435, 65, 47, 491, - 493, 494, 349, 383, 445, 709, 561, 440, 456, 460, - 441, 388, 430, 462, 70, 453, 710, 448, 446, 384, - 619, 620, 395, 647, 425, 500, 596, 595, 594, 593, - 592, 591, 590, 589, 352, 353, 354, 469, 470, 471, - 481, 474, 475, 476, 477, 478, 479, 480, 519, 520, - 711, 540, 542, 543, 612, 544, 541, 268, 737, 426, - 427, 271, 713, 714, 101, 715, 717, 716, 31, 718, - 727, 724, 725, 726, 622, 545, 609, 721, 611, 610, - 669, 670, 671, 672, 673, -481, -479, -398, 607, 309, - 701, 449, 606, 608, 443, 422, 733, 736, 447, 291, - 352, 353, 354, 517, 420, -269, -398, 737, -94, -17, - -16, -9, -215, -216, -226, 42, -283, -398, 458, -283, - 270, -407, 26, 499, -107, 500, 265, 266, 88, 80, - -398, -10, -121, -8, -128, -92, -213, 504, -405, -398, - 352, 352, 612, -405, 270, -400, 301, 480, -398, -537, - 276, -485, -457, 302, -484, -459, -487, -460, 35, 260, - 262, 261, 621, 298, 18, 447, 272, 16, 15, 448, - 284, 28, 29, 31, 17, 449, 451, 32, 452, 455, - 456, 457, 45, 461, 462, 291, 91, 99, 94, 669, - 670, 671, 672, 673, 309, -268, -398, -433, -425, 120, - -428, -420, -421, -423, -376, -575, -418, 88, 149, 150, - 157, 121, 740, -422, -518, 39, 123, 627, 631, 668, - 571, -368, -369, -370, -371, -372, -373, 613, -398, -576, - -574, 94, 104, 106, 110, 111, 109, 107, 171, 202, - 108, 95, 172, -216, 91, -596, 637, 643, -392, 660, - 683, 684, 685, 686, 659, 64, -544, -552, 269, -550, - 170, 208, 287, 204, 16, 155, 492, 205, 676, 677, - 678, 634, 656, 573, 574, 681, 638, 648, 663, 629, - 630, 632, 624, 625, 626, 628, 639, 641, 655, -553, - 651, 661, 662, 647, 679, 680, 724, 664, 665, 666, - 675, 674, 667, 669, 670, 671, 672, 673, 717, 93, - 92, 654, 653, 640, 635, 636, 642, 623, 633, 644, - 652, 657, 658, 431, 113, 432, 433, 563, 423, 83, - 434, 276, 499, 73, 435, 436, 437, 438, 439, 570, - 440, 74, 441, 430, 291, 482, 442, 207, 225, 576, - 575, 577, 567, 564, 562, 565, 566, 568, 569, 645, - 646, 650, -152, -154, 687, -652, -359, -653, 6, 7, - 8, 9, -654, 172, -643, 501, 617, 94, 563, 270, - 345, 420, 19, 723, 381, 605, 723, 381, 605, 359, - 182, 179, -471, 182, 119, 188, 187, 274, 182, -471, - -398, 185, 723, 184, 719, 612, 355, -447, -199, 420, - 482, 374, 100, 301, -451, -448, 603, -538, 349, 345, - 321, 271, 116, -200, 281, 280, 114, 563, 269, 459, - 340, 59, 61, -226, 275, -604, 597, -603, -398, -612, - -613, 257, 258, 259, 723, 544, 612, 728, 539, 433, - 102, 103, 719, 720, 30, 270, 444, 297, 537, 535, - 536, 540, 541, 542, 543, -72, -554, -536, 532, 531, - -411, 524, 530, 522, 534, 525, 421, 377, 374, 621, - 376, 381, 260, 713, 604, 598, -386, 466, 502, 560, - 561, 445, 503, 547, 549, 526, 113, 211, 208, 271, - 273, 270, 719, 612, 301, 420, 563, 482, 100, 374, - 270, -612, 728, 179, 547, 549, 501, 301, 480, 44, - -478, 492, -477, -479, 548, 559, 92, 93, 546, -386, - 113, 523, 523, -652, -359, -214, -216, -131, -602, 605, - 723, 612, 271, 420, 482, 301, 272, 270, 600, 603, - 273, 563, 269, 352, 444, 297, 374, 381, 100, 184, - 719, -220, -221, -222, 253, 254, 255, 72, 258, 256, - 69, 35, 36, 37, -1, 127, 739, -425, -425, -6, - 742, -6, -425, -398, -398, 174, -290, -294, -291, -293, - -292, 738, -296, -295, 208, 209, 170, 212, 218, 214, - 215, 216, 217, 219, 220, 221, 222, 223, 226, 227, - 224, 34, 225, 287, 204, 205, 206, 207, -299, 191, - 210, 615, 246, 192, 247, 193, 248, 194, 249, 168, - 169, 250, 195, 198, 199, 200, 201, 197, 228, 229, - 230, 231, 232, 233, 234, 235, 237, 236, 238, 239, - 240, 241, 242, 243, 244, 245, 173, -257, 94, 35, - 88, 173, 94, -652, -236, -237, 11, -246, 293, -283, - -275, 173, 740, 19, -283, -374, -398, 501, 130, -107, - 80, -107, 500, 80, -107, 500, 265, -605, -606, -607, - -609, 265, 500, 499, 266, 336, -126, 173, 309, 19, - -405, -405, -398, 86, -283, -459, 301, -485, -457, 39, - 85, 174, 274, 174, 85, 88, 445, 420, 482, 446, - 563, 270, 459, 273, 301, 460, 420, 482, 270, 273, - 563, 301, 420, 270, 273, 482, 301, 460, 420, 522, - 523, 273, 30, 450, 453, 454, 523, -558, 559, 174, - 119, 116, 117, 118, -425, 137, -440, 130, 131, 132, - 133, 134, 135, 136, 144, 143, 156, 149, 150, 151, - 152, 153, 154, 155, 145, 146, 147, 148, 140, 120, - 138, 142, 139, 122, 161, 160, -216, -425, -433, 64, - -423, -423, -423, -423, -398, -518, -430, -425, 88, 88, - 88, 88, 88, 173, 107, 94, 88, -425, 88, 88, + -190, -192, -149, -49, 290, 289, 41, 356, 357, 358, + 451, 288, 265, 267, 17, 34, 45, 426, -215, 88, + 611, 266, -217, 15, 743, -6, -3, -2, -162, -166, + -170, -173, -174, -171, -172, -4, -130, 123, 280, 707, + -398, 443, 708, 710, 709, 91, 99, -391, -393, 521, + 295, 447, 453, 705, 737, 740, 610, 612, 313, 627, + 628, 629, 630, 631, 632, 633, 634, 636, 637, 638, + 639, 640, 641, 642, 652, 653, 643, 644, 645, 646, + 647, 648, 649, 650, 654, 655, 656, 657, 658, 659, + 660, 661, 662, 663, 664, 665, 666, 667, 577, 578, + 685, 687, 688, 689, 690, 606, 635, 672, 680, 681, + 682, 424, 425, 618, 702, 742, 307, 331, 476, 337, + 344, 410, 177, 195, 191, 219, 210, 416, 363, 362, + 611, 186, 311, 349, 312, 98, 180, 560, 113, 533, + 505, 183, 369, 372, 370, 371, 326, 328, 330, 607, + 608, 437, 333, 605, 332, 334, 336, 609, 367, 427, + 206, 200, 325, 309, 198, 314, 417, 43, 315, 408, + 407, 224, 316, 317, 622, 529, 423, 535, 341, 55, + 503, 199, 329, 532, 701, 232, 236, 240, 241, 242, + 243, 244, 245, 246, 247, 248, 249, 551, 414, 396, + 397, 398, 552, 419, 168, 169, 537, 413, 554, 418, + 223, 226, 227, 228, 229, 230, 231, 287, 404, 405, + 420, 421, 422, 46, 620, 299, 555, 234, 732, 222, + 217, 563, 345, 343, 409, 221, 194, 216, 310, 68, + 238, 237, 239, 499, 500, 501, 502, 318, 319, 441, + 550, 213, 201, 428, 187, 25, 558, 294, 534, 454, + 373, 374, 320, 338, 346, 368, 233, 235, 301, 306, + 361, 415, 621, 507, 305, 542, 543, 342, 556, 197, + 298, 327, 293, 559, 733, 188, 456, 321, 181, 335, + 553, 735, 562, 67, 163, 193, 184, 723, 724, 284, + 686, 178, 303, 308, 703, 734, 322, 323, 324, 604, + 348, 347, 339, 185, 214, 300, 220, 204, 192, 215, + 179, 302, 561, 164, 699, 426, 486, 212, 209, 304, + 277, 704, 557, 536, 182, 490, 166, 207, 350, 693, + 694, 695, 698, 442, 403, 351, 352, 205, 291, 527, + 528, 355, 496, 391, 470, 506, 477, 471, 255, 256, + 359, 539, 541, 225, 696, 375, 376, 377, 531, 378, + 380, 381, 386, 446, 59, 61, 100, 103, 102, 738, + 739, 66, 32, 432, 435, 468, 472, 393, 700, 619, + 390, 394, 395, 436, 28, 488, 458, 492, 491, 51, + 52, 53, 56, 57, 58, 60, 62, 63, 54, 603, + 451, 465, 564, 48, 50, 461, 462, 30, 438, 487, + 509, 389, 489, 520, 49, 518, 519, 540, 29, 440, + 439, 65, 47, 495, 497, 498, 353, 387, 449, 713, + 565, 444, 460, 464, 445, 392, 434, 466, 70, 457, + 714, 452, 450, 388, 623, 624, 399, 651, 429, 504, + 600, 599, 598, 597, 596, 595, 594, 593, 356, 357, + 358, 473, 474, 475, 485, 478, 479, 480, 481, 482, + 483, 484, 523, 524, 715, 544, 546, 547, 616, 548, + 545, 272, 741, 430, 431, 275, 717, 718, 101, 719, + 721, 720, 31, 722, 731, 728, 729, 730, 626, 549, + 613, 725, 615, 614, 673, 674, 675, 676, 677, -481, + -479, -398, 611, 313, 705, 453, 610, 612, 447, 426, + 737, 740, 451, 295, 356, 357, 358, 521, 424, -269, + -398, 741, -94, -17, -16, -9, -215, -216, -226, 42, + -283, -398, 462, -283, 274, -407, 26, 503, -107, 504, + 269, 270, 88, 80, -398, -10, -121, -8, -128, -92, + -213, 508, -405, -398, 356, 356, 616, -405, 274, -400, + 305, 484, -398, -537, 280, -485, -457, 306, -484, -459, + -487, -460, 35, 264, 266, 265, 625, 302, 18, 451, + 276, 16, 15, 452, 288, 28, 29, 31, 17, 453, + 455, 32, 456, 459, 460, 461, 45, 465, 466, 295, + 91, 99, 94, 673, 674, 675, 676, 677, 313, -268, + -398, -433, -425, 120, -428, -420, -421, -423, -376, -575, + -418, 88, 149, 150, 157, 121, 744, -422, -518, 39, + 123, 631, 635, 672, 575, -368, -369, -370, -371, -372, + -373, 617, -398, -576, -574, 94, 104, 106, 110, 111, + 109, 107, 171, 202, 108, 95, 172, -216, 91, -596, + 641, 647, -392, 664, 687, 688, 689, 690, 663, 64, + -544, -552, 273, -550, 170, 208, 291, 204, 16, 155, + 496, 205, 680, 681, 682, 638, 660, 577, 578, 685, + 642, 652, 667, 633, 634, 636, 628, 629, 630, 632, + 643, 645, 659, -553, 655, 665, 666, 651, 683, 684, + 728, 668, 669, 670, 679, 678, 671, 673, 674, 675, + 676, 677, 721, 93, 92, 658, 657, 644, 639, 640, + 646, 627, 637, 648, 656, 661, 662, 435, 113, 436, + 437, 567, 427, 83, 438, 280, 503, 73, 439, 440, + 441, 442, 443, 574, 444, 74, 445, 434, 295, 486, + 446, 207, 225, 580, 579, 581, 571, 568, 566, 569, + 570, 572, 573, 649, 650, 654, -152, -154, 691, -652, + -359, -653, 6, 7, 8, 9, -654, 172, -643, 505, + 621, 94, 567, 274, 349, 424, 19, 727, 385, 609, + 727, 385, 609, 363, 182, 179, -471, 182, 119, 188, + 187, 278, 182, -471, -398, 185, 727, 184, 723, 616, + 359, -447, -199, 424, 486, 378, 100, 305, -451, -448, + 607, -538, 353, 349, 325, 275, 116, -200, 285, 284, + 114, 567, 273, 463, 344, 59, 61, -226, 279, -604, + 601, -603, -398, -612, -613, 261, 262, 263, 727, 548, + 616, 732, 543, 437, 102, 103, 723, 724, 30, 274, + 448, 301, 541, 539, 540, 544, 545, 546, 547, -72, + -554, -536, 536, 535, -411, 528, 534, 526, 538, 529, + 425, 381, 378, 625, 380, 385, 264, 717, 608, 602, + -386, 470, 506, 564, 565, 449, 507, 551, 553, 530, + 113, 211, 208, 275, 277, 274, 723, 616, 305, 424, + 567, 486, 100, 378, 274, -612, 732, 179, 551, 553, + 505, 305, 484, 44, -478, 496, -477, -479, 552, 563, + 92, 93, 550, -386, 113, 527, 527, -652, -359, -214, + -216, -131, -602, 609, 727, 616, 275, 424, 486, 305, + 276, 274, 604, 607, 277, 567, 273, 356, 448, 301, + 378, 385, 100, 184, 723, -220, -221, -222, 257, 258, + 259, 72, 262, 260, 69, 35, 36, 37, -1, 127, + 743, -425, -425, -6, 746, -6, -425, -398, -398, 174, + -290, -294, -291, -293, -292, 742, -296, -295, 208, 209, + 170, 212, 218, 214, 215, 216, 217, 219, 220, 221, + 222, 223, 226, 227, 228, 229, 230, 231, 224, 34, + 225, 291, 204, 205, 206, 207, -299, 191, 210, 619, + 250, 192, 251, 193, 252, 194, 253, 168, 169, 254, + 195, 198, 199, 200, 201, 197, 232, 233, 234, 235, + 236, 237, 238, 239, 241, 240, 242, 243, 244, 245, + 246, 247, 248, 249, 173, -257, 94, 35, 88, 173, + 94, -652, -236, -237, 11, -246, 297, -283, -275, 173, + 744, 19, -283, -374, -398, 505, 130, -107, 80, -107, + 504, 80, -107, 504, 269, -605, -606, -607, -609, 269, + 504, 503, 270, 340, -126, 173, 313, 19, -405, -405, + -398, 86, -283, -459, 305, -485, -457, 39, 85, 174, + 278, 174, 85, 88, 449, 424, 486, 450, 567, 274, + 463, 277, 305, 464, 424, 486, 274, 277, 567, 305, + 424, 274, 277, 486, 305, 464, 424, 526, 527, 277, + 30, 454, 457, 458, 527, -558, 563, 174, 119, 116, + 117, 118, -425, 137, -440, 130, 131, 132, 133, 134, + 135, 136, 144, 143, 156, 149, 150, 151, 152, 153, + 154, 155, 145, 146, 147, 148, 140, 120, 138, 142, + 139, 122, 161, 160, -216, -425, -433, 64, -423, -423, + -423, -423, -398, -518, -430, -425, 88, 88, 88, 88, + 88, 173, 107, 94, 88, -425, 88, 88, 88, 88, + 88, 88, 88, 88, 88, 88, 88, 88, 88, -551, + 88, 88, -437, -438, 88, 88, -418, -374, 88, 94, + 94, 88, 88, 88, 94, 88, 88, 88, -438, -438, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, - 88, -551, 88, 88, -437, -438, 88, 88, -418, -374, - 88, 94, 94, 88, 88, 88, 94, 88, 88, 88, - -438, -438, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, 88, - 88, 88, 88, 88, 88, 88, 88, 88, -237, 174, - -236, 88, -236, -237, -217, -216, 35, 36, 35, 36, - 35, 36, 35, 36, -655, 710, 88, 104, 734, 251, - -250, -398, -251, -398, -160, 19, 740, -398, 719, -635, - 35, 612, 375, 612, 612, 375, 612, 260, 18, 363, - 57, 364, 552, 14, 186, 187, 188, -398, 185, 274, - -398, -445, 276, -445, -445, -445, -267, -398, 297, 444, - 273, 600, 273, -200, -445, 19, -445, -445, -445, -445, - 272, -445, 26, 270, 270, 270, 270, -445, 570, 130, - 130, 62, -246, -226, 174, -604, -245, 88, -614, 190, - -636, -635, 545, 729, 730, 731, 85, -410, 138, 142, - -410, -355, 20, -355, 26, 26, 299, 299, 299, -410, - 339, -663, -664, 19, 140, -408, -664, -408, -408, -410, - -665, 272, 533, 46, 300, 299, -238, -239, 24, -238, - 527, 523, -502, 528, 529, -412, -664, -411, -410, -410, - -411, -410, -410, 380, -410, 35, 375, 376, 270, 273, - 563, 374, 714, -663, -663, 34, 34, -537, -537, -283, - -537, -398, 276, -460, -537, 598, -387, -398, -537, -537, - -537, -338, -339, -283, -615, 275, 731, -649, -648, 550, - -651, 552, 179, -479, 179, -479, 91, -459, 301, 301, - 174, 130, 26, -480, 130, 141, -479, -479, -480, -480, - -308, 44, -397, 170, -398, 94, -308, 44, -646, -645, - -283, -237, -217, -216, 89, 89, 89, 612, -537, -537, - -537, -537, -537, -537, -537, -538, -537, -537, -537, -537, - -537, -405, -258, -398, -269, 276, -537, 375, -537, -537, - -537, -218, -219, 151, -425, -398, -222, -3, -164, -163, - 124, 125, 127, 704, 439, 703, 707, 701, -479, 44, - -531, 164, 163, 88, -525, -527, 88, -526, 88, -526, - -526, -526, -526, -526, 88, 88, -528, 88, -528, -528, - -525, -529, 203, 88, -529, -530, 88, -530, -529, -398, - -506, 14, -431, -433, -398, 42, -237, -155, 42, -239, - 23, -548, 64, -213, 88, 34, 88, -398, 205, 184, - 718, 38, 100, 173, 104, 94, -126, -107, 80, -126, - -107, -107, 89, 174, -608, 110, 111, -610, 94, 223, - 214, -398, -124, 94, -574, -7, -12, -8, -10, -11, - -53, -92, -213, 606, 608, -577, -575, 88, 35, 491, - 85, 19, -486, 270, 563, 444, 297, 273, 420, -484, - -466, -463, -461, -397, -459, -462, -461, -489, -374, 523, - -156, 506, 505, 351, -425, -425, -425, -425, -425, 109, - 120, 399, 110, 111, -420, -441, 35, 347, 348, -421, - -421, -421, -421, -421, -421, -421, -421, -421, -421, -421, - -421, -423, -423, -429, -439, -518, 88, 140, 138, 142, - 139, 122, -423, -423, -421, -421, -288, -290, 163, 164, - -310, -397, 170, 89, 174, -425, -601, -600, 124, -425, - -425, -425, -425, -452, -454, -374, 88, -398, -421, -597, - -598, 578, 579, 580, 581, 582, 583, 584, 585, 586, - 587, 588, 435, 430, 436, 434, 423, 442, 437, 438, - 207, 595, 596, 589, 590, 591, 592, 593, 594, -431, - -431, -425, -597, -421, -431, -367, 36, 35, -433, -433, - -433, 89, -425, -611, 397, 396, 398, -241, -398, -431, - 89, 89, 89, 104, -433, -433, -431, -421, -431, -431, - -431, -431, -598, -598, -599, 287, 204, 206, 205, -367, - -367, -367, -367, 151, -433, -433, -367, -367, -367, -367, - 151, -367, -367, -367, -367, -367, -367, -367, -367, -367, - -367, -367, 89, 89, 89, 89, -425, 89, -425, -425, - -425, -425, -425, 151, -433, -238, -154, -556, -555, -425, - 44, -155, -239, -656, 711, 88, -374, -644, 94, 94, - 740, -160, 173, 19, 270, -160, 173, 719, 184, -160, - 563, 19, -398, -398, 94, 104, -398, 94, 104, 270, - 563, 270, 563, -283, -283, -283, 553, 554, 183, 187, - 186, -398, 185, -398, -398, 120, -398, -398, -398, 38, - -269, -258, -445, -445, -445, -619, -398, 95, 94, -467, - -464, -461, -398, -398, -457, -398, -387, -283, -445, -445, - -445, -445, -283, -319, 56, 57, 58, -461, -201, 59, - 60, -547, 64, -213, 88, 34, -246, -603, 38, -244, - -398, -615, -141, 26, 301, -355, -423, -423, -425, 420, - 563, 270, -461, 301, -663, -410, -410, -388, -387, -412, - -407, -412, -412, -355, -408, -410, -410, -425, -412, -408, - -355, -398, 523, -355, -355, -502, -387, -410, 94, -409, - -398, -409, -445, -387, -388, -388, -283, -283, -333, -340, - -334, -341, 293, 267, 428, 429, 263, 261, 11, 262, - -349, 340, -446, 571, -314, -315, 80, 45, -317, 291, - 468, 464, 303, 307, 98, 308, 501, 309, 272, 311, - 312, 313, 328, 330, 283, 314, 315, 316, 492, 317, - 178, 329, 318, 319, 320, 446, -309, 6, 382, 44, - 54, 55, 515, 514, 619, 14, 304, -398, 471, 608, - 34, 39, 263, 267, 262, -619, -617, 34, -398, 34, - -467, -461, -398, -398, 174, 274, -229, -231, -228, -224, - -225, -230, -358, -360, -227, 88, -283, -216, -398, -479, - 174, 551, 553, 554, -649, -480, -649, -480, 274, 35, - 491, -483, 491, 35, -457, -477, 547, 549, -472, 94, - 492, -462, -482, 85, 170, -555, -480, -480, -482, -482, - 160, 174, -647, 552, 553, 257, -238, 104, -637, -635, - -398, 612, -398, -285, -283, -619, -466, -457, -398, -537, - -285, -285, -285, -400, -400, 88, 173, 39, -398, -537, - -398, -398, -398, -354, 174, -353, 19, -399, -398, 38, - 94, 173, -165, -163, 126, -425, -6, 703, -425, -6, - -6, -425, -6, -425, -535, 166, -290, 104, 104, -377, - 94, -377, 104, -529, 104, 104, 622, 89, 94, -238, - 688, -240, 23, -235, -234, -425, -549, -434, -595, 687, - -248, 89, -241, -593, -594, -241, -247, -398, -275, 130, - 130, 130, 27, -537, -398, 26, -126, -107, -606, 173, - 174, -244, -486, -465, -462, -488, 151, -398, -473, 174, - 14, 743, 92, 274, -632, -631, 483, 89, 174, -559, - 275, 570, 94, 740, 499, 251, 252, 109, 399, 110, - 111, -518, -433, -429, -423, -423, -421, -421, -427, 288, - -427, 119, -298, 169, 168, -298, -425, 741, -424, -600, - 126, -425, 38, 174, 38, 174, 86, 174, 89, -525, - -425, 173, 174, 89, 89, 19, 19, 140, 89, -425, - 89, 89, 89, 89, 19, 19, -425, 89, 173, 89, - 89, 89, 89, 86, 89, 174, 89, 89, 89, 89, - 174, 174, 174, -433, -433, -425, -433, 89, 89, 89, - -425, -425, -425, -433, 89, -425, -425, -425, -425, -425, - -425, -425, -425, -425, -425, -244, -496, 518, -496, -496, - -496, 89, -496, 89, 174, 89, 174, 89, 89, 174, - 174, 174, 174, 89, -240, 88, 104, 174, 735, -381, - -380, 94, -161, 274, -398, 719, -398, -161, -398, -398, - 130, -161, -398, 719, 94, 94, -283, -387, -283, -387, - 614, 42, 42, 184, 188, 188, 187, -398, 94, 39, - 26, 26, 338, -136, 609, -268, 88, 88, -283, -283, - -283, -621, 469, -398, -633, 174, 44, -631, 563, -197, - 351, -449, 86, -204, 358, 19, 14, -283, -283, -283, - -283, -297, 38, -470, 85, -549, -248, 89, -593, -547, - 88, 89, 174, 19, -223, -284, -398, -143, 24, -398, - -460, -398, -398, -398, -458, 86, -398, -388, -355, -355, - -412, -355, -355, 174, 25, -410, -412, -412, -275, -408, - -275, 173, -275, -387, -524, 38, -245, 174, 23, 293, - -282, -395, -279, -281, 278, -415, -280, 281, -589, 279, - 277, 114, 282, 336, 115, 272, -395, -395, 278, -318, - 274, 38, -395, -336, 272, 402, 336, 279, 23, 293, - -335, 272, 115, -398, 278, 282, 279, 277, -394, 130, - -386, 160, 274, 46, 446, -394, 620, 293, -394, -394, - -394, -394, -394, -394, -394, 310, 310, -394, -394, -394, - -394, -394, -394, -394, -394, -394, -394, -394, 179, -394, - -394, -394, -394, -394, -394, 88, 305, 306, 338, 609, - 124, 622, 611, -460, 274, 538, 538, -622, 469, 34, - 426, 426, 427, -633, 422, 45, 34, -205, 420, -339, - -337, -409, 34, -361, -362, -363, -364, -366, -365, 71, - 75, 77, 81, 72, 73, 74, 526, 78, 83, 76, - 34, 174, -396, -401, 38, -398, 94, -396, -216, -231, - -229, -396, 88, -480, -648, -650, 555, 552, 558, -482, - -482, 104, 274, 88, 130, -482, -482, 44, -397, -645, - 559, 553, -240, -265, 721, 174, 85, -285, -259, -260, - -261, -262, -290, -374, 738, 209, 212, 214, 215, 216, - 217, 219, 220, 221, 222, 223, 226, 227, 224, 225, - 287, 204, 205, 206, 207, 191, 210, 615, 192, 193, - 194, 168, 169, 195, 198, 199, 200, 201, 197, 228, - 229, 230, 231, 232, 233, 234, 235, 237, 236, 238, - 239, 240, 241, 242, 243, 244, 245, -398, -269, 94, - 19, -265, -355, -219, -231, -398, 94, -398, 151, 127, - -6, 125, -169, -168, -167, 128, 701, 707, 127, 127, - 127, 89, 89, 89, 89, 174, 89, 89, 89, 174, - 89, 174, 104, -562, 528, -240, 94, -155, 664, 174, - -232, 40, 41, 174, 88, 89, 174, 64, 174, 130, - 89, 174, -425, -398, 94, -425, 205, 94, 173, 501, - -398, -575, 89, -488, 174, 274, 173, 173, -463, 449, - -397, -465, 23, 14, -374, 42, -381, 130, 740, -398, - 89, -427, -427, 119, -423, -420, 89, 127, -425, 125, - -288, -425, -288, -289, -295, 170, 208, 287, 207, 206, - 204, 163, 164, -308, -454, 614, -232, 89, -398, -433, - -425, -425, -421, 89, -425, -425, 19, -398, -308, -421, - -425, -425, -425, -237, -237, 89, 89, -495, -496, -495, - -495, 89, 89, 89, 89, -495, 89, 89, 89, 89, - 89, 89, 89, 89, 89, 89, 89, 88, -496, -496, - -425, -496, -425, -496, -496, -425, 104, 106, 104, 106, - -555, -155, -657, 66, 709, 65, 491, 109, 341, 174, - 104, 94, 741, 174, 130, 420, -398, 19, 173, 94, - -398, 94, 19, 270, -398, 19, 19, -283, -283, -283, - 188, 94, -634, 345, 420, 563, 270, 420, 345, 563, - 270, -507, 104, -137, 124, 94, 457, -270, -271, -272, - -273, -274, 140, 175, 176, -259, -245, 88, -245, -624, - 530, 471, 481, -394, 374, -417, -416, 422, 45, -542, - 492, 477, 478, -464, 301, -387, 151, -630, 101, 130, - 85, 386, 390, 392, 394, 393, 391, 387, 388, 389, - -443, -444, -442, -446, -387, 94, -617, 88, 88, -213, - 38, 138, -204, 358, 19, 88, 88, 38, -519, 371, - -290, 43, 89, 64, -1, -398, -283, -223, -398, 19, - 174, -616, 173, 104, -398, -457, -410, -355, -425, -425, - -355, -410, -410, -412, -398, -275, -519, -290, 38, -334, - 267, 262, -492, 338, 339, -493, -509, 341, -511, 88, - -287, -374, -280, -588, -589, -445, -398, 115, -588, 115, - 88, -287, -374, -374, -337, -374, -398, -398, -398, -398, - -344, -343, -374, -347, 35, -348, -398, -398, -398, -398, - 115, -398, 115, -313, 44, 51, 52, 53, -394, -394, - 211, -316, 44, 491, 493, 494, -347, 104, 104, 104, - 104, 94, 94, 94, -394, -394, 104, 94, -401, 94, - -590, 187, 48, 49, 104, 104, 104, 104, 44, 94, - -321, 44, 321, 325, 322, 323, 324, 94, 104, 44, - 104, 44, 104, 44, -398, 88, -591, -592, 94, -507, - 94, 88, 104, 94, 263, -460, 94, 85, -624, -394, - 426, -479, 130, 130, -417, -626, 98, 472, -626, -629, - 351, -207, 563, 35, -249, 267, 262, -617, -469, -468, - -374, -228, -228, -228, -228, -228, -228, 71, 82, 71, - -242, 88, 71, 76, 71, 76, 71, 76, 71, -363, - 71, 82, -469, -230, -245, -401, 89, -642, -641, -640, - -638, 79, 275, 80, -431, -482, 552, 556, 557, -465, - -413, 94, -472, -155, -283, -283, -540, 331, 332, 89, - 174, -290, -398, -357, 21, 173, 123, -6, -165, -167, - -425, -6, -425, 703, 439, 704, 94, 104, 104, -570, - 512, 507, 509, -155, -571, 499, 14, -234, -233, 47, - -434, -557, -556, 64, -213, -241, -549, -594, -555, -398, - 741, 741, 741, 741, 94, -398, 104, 19, -462, -457, - 151, 151, -398, 450, -473, 94, 470, 94, 270, 741, - 94, -381, -420, -425, 89, 38, 89, 89, -526, -526, - -525, -528, -525, -298, -298, 89, 88, -232, 89, 89, - 26, 89, 89, 89, 89, -425, 89, 89, 174, 174, - 89, -545, 572, -546, 649, -495, -495, -495, -495, -495, - -495, -495, -495, -495, -495, -495, -495, -495, -495, -495, - -495, -495, -436, -435, 293, 89, 174, 89, 174, 89, - 513, 716, 716, 513, 716, 716, 89, 174, -597, 174, - -389, 346, -389, -380, 94, -398, 94, 719, -398, 741, - 741, 719, -398, 94, -283, -387, -252, 529, -210, 124, - -211, 122, 46, 94, -398, 19, -398, -398, 338, -398, - 338, -398, -398, 94, -142, 622, 88, -139, 610, 94, - 89, 174, -374, 89, 38, -276, -277, -278, -287, -279, - -281, 38, -625, 98, -620, 94, -398, 95, -398, -626, - 172, 424, 44, 473, 474, 489, 419, 104, 104, 479, - -618, -398, -206, 270, 420, -206, -628, 55, 130, 94, - -283, -442, -386, 160, 312, -275, -398, 374, -352, -351, - -398, 94, -276, -213, -283, -283, 94, -276, -276, -213, - -520, 373, 23, 104, 150, 115, 64, -213, -549, 89, - -246, 86, 173, -231, -284, -398, 151, -355, -275, -355, - -355, -410, -520, -213, -504, 342, 88, -502, 88, -502, - 115, 387, -512, -510, 293, -342, 48, 50, -290, -586, - -398, -584, -586, -398, -584, -584, -445, -425, -342, -287, - 274, 34, 262, -345, 390, 384, 385, 390, 392, 394, - 393, -474, 337, 120, -474, 174, -232, 174, -398, -308, - -308, 34, 94, 94, -285, 89, 174, 130, 94, -139, - -138, -425, -214, -216, 274, 85, 270, -625, -620, 130, - -480, 94, 94, -626, 94, 94, -630, 130, -286, 270, - -387, 174, -249, -249, -355, 19, 174, 130, -254, -253, - 85, 86, -255, 85, -253, -253, 71, -243, 94, 71, - 71, 71, -355, -640, -639, 26, -589, -589, -589, 89, - 89, -256, 26, -261, 44, 374, -356, 22, 23, 151, - 127, 125, 127, 127, -398, 89, 89, -532, 689, -566, - -568, 507, 23, 23, -256, -572, 694, 94, 450, 48, - 49, 89, -549, 741, -457, -473, 492, -283, 174, 741, - -288, -327, 94, -425, 89, -425, -425, 89, 94, 89, - 94, -237, 23, -496, -425, -496, -425, -496, 89, 174, - 89, 89, 89, 174, 89, 89, -425, 89, -597, -390, - 205, 94, -390, -398, -398, 19, -399, -209, 274, -275, - -212, 369, 88, 365, -210, 184, 88, 94, -398, 19, - -398, -507, 338, -507, 338, 270, -398, -265, -140, 611, - 104, -138, 94, -450, 616, -272, -290, 268, -213, 89, - 174, -213, 94, -623, 483, -508, 379, 104, 44, 104, - 172, 475, -543, -198, 98, -285, 35, -249, -198, -627, - 98, 130, 740, 88, -394, -394, -394, -209, 374, -398, - 89, 174, -394, -394, 89, -209, -398, 89, 89, -306, - 14, -521, 292, 104, 150, 104, 150, 104, 17, 275, - -549, -396, -231, -398, -355, -616, 173, -355, -521, -494, - 343, 104, -421, 88, -421, 88, -503, 340, 88, 89, - 174, -398, -374, -303, -302, -300, 109, 120, 44, 464, - -301, 98, 160, 326, 329, 328, 304, 327, -332, -414, - 85, 682, 467, 384, 385, -446, 689, 602, 697, 38, - 277, 114, 115, 451, -415, 88, 88, 86, 346, 88, - 88, -586, 89, -342, -374, 44, -345, 44, -346, 408, - -455, -455, -455, -455, 337, -343, -398, 160, -308, 89, - -592, 94, 89, -460, 270, -398, -623, 94, -482, -628, - 94, -198, -285, -617, -237, -231, -468, -555, -425, 88, - -425, 89, 88, 71, 11, 21, 17, -418, -398, -425, - -433, 724, 726, 727, 276, -6, 704, 439, -323, 690, - 94, 23, 94, -564, 94, -562, 94, -433, -158, -320, - -386, 309, 89, -326, 140, 14, 89, 89, 89, -495, - -495, -498, -497, -501, 513, 338, 521, -433, 89, 89, - 94, 94, 89, 89, 94, 94, 94, 719, 420, -209, - 38, 457, 24, 628, 370, -244, 366, 367, 368, -398, - 94, -433, -214, 740, 374, -398, 19, 94, -507, 94, - -507, -398, 338, 38, 94, 89, 94, 94, -263, -290, - -202, 14, -306, -278, -202, 23, 14, 172, 423, 44, - 104, 44, 476, 94, -206, 130, 110, 111, -382, -383, - 94, -452, -308, -310, 94, -398, -351, -418, -418, -304, - -213, 38, -305, -349, -446, 374, -157, -156, -304, 88, - -522, 178, 104, 150, 104, 104, -469, -355, -355, -522, - -511, 23, 89, -489, 89, -489, 88, 130, -421, -510, - -513, 64, -300, 109, -421, 94, -310, -311, 44, 325, - 321, 130, 130, -312, 44, 305, 306, -322, 88, 336, - 17, 104, 211, 88, 698, 88, 115, 115, -283, -452, - -452, -587, 386, 387, 388, 395, 390, 391, 389, 392, - 393, 394, -587, -452, -452, 88, -475, -474, -421, -455, - 130, -456, 283, 400, 401, 98, 14, 384, 385, 405, - 404, 403, 409, 410, 414, 415, 411, 413, 412, 416, - 417, 418, 406, 407, 408, 423, 434, -394, 160, -398, - 173, -627, -238, -355, -244, -585, -398, 277, 23, 23, - -541, 14, 725, 88, 88, -398, -398, -378, 691, 104, - 94, 509, -570, -533, 692, -560, -502, -308, 130, 89, - 78, 615, 617, 89, -500, 122, 475, 479, -419, -422, - 104, 106, 202, 172, -496, -496, 89, 89, -398, -398, - -283, 94, 104, 89, 119, 119, 89, 89, -385, -384, - 94, -398, 374, -398, -265, 94, -265, 94, 338, -507, - -2, 616, -203, 63, 559, 94, 95, 470, 94, 95, - 104, 423, -198, 94, 741, 174, 130, 89, -508, -490, - 293, -213, 174, -349, -386, -398, -158, -490, -307, -350, - -398, 94, -539, 187, 372, 14, 104, 150, 104, -237, - -523, 187, 372, -493, 89, 89, 89, -489, 104, 89, - -517, -514, 88, -349, 295, 140, 94, 94, 104, 88, - -550, 34, 94, 38, -425, -453, 88, 89, 89, 89, - 89, -452, 110, 111, -394, -394, 94, 94, 383, -394, - -394, -394, -394, -394, -394, 88, 94, 94, -394, -394, - -394, -394, 130, -394, -394, -308, -394, 173, -398, 89, - 89, 174, 727, 88, -433, -433, 88, 23, -532, -534, - 693, 94, -569, 512, -563, -561, 507, 508, 509, 510, - 94, 616, 68, 618, -499, -500, 479, -419, -422, 687, - 519, 519, 519, 94, -398, 94, 741, 174, 130, -398, - 374, -265, -265, -507, 94, -266, -398, 336, 492, -383, - 94, -455, -491, 345, 23, -349, -394, -508, -491, 89, - 174, -394, -394, 372, 104, 150, 104, -238, 372, -505, - 344, 89, -517, -349, -516, -515, 343, 296, 88, 89, - -425, -437, -394, 89, 88, 89, -325, -324, 613, -452, - -455, 86, -455, 86, -455, 86, -455, 86, 89, 104, - 104, -398, 104, 104, 104, 104, 104, 104, -489, 104, - 104, 104, 104, 110, 111, 104, 104, -308, -398, -398, - 277, -153, 88, 89, 89, -379, -398, -564, -323, 94, - -573, 275, -567, -568, 511, -561, 23, 509, 23, 23, - -159, 174, 68, 119, 520, 520, 520, -210, -211, -210, - -211, -265, -384, 94, -398, 94, -265, -264, 38, 514, - 450, 23, -492, -308, -350, -418, -418, 104, 104, 89, - 174, -398, 292, 88, -432, -426, -425, 292, 89, -398, - -425, -476, 700, 699, -331, -329, -330, 85, 526, 334, - 335, 89, -587, -587, -587, -587, -332, 89, 89, 174, - -431, 89, 174, -378, -580, 88, 104, -566, -565, -567, - 23, -564, 23, -564, -564, 516, 14, -499, -210, -210, - -265, 94, -374, 88, -504, -515, -514, -432, 89, 174, - -474, 89, -330, 85, -329, 85, 18, 17, -455, -455, - -455, -455, 88, 89, -398, -583, 34, 89, -579, -578, - -375, -574, -398, 512, 513, 94, -564, 130, 617, -660, - -659, 715, -489, -494, 89, -426, -476, -328, 331, 332, - 34, 187, -328, -431, -582, -581, -376, 89, 174, 173, - 94, 618, 94, 89, -511, 109, 44, 333, 89, 174, - 130, -578, -398, -581, 44, -425, 173, -398, + 88, 88, 88, 88, 88, 88, -237, 174, -236, 88, + -236, -237, -217, -216, 35, 36, 35, 36, 35, 36, + 35, 36, -655, 714, 88, 104, 738, 255, -250, -398, + -251, -398, -160, 19, 744, -398, 723, -635, 35, 616, + 379, 616, 616, 379, 616, 264, 18, 367, 57, 368, + 556, 14, 186, 187, 188, -398, 185, 278, -398, -445, + 280, -445, -445, -445, -267, -398, 301, 448, 277, 604, + 277, -200, -445, 19, -445, -445, -445, -445, 276, -445, + 26, 274, 274, 274, 274, -445, 574, 130, 130, 62, + -246, -226, 174, -604, -245, 88, -614, 190, -636, -635, + 549, 733, 734, 735, 85, -410, 138, 142, -410, -355, + 20, -355, 26, 26, 303, 303, 303, -410, 343, -663, + -664, 19, 140, -408, -664, -408, -408, -410, -665, 276, + 537, 46, 304, 303, -238, -239, 24, -238, 531, 527, + -502, 532, 533, -412, -664, -411, -410, -410, -411, -410, + -410, 384, -410, 35, 379, 380, 274, 277, 567, 378, + 718, -663, -663, 34, 34, -537, -537, -283, -537, -398, + 280, -460, -537, 602, -387, -398, -537, -537, -537, -338, + -339, -283, -615, 279, 735, -649, -648, 554, -651, 556, + 179, -479, 179, -479, 91, -459, 305, 305, 174, 130, + 26, -480, 130, 141, -479, -479, -480, -480, -308, 44, + -397, 170, -398, 94, -308, 44, -646, -645, -283, -237, + -217, -216, 89, 89, 89, 616, -537, -537, -537, -537, + -537, -537, -537, -538, -537, -537, -537, -537, -537, -405, + -258, -398, -269, 280, -537, 379, -537, -537, -537, -218, + -219, 151, -425, -398, -222, -3, -164, -163, 124, 125, + 127, 708, 443, 707, 711, 705, -479, 44, -531, 164, + 163, 88, -525, -527, 88, -526, 88, -526, -526, -526, + -526, -526, -526, -526, -526, -526, 88, 88, -528, 88, + -528, -528, -525, -529, 203, 88, -529, -530, 88, -530, + -529, -398, -506, 14, -431, -433, -398, 42, -237, -155, + 42, -239, 23, -548, 64, -213, 88, 34, 88, -398, + 205, 184, 722, 38, 100, 173, 104, 94, -126, -107, + 80, -126, -107, -107, 89, 174, -608, 110, 111, -610, + 94, 223, 214, -398, -124, 94, -574, -7, -12, -8, + -10, -11, -53, -92, -213, 610, 612, -577, -575, 88, + 35, 495, 85, 19, -486, 274, 567, 448, 301, 277, + 424, -484, -466, -463, -461, -397, -459, -462, -461, -489, + -374, 527, -156, 510, 509, 355, -425, -425, -425, -425, + -425, 109, 120, 403, 110, 111, -420, -441, 35, 351, + 352, -421, -421, -421, -421, -421, -421, -421, -421, -421, + -421, -421, -421, -423, -423, -429, -439, -518, 88, 140, + 138, 142, 139, 122, -423, -423, -421, -421, -288, -290, + 163, 164, -310, -397, 170, 89, 174, -425, -601, -600, + 124, -425, -425, -425, -425, -452, -454, -374, 88, -398, + -421, -597, -598, 582, 583, 584, 585, 586, 587, 588, + 589, 590, 591, 592, 439, 434, 440, 438, 427, 446, + 441, 442, 207, 599, 600, 593, 594, 595, 596, 597, + 598, -431, -431, -425, -597, -421, -431, -367, 36, 35, + -433, -433, -433, 89, -425, -611, 401, 400, 402, -241, + -398, -431, 89, 89, 89, 104, -433, -433, -431, -421, + -431, -431, -431, -431, -598, -598, -599, 291, 204, 206, + 205, -367, -367, -367, -367, 151, -433, -433, -367, -367, + -367, -367, 151, -367, -367, -367, -367, -367, -367, -367, + -367, -367, -367, -367, 89, 89, 89, 89, -425, 89, + -425, -425, -425, -425, -425, 151, -433, -238, -154, -556, + -555, -425, 44, -155, -239, -656, 715, 88, -374, -644, + 94, 94, 744, -160, 173, 19, 274, -160, 173, 723, + 184, -160, 567, 19, -398, -398, 94, 104, -398, 94, + 104, 274, 567, 274, 567, -283, -283, -283, 557, 558, + 183, 187, 186, -398, 185, -398, -398, 120, -398, -398, + -398, 38, -269, -258, -445, -445, -445, -619, -398, 95, + 94, -467, -464, -461, -398, -398, -457, -398, -387, -283, + -445, -445, -445, -445, -283, -319, 56, 57, 58, -461, + -201, 59, 60, -547, 64, -213, 88, 34, -246, -603, + 38, -244, -398, -615, -141, 26, 305, -355, -423, -423, + -425, 424, 567, 274, -461, 305, -663, -410, -410, -388, + -387, -412, -407, -412, -412, -355, -408, -410, -410, -425, + -412, -408, -355, -398, 527, -355, -355, -502, -387, -410, + 94, -409, -398, -409, -445, -387, -388, -388, -283, -283, + -333, -340, -334, -341, 297, 271, 432, 433, 267, 265, + 11, 266, -349, 344, -446, 575, -314, -315, 80, 45, + -317, 295, 472, 468, 307, 311, 98, 312, 505, 313, + 276, 315, 316, 317, 332, 334, 287, 318, 319, 320, + 496, 321, 178, 333, 322, 323, 324, 450, -309, 6, + 386, 44, 54, 55, 519, 518, 623, 14, 308, -398, + 475, 612, 34, 39, 267, 271, 266, -619, -617, 34, + -398, 34, -467, -461, -398, -398, 174, 278, -229, -231, + -228, -224, -225, -230, -358, -360, -227, 88, -283, -216, + -398, -479, 174, 555, 557, 558, -649, -480, -649, -480, + 278, 35, 495, -483, 495, 35, -457, -477, 551, 553, + -472, 94, 496, -462, -482, 85, 170, -555, -480, -480, + -482, -482, 160, 174, -647, 556, 557, 261, -238, 104, + -637, -635, -398, 616, -398, -285, -283, -619, -466, -457, + -398, -537, -285, -285, -285, -400, -400, 88, 173, 39, + -398, -537, -398, -398, -398, -354, 174, -353, 19, -399, + -398, 38, 94, 173, -165, -163, 126, -425, -6, 707, + -425, -6, -6, -425, -6, -425, -535, 166, -290, 104, + 104, -377, 94, -377, 104, -529, 104, 104, 626, 89, + 94, -238, 692, -240, 23, -235, -234, -425, -549, -434, + -595, 691, -248, 89, -241, -593, -594, -241, -247, -398, + -275, 130, 130, 130, 27, -537, -398, 26, -126, -107, + -606, 173, 174, -244, -486, -465, -462, -488, 151, -398, + -473, 174, 14, 747, 92, 278, -632, -631, 487, 89, + 174, -559, 279, 574, 94, 744, 503, 255, 256, 109, + 403, 110, 111, -518, -433, -429, -423, -423, -421, -421, + -427, 292, -427, 119, -298, 169, 168, -298, -425, 745, + -424, -600, 126, -425, 38, 174, 38, 174, 86, 174, + 89, -525, -425, 173, 174, 89, 89, 19, 19, 140, + 89, -425, 89, 89, 89, 89, 19, 19, -425, 89, + 173, 89, 89, 89, 89, 86, 89, 174, 89, 89, + 89, 89, 174, 174, 174, -433, -433, -425, -433, 89, + 89, 89, -425, -425, -425, -433, 89, -425, -425, -425, + -425, -425, -425, -425, -425, -425, -425, -244, -496, 522, + -496, -496, -496, 89, -496, 89, 174, 89, 174, 89, + 89, 174, 174, 174, 174, 89, -240, 88, 104, 174, + 739, -381, -380, 94, -161, 278, -398, 723, -398, -161, + -398, -398, 130, -161, -398, 723, 94, 94, -283, -387, + -283, -387, 618, 42, 42, 184, 188, 188, 187, -398, + 94, 39, 26, 26, 342, -136, 613, -268, 88, 88, + -283, -283, -283, -621, 473, -398, -633, 174, 44, -631, + 567, -197, 355, -449, 86, -204, 362, 19, 14, -283, + -283, -283, -283, -297, 38, -470, 85, -549, -248, 89, + -593, -547, 88, 89, 174, 19, -223, -284, -398, -143, + 24, -398, -460, -398, -398, -398, -458, 86, -398, -388, + -355, -355, -412, -355, -355, 174, 25, -410, -412, -412, + -275, -408, -275, 173, -275, -387, -524, 38, -245, 174, + 23, 297, -282, -395, -279, -281, 282, -415, -280, 285, + -589, 283, 281, 114, 286, 340, 115, 276, -395, -395, + 282, -318, 278, 38, -395, -336, 276, 406, 340, 283, + 23, 297, -335, 276, 115, -398, 282, 286, 283, 281, + -394, 130, -386, 160, 278, 46, 450, -394, 624, 297, + -394, -394, -394, -394, -394, -394, -394, 314, 314, -394, + -394, -394, -394, -394, -394, -394, -394, -394, -394, -394, + 179, -394, -394, -394, -394, -394, -394, 88, 309, 310, + 342, 613, 124, 626, 615, -460, 278, 542, 542, -622, + 473, 34, 430, 430, 431, -633, 426, 45, 34, -205, + 424, -339, -337, -409, 34, -361, -362, -363, -364, -366, + -365, 71, 75, 77, 81, 72, 73, 74, 530, 78, + 83, 76, 34, 174, -396, -401, 38, -398, 94, -396, + -216, -231, -229, -396, 88, -480, -648, -650, 559, 556, + 562, -482, -482, 104, 278, 88, 130, -482, -482, 44, + -397, -645, 563, 557, -240, -265, 725, 174, 85, -285, + -259, -260, -261, -262, -290, -374, 742, 209, 212, 214, + 215, 216, 217, 219, 220, 221, 222, 223, 226, 227, + 228, 229, 230, 231, 224, 225, 291, 204, 205, 206, + 207, 191, 210, 619, 192, 193, 194, 168, 169, 195, + 198, 199, 200, 201, 197, 232, 233, 234, 235, 236, + 237, 238, 239, 241, 240, 242, 243, 244, 245, 246, + 247, 248, 249, -398, -269, 94, 19, -265, -355, -219, + -231, -398, 94, -398, 151, 127, -6, 125, -169, -168, + -167, 128, 705, 711, 127, 127, 127, 89, 89, 89, + 89, 174, 89, 89, 89, 174, 89, 174, 104, -562, + 532, -240, 94, -155, 668, 174, -232, 40, 41, 174, + 88, 89, 174, 64, 174, 130, 89, 174, -425, -398, + 94, -425, 205, 94, 173, 505, -398, -575, 89, -488, + 174, 278, 173, 173, -463, 453, -397, -465, 23, 14, + -374, 42, -381, 130, 744, -398, 89, -427, -427, 119, + -423, -420, 89, 127, -425, 125, -288, -425, -288, -289, + -295, 170, 208, 291, 207, 206, 204, 163, 164, -308, + -454, 618, -232, 89, -398, -433, -425, -425, -421, 89, + -425, -425, 19, -398, -308, -421, -425, -425, -425, -237, + -237, 89, 89, -495, -496, -495, -495, 89, 89, 89, + 89, -495, 89, 89, 89, 89, 89, 89, 89, 89, + 89, 89, 89, 88, -496, -496, -425, -496, -425, -496, + -496, -425, 104, 106, 104, 106, -555, -155, -657, 66, + 713, 65, 495, 109, 345, 174, 104, 94, 745, 174, + 130, 424, -398, 19, 173, 94, -398, 94, 19, 274, + -398, 19, 19, -283, -283, -283, 188, 94, -634, 349, + 424, 567, 274, 424, 349, 567, 274, -507, 104, -137, + 124, 94, 461, -270, -271, -272, -273, -274, 140, 175, + 176, -259, -245, 88, -245, -624, 534, 475, 485, -394, + 378, -417, -416, 426, 45, -542, 496, 481, 482, -464, + 305, -387, 151, -630, 101, 130, 85, 390, 394, 396, + 398, 397, 395, 391, 392, 393, -443, -444, -442, -446, + -387, 94, -617, 88, 88, -213, 38, 138, -204, 362, + 19, 88, 88, 38, -519, 375, -290, 43, 89, 64, + -1, -398, -283, -223, -398, 19, 174, -616, 173, 104, + -398, -457, -410, -355, -425, -425, -355, -410, -410, -412, + -398, -275, -519, -290, 38, -334, 271, 266, -492, 342, + 343, -493, -509, 345, -511, 88, -287, -374, -280, -588, + -589, -445, -398, 115, -588, 115, 88, -287, -374, -374, + -337, -374, -398, -398, -398, -398, -344, -343, -374, -347, + 35, -348, -398, -398, -398, -398, 115, -398, 115, -313, + 44, 51, 52, 53, -394, -394, 211, -316, 44, 495, + 497, 498, -347, 104, 104, 104, 104, 94, 94, 94, + -394, -394, 104, 94, -401, 94, -590, 187, 48, 49, + 104, 104, 104, 104, 44, 94, -321, 44, 325, 329, + 326, 327, 328, 94, 104, 44, 104, 44, 104, 44, + -398, 88, -591, -592, 94, -507, 94, 88, 104, 94, + 267, -460, 94, 85, -624, -394, 430, -479, 130, 130, + -417, -626, 98, 476, -626, -629, 355, -207, 567, 35, + -249, 271, 266, -617, -469, -468, -374, -228, -228, -228, + -228, -228, -228, 71, 82, 71, -242, 88, 71, 76, + 71, 76, 71, 76, 71, -363, 71, 82, -469, -230, + -245, -401, 89, -642, -641, -640, -638, 79, 279, 80, + -431, -482, 556, 560, 561, -465, -413, 94, -472, -155, + -283, -283, -540, 335, 336, 89, 174, -290, -398, -357, + 21, 173, 123, -6, -165, -167, -425, -6, -425, 707, + 443, 708, 94, 104, 104, -570, 516, 511, 513, -155, + -571, 503, 14, -234, -233, 47, -434, -557, -556, 64, + -213, -241, -549, -594, -555, -398, 745, 745, 745, 745, + 94, -398, 104, 19, -462, -457, 151, 151, -398, 454, + -473, 94, 474, 94, 274, 745, 94, -381, -420, -425, + 89, 38, 89, 89, -526, -526, -525, -528, -525, -298, + -298, 89, 88, -232, 89, 89, 26, 89, 89, 89, + 89, -425, 89, 89, 174, 174, 89, -545, 576, -546, + 653, -495, -495, -495, -495, -495, -495, -495, -495, -495, + -495, -495, -495, -495, -495, -495, -495, -495, -436, -435, + 297, 89, 174, 89, 174, 89, 517, 720, 720, 517, + 720, 720, 89, 174, -597, 174, -389, 350, -389, -380, + 94, -398, 94, 723, -398, 745, 745, 723, -398, 94, + -283, -387, -252, 533, -210, 124, -211, 122, 46, 94, + -398, 19, -398, -398, 342, -398, 342, -398, -398, 94, + -142, 626, 88, -139, 614, 94, 89, 174, -374, 89, + 38, -276, -277, -278, -287, -279, -281, 38, -625, 98, + -620, 94, -398, 95, -398, -626, 172, 428, 44, 477, + 478, 493, 423, 104, 104, 483, -618, -398, -206, 274, + 424, -206, -628, 55, 130, 94, -283, -442, -386, 160, + 316, -275, -398, 378, -352, -351, -398, 94, -276, -213, + -283, -283, 94, -276, -276, -213, -520, 377, 23, 104, + 150, 115, 64, -213, -549, 89, -246, 86, 173, -231, + -284, -398, 151, -355, -275, -355, -355, -410, -520, -213, + -504, 346, 88, -502, 88, -502, 115, 391, -512, -510, + 297, -342, 48, 50, -290, -586, -398, -584, -586, -398, + -584, -584, -445, -425, -342, -287, 278, 34, 266, -345, + 394, 388, 389, 394, 396, 398, 397, -474, 341, 120, + -474, 174, -232, 174, -398, -308, -308, 34, 94, 94, + -285, 89, 174, 130, 94, -139, -138, -425, -214, -216, + 278, 85, 274, -625, -620, 130, -480, 94, 94, -626, + 94, 94, -630, 130, -286, 274, -387, 174, -249, -249, + -355, 19, 174, 130, -254, -253, 85, 86, -255, 85, + -253, -253, 71, -243, 94, 71, 71, 71, -355, -640, + -639, 26, -589, -589, -589, 89, 89, -256, 26, -261, + 44, 378, -356, 22, 23, 151, 127, 125, 127, 127, + -398, 89, 89, -532, 693, -566, -568, 511, 23, 23, + -256, -572, 698, 94, 454, 48, 49, 89, -549, 745, + -457, -473, 496, -283, 174, 745, -288, -327, 94, -425, + 89, -425, -425, 89, 94, 89, 94, -237, 23, -496, + -425, -496, -425, -496, 89, 174, 89, 89, 89, 174, + 89, 89, -425, 89, -597, -390, 205, 94, -390, -398, + -398, 19, -399, -209, 278, -275, -212, 373, 88, 369, + -210, 184, 88, 94, -398, 19, -398, -507, 342, -507, + 342, 274, -398, -265, -140, 615, 104, -138, 94, -450, + 620, -272, -290, 272, -213, 89, 174, -213, 94, -623, + 487, -508, 383, 104, 44, 104, 172, 479, -543, -198, + 98, -285, 35, -249, -198, -627, 98, 130, 744, 88, + -394, -394, -394, -209, 378, -398, 89, 174, -394, -394, + 89, -209, -398, 89, 89, -306, 14, -521, 296, 104, + 150, 104, 150, 104, 17, 279, -549, -396, -231, -398, + -355, -616, 173, -355, -521, -494, 347, 104, -421, 88, + -421, 88, -503, 344, 88, 89, 174, -398, -374, -303, + -302, -300, 109, 120, 44, 468, -301, 98, 160, 330, + 333, 332, 308, 331, -332, -414, 85, 686, 471, 388, + 389, -446, 693, 606, 701, 38, 281, 114, 115, 455, + -415, 88, 88, 86, 350, 88, 88, -586, 89, -342, + -374, 44, -345, 44, -346, 412, -455, -455, -455, -455, + 341, -343, -398, 160, -308, 89, -592, 94, 89, -460, + 274, -398, -623, 94, -482, -628, 94, -198, -285, -617, + -237, -231, -468, -555, -425, 88, -425, 89, 88, 71, + 11, 21, 17, -418, -398, -425, -433, 728, 730, 731, + 280, -6, 708, 443, -323, 694, 94, 23, 94, -564, + 94, -562, 94, -433, -158, -320, -386, 313, 89, -326, + 140, 14, 89, 89, 89, -495, -495, -498, -497, -501, + 517, 342, 525, -433, 89, 89, 94, 94, 89, 89, + 94, 94, 94, 723, 424, -209, 38, 461, 24, 632, + 374, -244, 370, 371, 372, -398, 94, -433, -214, 744, + 378, -398, 19, 94, -507, 94, -507, -398, 342, 38, + 94, 89, 94, 94, -263, -290, -202, 14, -306, -278, + -202, 23, 14, 172, 427, 44, 104, 44, 480, 94, + -206, 130, 110, 111, -382, -383, 94, -452, -308, -310, + 94, -398, -351, -418, -418, -304, -213, 38, -305, -349, + -446, 378, -157, -156, -304, 88, -522, 178, 104, 150, + 104, 104, -469, -355, -355, -522, -511, 23, 89, -489, + 89, -489, 88, 130, -421, -510, -513, 64, -300, 109, + -421, 94, -310, -311, 44, 329, 325, 130, 130, -312, + 44, 309, 310, -322, 88, 340, 17, 104, 211, 88, + 702, 88, 115, 115, -283, -452, -452, -587, 390, 391, + 392, 399, 394, 395, 393, 396, 397, 398, -587, -452, + -452, 88, -475, -474, -421, -455, 130, -456, 287, 404, + 405, 98, 14, 388, 389, 409, 408, 407, 413, 414, + 418, 419, 415, 417, 416, 420, 421, 422, 410, 411, + 412, 427, 438, -394, 160, -398, 173, -627, -238, -355, + -244, -585, -398, 281, 23, 23, -541, 14, 729, 88, + 88, -398, -398, -378, 695, 104, 94, 513, -570, -533, + 696, -560, -502, -308, 130, 89, 78, 619, 621, 89, + -500, 122, 479, 483, -419, -422, 104, 106, 202, 172, + -496, -496, 89, 89, -398, -398, -283, 94, 104, 89, + 119, 119, 89, 89, -385, -384, 94, -398, 378, -398, + -265, 94, -265, 94, 342, -507, -2, 620, -203, 63, + 563, 94, 95, 474, 94, 95, 104, 427, -198, 94, + 745, 174, 130, 89, -508, -490, 297, -213, 174, -349, + -386, -398, -158, -490, -307, -350, -398, 94, -539, 187, + 376, 14, 104, 150, 104, -237, -523, 187, 376, -493, + 89, 89, 89, -489, 104, 89, -517, -514, 88, -349, + 299, 140, 94, 94, 104, 88, -550, 34, 94, 38, + -425, -453, 88, 89, 89, 89, 89, -452, 110, 111, + -394, -394, 94, 94, 387, -394, -394, -394, -394, -394, + -394, 88, 94, 94, -394, -394, -394, -394, 130, -394, + -394, -308, -394, 173, -398, 89, 89, 174, 731, 88, + -433, -433, 88, 23, -532, -534, 697, 94, -569, 516, + -563, -561, 511, 512, 513, 514, 94, 620, 68, 622, + -499, -500, 483, -419, -422, 691, 523, 523, 523, 94, + -398, 94, 745, 174, 130, -398, 378, -265, -265, -507, + 94, -266, -398, 340, 496, -383, 94, -455, -491, 349, + 23, -349, -394, -508, -491, 89, 174, -394, -394, 376, + 104, 150, 104, -238, 376, -505, 348, 89, -517, -349, + -516, -515, 347, 300, 88, 89, -425, -437, -394, 89, + 88, 89, -325, -324, 617, -452, -455, 86, -455, 86, + -455, 86, -455, 86, 89, 104, 104, -398, 104, 104, + 104, 104, 104, 104, -489, 104, 104, 104, 104, 110, + 111, 104, 104, -308, -398, -398, 281, -153, 88, 89, + 89, -379, -398, -564, -323, 94, -573, 279, -567, -568, + 515, -561, 23, 513, 23, 23, -159, 174, 68, 119, + 524, 524, 524, -210, -211, -210, -211, -265, -384, 94, + -398, 94, -265, -264, 38, 518, 454, 23, -492, -308, + -350, -418, -418, 104, 104, 89, 174, -398, 296, 88, + -432, -426, -425, 296, 89, -398, -425, -476, 704, 703, + -331, -329, -330, 85, 530, 338, 339, 89, -587, -587, + -587, -587, -332, 89, 89, 174, -431, 89, 174, -378, + -580, 88, 104, -566, -565, -567, 23, -564, 23, -564, + -564, 520, 14, -499, -210, -210, -265, 94, -374, 88, + -504, -515, -514, -432, 89, 174, -474, 89, -330, 85, + -329, 85, 18, 17, -455, -455, -455, -455, 88, 89, + -398, -583, 34, 89, -579, -578, -375, -574, -398, 516, + 517, 94, -564, 130, 621, -660, -659, 719, -489, -494, + 89, -426, -476, -328, 335, 336, 34, 187, -328, -431, + -582, -581, -376, 89, 174, 173, 94, 622, 94, 89, + -511, 109, 44, 337, 89, 174, 130, -578, -398, -581, + 44, -425, 173, -398, } var yyDef = [...]int{ @@ -11475,451 +11572,453 @@ var yyDef = [...]int{ 436, -2, 0, 0, 794, 0, 0, 0, 878, 0, 0, 0, 923, 941, 23, 0, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 0, 0, 19, - 0, 19, 0, 0, 0, 1566, 1567, 1568, 1569, 2459, - 2429, -2, 2177, 2137, 2353, 2354, 2244, 2258, 2130, 2506, - 2507, 2508, 2509, 2510, 2511, 2512, 2513, 2514, 2515, 2516, - 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, 2525, 2526, - 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, 2536, - 2537, 2538, 2539, 2540, 2541, 2542, 2543, 2544, 2545, 2546, - 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, - 2557, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2090, 2091, - 2092, 2093, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2101, - 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, 2110, 2111, - 2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, - 2122, 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2131, 2132, - 2133, 2134, 2135, 2136, 2138, 2139, 2140, 2141, 2142, 2143, - 2144, 2145, 2146, 2147, 2148, 2149, 2150, 2151, 2152, 2153, - 2154, 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2162, 2163, - 2164, 2165, 2166, 2167, 2168, 2169, 2170, 2171, 2172, 2173, - 2174, 2175, 2176, 2178, 2179, 2180, 2181, 2182, 2183, 2184, - 2185, 2186, 2187, 2188, 2189, 2190, 2191, 2192, 2193, 2194, - 2195, 2196, 2197, 2198, 2199, 2200, 2201, 2202, 2203, 2204, - 2205, 2206, 2207, 2208, 2209, 2210, 2211, 2212, 2213, 2214, - 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2222, 2223, 2224, - 2225, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, 2234, - 2235, 2236, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2245, - 2246, 2247, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, - 2256, 2257, 2260, 2261, 2262, 2263, 2264, 2265, 2266, 2267, - 2268, 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2277, - 2278, 2279, 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, - 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, - 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, 2306, 2307, - 2308, 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, 2317, - 2318, 2319, 2320, 2321, 2322, 2323, 2324, 2325, 2326, 2327, - 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, - 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, 2346, 2347, - 2348, 2349, 2350, 2351, 2352, 2355, 2356, 2357, 2358, 2359, - 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, - 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, 2378, 2379, - 2380, 2381, 2382, 2383, 2384, 2385, -2, 2387, 2388, 2389, - 2390, 2391, 2392, 2393, 2394, 2395, 2396, 2397, 2398, 2399, - 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, - 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, - 2420, 2421, 2422, 2423, 2424, 2425, 2426, 2427, 2428, 2430, - 2431, 2432, 2433, 2434, 2435, 2436, 2437, 2438, 2439, 2440, - 2441, 2442, 2443, 2444, -2, -2, -2, 2448, 2449, 2450, - 2451, 2452, 2453, 2454, 2455, 2456, 2457, 2458, 2460, 2461, - 2462, 2463, 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, - 2472, 2473, 2474, 2475, 2476, 2477, 2478, 2479, 2480, 2481, - 2482, 2483, 2484, 2485, 2486, 2487, 2488, 2489, 2490, 2491, - 2492, 2493, 2494, 2495, 2496, 0, 334, 332, 2102, 2130, - 2137, 2177, 2244, 2258, 2259, 2299, 2353, 2354, 2386, 2429, - 2445, 2446, 2447, 2459, 0, 0, 1094, 0, 371, 783, - 784, 811, 878, 906, 844, 0, 849, 1511, 0, 740, - 0, 411, 0, 2154, 415, 2436, 0, 0, 0, 0, - 737, 405, 406, 407, 408, 409, 410, 0, 0, 1053, - 0, 0, 2466, 401, 0, 365, 2246, 2458, 1570, 0, - 0, 0, 0, 0, 221, 1229, 223, 1231, 227, 235, - 0, 0, 0, 240, 241, 244, 245, 246, 247, 248, - 0, 252, 0, 254, 257, 0, 259, 260, 0, 263, - 264, 265, 0, 275, 276, 277, 1232, 1233, 1234, 1235, - 1236, 1237, 1238, 1239, -2, 150, 1092, 2036, 1920, 0, - 1927, 1940, 1951, 1660, 1661, 1662, 1663, 0, 0, 0, - 0, 0, 0, 1671, 1672, 0, 1715, 2510, 2553, 2554, - 0, 1681, 1682, 1683, 1684, 1685, 1686, 0, 161, 173, - 174, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 0, 1981, - 1982, 1983, 0, 1645, 1566, 0, 2519, 2527, 0, 2541, - 2548, 2549, 2550, 2551, 2540, 0, 0, 1876, 0, 1866, - 0, 0, -2, -2, 0, 0, 2326, -2, 2555, 2556, - 2557, 2516, 2537, 2545, 2546, 2547, 2520, 2521, 2544, 2512, - 2513, 2514, 2507, 2508, 2509, 2511, 2523, 2525, 2536, 0, - 2532, 2542, 2543, 2434, 0, 0, 2483, 0, 0, 0, - 0, 0, 0, 2492, 2493, 2494, 2495, 2496, 2478, 175, - 176, -2, -2, -2, -2, -2, -2, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, 1887, -2, 1889, - -2, 1891, -2, 1893, -2, -2, -2, -2, 1898, 1899, - -2, 1901, -2, -2, -2, -2, -2, -2, -2, 1878, - 1879, 1880, 1881, 1870, 1871, 1872, 1873, 1874, 1875, -2, - -2, -2, 906, 1001, 0, 906, 0, 879, 928, 931, - 934, 937, 882, 0, 0, 123, 124, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 360, 361, 349, 351, 0, 355, 0, 0, 351, 348, - 342, 0, 1294, 1294, 1294, 1294, 0, 0, 0, 1294, - 1294, 1294, 1294, 1294, 0, 1294, 0, 0, 0, 0, - 0, 1294, 0, 1130, 1241, 1242, 1243, 1292, 1293, 1397, - 0, 0, 0, 844, 0, 892, 0, 894, 897, 799, - 795, 796, 797, 798, 77, 642, 0, 0, 0, 717, - 717, 966, 966, 0, 660, 0, 0, 0, 717, 0, - 674, 666, 0, 0, 0, 717, 0, 0, 899, 899, - 0, 720, 727, 717, 717, -2, 717, 717, 0, 712, - 717, 0, 0, 0, 1308, 680, 681, 682, 666, 666, - 685, 686, 687, 697, 698, 728, 2078, 0, 0, 576, - 576, 0, 576, 0, 0, 576, 0, 576, 576, 576, - 0, 801, 2199, 2294, 2171, 2264, 2112, 2246, 2458, 0, - 307, 2326, 312, 0, 2176, 2202, 0, 0, 2221, 0, - -2, 0, 388, 906, 0, 0, 878, 0, 0, 0, - 576, 576, 576, 576, 576, 576, 576, 1396, 576, 576, - 576, 576, 576, 0, 0, 0, 576, 0, 576, 576, - 576, 0, 942, 943, 945, 946, 947, 948, 949, 950, - 951, 952, 953, 954, 5, 6, 19, 0, 0, 0, - 0, 0, 0, 129, 128, 0, 2037, 2073, 1986, 1987, - 1988, 0, 2060, 1991, 2064, 2064, 2064, 2064, 2021, 2022, - 2023, 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2064, 2064, - 0, 0, 2035, 2012, 2062, 2062, 2062, 2060, 2039, 1992, - 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, - 2003, 2004, 2005, 2067, 2067, 2070, 2070, 2067, 2040, 2041, - 2042, 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050, 2051, - 2052, 2053, 2054, 2055, 2056, 2057, 0, 453, 451, 452, - 1916, 0, 0, 906, -2, 0, 0, 0, 0, 848, - 1509, 0, 0, 0, 741, 412, 1571, 0, 0, 416, - 0, 417, 0, 0, 419, 0, 0, 0, 441, 0, - 444, 427, 428, 429, 430, 431, 423, 0, 201, 0, - 403, 404, 400, 0, 0, 367, 0, 0, 0, 577, - 0, 0, 0, 0, 0, 0, 232, 228, 236, 239, - 249, 256, 0, 268, 270, 273, 229, 237, 242, 243, - 250, 271, 230, 233, 234, 238, 272, 274, 231, 251, - 255, 269, 253, 258, 261, 262, 267, 0, 202, 0, - 0, 0, 0, 0, 1926, 0, 0, 1959, 1960, 1961, - 1962, 1963, 1964, 1965, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, -2, 1920, 0, 0, - 1666, 1667, 1668, 1669, 0, 1673, 0, 1716, 0, 0, - 0, 0, 0, 0, 1980, 1984, 0, 0, 1916, 1916, - 0, 0, 1916, 1912, 0, 0, 0, 0, 0, 0, - 1916, 1849, 0, 0, 1851, 1867, 0, 0, 1853, 1854, - 0, 1857, 1858, 1916, 0, 1916, 1862, 1916, 1916, 1916, - 1843, 1844, 0, 0, 0, 1912, 1912, 1912, 1912, 0, - 0, 1912, 1912, 1912, 1912, 1912, 1912, 1912, 1912, 1912, - 1912, 1912, 1912, 1912, 1912, 1912, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, - 907, 0, -2, 0, 925, 927, 929, 930, 932, 933, - 935, 936, 938, 939, 884, 0, 0, 125, 0, 0, - 0, 106, 0, 0, 104, 0, 0, 0, 0, 75, - 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 353, 0, 358, 344, 2287, 0, - 343, 0, 0, 0, 0, 0, 0, 1091, 0, 0, - 1294, 1294, 1294, 1131, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1294, 1294, 1294, 1294, 0, 1314, 0, - 0, 0, 0, 844, 0, 893, 0, 0, 801, 800, - 74, 78, 644, 648, 649, 650, 0, 966, 0, 0, - 653, 654, 0, 655, 0, 0, 666, 717, 717, 672, - 673, 668, 667, 723, 724, 720, 0, 720, 720, 966, - 0, 691, 692, 693, 717, 717, 699, 900, 0, 700, - 701, 720, 0, 725, 726, 966, 0, 0, 966, 966, - 0, 709, 710, 0, 713, 717, 0, 716, 0, 0, - 1294, 0, 733, 668, 668, 2079, 2080, 0, 0, 1305, - 0, 0, 0, 0, 0, 0, 0, 736, 0, 0, - 0, 471, 472, 0, 0, 802, 0, 286, 290, 0, - 293, 0, 2294, 0, 2294, 0, 0, 300, 0, 0, - 0, 0, 0, 0, 330, 331, 0, 0, 0, 0, - 321, 324, 1503, 1504, 1226, 1227, 325, 326, 380, 381, - 0, 899, 924, 926, 920, 921, 922, 0, 0, 0, - 0, 0, 0, 0, 0, 576, 0, 0, 0, 0, - 0, 777, 0, 1109, 779, 0, 0, 576, 0, 0, - 0, 974, 968, 970, 1048, 161, 944, 8, 146, 143, - 0, 19, 0, 0, 19, 19, 0, 19, 335, 0, - 2076, 2074, 2075, 0, 1990, 2061, 0, 2017, 0, 2018, - 2019, 2020, 2031, 2032, 0, 0, 2013, 0, 2014, 2015, - 2016, 2006, 2067, 0, 2008, 2009, 0, 2010, 2011, 333, - 450, 0, 0, 1917, 1095, 0, 899, 876, 0, 904, - 0, 803, 836, 805, 0, 825, 0, 1511, 0, 0, - 0, 0, 576, 0, 413, 0, 424, 418, 0, 425, - 420, 421, 0, 0, 443, 445, 446, 447, 448, 432, - 433, 738, 397, 398, 399, 389, 390, 391, 392, 393, - 394, 395, 396, 0, 0, 402, 171, 0, 368, 369, - 0, 0, 0, 215, 216, 217, 218, 219, 220, 222, - 206, 766, 768, 1218, 1230, 0, 1221, 0, 225, 266, - 198, 0, 0, 0, 1921, 1922, 1923, 1924, 1925, 1930, - 0, 1932, 1934, 1936, 1938, 0, 1956, -2, -2, 1646, - 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 1656, - 1657, 1658, 1659, 1941, 1954, 1955, 0, 0, 0, 0, - 0, 0, 1952, 1952, 1947, 0, 1678, 1720, 1732, 1732, - 1687, 1505, 1506, 1664, 0, 0, 1713, 1717, 0, 0, - 0, 0, 0, 0, 1273, 2060, 0, 162, 1951, 1911, - 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, 1819, - 1820, 1821, 1822, 1823, 1824, 1825, 1826, 1827, 1828, 1829, - 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, 1838, 0, - 0, 1920, 0, 0, 0, 0, 1913, 1914, 0, 0, - 0, 1798, 0, 0, 1804, 1805, 1806, 0, 831, 0, - 1877, 1850, 1868, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1839, 1840, 1841, 1842, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1000, 1002, 0, 840, 842, - 843, 873, 904, 880, 0, 0, 0, 121, 126, 0, - 1364, 112, 0, 0, 0, 112, 0, 0, 0, 112, - 0, 0, 0, 82, 1202, 1309, 83, 1201, 1311, 0, - 0, 0, 0, 0, 0, 0, 362, 363, 0, 0, - 357, 345, 2287, 347, 0, 0, 0, 0, 1078, 0, - 0, 0, 0, 0, 0, 0, 1146, 1147, 0, 574, - 1212, 0, 0, 0, 1228, 1277, 1290, 0, 0, 0, - 0, 0, 1370, 1132, 1137, 1138, 1139, 1133, 1134, 1140, - 1141, 822, 836, 817, 0, 825, 0, 895, 0, 0, - 1017, 0, 646, 0, 0, 652, 718, 719, 967, 656, - 0, 0, 663, 2246, 668, 966, 966, 675, 669, 676, - 722, 677, 678, 679, 720, 966, 966, 901, 717, 720, - 702, 721, 720, 1511, 706, 0, 711, 714, 715, 1511, - 734, 1511, 0, 732, 683, 684, 1372, 897, 469, 470, - 475, 477, 0, 536, 536, 536, 519, 536, 0, 0, - 507, 2081, 0, 0, 0, 0, 516, 2081, 0, 0, - 2081, 2081, 2081, 2081, 2081, 2081, 2081, 0, 0, 2081, - 2081, 2081, 2081, 2081, 2081, 2081, 2081, 2081, 2081, 2081, - 0, 2081, 2081, 2081, 2081, 2081, 1489, 2081, 0, 1306, - 526, 527, 528, 529, 534, 535, 0, 0, 480, 481, - 0, 0, 0, 0, 0, 569, 0, 0, 1145, 0, - 574, 0, 0, 1190, 0, 0, 979, 0, 980, 981, - 982, 977, 1019, 1043, 1043, 0, 1043, 1023, 1511, 0, - 0, 0, 298, 299, 287, 0, 288, 0, 0, 301, - 302, 0, 304, 305, 306, 313, 2171, 2264, 308, 310, - 0, 0, 314, 327, 328, 329, 0, 0, 319, 320, - 0, 0, 383, 384, 386, 0, 904, 1310, 1296, 79, - 80, 2466, 762, 763, 1507, 764, 765, 769, 0, 0, - 772, 773, 774, 775, 776, 1111, 0, 0, 1199, 0, - 1203, 1205, 1296, 966, 0, 975, 0, 971, 1049, 0, - 1051, 0, 0, 144, 19, 0, 137, 134, 0, 0, - 0, 0, 0, 2038, 1985, 2077, 0, 0, 0, 0, - 2058, 0, 0, 2007, 0, 0, 0, 127, 856, 904, - 0, 850, 0, 908, 909, 912, 804, 833, 0, 837, - 0, 0, 829, 809, 826, 0, 0, 846, 1510, 0, - 0, 0, 0, 0, 1572, 0, 426, 422, 442, 0, - 0, 0, 0, 209, 1215, 0, 210, 214, 204, 0, - 0, 0, 1220, 0, 1217, 1222, 0, 224, 0, 0, - 199, 200, 1355, 1364, 0, 0, 0, 1931, 1933, 1935, - 1937, 1939, 0, 1942, 1952, 1952, 1948, 0, 1943, 0, - 1945, 0, 1721, 1733, 1734, 1722, 1921, 1670, 0, 1718, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, - 0, 0, 0, 1786, 1788, 0, 0, 0, 1793, 0, - 1795, 1796, 1797, 1799, 0, 0, 0, 1803, 0, 1848, - 1869, 1852, 1855, 0, 1859, 0, 1861, 1863, 1864, 1865, - 0, 0, 0, 906, 906, 0, 0, 1757, 1757, 1757, - 0, 0, 0, 0, 1757, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1690, 0, 1691, 1692, - 1693, 0, 1695, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1003, 850, 0, 0, 0, 0, 0, - 1362, 0, 102, 0, 107, 0, 0, 103, 108, 0, - 0, 105, 0, 0, 114, 84, 0, 0, 1317, 1318, - 0, 0, 0, 364, 352, 354, 0, 346, 0, 1295, - 0, 0, 0, 1082, 0, 0, -2, 1111, 897, 0, - 897, 1157, 2081, 0, 578, 0, 0, 1214, 0, 1179, - 0, 0, 0, -2, 0, 0, 0, 1290, 0, 0, - 0, 1374, 0, 812, 0, 816, 0, 0, 821, 813, - 23, 898, 0, 0, 0, 788, 792, 643, 0, 645, - 651, 659, 657, 0, 661, 0, 662, 717, 670, 671, - 966, 694, 695, 0, 0, 966, 717, 717, 705, 720, - 729, 0, 730, 1511, 1374, 0, 0, 1305, 1440, 1408, - 497, 0, 1524, 1525, 537, 0, 1531, 1540, 1294, 1610, - 0, 1540, 0, 0, 1542, 1543, 0, 0, 0, 0, - 520, 521, 0, 506, 0, 0, 0, 0, 0, 0, - 505, 0, 0, 547, 0, 0, 0, 0, 0, 2082, - 2081, 2081, 0, 514, 515, 0, 518, 0, 0, 0, - 0, 0, 0, 0, 0, 2081, 2081, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1480, 0, - 0, 0, 0, 0, 0, 0, 1495, 1496, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1157, 2081, 0, - 0, 0, 0, 578, 1209, 1209, 1177, 1195, 0, 473, - 474, 544, 0, 0, 0, 0, 0, 0, 0, 1009, - 0, 0, 0, 1008, 0, 0, 0, 0, 0, 0, - 0, 0, 897, 1044, 0, 1046, 1047, 1021, -2, 0, - 979, 1026, 1916, 0, 291, 292, 0, 0, 297, 315, - 317, 289, 0, 0, 0, 316, 318, 322, 323, 382, - 385, 387, 850, 76, 1297, 0, 0, 1398, 0, 1112, - 1113, 1115, 1116, 0, 2087, -2, -2, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, -2, 2145, -2, + 0, 19, 0, 0, 0, 1566, 1567, 1568, 1569, 2467, + 2437, -2, 2181, 2141, 2361, 2362, 2252, 2266, 2134, 2514, + 2515, 2516, 2517, 2518, 2519, 2520, 2521, 2522, 2523, 2524, + 2525, 2526, 2527, 2528, 2529, 2530, 2531, 2532, 2533, 2534, + 2535, 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, 2544, + 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, + 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, + 2565, 2087, 2088, 2089, 2090, 2091, 2092, 2093, 2094, 2095, + 2096, 2097, 2098, 2099, 2100, 2101, 2102, 2103, 2104, 2105, + 2106, 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2114, 2115, + 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2123, 2124, 2125, + 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2135, 2136, + 2137, 2138, 2139, 2140, 2142, 2143, 2144, 2145, 2146, 2147, + 2148, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, + 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, + 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, 2176, 2177, + 2178, 2179, 2180, 2182, 2183, 2184, 2185, 2186, 2187, 2188, + 2189, 2190, 2191, 2192, 2193, 2194, 2195, 2196, 2197, 2198, + 2199, 2200, 2201, 2202, 2203, 2204, 2205, 2206, 2207, 2208, + 2209, 2210, 2211, 2212, 2213, 2214, 2215, 2216, 2217, 2218, + 2219, 2220, 2221, 2222, 2223, 2224, 2225, 2226, 2227, 2228, + 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, 2237, 2238, + 2239, 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, 2248, + 2249, 2250, 2251, 2253, 2254, 2255, 2256, 2257, 2258, 2259, + 2260, 2261, 2262, 2263, 2264, 2265, 2268, 2269, 2270, 2271, + 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, + 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, 2290, 2291, + 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, 2301, + 2302, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, + 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, + 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, 2330, 2331, + 2332, 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, 2341, + 2342, 2343, 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, + 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2363, + 2364, 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, 2373, + 2374, 2375, 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, + 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, 2392, 2393, + -2, 2395, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, + 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, + 2414, 2415, 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, + 2424, 2425, 2426, 2427, 2428, 2429, 2430, 2431, 2432, 2433, + 2434, 2435, 2436, 2438, 2439, 2440, 2441, 2442, 2443, 2444, + 2445, 2446, 2447, 2448, 2449, 2450, 2451, 2452, -2, -2, + -2, 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, 2464, + 2465, 2466, 2468, 2469, 2470, 2471, 2472, 2473, 2474, 2475, + 2476, 2477, 2478, 2479, 2480, 2481, 2482, 2483, 2484, 2485, + 2486, 2487, 2488, 2489, 2490, 2491, 2492, 2493, 2494, 2495, + 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, 2504, 0, + 334, 332, 2106, 2134, 2141, 2181, 2252, 2266, 2267, 2307, + 2361, 2362, 2394, 2437, 2453, 2454, 2455, 2467, 0, 0, + 1094, 0, 371, 783, 784, 811, 878, 906, 844, 0, + 849, 1511, 0, 740, 0, 411, 0, 2158, 415, 2444, + 0, 0, 0, 0, 737, 405, 406, 407, 408, 409, + 410, 0, 0, 1053, 0, 0, 2474, 401, 0, 365, + 2254, 2466, 1570, 0, 0, 0, 0, 0, 221, 1229, + 223, 1231, 227, 235, 0, 0, 0, 240, 241, 244, + 245, 246, 247, 248, 0, 252, 0, 254, 257, 0, + 259, 260, 0, 263, 264, 265, 0, 275, 276, 277, + 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, -2, 150, + 1092, 2040, 1920, 0, 1927, 1940, 1951, 1660, 1661, 1662, + 1663, 0, 0, 0, 0, 0, 0, 1671, 1672, 0, + 1715, 2518, 2561, 2562, 0, 1681, 1682, 1683, 1684, 1685, + 1686, 0, 161, 173, 174, 1973, 1974, 1975, 1976, 1977, + 1978, 1979, 0, 1981, 1982, 1983, 0, 1645, 1566, 0, + 2527, 2535, 0, 2549, 2556, 2557, 2558, 2559, 2548, 0, + 0, 1876, 0, 1866, 0, 0, -2, -2, 0, 0, + 2334, -2, 2563, 2564, 2565, 2524, 2545, 2553, 2554, 2555, + 2528, 2529, 2552, 2520, 2521, 2522, 2515, 2516, 2517, 2519, + 2531, 2533, 2544, 0, 2540, 2550, 2551, 2442, 0, 0, + 2491, 0, 0, 0, 0, 0, 0, 2500, 2501, 2502, + 2503, 2504, 2486, 175, 176, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, 1887, -2, 1889, -2, 1891, -2, 1893, -2, -2, + -2, -2, 1898, 1899, -2, 1901, -2, -2, -2, -2, + -2, -2, -2, 1878, 1879, 1880, 1881, 1870, 1871, 1872, + 1873, 1874, 1875, -2, -2, -2, 906, 1001, 0, 906, + 0, 879, 928, 931, 934, 937, 882, 0, 0, 123, + 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 360, 361, 349, 351, 0, 355, + 0, 0, 351, 348, 342, 0, 1294, 1294, 1294, 1294, + 0, 0, 0, 1294, 1294, 1294, 1294, 1294, 0, 1294, + 0, 0, 0, 0, 0, 1294, 0, 1130, 1241, 1242, + 1243, 1292, 1293, 1397, 0, 0, 0, 844, 0, 892, + 0, 894, 897, 799, 795, 796, 797, 798, 77, 642, + 0, 0, 0, 717, 717, 966, 966, 0, 660, 0, + 0, 0, 717, 0, 674, 666, 0, 0, 0, 717, + 0, 0, 899, 899, 0, 720, 727, 717, 717, -2, + 717, 717, 0, 712, 717, 0, 0, 0, 1308, 680, + 681, 682, 666, 666, 685, 686, 687, 697, 698, 728, + 2082, 0, 0, 576, 576, 0, 576, 0, 0, 576, + 0, 576, 576, 576, 0, 801, 2207, 2302, 2175, 2272, + 2116, 2254, 2466, 0, 307, 2334, 312, 0, 2180, 2210, + 0, 0, 2229, 0, -2, 0, 388, 906, 0, 0, + 878, 0, 0, 0, 576, 576, 576, 576, 576, 576, + 576, 1396, 576, 576, 576, 576, 576, 0, 0, 0, + 576, 0, 576, 576, 576, 0, 942, 943, 945, 946, + 947, 948, 949, 950, 951, 952, 953, 954, 5, 6, + 19, 0, 0, 0, 0, 0, 0, 129, 128, 0, + 2041, 2077, 1986, 1987, 1988, 0, 2064, 1991, 2068, 2068, + 2068, 2068, 2021, 2022, 2023, 2024, 2025, 2026, 2027, 2028, + 2029, 2030, 2068, 2068, 2068, 2068, 2068, 2068, 0, 0, + 2039, 2012, 2066, 2066, 2066, 2064, 2043, 1992, 1993, 1994, + 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, + 2005, 2071, 2071, 2074, 2074, 2071, 2044, 2045, 2046, 2047, + 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2057, + 2058, 2059, 2060, 2061, 0, 453, 451, 452, 1916, 0, + 0, 906, -2, 0, 0, 0, 0, 848, 1509, 0, + 0, 0, 741, 412, 1571, 0, 0, 416, 0, 417, + 0, 0, 419, 0, 0, 0, 441, 0, 444, 427, + 428, 429, 430, 431, 423, 0, 201, 0, 403, 404, + 400, 0, 0, 367, 0, 0, 0, 577, 0, 0, + 0, 0, 0, 0, 232, 228, 236, 239, 249, 256, + 0, 268, 270, 273, 229, 237, 242, 243, 250, 271, + 230, 233, 234, 238, 272, 274, 231, 251, 255, 269, + 253, 258, 261, 262, 267, 0, 202, 0, 0, 0, + 0, 0, 1926, 0, 0, 1959, 1960, 1961, 1962, 1963, + 1964, 1965, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -2, 1920, 0, 0, 1666, 1667, + 1668, 1669, 0, 1673, 0, 1716, 0, 0, 0, 0, + 0, 0, 1980, 1984, 0, 0, 1916, 1916, 0, 0, + 1916, 1912, 0, 0, 0, 0, 0, 0, 1916, 1849, + 0, 0, 1851, 1867, 0, 0, 1853, 1854, 0, 1857, + 1858, 1916, 0, 1916, 1862, 1916, 1916, 1916, 1843, 1844, + 0, 0, 0, 1912, 1912, 1912, 1912, 0, 0, 1912, + 1912, 1912, 1912, 1912, 1912, 1912, 1912, 1912, 1912, 1912, + 1912, 1912, 1912, 1912, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 899, 0, 907, 0, + -2, 0, 925, 927, 929, 930, 932, 933, 935, 936, + 938, 939, 884, 0, 0, 125, 0, 0, 0, 106, + 0, 0, 104, 0, 0, 0, 0, 75, 81, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 353, 0, 358, 344, 2295, 0, 343, 0, + 0, 0, 0, 0, 0, 1091, 0, 0, 1294, 1294, + 1294, 1131, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1294, 1294, 1294, 1294, 0, 1314, 0, 0, 0, + 0, 844, 0, 893, 0, 0, 801, 800, 74, 78, + 644, 648, 649, 650, 0, 966, 0, 0, 653, 654, + 0, 655, 0, 0, 666, 717, 717, 672, 673, 668, + 667, 723, 724, 720, 0, 720, 720, 966, 0, 691, + 692, 693, 717, 717, 699, 900, 0, 700, 701, 720, + 0, 725, 726, 966, 0, 0, 966, 966, 0, 709, + 710, 0, 713, 717, 0, 716, 0, 0, 1294, 0, + 733, 668, 668, 2083, 2084, 0, 0, 1305, 0, 0, + 0, 0, 0, 0, 0, 736, 0, 0, 0, 471, + 472, 0, 0, 802, 0, 286, 290, 0, 293, 0, + 2302, 0, 2302, 0, 0, 300, 0, 0, 0, 0, + 0, 0, 330, 331, 0, 0, 0, 0, 321, 324, + 1503, 1504, 1226, 1227, 325, 326, 380, 381, 0, 899, + 924, 926, 920, 921, 922, 0, 0, 0, 0, 0, + 0, 0, 0, 576, 0, 0, 0, 0, 0, 777, + 0, 1109, 779, 0, 0, 576, 0, 0, 0, 974, + 968, 970, 1048, 161, 944, 8, 146, 143, 0, 19, + 0, 0, 19, 19, 0, 19, 335, 0, 2080, 2078, + 2079, 0, 1990, 2065, 0, 2017, 0, 2018, 2019, 2020, + 2031, 2032, 2033, 2034, 2035, 2036, 0, 0, 2013, 0, + 2014, 2015, 2016, 2006, 2071, 0, 2008, 2009, 0, 2010, + 2011, 333, 450, 0, 0, 1917, 1095, 0, 899, 876, + 0, 904, 0, 803, 836, 805, 0, 825, 0, 1511, + 0, 0, 0, 0, 576, 0, 413, 0, 424, 418, + 0, 425, 420, 421, 0, 0, 443, 445, 446, 447, + 448, 432, 433, 738, 397, 398, 399, 389, 390, 391, + 392, 393, 394, 395, 396, 0, 0, 402, 171, 0, + 368, 369, 0, 0, 0, 215, 216, 217, 218, 219, + 220, 222, 206, 766, 768, 1218, 1230, 0, 1221, 0, + 225, 266, 198, 0, 0, 0, 1921, 1922, 1923, 1924, + 1925, 1930, 0, 1932, 1934, 1936, 1938, 0, 1956, -2, + -2, 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, 1654, + 1655, 1656, 1657, 1658, 1659, 1941, 1954, 1955, 0, 0, + 0, 0, 0, 0, 1952, 1952, 1947, 0, 1678, 1720, + 1732, 1732, 1687, 1505, 1506, 1664, 0, 0, 1713, 1717, + 0, 0, 0, 0, 0, 0, 1273, 2064, 0, 162, + 1951, 1911, 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, + 1818, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, 1827, + 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, + 1838, 0, 0, 1920, 0, 0, 0, 0, 1913, 1914, + 0, 0, 0, 1798, 0, 0, 1804, 1805, 1806, 0, + 831, 0, 1877, 1850, 1868, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1839, 1840, 1841, + 1842, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1000, 1002, 0, + 840, 842, 843, 873, 904, 880, 0, 0, 0, 121, + 126, 0, 1364, 112, 0, 0, 0, 112, 0, 0, + 0, 112, 0, 0, 0, 82, 1202, 1309, 83, 1201, + 1311, 0, 0, 0, 0, 0, 0, 0, 362, 363, + 0, 0, 357, 345, 2295, 347, 0, 0, 0, 0, + 1078, 0, 0, 0, 0, 0, 0, 0, 1146, 1147, + 0, 574, 1212, 0, 0, 0, 1228, 1277, 1290, 0, + 0, 0, 0, 0, 1370, 1132, 1137, 1138, 1139, 1133, + 1134, 1140, 1141, 822, 836, 817, 0, 825, 0, 895, + 0, 0, 1017, 0, 646, 0, 0, 652, 718, 719, + 967, 656, 0, 0, 663, 2254, 668, 966, 966, 675, + 669, 676, 722, 677, 678, 679, 720, 966, 966, 901, + 717, 720, 702, 721, 720, 1511, 706, 0, 711, 714, + 715, 1511, 734, 1511, 0, 732, 683, 684, 1372, 897, + 469, 470, 475, 477, 0, 536, 536, 536, 519, 536, + 0, 0, 507, 2085, 0, 0, 0, 0, 516, 2085, + 0, 0, 2085, 2085, 2085, 2085, 2085, 2085, 2085, 0, + 0, 2085, 2085, 2085, 2085, 2085, 2085, 2085, 2085, 2085, + 2085, 2085, 0, 2085, 2085, 2085, 2085, 2085, 1489, 2085, + 0, 1306, 526, 527, 528, 529, 534, 535, 0, 0, + 480, 481, 0, 0, 0, 0, 0, 569, 0, 0, + 1145, 0, 574, 0, 0, 1190, 0, 0, 979, 0, + 980, 981, 982, 977, 1019, 1043, 1043, 0, 1043, 1023, + 1511, 0, 0, 0, 298, 299, 287, 0, 288, 0, + 0, 301, 302, 0, 304, 305, 306, 313, 2175, 2272, + 308, 310, 0, 0, 314, 327, 328, 329, 0, 0, + 319, 320, 0, 0, 383, 384, 386, 0, 904, 1310, + 1296, 79, 80, 2474, 762, 763, 1507, 764, 765, 769, + 0, 0, 772, 773, 774, 775, 776, 1111, 0, 0, + 1199, 0, 1203, 1205, 1296, 966, 0, 975, 0, 971, + 1049, 0, 1051, 0, 0, 144, 19, 0, 137, 134, + 0, 0, 0, 0, 0, 2042, 1985, 2081, 0, 0, + 0, 0, 2062, 0, 0, 2007, 0, 0, 0, 127, + 856, 904, 0, 850, 0, 908, 909, 912, 804, 833, + 0, 837, 0, 0, 829, 809, 826, 0, 0, 846, + 1510, 0, 0, 0, 0, 0, 1572, 0, 426, 422, + 442, 0, 0, 0, 0, 209, 1215, 0, 210, 214, + 204, 0, 0, 0, 1220, 0, 1217, 1222, 0, 224, + 0, 0, 199, 200, 1355, 1364, 0, 0, 0, 1931, + 1933, 1935, 1937, 1939, 0, 1942, 1952, 1952, 1948, 0, + 1943, 0, 1945, 0, 1721, 1733, 1734, 1722, 1921, 1670, + 0, 1718, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 912, 0, 0, 0, 1786, 1788, 0, 0, 0, + 1793, 0, 1795, 1796, 1797, 1799, 0, 0, 0, 1803, + 0, 1848, 1869, 1852, 1855, 0, 1859, 0, 1861, 1863, + 1864, 1865, 0, 0, 0, 906, 906, 0, 0, 1757, + 1757, 1757, 0, 0, 0, 0, 1757, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1690, 0, + 1691, 1692, 1693, 0, 1695, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1003, 850, 0, 0, 0, + 0, 0, 1362, 0, 102, 0, 107, 0, 0, 103, + 108, 0, 0, 105, 0, 0, 114, 84, 0, 0, + 1317, 1318, 0, 0, 0, 364, 352, 354, 0, 346, + 0, 1295, 0, 0, 0, 1082, 0, 0, -2, 1111, + 897, 0, 897, 1157, 2085, 0, 578, 0, 0, 1214, + 0, 1179, 0, 0, 0, -2, 0, 0, 0, 1290, + 0, 0, 0, 1374, 0, 812, 0, 816, 0, 0, + 821, 813, 23, 898, 0, 0, 0, 788, 792, 643, + 0, 645, 651, 659, 657, 0, 661, 0, 662, 717, + 670, 671, 966, 694, 695, 0, 0, 966, 717, 717, + 705, 720, 729, 0, 730, 1511, 1374, 0, 0, 1305, + 1440, 1408, 497, 0, 1524, 1525, 537, 0, 1531, 1540, + 1294, 1610, 0, 1540, 0, 0, 1542, 1543, 0, 0, + 0, 0, 520, 521, 0, 506, 0, 0, 0, 0, + 0, 0, 505, 0, 0, 547, 0, 0, 0, 0, + 0, 2086, 2085, 2085, 0, 514, 515, 0, 518, 0, + 0, 0, 0, 0, 0, 0, 0, 2085, 2085, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1480, 0, 0, 0, 0, 0, 0, 0, 1495, 1496, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, + 2085, 0, 0, 0, 0, 578, 1209, 1209, 1177, 1195, + 0, 473, 474, 544, 0, 0, 0, 0, 0, 0, + 0, 1009, 0, 0, 0, 1008, 0, 0, 0, 0, + 0, 0, 0, 0, 897, 1044, 0, 1046, 1047, 1021, + -2, 0, 979, 1026, 1916, 0, 291, 292, 0, 0, + 297, 315, 317, 289, 0, 0, 0, 316, 318, 322, + 323, 382, 385, 387, 850, 76, 1297, 0, 0, 1398, + 0, 1112, 1113, 1115, 1116, 0, 2091, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, 2149, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - -2, -2, -2, -2, -2, -2, -2, 1110, 780, 1200, - 0, 1207, 957, 969, 976, 1050, 1052, 162, 972, 0, - 147, 19, 146, 138, 139, 0, 19, 0, 0, 0, - 0, 1989, 2066, 2065, 2033, 0, 2034, 2063, 2068, 0, - 2071, 0, 454, 860, 0, 850, 852, 877, 0, 0, - 915, 913, 914, 836, 838, 0, 0, 836, 0, 0, - 845, 0, 0, 0, 0, 0, 0, 1204, 0, 0, - 739, 172, 449, 0, 0, 0, 0, 0, 767, 0, - 1219, 206, 0, 0, 226, 0, 0, 0, 1364, 1359, - 1915, 1944, 1946, 0, 1953, 1949, 1665, 1674, 1714, 0, - 0, 0, 0, 0, 1723, 2064, 2064, 1726, 2060, 2062, - 2060, 1732, 1732, 0, 1274, 0, 1275, 912, 163, 0, - 0, 0, 0, 1794, 0, 0, 0, 832, 0, 0, - 0, 0, 0, 1753, 1755, 1757, 1757, 1764, 1758, 1765, - 1766, 1757, 1757, 1757, 1757, 1771, 1757, 1757, 1757, 1757, - 1757, 1757, 1757, 1757, 1757, 1757, 1757, 1751, 1694, 1696, - 0, 1699, 0, 1702, 1703, 0, 0, 0, 1974, 1975, - 841, 874, 0, 0, 887, 888, 889, 890, 891, 0, - 0, 65, 65, 1364, 0, 0, 0, 0, 0, 120, - 0, 0, 0, 0, 0, 0, 0, 1326, 1334, 0, - 356, 0, 85, 86, 88, 0, 0, 0, 0, 0, - 0, 0, 101, 1086, 0, 1080, 0, 0, 1097, 1098, - 1100, 0, 1103, 1104, 1105, 0, 0, 1517, 0, 1161, - 1158, 1159, 1160, 0, 0, 1209, 579, 580, 581, 582, - 0, 0, 0, 1213, 0, 0, 0, 1170, 0, 0, - 0, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, - 1287, -2, 1300, 0, 1511, 0, 0, 0, 1517, 1346, - 0, 0, 1351, 0, 0, 1517, 1517, 0, 1382, 0, - 1371, 0, 0, 836, 0, 1018, 844, 0, -2, 0, - 0, 790, 0, 647, 658, 664, 966, 688, 902, 903, - 1511, 966, 966, 717, 735, 731, 1382, 1373, 0, 476, - 536, 0, 1428, 0, 0, 1434, 0, 1441, 490, 0, - 538, 0, 1530, 1560, 1541, 1560, 1611, 1560, 1560, 1294, - 0, 538, 0, 0, 508, 0, 0, 0, 0, 0, - 504, 541, 912, 491, 493, 494, 495, 545, 546, 548, - 0, 550, 551, 510, 522, 523, 524, 525, 0, 0, - 0, 517, 530, 531, 532, 533, 492, 1457, 1458, 1459, - 1462, 1463, 1464, 1465, 0, 0, 1468, 1469, 1470, 1471, - 1472, 1557, 1558, 1559, 1473, 1474, 1475, 1476, 1477, 1478, - 1479, 1497, 1498, 1499, 1500, 1501, 1502, 1481, 1482, 1483, - 1484, 1485, 1486, 1487, 1488, 0, 0, 1492, 0, 0, - 1080, 0, 484, 485, 0, 487, 0, 0, 1161, 0, - 0, 0, 0, 0, 1209, 572, 0, 0, 573, 1179, - 0, 1197, 0, 1191, 1192, 0, 0, 814, 966, 375, - 0, 1013, 1004, 0, 986, 0, 988, 1010, 989, 1011, - 0, 0, 993, 0, 995, 0, 997, 0, 991, 992, - 999, 990, 966, 978, 1020, 1045, 1022, 1025, 1027, 1028, - 1034, 0, 0, 0, 0, 285, 294, 295, 296, 303, - 0, 598, 309, 918, 1508, 770, 771, 1399, 1400, 778, - 0, 1117, 0, 955, 0, 0, 142, 145, 0, 140, - 0, 0, 0, 0, 132, 130, 2059, 0, 0, 862, - 186, 0, 0, 918, 854, 0, 0, 910, 911, 0, - 834, 0, 839, 836, 808, 830, 807, 827, 828, 847, - 1512, 1513, 1514, 1515, 0, 1573, 414, 0, 1216, 206, - 211, 212, 213, 207, 205, 1223, 0, 1225, 0, 1357, - 0, 0, 1950, 1719, 1675, 0, 1677, 1679, 1724, 1725, - 1727, 1728, 1729, 1730, 1731, 1680, 0, 1276, 1787, 1789, - 0, 1791, 1792, 1800, 1801, 0, 1856, 1860, 0, 0, - 1847, 0, 0, 0, 0, 1762, 1763, 1767, 1768, 1769, - 1770, 1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1780, - 1781, 1782, 906, 1752, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 885, 0, 0, 0, - 67, 0, 67, 1363, 1365, 113, 115, 0, 109, 110, - 111, 0, 0, 1048, 1340, 1511, 1328, 0, 1320, 0, - 1334, 0, 0, 0, 87, 0, 89, 0, 2249, 0, - 0, 0, 0, 1296, 1088, 0, 0, 1079, 0, 1090, - 1106, 1102, 0, 0, 0, 0, 1518, 1519, 1521, 1522, - 1523, 0, 1128, 0, 0, 1149, 1150, 1151, 1175, 1163, - 0, 584, 585, 0, 0, 0, 597, 593, 594, 595, - 575, 1208, 1186, 0, 0, 1186, 1173, 0, 0, 1185, - 0, 1301, 2081, 2081, 2081, 1340, 0, 0, 0, 1442, - 2081, 2081, 0, 1348, 1350, 1340, 0, 0, 0, 1446, - 1385, 0, 0, 1376, 0, 0, 836, 820, 819, 896, - 1043, 0, 0, 966, 789, 792, 793, 665, 703, 707, - 704, 966, 1385, 468, 1406, 0, 0, 0, 0, 0, - 1438, 0, 0, 1410, 0, 509, 539, 0, -2, 0, - 1561, 0, 1544, 1561, 0, 0, 1560, 0, 498, 538, - 0, 0, 0, 552, 0, 560, 561, 1245, 1245, 1245, - 1245, 558, 1606, 0, 559, 0, 543, 0, 549, 1460, - 1461, 0, 1466, 1467, 0, 1491, 0, 0, 479, 482, - 0, 1084, 1085, -2, 0, 0, 0, 564, 0, 0, - 0, 565, 566, 571, 1210, 1211, 1170, 0, 1186, 0, - 1196, 0, 1193, 1194, 906, 0, 0, 0, 983, 1014, - 0, 0, 984, 0, 985, 987, 1012, 0, 1006, 994, - 996, 998, 373, 1029, 0, 0, 1031, 1032, 1033, 1024, - 311, 872, 0, 1114, 0, 0, 940, 0, 0, 973, - 0, 19, 0, 0, 135, 2069, 2072, 864, 0, 861, - 187, 0, 0, 0, 875, 856, 0, 853, 0, 916, - 917, 835, 806, 1516, 208, 203, 1224, 1367, 0, 1358, - 0, 1630, 1689, 0, 1802, 0, 0, 1757, 1754, 1757, - 1756, 1748, 0, 1697, 0, 1700, 0, 1704, 1705, 0, - 1707, 1708, 1709, 0, 1711, 1712, 0, 883, 0, 63, - 0, 66, 64, 0, 0, 0, 119, 1315, 0, 1340, - 1319, 0, 0, 0, 1321, 0, 0, 0, 0, 0, - 90, 0, 0, 0, 0, 0, 0, 99, 0, 0, - 1087, 0, 1081, 0, 0, 1099, 1101, 0, 1135, 1446, - 0, 1135, 1162, 1148, 0, 1129, 0, 0, 586, 587, - 0, 590, 596, 1164, 0, 0, 1167, 1168, 1166, 1169, - 0, 0, 1183, 0, 0, 0, 0, 1288, 0, 1291, - 1307, 0, 0, 0, -2, 1352, 0, 0, -2, 1345, - 0, 1391, 0, 1383, 0, 1375, 0, 1378, 0, 824, - 818, 966, 966, -2, 786, 791, 0, 708, 1391, 1408, - 0, 1429, 0, 0, 0, 0, 0, 0, 0, 1409, - 0, 1422, 540, 1562, -2, 1576, 1578, 0, 1306, 1581, - 1582, 0, 0, 0, 0, 0, 0, 1637, 1590, 0, - 0, 0, 1595, 1596, 1597, 0, 0, 1600, 0, 0, - 0, 1968, 1969, 0, 1609, 0, 0, 0, 0, 0, - 0, 0, 1538, 499, 500, 0, 502, 503, 1245, 0, - 554, 555, 556, 557, 1607, 542, 496, 2081, 512, 1490, - 1493, 1494, 483, 486, 0, 0, 570, 567, 568, 1173, - 1178, 1189, 1198, 815, 899, 966, 376, 377, 1015, 0, - 1005, 1007, 1038, 1035, 0, 0, 919, 1118, 1206, 956, - 964, 2483, 2485, 2482, 136, 141, 0, 0, 866, 0, - 863, 0, 857, 859, 197, 860, 855, 905, 157, 189, - 0, 0, 1676, 0, 0, 0, 1790, 1845, 1846, 1760, - 1761, 0, 1749, 0, 1743, 1744, 1745, 1750, 0, 0, - 0, 0, 886, 881, 68, 117, 116, 0, 0, 1316, - 0, 0, 0, 1332, 1333, 0, 1335, 1336, 1337, 0, - 0, 0, 0, 72, 0, 0, 0, 1296, 0, 1296, - 0, 0, 0, 0, 1089, 1083, 1093, 1107, 0, 1120, - 1127, 1142, 1312, 1520, 1126, 0, 0, 0, 583, 588, - 0, 591, 592, 1187, 1186, 0, 1171, 1172, 0, 1181, - 0, 0, 1302, 1303, 1304, 1175, 1443, 1444, 1445, 1401, - 1347, 0, -2, 1454, 0, 0, 1343, 1367, 1401, 0, - 1379, 0, 1386, 0, 1384, 1377, 823, 906, 787, 1388, - 478, 1440, 1430, 0, 1432, 0, 0, 0, 0, 1411, - -2, 0, 1577, 1579, 1580, 1583, 1584, 1585, 1642, 1643, - 1644, 0, 0, 1588, 1639, 1640, 1641, 1589, 0, 0, - 0, 1594, 0, 0, 0, 0, 1966, 1967, 1635, 0, - 0, 1545, 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, - 1555, 1556, 1546, 0, 0, 0, 1537, 1539, 501, 553, - 0, 1246, 2081, 2081, 0, 0, 0, 1252, 1253, 2081, - 2081, 2081, 2081, 2081, 2081, 0, 0, 0, 2081, 2081, - 2081, 2081, 1267, 1268, 0, 2081, 2081, 0, 2081, 0, - 0, 1188, 372, 374, 0, 0, 1039, 1041, 1036, 1037, - 958, 0, 0, 0, 0, 131, 133, 148, 0, 865, - 188, 0, 862, 159, 0, 180, 0, 1368, 0, 1688, - 0, 0, 0, 1759, 1746, 0, 0, 0, 0, 0, - 1970, 1971, 1972, 0, 1698, 1701, 1706, 1710, 0, 1341, - 1329, 1330, 1331, 1327, 0, 0, 1338, 1339, 0, 70, - 0, 93, 0, 0, 94, 1296, 95, 1296, 0, 0, - 1077, 0, 0, 1143, 1144, 1152, 1153, 0, 1155, 1156, - 1176, 589, 1165, 1174, 1180, 1183, 0, 1245, 1289, 1403, - 0, 1349, 1305, 1456, 2081, 1175, 1354, 1403, 0, 1448, - 2081, 2081, 1369, 0, 1381, 0, 1393, 0, 1387, 899, - 467, 0, 1390, 1426, 1431, 1433, 1435, 0, 1439, 1437, - 1412, -2, 0, 1420, 0, 0, 1586, 1587, 0, 0, - 1866, 2081, 0, 0, 0, 1625, 0, 1245, 1245, 1245, - 1245, 0, 562, 563, 0, 0, 1249, 1250, 0, 0, - 0, 0, 0, 0, 0, 0, 1261, 1262, 0, 0, - 0, 0, 0, 0, 0, 511, 0, 0, 489, 1016, - 1030, 0, 965, 0, 0, 0, 0, 0, 864, 149, - 0, 158, 177, 0, 190, 191, 0, 0, 0, 0, - 1360, 0, 1633, 1634, 0, 1735, 0, 0, 0, 1739, - 1740, 1741, 1742, 118, 1334, 1334, 1296, 72, 0, 92, - 0, 96, 97, 0, 1296, 0, 1119, 0, 1154, 1182, - 1184, 1244, 1342, 0, 1440, 1455, 0, 1353, 1344, 1447, - 0, 0, 0, 1380, 1392, 0, 1395, 785, 1389, 1407, - 0, 1436, 1413, 1421, 0, 1416, 0, 0, 0, 1638, - 0, 1593, 0, 1599, 0, 1603, 1613, 1626, 0, 0, - 1526, 0, 1528, 0, 1532, 0, 1534, 0, 0, 1247, - 1248, 1251, 1254, 1255, 1256, 1257, 1258, 1259, 0, 1263, - 1264, 1265, 1266, 1269, 1270, 1271, 1272, 513, 488, 1040, - 1042, 0, 1916, 960, 961, 0, 868, 858, 866, 160, - 164, 0, 186, 183, 0, 192, 0, 0, 0, 0, - 1356, 0, 1631, 0, 1736, 1737, 1738, 1322, 1334, 1323, - 1334, 69, 71, 73, 91, 1296, 98, 0, 1121, 1122, - 1136, 0, 1428, 1460, 1449, 1450, 1451, 1394, 1427, 1415, - 0, -2, 1423, 0, 0, 1918, 1928, 1929, 1591, 1598, - 0, 1602, 1604, 1605, 1612, 1614, 1615, 0, 1627, 1628, - 1629, 1636, 1245, 1245, 1245, 1245, 1536, 1260, 959, 0, - 0, 867, 0, 851, 151, 0, 0, 181, 182, 184, - 0, 193, 0, 195, 196, 0, 0, 1747, 1324, 1325, - 100, 1123, 1404, 0, 1406, 1417, -2, 0, 1425, 0, - 1592, 1603, 1616, 0, 1617, 0, 0, 0, 1527, 1529, - 1533, 1535, 1916, 962, 869, 1366, 0, 165, 0, 167, - 169, 170, 1563, 178, 179, 185, 194, 0, 0, 1108, - 1124, 0, 0, 1408, 1424, 1919, 1601, 1618, 1620, 1621, - 0, 0, 1619, 0, 152, 153, 0, 166, 0, 0, - 1361, 1632, 1125, 1405, 1402, 1622, 1624, 1623, 963, 0, - 0, 168, 1564, 154, 155, 156, 0, 1565, + -2, -2, -2, 1110, 780, 1200, 0, 1207, 957, 969, + 976, 1050, 1052, 162, 972, 0, 147, 19, 146, 138, + 139, 0, 19, 0, 0, 0, 0, 1989, 2070, 2069, + 2037, 0, 2038, 2067, 2072, 0, 2075, 0, 454, 860, + 0, 850, 852, 877, 0, 0, 915, 913, 914, 836, + 838, 0, 0, 836, 0, 0, 845, 0, 0, 0, + 0, 0, 0, 1204, 0, 0, 739, 172, 449, 0, + 0, 0, 0, 0, 767, 0, 1219, 206, 0, 0, + 226, 0, 0, 0, 1364, 1359, 1915, 1944, 1946, 0, + 1953, 1949, 1665, 1674, 1714, 0, 0, 0, 0, 0, + 1723, 2068, 2068, 1726, 2064, 2066, 2064, 1732, 1732, 0, + 1274, 0, 1275, 912, 163, 0, 0, 0, 0, 1794, + 0, 0, 0, 832, 0, 0, 0, 0, 0, 1753, + 1755, 1757, 1757, 1764, 1758, 1765, 1766, 1757, 1757, 1757, + 1757, 1771, 1757, 1757, 1757, 1757, 1757, 1757, 1757, 1757, + 1757, 1757, 1757, 1751, 1694, 1696, 0, 1699, 0, 1702, + 1703, 0, 0, 0, 1974, 1975, 841, 874, 0, 0, + 887, 888, 889, 890, 891, 0, 0, 65, 65, 1364, + 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, + 0, 0, 0, 1326, 1334, 0, 356, 0, 85, 86, + 88, 0, 0, 0, 0, 0, 0, 0, 101, 1086, + 0, 1080, 0, 0, 1097, 1098, 1100, 0, 1103, 1104, + 1105, 0, 0, 1517, 0, 1161, 1158, 1159, 1160, 0, + 0, 1209, 579, 580, 581, 582, 0, 0, 0, 1213, + 0, 0, 0, 1170, 0, 0, 0, 1278, 1279, 1280, + 1281, 1282, 1283, 1284, 1285, 1286, 1287, -2, 1300, 0, + 1511, 0, 0, 0, 1517, 1346, 0, 0, 1351, 0, + 0, 1517, 1517, 0, 1382, 0, 1371, 0, 0, 836, + 0, 1018, 844, 0, -2, 0, 0, 790, 0, 647, + 658, 664, 966, 688, 902, 903, 1511, 966, 966, 717, + 735, 731, 1382, 1373, 0, 476, 536, 0, 1428, 0, + 0, 1434, 0, 1441, 490, 0, 538, 0, 1530, 1560, + 1541, 1560, 1611, 1560, 1560, 1294, 0, 538, 0, 0, + 508, 0, 0, 0, 0, 0, 504, 541, 912, 491, + 493, 494, 495, 545, 546, 548, 0, 550, 551, 510, + 522, 523, 524, 525, 0, 0, 0, 517, 530, 531, + 532, 533, 492, 1457, 1458, 1459, 1462, 1463, 1464, 1465, + 0, 0, 1468, 1469, 1470, 1471, 1472, 1557, 1558, 1559, + 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1497, 1498, 1499, + 1500, 1501, 1502, 1481, 1482, 1483, 1484, 1485, 1486, 1487, + 1488, 0, 0, 1492, 0, 0, 1080, 0, 484, 485, + 0, 487, 0, 0, 1161, 0, 0, 0, 0, 0, + 1209, 572, 0, 0, 573, 1179, 0, 1197, 0, 1191, + 1192, 0, 0, 814, 966, 375, 0, 1013, 1004, 0, + 986, 0, 988, 1010, 989, 1011, 0, 0, 993, 0, + 995, 0, 997, 0, 991, 992, 999, 990, 966, 978, + 1020, 1045, 1022, 1025, 1027, 1028, 1034, 0, 0, 0, + 0, 285, 294, 295, 296, 303, 0, 598, 309, 918, + 1508, 770, 771, 1399, 1400, 778, 0, 1117, 0, 955, + 0, 0, 142, 145, 0, 140, 0, 0, 0, 0, + 132, 130, 2063, 0, 0, 862, 186, 0, 0, 918, + 854, 0, 0, 910, 911, 0, 834, 0, 839, 836, + 808, 830, 807, 827, 828, 847, 1512, 1513, 1514, 1515, + 0, 1573, 414, 0, 1216, 206, 211, 212, 213, 207, + 205, 1223, 0, 1225, 0, 1357, 0, 0, 1950, 1719, + 1675, 0, 1677, 1679, 1724, 1725, 1727, 1728, 1729, 1730, + 1731, 1680, 0, 1276, 1787, 1789, 0, 1791, 1792, 1800, + 1801, 0, 1856, 1860, 0, 0, 1847, 0, 0, 0, + 0, 1762, 1763, 1767, 1768, 1769, 1770, 1772, 1773, 1774, + 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 906, 1752, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 885, 0, 0, 0, 67, 0, 67, 1363, + 1365, 113, 115, 0, 109, 110, 111, 0, 0, 1048, + 1340, 1511, 1328, 0, 1320, 0, 1334, 0, 0, 0, + 87, 0, 89, 0, 2257, 0, 0, 0, 0, 1296, + 1088, 0, 0, 1079, 0, 1090, 1106, 1102, 0, 0, + 0, 0, 1518, 1519, 1521, 1522, 1523, 0, 1128, 0, + 0, 1149, 1150, 1151, 1175, 1163, 0, 584, 585, 0, + 0, 0, 597, 593, 594, 595, 575, 1208, 1186, 0, + 0, 1186, 1173, 0, 0, 1185, 0, 1301, 2085, 2085, + 2085, 1340, 0, 0, 0, 1442, 2085, 2085, 0, 1348, + 1350, 1340, 0, 0, 0, 1446, 1385, 0, 0, 1376, + 0, 0, 836, 820, 819, 896, 1043, 0, 0, 966, + 789, 792, 793, 665, 703, 707, 704, 966, 1385, 468, + 1406, 0, 0, 0, 0, 0, 1438, 0, 0, 1410, + 0, 509, 539, 0, -2, 0, 1561, 0, 1544, 1561, + 0, 0, 1560, 0, 498, 538, 0, 0, 0, 552, + 0, 560, 561, 1245, 1245, 1245, 1245, 558, 1606, 0, + 559, 0, 543, 0, 549, 1460, 1461, 0, 1466, 1467, + 0, 1491, 0, 0, 479, 482, 0, 1084, 1085, -2, + 0, 0, 0, 564, 0, 0, 0, 565, 566, 571, + 1210, 1211, 1170, 0, 1186, 0, 1196, 0, 1193, 1194, + 906, 0, 0, 0, 983, 1014, 0, 0, 984, 0, + 985, 987, 1012, 0, 1006, 994, 996, 998, 373, 1029, + 0, 0, 1031, 1032, 1033, 1024, 311, 872, 0, 1114, + 0, 0, 940, 0, 0, 973, 0, 19, 0, 0, + 135, 2073, 2076, 864, 0, 861, 187, 0, 0, 0, + 875, 856, 0, 853, 0, 916, 917, 835, 806, 1516, + 208, 203, 1224, 1367, 0, 1358, 0, 1630, 1689, 0, + 1802, 0, 0, 1757, 1754, 1757, 1756, 1748, 0, 1697, + 0, 1700, 0, 1704, 1705, 0, 1707, 1708, 1709, 0, + 1711, 1712, 0, 883, 0, 63, 0, 66, 64, 0, + 0, 0, 119, 1315, 0, 1340, 1319, 0, 0, 0, + 1321, 0, 0, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 99, 0, 0, 1087, 0, 1081, 0, + 0, 1099, 1101, 0, 1135, 1446, 0, 1135, 1162, 1148, + 0, 1129, 0, 0, 586, 587, 0, 590, 596, 1164, + 0, 0, 1167, 1168, 1166, 1169, 0, 0, 1183, 0, + 0, 0, 0, 1288, 0, 1291, 1307, 0, 0, 0, + -2, 1352, 0, 0, -2, 1345, 0, 1391, 0, 1383, + 0, 1375, 0, 1378, 0, 824, 818, 966, 966, -2, + 786, 791, 0, 708, 1391, 1408, 0, 1429, 0, 0, + 0, 0, 0, 0, 0, 1409, 0, 1422, 540, 1562, + -2, 1576, 1578, 0, 1306, 1581, 1582, 0, 0, 0, + 0, 0, 0, 1637, 1590, 0, 0, 0, 1595, 1596, + 1597, 0, 0, 1600, 0, 0, 0, 1968, 1969, 0, + 1609, 0, 0, 0, 0, 0, 0, 0, 1538, 499, + 500, 0, 502, 503, 1245, 0, 554, 555, 556, 557, + 1607, 542, 496, 2085, 512, 1490, 1493, 1494, 483, 486, + 0, 0, 570, 567, 568, 1173, 1178, 1189, 1198, 815, + 899, 966, 376, 377, 1015, 0, 1005, 1007, 1038, 1035, + 0, 0, 919, 1118, 1206, 956, 964, 2491, 2493, 2490, + 136, 141, 0, 0, 866, 0, 863, 0, 857, 859, + 197, 860, 855, 905, 157, 189, 0, 0, 1676, 0, + 0, 0, 1790, 1845, 1846, 1760, 1761, 0, 1749, 0, + 1743, 1744, 1745, 1750, 0, 0, 0, 0, 886, 881, + 68, 117, 116, 0, 0, 1316, 0, 0, 0, 1332, + 1333, 0, 1335, 1336, 1337, 0, 0, 0, 0, 72, + 0, 0, 0, 1296, 0, 1296, 0, 0, 0, 0, + 1089, 1083, 1093, 1107, 0, 1120, 1127, 1142, 1312, 1520, + 1126, 0, 0, 0, 583, 588, 0, 591, 592, 1187, + 1186, 0, 1171, 1172, 0, 1181, 0, 0, 1302, 1303, + 1304, 1175, 1443, 1444, 1445, 1401, 1347, 0, -2, 1454, + 0, 0, 1343, 1367, 1401, 0, 1379, 0, 1386, 0, + 1384, 1377, 823, 906, 787, 1388, 478, 1440, 1430, 0, + 1432, 0, 0, 0, 0, 1411, -2, 0, 1577, 1579, + 1580, 1583, 1584, 1585, 1642, 1643, 1644, 0, 0, 1588, + 1639, 1640, 1641, 1589, 0, 0, 0, 1594, 0, 0, + 0, 0, 1966, 1967, 1635, 0, 0, 1545, 1547, 1548, + 1549, 1550, 1551, 1552, 1553, 1554, 1555, 1556, 1546, 0, + 0, 0, 1537, 1539, 501, 553, 0, 1246, 2085, 2085, + 0, 0, 0, 1252, 1253, 2085, 2085, 2085, 2085, 2085, + 2085, 0, 0, 0, 2085, 2085, 2085, 2085, 1267, 1268, + 0, 2085, 2085, 0, 2085, 0, 0, 1188, 372, 374, + 0, 0, 1039, 1041, 1036, 1037, 958, 0, 0, 0, + 0, 131, 133, 148, 0, 865, 188, 0, 862, 159, + 0, 180, 0, 1368, 0, 1688, 0, 0, 0, 1759, + 1746, 0, 0, 0, 0, 0, 1970, 1971, 1972, 0, + 1698, 1701, 1706, 1710, 0, 1341, 1329, 1330, 1331, 1327, + 0, 0, 1338, 1339, 0, 70, 0, 93, 0, 0, + 94, 1296, 95, 1296, 0, 0, 1077, 0, 0, 1143, + 1144, 1152, 1153, 0, 1155, 1156, 1176, 589, 1165, 1174, + 1180, 1183, 0, 1245, 1289, 1403, 0, 1349, 1305, 1456, + 2085, 1175, 1354, 1403, 0, 1448, 2085, 2085, 1369, 0, + 1381, 0, 1393, 0, 1387, 899, 467, 0, 1390, 1426, + 1431, 1433, 1435, 0, 1439, 1437, 1412, -2, 0, 1420, + 0, 0, 1586, 1587, 0, 0, 1866, 2085, 0, 0, + 0, 1625, 0, 1245, 1245, 1245, 1245, 0, 562, 563, + 0, 0, 1249, 1250, 0, 0, 0, 0, 0, 0, + 0, 0, 1261, 1262, 0, 0, 0, 0, 0, 0, + 0, 511, 0, 0, 489, 1016, 1030, 0, 965, 0, + 0, 0, 0, 0, 864, 149, 0, 158, 177, 0, + 190, 191, 0, 0, 0, 0, 1360, 0, 1633, 1634, + 0, 1735, 0, 0, 0, 1739, 1740, 1741, 1742, 118, + 1334, 1334, 1296, 72, 0, 92, 0, 96, 97, 0, + 1296, 0, 1119, 0, 1154, 1182, 1184, 1244, 1342, 0, + 1440, 1455, 0, 1353, 1344, 1447, 0, 0, 0, 1380, + 1392, 0, 1395, 785, 1389, 1407, 0, 1436, 1413, 1421, + 0, 1416, 0, 0, 0, 1638, 0, 1593, 0, 1599, + 0, 1603, 1613, 1626, 0, 0, 1526, 0, 1528, 0, + 1532, 0, 1534, 0, 0, 1247, 1248, 1251, 1254, 1255, + 1256, 1257, 1258, 1259, 0, 1263, 1264, 1265, 1266, 1269, + 1270, 1271, 1272, 513, 488, 1040, 1042, 0, 1916, 960, + 961, 0, 868, 858, 866, 160, 164, 0, 186, 183, + 0, 192, 0, 0, 0, 0, 1356, 0, 1631, 0, + 1736, 1737, 1738, 1322, 1334, 1323, 1334, 69, 71, 73, + 91, 1296, 98, 0, 1121, 1122, 1136, 0, 1428, 1460, + 1449, 1450, 1451, 1394, 1427, 1415, 0, -2, 1423, 0, + 0, 1918, 1928, 1929, 1591, 1598, 0, 1602, 1604, 1605, + 1612, 1614, 1615, 0, 1627, 1628, 1629, 1636, 1245, 1245, + 1245, 1245, 1536, 1260, 959, 0, 0, 867, 0, 851, + 151, 0, 0, 181, 182, 184, 0, 193, 0, 195, + 196, 0, 0, 1747, 1324, 1325, 100, 1123, 1404, 0, + 1406, 1417, -2, 0, 1425, 0, 1592, 1603, 1616, 0, + 1617, 0, 0, 0, 1527, 1529, 1533, 1535, 1916, 962, + 869, 1366, 0, 165, 0, 167, 169, 170, 1563, 178, + 179, 185, 194, 0, 0, 1108, 1124, 0, 0, 1408, + 1424, 1919, 1601, 1618, 1620, 1621, 0, 0, 1619, 0, + 152, 153, 0, 166, 0, 0, 1361, 1632, 1125, 1405, + 1402, 1622, 1624, 1623, 963, 0, 0, 168, 1564, 154, + 155, 156, 0, 1565, } var yyTok1 = [...]int{ @@ -11928,14 +12027,14 @@ var yyTok1 = [...]int{ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 121, 3, 3, 3, 154, 144, 3, 88, 89, 151, 149, 174, 150, 173, 152, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 742, 739, - 131, 130, 132, 3, 743, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 746, 743, + 131, 130, 132, 3, 747, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 156, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 740, 143, 741, 157, + 3, 3, 3, 744, 143, 745, 157, } var yyTok2 = [...]int{ @@ -12060,7 +12159,8 @@ var yyTok3 = [...]int{ 58045, 720, 58046, 721, 58047, 722, 58048, 723, 58049, 724, 58050, 725, 58051, 726, 58052, 727, 58053, 728, 58054, 729, 58055, 730, 58056, 731, 58057, 732, 58058, 733, 58059, 734, - 58060, 735, 58061, 736, 58062, 737, 58063, 738, 0, + 58060, 735, 58061, 736, 58062, 737, 58063, 738, 58064, 739, + 58065, 740, 58066, 741, 58067, 742, 0, } var yyErrorMessages = [...]struct { @@ -29862,9 +29962,77 @@ yydefault: } yyVAL.union = yyLOCAL case 2033: - yyDollar = yyS[yypt-4 : yypt+1] + yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL *tree.T //line mysql_sql.y:13600 + { + locale := "" + yyLOCAL = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: yyDollar[1].str, + DisplayWith: yyDollar[2].lengthOptUnion(), + Oid: uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } + yyVAL.union = yyLOCAL + case 2034: + yyDollar = yyS[yypt-2 : yypt+1] + var yyLOCAL *tree.T +//line mysql_sql.y:13613 + { + locale := "" + yyLOCAL = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: yyDollar[1].str, + DisplayWith: yyDollar[2].lengthOptUnion(), + Oid: uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } + yyVAL.union = yyLOCAL + case 2035: + yyDollar = yyS[yypt-2 : yypt+1] + var yyLOCAL *tree.T +//line mysql_sql.y:13626 + { + locale := "" + yyLOCAL = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: yyDollar[1].str, + DisplayWith: yyDollar[2].lengthOptUnion(), + Oid: uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } + yyVAL.union = yyLOCAL + case 2036: + yyDollar = yyS[yypt-2 : yypt+1] + var yyLOCAL *tree.T +//line mysql_sql.y:13639 + { + locale := "" + yyLOCAL = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: yyDollar[1].str, + DisplayWith: yyDollar[2].lengthOptUnion(), + Oid: uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } + yyVAL.union = yyLOCAL + case 2037: + yyDollar = yyS[yypt-4 : yypt+1] + var yyLOCAL *tree.T +//line mysql_sql.y:13652 { locale := "" yyLOCAL = &tree.T{ @@ -29878,10 +30046,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2034: + case 2038: yyDollar = yyS[yypt-4 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13613 +//line mysql_sql.y:13665 { locale := "" yyLOCAL = &tree.T{ @@ -29895,10 +30063,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2035: + case 2039: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13626 +//line mysql_sql.y:13678 { locale := "" yyLOCAL = &tree.T{ @@ -29912,20 +30080,20 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2036: + case 2040: yyDollar = yyS[yypt-2 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:13641 +//line mysql_sql.y:13693 { yyLOCAL = &tree.Do{ Exprs: yyDollar[2].exprsUnion(), } } yyVAL.union = yyLOCAL - case 2037: + case 2041: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:13649 +//line mysql_sql.y:13701 { yyLOCAL = &tree.Declare{ Variables: yyDollar[2].strsUnion(), @@ -29934,10 +30102,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2038: + case 2042: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.Statement -//line mysql_sql.y:13658 +//line mysql_sql.y:13710 { yyLOCAL = &tree.Declare{ Variables: yyDollar[2].strsUnion(), @@ -29946,83 +30114,83 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2039: + case 2043: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL *tree.T -//line mysql_sql.y:13668 +//line mysql_sql.y:13720 { yyLOCAL = tree.NewSpatialType(yyDollar[1].str) } yyVAL.union = yyLOCAL - case 2058: + case 2062: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:13696 +//line mysql_sql.y:13748 { yyLOCAL = make([]string, 0, 4) yyLOCAL = append(yyLOCAL, yyDollar[1].str) } yyVAL.union = yyLOCAL - case 2059: + case 2063: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL []string -//line mysql_sql.y:13701 +//line mysql_sql.y:13753 { yyLOCAL = append(yyDollar[1].strsUnion(), yyDollar[3].str) } yyVAL.union = yyLOCAL - case 2060: + case 2064: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13707 +//line mysql_sql.y:13759 { yyLOCAL = 0 } yyVAL.union = yyLOCAL - case 2062: + case 2066: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13714 +//line mysql_sql.y:13766 { yyLOCAL = 0 } yyVAL.union = yyLOCAL - case 2063: + case 2067: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13718 +//line mysql_sql.y:13770 { yyLOCAL = int32(yyDollar[2].item.(int64)) } yyVAL.union = yyLOCAL - case 2064: + case 2068: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13723 +//line mysql_sql.y:13775 { yyLOCAL = int32(-1) } yyVAL.union = yyLOCAL - case 2065: + case 2069: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13727 +//line mysql_sql.y:13779 { yyLOCAL = int32(yyDollar[2].item.(int64)) } yyVAL.union = yyLOCAL - case 2066: + case 2070: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL int32 -//line mysql_sql.y:13733 +//line mysql_sql.y:13785 { yyLOCAL = tree.GetDisplayWith(int32(yyDollar[2].item.(int64))) } yyVAL.union = yyLOCAL - case 2067: + case 2071: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13739 +//line mysql_sql.y:13791 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.NotDefineDisplayWidth, @@ -30030,10 +30198,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2068: + case 2072: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13746 +//line mysql_sql.y:13798 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.GetDisplayWith(int32(yyDollar[2].item.(int64))), @@ -30041,10 +30209,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2069: + case 2073: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13753 +//line mysql_sql.y:13805 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.GetDisplayWith(int32(yyDollar[2].item.(int64))), @@ -30052,10 +30220,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2070: + case 2074: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13762 +//line mysql_sql.y:13814 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: 38, // this is the default precision for decimal @@ -30063,10 +30231,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2071: + case 2075: yyDollar = yyS[yypt-3 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13769 +//line mysql_sql.y:13821 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.GetDisplayWith(int32(yyDollar[2].item.(int64))), @@ -30074,10 +30242,10 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2072: + case 2076: yyDollar = yyS[yypt-5 : yypt+1] var yyLOCAL tree.LengthScaleOpt -//line mysql_sql.y:13776 +//line mysql_sql.y:13828 { yyLOCAL = tree.LengthScaleOpt{ DisplayWith: tree.GetDisplayWith(int32(yyDollar[2].item.(int64))), @@ -30085,52 +30253,52 @@ yydefault: } } yyVAL.union = yyLOCAL - case 2073: + case 2077: yyDollar = yyS[yypt-0 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:13785 +//line mysql_sql.y:13837 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 2074: + case 2078: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:13789 +//line mysql_sql.y:13841 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 2075: + case 2079: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:13793 +//line mysql_sql.y:13845 { yyLOCAL = false } yyVAL.union = yyLOCAL - case 2076: + case 2080: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:13799 +//line mysql_sql.y:13851 { } - case 2077: + case 2081: yyDollar = yyS[yypt-1 : yypt+1] var yyLOCAL bool -//line mysql_sql.y:13801 +//line mysql_sql.y:13853 { yyLOCAL = true } yyVAL.union = yyLOCAL - case 2081: + case 2085: yyDollar = yyS[yypt-0 : yypt+1] -//line mysql_sql.y:13811 +//line mysql_sql.y:13863 { yyVAL.str = "" } - case 2082: + case 2086: yyDollar = yyS[yypt-1 : yypt+1] -//line mysql_sql.y:13815 +//line mysql_sql.y:13867 { yyVAL.str = string(yyDollar[1].str) } diff --git a/pkg/sql/parsers/dialect/mysql/mysql_sql.y b/pkg/sql/parsers/dialect/mysql/mysql_sql.y index f3d00d1868fbf..3faa761b9cb9b 100644 --- a/pkg/sql/parsers/dialect/mysql/mysql_sql.y +++ b/pkg/sql/parsers/dialect/mysql/mysql_sql.y @@ -379,7 +379,7 @@ func sqlTaskInt64(v any) int64 { %token TIME TIMESTAMP DATETIME YEAR %token CHAR VARCHAR BOOL CHARACTER VARBINARY NCHAR %token TEXT TINYTEXT MEDIUMTEXT LONGTEXT DATALINK -%token BLOB TINYBLOB MEDIUMBLOB LONGBLOB JSON ENUM UUID VECF32 VECF64 +%token BLOB TINYBLOB MEDIUMBLOB LONGBLOB JSON ENUM UUID VECF32 VECF64 VECBF16 VECF16 VECINT8 VECUINT8 %token GEOMETRY POINT LINESTRING POLYGON GEOMETRYCOLLECTION MULTIPOINT MULTILINESTRING MULTIPOLYGON %token GEOMETRY32 GEOGRAPHY GEOGRAPHY32 POINT32 LINESTRING32 POLYGON32 GEOMETRYCOLLECTION32 MULTIPOINT32 MULTILINESTRING32 MULTIPOLYGON32 %token INT1 INT2 INT3 INT4 INT8 S3OPTION STAGEOPTION @@ -13597,6 +13597,58 @@ char_type: }, } } +| VECBF16 length_option_opt + { + locale := "" + $$ = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: $1, + DisplayWith: $2, + Oid:uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } +| VECF16 length_option_opt + { + locale := "" + $$ = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: $1, + DisplayWith: $2, + Oid:uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } +| VECINT8 length_option_opt + { + locale := "" + $$ = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: $1, + DisplayWith: $2, + Oid:uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } +| VECUINT8 length_option_opt + { + locale := "" + $$ = &tree.T{ + InternalType: tree.InternalType{ + Family: tree.ArrayFamily, + Locale: &locale, + FamilyString: $1, + DisplayWith: $2, + Oid:uint32(defines.MYSQL_TYPE_VARCHAR), + }, + } + } | ENUM '(' enum_values ')' { locale := "" @@ -14115,6 +14167,10 @@ non_reserved_keyword: | JSON | VECF32 | VECF64 +| VECBF16 +| VECF16 +| VECINT8 +| VECUINT8 | KEY_BLOCK_SIZE | LISTS | OP_TYPE diff --git a/pkg/sql/parsers/dialect/mysql/mysql_sql_test.go b/pkg/sql/parsers/dialect/mysql/mysql_sql_test.go index 97faaffd8a1c1..aca1723730535 100644 --- a/pkg/sql/parsers/dialect/mysql/mysql_sql_test.go +++ b/pkg/sql/parsers/dialect/mysql/mysql_sql_test.go @@ -3337,6 +3337,50 @@ var ( input: "create table t1 (id bigint primary key, embedding vecf32(3), payload json, tags array(varchar(20)))", output: "create table t1 (id bigint primary key, embedding vecf32(3), payload json, tags array(varchar(20)))", }, + { + input: "create table t1(a vecbf16(3), b vecf16(3), c vecint8(3))", + output: "create table t1 (a vecbf16(3), b vecf16(3), c vecint8(3))", + }, + { + input: "create table t1(a vecbf16(128), b vecf16(65535), c vecint8(1))", + output: "create table t1 (a vecbf16(128), b vecf16(65535), c vecint8(1))", + }, + { + input: "create table t1(a vecuint8(3))", + output: "create table t1 (a vecuint8(3))", + }, + { + input: "create table t1(a vecuint8(128), b vecuint8(65535), c vecuint8(1))", + output: "create table t1 (a vecuint8(128), b vecuint8(65535), c vecuint8(1))", + }, + { + input: "select cast('[1,2,3]' as vecbf16(3))", + output: "select cast([1,2,3] as vecbf16(3))", + }, + { + input: "select cast('[1,2,3]' as vecf16(3))", + output: "select cast([1,2,3] as vecf16(3))", + }, + { + input: "select cast('[1,2,3]' as vecint8(3))", + output: "select cast([1,2,3] as vecint8(3))", + }, + { + input: "select cast(b as vecint8(3)) from t1", + output: "select cast(b as vecint8(3)) from t1", + }, + { + input: "select cast('[1,2,3]' as vecuint8(3))", + output: "select cast([1,2,3] as vecuint8(3))", + }, + { + input: "select cast(b as vecuint8(3)) from t1", + output: "select cast(b as vecuint8(3)) from t1", + }, + { + input: "select l2_distance(a, b) from t1", + output: "select l2_distance(a, b) from t1", + }, { input: "alter table tbl1 drop constraint fk_name", output: "alter table tbl1 drop foreign key fk_name", diff --git a/pkg/sql/parsers/tree/types.go b/pkg/sql/parsers/tree/types.go index 8ee2d778c36c3..837e0eb777a55 100644 --- a/pkg/sql/parsers/tree/types.go +++ b/pkg/sql/parsers/tree/types.go @@ -218,7 +218,7 @@ func (node *InternalType) Format(ctx *FmtCtx) { ctx.WriteString(strconv.FormatInt(int64(node.DisplayWith), 10)) ctx.WriteByte(')') } - case "vecf32", "vecf64": + case "vecf32", "vecf64", "vecbf16", "vecf16", "vecint8", "vecuint8": if node.DisplayWith >= 0 { // Prints 'vecf32(4)' ctx.WriteByte('(') diff --git a/pkg/sql/plan/apply_indices_cagra.go b/pkg/sql/plan/apply_indices_cagra.go index 52b64d53baa00..02d6873602f00 100644 --- a/pkg/sql/plan/apply_indices_cagra.go +++ b/pkg/sql/plan/apply_indices_cagra.go @@ -34,6 +34,7 @@ type cagraIndexContext struct { vecLitArg *plan.Expr origFuncName string partPos int32 + partType plan.Type pkPos int32 pkType plan.Type params string @@ -84,6 +85,7 @@ func (builder *QueryBuilder) prepareCagraIndexContext(vecCtx *vectorSortContext, keyPart := idxDef.Parts[0] partPos := vecCtx.scanNode.TableDef.Name2ColIndex[keyPart] + partType := vecCtx.scanNode.TableDef.Cols[partPos].Typ _, vecLitArg, found := builder.getArgsFromDistFn(vecCtx.distFnExpr, partPos) if !found { return nil, nil @@ -114,6 +116,7 @@ func (builder *QueryBuilder) prepareCagraIndexContext(vecCtx *vectorSortContext, vecLitArg: vecLitArg, origFuncName: origFuncName, partPos: partPos, + partType: partType, pkPos: pkPos, pkType: pkType, params: idxDef.IndexAlgoParams, @@ -142,7 +145,7 @@ func (builder *QueryBuilder) applyIndicesForSortUsingCagra(nodeID int32, vecCtx return nodeID, err } - tblCfgStr := fmt.Sprintf(`{"db": "%s", "src": "%s", "metadata":"%s", "index":"%s", "threads_search": %d, "orig_func_name": "%s", "batch_window": %d, "gpu_multi_simulation": %d}`, + tblCfgStr := fmt.Sprintf(`{"db": "%s", "src": "%s", "metadata":"%s", "index":"%s", "threads_search": %d, "orig_func_name": "%s", "batch_window": %d, "gpu_multi_simulation": %d, "parttype": %d}`, scanNode.ObjRef.SchemaName, scanNode.TableDef.Name, cagraCtx.metaDef.IndexTableName, @@ -150,7 +153,8 @@ func (builder *QueryBuilder) applyIndicesForSortUsingCagra(nodeID int32, vecCtx cagraCtx.nThread, cagraCtx.origFuncName, cagraCtx.batchWindow, - cagraCtx.gpuMultiSim) + cagraCtx.gpuMultiSim, + cagraCtx.partType.Id) // Predicate pushdown on INCLUDE columns and the primary key: peel // filters that reference only INCLUDE columns (or the PK, routed to diff --git a/pkg/sql/plan/apply_indices_hnsw.go b/pkg/sql/plan/apply_indices_hnsw.go index 1eef0876f1dad..608f0db9c0d40 100644 --- a/pkg/sql/plan/apply_indices_hnsw.go +++ b/pkg/sql/plan/apply_indices_hnsw.go @@ -305,7 +305,10 @@ func (builder *QueryBuilder) getArgsFromDistFn(distFnExpr *plan.Function, partPo } distFnArgs := distFnExpr.Args - if distFnArgs[0].Typ.GetId() != int32(types.T_array_float32) && distFnArgs[0].Typ.GetId() != int32(types.T_array_float64) { + // Accept any vector element type (f32/f64 and the narrow bf16/f16/int8/uint8), + // so a direct ivf index on a narrow-base column also pushes down rather than + // brute-forcing. + if !types.T(distFnArgs[0].Typ.GetId()).IsArrayRelate() { return } diff --git a/pkg/sql/plan/apply_indices_ivfpq.go b/pkg/sql/plan/apply_indices_ivfpq.go index 410aa276e4b7b..1c66c65c875d9 100644 --- a/pkg/sql/plan/apply_indices_ivfpq.go +++ b/pkg/sql/plan/apply_indices_ivfpq.go @@ -34,6 +34,7 @@ type ivfpqIndexContext struct { vecLitArg *plan.Expr origFuncName string partPos int32 + partType plan.Type pkPos int32 pkType plan.Type params string @@ -82,6 +83,7 @@ func (builder *QueryBuilder) prepareIvfpqIndexContext(vecCtx *vectorSortContext, keyPart := idxDef.Parts[0] partPos := vecCtx.scanNode.TableDef.Name2ColIndex[keyPart] + partType := vecCtx.scanNode.TableDef.Cols[partPos].Typ _, vecLitArg, found := builder.getArgsFromDistFn(vecCtx.distFnExpr, partPos) if !found { return nil, nil @@ -119,6 +121,7 @@ func (builder *QueryBuilder) prepareIvfpqIndexContext(vecCtx *vectorSortContext, vecLitArg: vecLitArg, origFuncName: origFuncName, partPos: partPos, + partType: partType, pkPos: pkPos, pkType: pkType, params: idxDef.IndexAlgoParams, @@ -148,7 +151,7 @@ func (builder *QueryBuilder) applyIndicesForSortUsingIvfpq(nodeID int32, vecCtx return nodeID, err } - tblCfgStr := fmt.Sprintf(`{"db": "%s", "src": "%s", "metadata":"%s", "index":"%s", "threads_search": %d, "orig_func_name": "%s", "batch_window": %d, "nprobe": %d, "gpu_multi_simulation": %d}`, + tblCfgStr := fmt.Sprintf(`{"db": "%s", "src": "%s", "metadata":"%s", "index":"%s", "threads_search": %d, "orig_func_name": "%s", "batch_window": %d, "nprobe": %d, "gpu_multi_simulation": %d, "parttype": %d}`, scanNode.ObjRef.SchemaName, scanNode.TableDef.Name, ivfpqCtx.metaDef.IndexTableName, @@ -157,7 +160,8 @@ func (builder *QueryBuilder) applyIndicesForSortUsingIvfpq(nodeID int32, vecCtx ivfpqCtx.origFuncName, ivfpqCtx.batchWindow, ivfpqCtx.nProbe, - ivfpqCtx.gpuMultiSim) + ivfpqCtx.gpuMultiSim, + ivfpqCtx.partType.Id) // Predicate pushdown on INCLUDE columns and the primary key: peel // filters that reference only INCLUDE columns (or the PK, routed to diff --git a/pkg/sql/plan/build_index_util.go b/pkg/sql/plan/build_index_util.go index 7810cbc5e0920..7a0ad35c6d14f 100644 --- a/pkg/sql/plan/build_index_util.go +++ b/pkg/sql/plan/build_index_util.go @@ -21,6 +21,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" + indexplugin "github.com/matrixorigin/matrixone/pkg/indexplugin" + catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" ) @@ -176,6 +178,10 @@ func indexColumnCheckKind(indexType tree.IndexType) string { return "ivfflat" case tree.INDEX_TYPE_HNSW: return "hnsw" + case tree.INDEX_TYPE_CAGRA: + return "cagra" + case tree.INDEX_TYPE_IVFPQ: + return "ivfpq" case tree.INDEX_TYPE_RTREE: return "rtree" default: @@ -205,8 +211,17 @@ func checkIndexColumnSupportability(ctx context.Context, col *ColDef, keyPart *t return moerr.NewNotSupported(ctx, fmt.Sprintf("DATALINK column '%s' cannot be in index", colName)) case int32(types.T_json): return moerr.NewNotSupported(ctx, fmt.Sprintf("JSON column '%s' cannot be in index", colName)) - case int32(types.T_array_float32), int32(types.T_array_float64): - if indexKind == "ivfflat" || indexKind == "hnsw" { + case int32(types.T_array_float32), int32(types.T_array_float64), + int32(types.T_array_float16), int32(types.T_array_bf16), + int32(types.T_array_int8), int32(types.T_array_uint8): + // A vector column is valid only as the key of a vector index, AND only if + // that algorithm supports this element type. Delegate to the plugin's + // catalog hook (SupportedVectorTypes) rather than hardcoding — each algo + // differs (ivfflat: f32/f64/f16/bf16/int8/uint8; cagra/ivfpq: f32/f16 only; + // hnsw: f32/f64). Non-vector index kinds (secondary/primary/unique/rtree) + // have no plugin, so the vector column is rejected. + if p, ok := indexplugin.Get(indexKind); ok && + catalogplugin.SupportsVectorType(p.Catalog(), types.T(col.Typ.Id)) { return nil } return moerr.NewNotSupported(ctx, fmt.Sprintf("VECTOR column '%s' cannot be in index", colName)) diff --git a/pkg/sql/plan/build_index_util_test.go b/pkg/sql/plan/build_index_util_test.go index 98fc1b6d280ed..e98e39680dafd 100644 --- a/pkg/sql/plan/build_index_util_test.go +++ b/pkg/sql/plan/build_index_util_test.go @@ -152,10 +152,20 @@ func TestCheckIndexColumnSupportability(t *testing.T) { require.Error(t, checkIndexColumnSupportability(ctx, colOf(types.T_json), keyPart, "secondary")) }) - t.Run("vector only allowed for ivfflat and hnsw", func(t *testing.T) { - require.NoError(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_float32), keyPart, "ivfflat")) + t.Run("vector type support is delegated to the plugin per algo", func(t *testing.T) { + // ivfflat accepts every vector element type (f32/f64 + narrow f16/bf16/int8/uint8). + for _, ty := range []types.T{ + types.T_array_float32, types.T_array_float64, types.T_array_float16, + types.T_array_bf16, types.T_array_int8, types.T_array_uint8, + } { + require.NoError(t, checkIndexColumnSupportability(ctx, colOf(ty), keyPart, "ivfflat")) + } require.NoError(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_float64), keyPart, "hnsw")) + // A vector column in a non-vector index kind has no plugin → rejected, + // for both wide and narrow element types. require.Error(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_float32), keyPart, "secondary")) + require.Error(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_int8), keyPart, "secondary")) + require.Error(t, checkIndexColumnSupportability(ctx, colOf(types.T_array_float16), keyPart, "unique")) }) t.Run("enum rejected only in primary key", func(t *testing.T) { diff --git a/pkg/sql/plan/build_show_util.go b/pkg/sql/plan/build_show_util.go index fbbd3d32f4d5d..5bc8fb7d7e6ce 100644 --- a/pkg/sql/plan/build_show_util.go +++ b/pkg/sql/plan/build_show_util.go @@ -938,7 +938,7 @@ func FormatColType(colType plan.Type) string { case types.T_bit, types.T_char, types.T_varchar, types.T_binary, types.T_varbinary: suffix = fmt.Sprintf("(%d)", colType.Width) - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: suffix = fmt.Sprintf("(%d)", colType.Width) } diff --git a/pkg/sql/plan/build_show_util_test.go b/pkg/sql/plan/build_show_util_test.go index a04325ce598ad..54df434667040 100644 --- a/pkg/sql/plan/build_show_util_test.go +++ b/pkg/sql/plan/build_show_util_test.go @@ -441,6 +441,17 @@ func TestFormatColTypeArrayMetadata(t *testing.T) { })) } +func TestFormatColTypeVector(t *testing.T) { + // Every vector type must round-trip its dimension in SHOW CREATE, not just + // f32/f64 (the narrow types were previously missing the (N) suffix). + require.Equal(t, "VECF32(3)", FormatColType(plan.Type{Id: int32(types.T_array_float32), Width: 3})) + require.Equal(t, "VECF64(3)", FormatColType(plan.Type{Id: int32(types.T_array_float64), Width: 3})) + require.Equal(t, "VECBF16(3)", FormatColType(plan.Type{Id: int32(types.T_array_bf16), Width: 3})) + require.Equal(t, "VECF16(3)", FormatColType(plan.Type{Id: int32(types.T_array_float16), Width: 3})) + require.Equal(t, "VECINT8(3)", FormatColType(plan.Type{Id: int32(types.T_array_int8), Width: 3})) + require.Equal(t, "VECUINT8(3)", FormatColType(plan.Type{Id: int32(types.T_array_uint8), Width: 3})) +} + // TestShowCreateExternalWriteFilePattern ensures SHOW CREATE TABLE formatting // keeps WRITE_FILE_PATTERN for writable external tables, in both the INFILE // and the URL s3option forms; without it the recreated table silently degrades diff --git a/pkg/sql/plan/build_util.go b/pkg/sql/plan/build_util.go index a2b4175c45bdc..aceb5aefc89a5 100644 --- a/pkg/sql/plan/build_util.go +++ b/pkg/sql/plan/build_util.go @@ -157,7 +157,7 @@ func getTypeFromAst(ctx context.Context, typ tree.ResolvableTypeReference) (plan // create table t1(a char) -> DisplayWith = -1；but get width=1 in MySQL and PgSQL if fstr == "char" || fstr == "binary" { width = 1 - } else if fstr == "vecf32" || fstr == "vecf64" { + } else if fstr == types.ArrayFloat32SQLName || fstr == types.ArrayFloat64SQLName || fstr == types.ArrayBF16SQLName || fstr == types.ArrayFloat16SQLName || fstr == types.ArrayInt8SQLName || fstr == types.ArrayUint8SQLName { width = types.MaxArrayDimension } else { width = types.MaxVarcharLen @@ -168,7 +168,7 @@ func getTypeFromAst(ctx context.Context, typ tree.ResolvableTypeReference) (plan return plan.Type{}, moerr.NewOutOfRangef(ctx, fstr, " typeLen is over the MaxCharLen: %v", types.MaxCharLen) } else if (fstr == "varchar" || fstr == "varbinary") && width > types.MaxVarcharLen { return plan.Type{}, moerr.NewOutOfRangef(ctx, fstr, " typeLen is over the MaxVarcharLen: %v", types.MaxVarcharLen) - } else if fstr == "vecf32" || fstr == "vecf64" { + } else if fstr == types.ArrayFloat32SQLName || fstr == types.ArrayFloat64SQLName || fstr == types.ArrayBF16SQLName || fstr == types.ArrayFloat16SQLName || fstr == types.ArrayInt8SQLName || fstr == types.ArrayUint8SQLName { if width > types.MaxArrayDimension { return plan.Type{}, moerr.NewOutOfRangef(ctx, fstr, " typeLen is over the MaxVectorLen : %v", types.MaxArrayDimension) } @@ -183,10 +183,18 @@ func getTypeFromAst(ctx context.Context, typ tree.ResolvableTypeReference) (plan return plan.Type{Id: int32(types.T_binary), Width: width}, nil case "varchar": return plan.Type{Id: int32(types.T_varchar), Width: width}, nil - case "vecf32": + case types.ArrayFloat32SQLName: return plan.Type{Id: int32(types.T_array_float32), Width: width}, nil - case "vecf64": + case types.ArrayFloat64SQLName: return plan.Type{Id: int32(types.T_array_float64), Width: width}, nil + case types.ArrayBF16SQLName: + return plan.Type{Id: int32(types.T_array_bf16), Width: width}, nil + case types.ArrayFloat16SQLName: + return plan.Type{Id: int32(types.T_array_float16), Width: width}, nil + case types.ArrayInt8SQLName: + return plan.Type{Id: int32(types.T_array_int8), Width: width}, nil + case types.ArrayUint8SQLName: + return plan.Type{Id: int32(types.T_array_uint8), Width: width}, nil } // varbinary return plan.Type{Id: int32(types.T_varbinary), Width: width}, nil diff --git a/pkg/sql/plan/function/func_binary.go b/pkg/sql/plan/function/func_binary.go index f823e2f705072..a8140ff5ebe16 100644 --- a/pkg/sql/plan/function/func_binary.go +++ b/pkg/sql/plan/function/func_binary.go @@ -11543,6 +11543,73 @@ func CosineDistanceArray[T types.RealNumbers](ivecs []*vector.Vector, result vec }, selectList) } +// arrayDistanceNarrow computes a binary vector distance for the narrow element +// types (bf16/f16/int8/uint8) using the NATIVE metric kernel for T — int8/uint8 +// run the INTEGER kernels (int32/int64 accumulate, no float upcast), bf16/f16 run +// the fused decode-to-float32 kernels (no intermediate []float32 materialized). +// This is the same kernel ivfflat's brute-force centroid scan uses, so the SQL +// re-rank (l2_distance over a narrow entries column — the hot path) no longer +// detours through the float32 bridge. It deliberately bypasses +// batchArrayDistanceSync (the GPU/usearch path), which only supports native float +// element types. +// +// m selects the kernel; sqrtResult sqrts the result for TRUE L2 (the kernel +// returns squared L2 for Metric_L2Distance, matching ResolveDistanceFn). The int8 +// squared sum is exact in int64, so sqrt-in-float64 is at least as accurate as the +// old float32 bridge and preserves ranking order. +func arrayDistanceNarrow[T types.ArrayElement]( + ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList, + m metric.MetricType, sqrtResult bool) error { + kernel, err := metric.ResolveDistanceFn[T, float64](m) + if err != nil { + return err + } + return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (float64, error) { + d, e := kernel(types.BytesToArray[T](v1), types.BytesToArray[T](v2)) + if e != nil { + return 0, e + } + if sqrtResult { + d = math.Sqrt(d) + } + return d, nil + }, selectList) +} + +// arrayDistanceViaF32 is retained only for cosine_similarity, whose float32 +// downcast corner-case handling (see moarray.CosineSimilarity) has no integer- +// kernel equivalent. Operands are upcast to []float32 and run through the f32 +// kernel. +func arrayDistanceViaF32[T types.ArrayElement]( + ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList, + kernel func(v1, v2 []float32) (float64, error)) error { + return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (float64, error) { + f1 := types.ToFloat32Array[T](types.BytesToArray[T](v1)) + f2 := types.ToFloat32Array[T](types.BytesToArray[T](v2)) + return kernel(f1, f2) + }, selectList) +} + +func L2DistanceArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceNarrow[T](ivecs, result, proc, length, selectList, metric.Metric_L2Distance, true) +} + +func L2DistanceSqArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceNarrow[T](ivecs, result, proc, length, selectList, metric.Metric_L2sqDistance, false) +} + +func InnerProductArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceNarrow[T](ivecs, result, proc, length, selectList, metric.Metric_InnerProduct, false) +} + +func CosineDistanceArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceNarrow[T](ivecs, result, proc, length, selectList, metric.Metric_CosineDistance, false) +} + +func CosineSimilarityArrayViaF32[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { + return arrayDistanceViaF32[T](ivecs, result, proc, length, selectList, moarray.CosineSimilarity[float32]) +} + func castBinaryArrayToInt(array []uint8) int64 { var result int64 for i, value := range array { diff --git a/pkg/sql/plan/function/func_cast.go b/pkg/sql/plan/function/func_cast.go index 8bcb1f68408df..81b000ea18ae8 100644 --- a/pkg/sql/plan/function/func_cast.go +++ b/pkg/sql/plan/function/func_cast.go @@ -32,7 +32,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vm/process" "golang.org/x/exp/constraints" ) @@ -52,6 +51,7 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_time, types.T_timestamp, types.T_year, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32, }, @@ -296,6 +296,7 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_char, types.T_varchar, types.T_blob, types.T_text, types.T_binary, types.T_varbinary, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32, types.T_TS, }, @@ -344,6 +345,7 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_char, types.T_varchar, types.T_blob, types.T_text, types.T_binary, types.T_varbinary, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32, }, @@ -362,6 +364,7 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_char, types.T_varchar, types.T_blob, types.T_text, types.T_binary, types.T_varbinary, types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry, types.T_geometry32, }, types.T_geometry: { @@ -424,9 +427,27 @@ var supportedTypeCast = map[types.T][]types.T{ types.T_array_float32: { types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, }, types.T_array_float64: { types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + }, + types.T_array_bf16: { + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + }, + types.T_array_float16: { + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + }, + types.T_array_int8: { + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + }, + types.T_array_uint8: { + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, }, } @@ -517,7 +538,7 @@ func NewCast(parameters []*vector.Vector, result vector.FunctionResultWrapper, p case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_blob, types.T_text, types.T_datalink, types.T_geometry, types.T_geometry32: s := vector.GenerateFunctionStrParameter(from) err = strTypeToOthers(proc, s, *toType, result, length, selectList) - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: //NOTE: Don't mix T_array and T_varchar. // T_varchar will have "[1,2,3]" string // T_array will have "@@@#@!#@!@#!" binary. @@ -636,7 +657,7 @@ func scalarNullToOthers(ctx context.Context, return appendNulls[uint64](result, length, selectList) case types.T_char, types.T_varchar, types.T_blob, types.T_binary, types.T_varbinary, types.T_text, types.T_json, - types.T_array_float32, types.T_array_float64, types.T_datalink, types.T_geometry: + types.T_array_float32, types.T_array_float64, types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink, types.T_geometry: return appendNulls[types.Varlena](result, length, selectList) case types.T_float32: return appendNulls[float32](result, length, selectList) @@ -1877,6 +1898,18 @@ func strTypeToOthers(proc *process.Process, case types.T_array_float64: rs := vector.MustFunctionResult[types.Varlena](result) return blobToArray[float64](ctx, source, rs, length, toType) + case types.T_array_bf16: + rs := vector.MustFunctionResult[types.Varlena](result) + return blobToArray[types.BF16](ctx, source, rs, length, toType) + case types.T_array_float16: + rs := vector.MustFunctionResult[types.Varlena](result) + return blobToArray[types.Float16](ctx, source, rs, length, toType) + case types.T_array_int8: + rs := vector.MustFunctionResult[types.Varlena](result) + return blobToArray[int8](ctx, source, rs, length, toType) + case types.T_array_uint8: + rs := vector.MustFunctionResult[types.Varlena](result) + return blobToArray[uint8](ctx, source, rs, length, toType) // NOTE 1: don't add `switch default` and panic here. If `T_blob` to `ARRAY` is not required, // then continue to the `str` to `Other` code. // NOTE 2: don't create a switch T_blob case in NewCast() as @@ -1959,6 +1992,18 @@ func strTypeToOthers(proc *process.Process, case types.T_array_float64: rs := vector.MustFunctionResult[types.Varlena](result) return strToArray[float64](ctx, source, rs, length, toType) + case types.T_array_bf16: + rs := vector.MustFunctionResult[types.Varlena](result) + return strToArray[types.BF16](ctx, source, rs, length, toType) + case types.T_array_float16: + rs := vector.MustFunctionResult[types.Varlena](result) + return strToArray[types.Float16](ctx, source, rs, length, toType) + case types.T_array_int8: + rs := vector.MustFunctionResult[types.Varlena](result) + return strToArray[int8](ctx, source, rs, length, toType) + case types.T_array_uint8: + rs := vector.MustFunctionResult[types.Varlena](result) + return strToArray[uint8](ctx, source, rs, length, toType) case types.T_year: rs := vector.MustFunctionResult[types.MoYear](result) return strToYear(ctx, source, rs, length, selectList) @@ -1975,24 +2020,45 @@ func arrayTypeToOthers(proc *process.Process, switch fromType.Oid { case types.T_array_float32: - switch toType.Oid { - case types.T_array_float32: - return arrayToArray[float32, float32](proc.Ctx, source, rs, length, toType) - case types.T_array_float64: - return arrayToArray[float32, float64](proc.Ctx, source, rs, length, toType) - } + return arrayToArrayDispatch[float32](proc, source, rs, length, toType) case types.T_array_float64: - switch toType.Oid { - case types.T_array_float32: - return arrayToArray[float64, float32](proc.Ctx, source, rs, length, toType) - case types.T_array_float64: - return arrayToArray[float64, float64](proc.Ctx, source, rs, length, toType) - } + return arrayToArrayDispatch[float64](proc, source, rs, length, toType) + case types.T_array_bf16: + return arrayToArrayDispatch[types.BF16](proc, source, rs, length, toType) + case types.T_array_float16: + return arrayToArrayDispatch[types.Float16](proc, source, rs, length, toType) + case types.T_array_int8: + return arrayToArrayDispatch[int8](proc, source, rs, length, toType) + case types.T_array_uint8: + return arrayToArrayDispatch[uint8](proc, source, rs, length, toType) } return moerr.NewInternalError(ctx, fmt.Sprintf("unsupported cast from %s to %s", fromType, toType)) } +// arrayToArrayDispatch resolves the target element type for a vector->vector +// cast whose source element type I is already known, then runs the float32 +// bridge in arrayToArray. Covers all 25 (5x5) vector-pair casts. +func arrayToArrayDispatch[I types.ArrayElement](proc *process.Process, + source vector.FunctionParameterWrapper[types.Varlena], + rs *vector.FunctionResult[types.Varlena], length int, toType types.Type) error { + switch toType.Oid { + case types.T_array_float32: + return arrayToArray[I, float32](proc.Ctx, source, rs, length, toType) + case types.T_array_float64: + return arrayToArray[I, float64](proc.Ctx, source, rs, length, toType) + case types.T_array_bf16: + return arrayToArray[I, types.BF16](proc.Ctx, source, rs, length, toType) + case types.T_array_float16: + return arrayToArray[I, types.Float16](proc.Ctx, source, rs, length, toType) + case types.T_array_int8: + return arrayToArray[I, int8](proc.Ctx, source, rs, length, toType) + case types.T_array_uint8: + return arrayToArray[I, uint8](proc.Ctx, source, rs, length, toType) + } + return moerr.NewInternalError(proc.Ctx, fmt.Sprintf("unsupported cast to %s", toType)) +} + func uuidToOthers(ctx context.Context, source vector.FunctionParameterWrapper[types.Uuid], toType types.Type, result vector.FunctionResultWrapper, length int, selectList *FunctionSelectList) error { @@ -5636,7 +5702,7 @@ func strToBit( return nil } -func strToArray[T types.RealNumbers]( +func strToArray[T types.ArrayElement]( _ context.Context, from vector.FunctionParameterWrapper[types.Varlena], to *vector.FunctionResult[types.Varlena], length int, _ types.Type) error { @@ -5672,7 +5738,7 @@ func strToArray[T types.RealNumbers]( return nil } -func blobToArray[T types.RealNumbers]( +func blobToArray[T types.ArrayElement]( _ context.Context, from vector.FunctionParameterWrapper[types.Varlena], to *vector.FunctionResult[types.Varlena], length int, _ types.Type) error { @@ -5702,7 +5768,7 @@ func blobToArray[T types.RealNumbers]( return nil } -func arrayToArray[I types.RealNumbers, O types.RealNumbers]( +func arrayToArray[I types.ArrayElement, O types.ArrayElement]( _ context.Context, from vector.FunctionParameterWrapper[types.Varlena], to *vector.FunctionResult[types.Varlena], length int, _ types.Type) error { @@ -5723,18 +5789,20 @@ func arrayToArray[I types.RealNumbers, O types.RealNumbers]( // cases b/b and b+sqrt(b) fails. if from.GetType().Oid == to.GetType().Oid { - // Eg:- VECF32(3) --> VECF32(3) + // Eg:- VECF32(3) --> VECF32(3): identical byte layout, copy as-is. if err := to.AppendBytes(v, false); err != nil { return err } } else { - // Eg:- VECF32(3) --> VECF64(3) + // Eg:- VECF32(3) --> VECF64(3), VECF32 --> VECINT8, etc. + // All 25 vector-pair casts route through the float32 bridge: + // upcast the source element type to []float32, then narrow to the + // target element type (int8 rounds+clamps; bf16/f16 round-to-even). + // This replaces moarray.Cast[I,O], which only handled float pairs. _v := types.BytesToArray[I](v) - cast, err := moarray.Cast[I, O](_v) - if err != nil { - return err - } - bytes := types.ArrayToBytes[O](cast) + f32 := types.ToFloat32Array[I](_v) + out := types.FromFloat32Array[O](f32) + bytes := types.ArrayToBytes[O](out) if err := to.AppendBytes(bytes, false); err != nil { return err } diff --git a/pkg/sql/plan/function/func_compare.go b/pkg/sql/plan/function/func_compare.go index d4eed39103d37..b5b692bd97634 100644 --- a/pkg/sql/plan/function/func_compare.go +++ b/pkg/sql/plan/function/func_compare.go @@ -44,6 +44,7 @@ func otherCompareOperatorSupports(typ1, typ2 types.Type) bool { case types.T_uuid: case types.T_Rowid: case types.T_array_float32, types.T_array_float64: + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: case types.T_year: default: return false @@ -82,6 +83,7 @@ func equalAndNotEqualOperatorSupports(typ1, typ2 types.Type) bool { case types.T_uuid: case types.T_Rowid: case types.T_array_float32, types.T_array_float64: + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: case types.T_enum: case types.T_year: default: @@ -237,6 +239,30 @@ func nullSafeEqualFn(parameters []*vector.Vector, result vector.FunctionResultWr _v2 := types.BytesToArray[float64](v2) return types.ArrayCompare[float64](_v1, _v2) == 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixedNullSafe(parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) == 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixedNullSafe(parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) == 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixedNullSafe(parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) == 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixedNullSafe(parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) == 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixedNullSafe[types.Date](parameters, rs, proc, length, func(a, b types.Date) bool { return a == b @@ -375,6 +401,22 @@ func equalFn(parameters []*vector.Vector, result vector.FunctionResultWrapper, p return types.ArrayCompare[float64](_v1, _v2) == 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + return types.ArrayElementCompare[types.BF16](types.BytesToArray[types.BF16](v1), types.BytesToArray[types.BF16](v2)) == 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + return types.ArrayElementCompare[types.Float16](types.BytesToArray[types.Float16](v1), types.BytesToArray[types.Float16](v2)) == 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + return types.ArrayElementCompare[int8](types.BytesToArray[int8](v1), types.BytesToArray[int8](v2)) == 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + return types.ArrayElementCompare[uint8](types.BytesToArray[uint8](v1), types.BytesToArray[uint8](v2)) == 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a == b @@ -761,6 +803,30 @@ func greatThanFn(parameters []*vector.Vector, result vector.FunctionResultWrappe return types.ArrayCompare[float64](_v1, _v2) > 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) > 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) > 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) > 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) > 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a > b @@ -888,6 +954,30 @@ func greatEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrapp return types.ArrayCompare[float64](_v1, _v2) >= 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) >= 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) >= 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) >= 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) >= 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a >= b @@ -1015,6 +1105,30 @@ func notEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrapper return types.ArrayCompare[float64](_v1, _v2) != 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) != 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) != 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) != 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) != 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a != b @@ -1142,6 +1256,30 @@ func lessThanFn(parameters []*vector.Vector, result vector.FunctionResultWrapper return types.ArrayCompare[float64](_v1, _v2) < 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) < 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) < 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) < 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) < 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a < b @@ -1269,6 +1407,30 @@ func lessEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrappe return types.ArrayCompare[float64](_v1, _v2) <= 0 }, selectList) + case types.T_array_bf16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.BF16](v1) + _v2 := types.BytesToArray[types.BF16](v2) + return types.ArrayElementCompare[types.BF16](_v1, _v2) <= 0 + }, selectList) + case types.T_array_float16: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[types.Float16](v1) + _v2 := types.BytesToArray[types.Float16](v2) + return types.ArrayElementCompare[types.Float16](_v1, _v2) <= 0 + }, selectList) + case types.T_array_int8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[int8](v1) + _v2 := types.BytesToArray[int8](v2) + return types.ArrayElementCompare[int8](_v1, _v2) <= 0 + }, selectList) + case types.T_array_uint8: + return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { + _v1 := types.BytesToArray[uint8](v1) + _v2 := types.BytesToArray[uint8](v2) + return types.ArrayElementCompare[uint8](_v1, _v2) <= 0 + }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { return a <= b diff --git a/pkg/sql/plan/function/func_testcase.go b/pkg/sql/plan/function/func_testcase.go index 0bb7b7958e596..de70243dfe5fd 100644 --- a/pkg/sql/plan/function/func_testcase.go +++ b/pkg/sql/plan/function/func_testcase.go @@ -15,6 +15,7 @@ package function import ( + "bytes" "fmt" "strings" @@ -715,6 +716,27 @@ func (fc *FunctionTestCase) Run() (succeed bool, errInfo string) { i+1, types.BytesToArray[float64](want), types.BytesToArray[float64](get)) } } + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: + // Narrow vector types compare byte-exact (their stored representation is + // the comparison ground truth; ArrayCompare only covers float32/float64). + r := vector.GenerateFunctionStrParameter(v) + s := vector.GenerateFunctionStrParameter(vExpected) + for i = 0; i < uint64(fc.fnLength); i++ { + want, null1 := s.GetStrValue(i) + get, null2 := r.GetStrValue(i) + if null1 { + if null2 { + continue + } + return false, fmt.Sprintf("the %dth row expected NULL, but get not null", i+1) + } + if null2 { + return false, fmt.Sprintf("the %dth row expected %v, but get NULL", i+1, want) + } + if !bytes.Equal(want, get) { + return false, fmt.Sprintf("the %dth row expected %v, but get %v", i+1, want, get) + } + } case types.T_uuid: r := vector.GenerateFunctionFixedTypeParameter[types.Uuid](v) s := vector.GenerateFunctionFixedTypeParameter[types.Uuid](vExpected) @@ -917,6 +939,18 @@ func newVectorByType(mp *mpool.MPool, typ types.Type, val any, nsp *nulls.Nulls) case types.T_array_float64: values := val.([][]float64) vector.AppendArrayList[float64](vec, values, nil, mp) + case types.T_array_bf16: + values := val.([][]types.BF16) + vector.AppendArrayList[types.BF16](vec, values, nil, mp) + case types.T_array_float16: + values := val.([][]types.Float16) + vector.AppendArrayList[types.Float16](vec, values, nil, mp) + case types.T_array_int8: + values := val.([][]int8) + vector.AppendArrayList[int8](vec, values, nil, mp) + case types.T_array_uint8: + values := val.([][]uint8) + vector.AppendArrayList[uint8](vec, values, nil, mp) case types.T_uuid: values := val.([]types.Uuid) vector.AppendFixedList(vec, values, nil, mp) diff --git a/pkg/sql/plan/function/func_unary.go b/pkg/sql/plan/function/func_unary.go index 7ba49577bbbe8..91995cb284eaa 100644 --- a/pkg/sql/plan/function/func_unary.go +++ b/pkg/sql/plan/function/func_unary.go @@ -233,7 +233,7 @@ var ( } ) -func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { +func NormalizeL2Array[T types.ArrayElement](parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { source := vector.GenerateFunctionStrParameter(parameters[0]) rs := vector.MustFunctionResult[types.Varlena](result) @@ -290,6 +290,14 @@ func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result v *outArrayF64Ptr = outArrayF64 arrayF64Pool.Put(outArrayF64Ptr) + case types.T_array_bf16: + _ = appendNormalizedNarrowArray[types.BF16](rs, data) + case types.T_array_float16: + _ = appendNormalizedNarrowArray[types.Float16](rs, data) + case types.T_array_int8: + _ = appendNormalizedNarrowArray[int8](rs, data) + case types.T_array_uint8: + _ = appendNormalizedNarrowArray[uint8](rs, data) } } @@ -297,6 +305,17 @@ func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result v return nil } +// appendNormalizedNarrowArray normalizes a narrow-typed vector (bf16/f16/int8) +// by upcasting to float32, normalizing in float32, then narrowing back to T. +// int8 normalization is mostly degenerate (unit vectors round to 0/±1) but is +// supported for completeness. +func appendNormalizedNarrowArray[T types.ArrayElement](rs *vector.FunctionResult[types.Varlena], data []byte) error { + in := types.ToFloat32Array[T](types.BytesToArray[T](data)) + out := make([]float32, len(in)) + _ = moarray.NormalizeL2(in, out) + return rs.AppendBytes(types.ArrayToBytes[T](types.FromFloat32Array[T](out)), false) +} + func L1NormArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { return opUnaryBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(in []byte) (float64, error) { _in := types.BytesToArray[T](in) @@ -311,7 +330,7 @@ func L2NormArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.Func }, selectList) } -func VectorDimsArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { +func VectorDimsArray[T types.ArrayElement](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { return opUnaryBytesToFixed[int64](ivecs, result, proc, length, func(in []byte) (out int64) { _in := types.BytesToArray[T](in) return int64(len(_in)) @@ -634,6 +653,12 @@ func bitCountFromFloat[T constraints.Float](v T, proc *process.Process) (uint64, if rounded >= ULLONG_MAX_DOUBLE { return bitCountFromUint64(uint64(math.MaxUint64)), nil } + // Converting a negative float directly to uint64 is undefined in Go (the + // result is implementation-specific: two's-complement on amd64, 0 on arm64), + // so route negatives through int64 first and reinterpret the bit pattern. + if rounded < 0 { + return bitCountFromSignedInt64Pattern(int64(rounded)), nil + } return bitCountFromUint64(uint64(rounded)), nil } @@ -5688,7 +5713,7 @@ func FromBase64(parameters []*vector.Vector, result vector.FunctionResultWrapper // VecFromBase64 decodes a base64-encoded string into a vector (vecf32 or vecf64). // The base64 payload must be the raw little-endian bytes of the vector elements, // as produced by to_base64(vecf32_col) or to_base64(vecf64_col). -func VecFromBase64[T types.RealNumbers](parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { +func VecFromBase64[T types.ArrayElement](parameters []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { source := vector.GenerateFunctionStrParameter(parameters[0]) rs := vector.MustFunctionResult[types.Varlena](result) @@ -5698,6 +5723,14 @@ func VecFromBase64[T types.RealNumbers](parameters []*vector.Vector, result vect elemSize = 4 case float64: elemSize = 8 + case types.BF16, types.Float16: + elemSize = 2 + case int8, uint8: + elemSize = 1 + default: + // Guard: an unhandled element type would leave elemSize==0 and panic at + // the `n % elemSize` check below. Fail explicitly instead. + return moerr.NewInternalErrorNoCtx("vec_from_base64: unsupported vector element type") } // Pre-extend area: peek at the first non-null input to estimate per-row decoded size. @@ -5731,11 +5764,11 @@ func VecFromBase64[T types.RealNumbers](parameters []*vector.Vector, result vect } n, err := base64.StdEncoding.Decode(buf, data) if err != nil { - return moerr.NewInternalErrorNoCtxf("vecf%d_from_base64: invalid base64 input", elemSize*8) + return moerr.NewInternalErrorNoCtx("vec_from_base64: invalid base64 input") } if n%elemSize != 0 { - return moerr.NewInternalErrorNoCtxf("vecf%d_from_base64: decoded length %d is not a multiple of %d bytes", elemSize*8, n, elemSize) + return moerr.NewInternalErrorNoCtxf("vec_from_base64: decoded length %d is not a multiple of %d bytes", n, elemSize) } if err = rs.AppendBytes(buf[:n], false); err != nil { diff --git a/pkg/sql/plan/function/func_unary_test.go b/pkg/sql/plan/function/func_unary_test.go index 35b2da0038bb1..5fa4f3fe07d21 100644 --- a/pkg/sql/plan/function/func_unary_test.go +++ b/pkg/sql/plan/function/func_unary_test.go @@ -4507,6 +4507,55 @@ func TestVecFromBase64(t *testing.T) { require.True(t, s, fmt.Sprintf("vecf64 case failed: %s", info)) } +// TestVecFromBase64Narrow exercises VecFromBase64's narrow elemSize branches +// (int8=1, bf16/f16=2) and its error paths via the function-UT harness. +func TestVecFromBase64Narrow(t *testing.T) { + proc := testutil.NewProcess(t) + + mkInput := func(b64 string) []FunctionTestInput { + return []FunctionTestInput{NewFunctionTestInput(types.T_varchar.ToType(), []string{b64}, []bool{})} + } + runCase := func(in []FunctionTestInput, res FunctionTestResult, fn fEvalFn) (bool, string) { + fcTC := NewFunctionTestCase(proc, in, res, fn) + return fcTC.Run() + } + + // int8 roundtrip (elemSize 1). + i8 := []int8{1, -2, 127, -128} + ok, info := runCase(mkInput(types.ArrayToBase64(i8)), + NewFunctionTestResult(types.T_array_int8.ToType(), false, [][]int8{i8}, []bool{}), VecFromBase64[int8]) + require.Truef(t, ok, "vecint8 roundtrip: %s", info) + + // uint8 roundtrip (elemSize 1). Regression: with the uint8 case missing from + // the decoder, elemSize was 0 and `n % elemSize` panicked (divide by zero). + u8 := []uint8{0, 255, 128, 1} + ok, info = runCase(mkInput(types.ArrayToBase64(u8)), + NewFunctionTestResult(types.T_array_uint8.ToType(), false, [][]uint8{u8}, []bool{}), VecFromBase64[uint8]) + require.Truef(t, ok, "vecuint8 roundtrip: %s", info) + + // bf16 roundtrip (elemSize 2). + bf := types.Float32ToBF16Slice([]float32{1.5, -2.25, 0, 8}) + ok, info = runCase(mkInput(types.ArrayToBase64(bf)), + NewFunctionTestResult(types.T_array_bf16.ToType(), false, [][]types.BF16{bf}, []bool{}), VecFromBase64[types.BF16]) + require.Truef(t, ok, "vecbf16 roundtrip: %s", info) + + // f16 roundtrip. + f16 := types.Float32ToFloat16Slice([]float32{1.5, -2.25, 0, 8}) + ok, info = runCase(mkInput(types.ArrayToBase64(f16)), + NewFunctionTestResult(types.T_array_float16.ToType(), false, [][]types.Float16{f16}, []bool{}), VecFromBase64[types.Float16]) + require.Truef(t, ok, "vecf16 roundtrip: %s", info) + + // invalid base64 -> error. + ok, info = runCase(mkInput("!!!not-base64!!!"), + NewFunctionTestResult(types.T_array_int8.ToType(), true, [][]int8{nil}, []bool{}), VecFromBase64[int8]) + require.Truef(t, ok, "invalid base64 should error: %s", info) + + // "AQID" decodes to 3 bytes, not a multiple of 2 (bf16 elemSize) -> error. + ok, info = runCase(mkInput("AQID"), + NewFunctionTestResult(types.T_array_bf16.ToType(), true, [][]types.BF16{nil}, []bool{}), VecFromBase64[types.BF16]) + require.Truef(t, ok, "odd length should error: %s", info) +} + func initValidatePasswordStrengthTestCase() []tcTemp { return []tcTemp{ { diff --git a/pkg/sql/plan/function/func_vecnarrow_test.go b/pkg/sql/plan/function/func_vecnarrow_test.go new file mode 100644 index 0000000000000..cdd24f80074ae --- /dev/null +++ b/pkg/sql/plan/function/func_vecnarrow_test.go @@ -0,0 +1,122 @@ +// Copyright 2021 - 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package function + +import ( + "fmt" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/stretchr/testify/require" +) + +// l2 of [1,2,3] vs [4,6,8]: sqrt(9+16+25)=sqrt(50)=7.0710678118654755 +// (the distance is computed in float64; the framework's InEpsilonF64 1e-9 +// tolerance absorbs cross-platform variance). +func TestL2DistanceNarrowArray(t *testing.T) { + proc := testutil.NewProcess(t) + + // int8: exact integer values, so distance matches float reference exactly. + t.Run("int8", func(t *testing.T) { + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{{1, 2, 3}}, []bool{false}), + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{{4, 6, 8}}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{7.0710678118654755}, []bool{false}), + L2DistanceArrayViaF32[int8]) + s, info := tc.Run() + require.True(t, s, info) + }) + + // uint8: exact unsigned integer values, distance matches the float reference. + t.Run("uint8", func(t *testing.T) { + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{{1, 2, 3}}, []bool{false}), + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{{4, 6, 8}}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{7.0710678118654755}, []bool{false}), + L2DistanceArrayViaF32[uint8]) + s, info := tc.Run() + require.True(t, s, info) + }) + + // bf16: small integers are exactly representable in bf16, so still exact. + t.Run("bf16", func(t *testing.T) { + mk := func(vs ...float32) []types.BF16 { + out := make([]types.BF16, len(vs)) + for i, v := range vs { + out[i] = types.BF16FromFloat32(v) + } + return out + } + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_bf16.ToType(), [][]types.BF16{mk(1, 2, 3)}, []bool{false}), + NewFunctionTestInput(types.T_array_bf16.ToType(), [][]types.BF16{mk(4, 6, 8)}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{7.0710678118654755}, []bool{false}), + L2DistanceArrayViaF32[types.BF16]) + s, info := tc.Run() + require.True(t, s, info) + }) + + // float16: same exact small integers. + t.Run("f16", func(t *testing.T) { + mk := func(vs ...float32) []types.Float16 { + out := make([]types.Float16, len(vs)) + for i, v := range vs { + out[i] = types.Float16FromFloat32(v) + } + return out + } + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_float16.ToType(), [][]types.Float16{mk(1, 2, 3)}, []bool{false}), + NewFunctionTestInput(types.T_array_float16.ToType(), [][]types.Float16{mk(4, 6, 8)}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{7.0710678118654755}, []bool{false}), + L2DistanceArrayViaF32[types.Float16]) + s, info := tc.Run() + require.True(t, s, info) + }) +} + +// Sanity: inner_product of [1,2,3]·[4,5,6] = 4+10+18 = 32. +func TestInnerProductNarrowArray(t *testing.T) { + proc := testutil.NewProcess(t) + tc := NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{{1, 2, 3}}, []bool{false}), + NewFunctionTestInput(types.T_array_int8.ToType(), [][]int8{{4, 5, 6}}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{-32}, []bool{false}), + InnerProductArrayViaF32[int8]) + s, info := tc.Run() + require.True(t, s, fmt.Sprintf("inner_product int8: %s", info)) + + // uint8 sibling: same dot product over unsigned values. + tc = NewFunctionTestCase(proc, + []FunctionTestInput{ + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{{1, 2, 3}}, []bool{false}), + NewFunctionTestInput(types.T_array_uint8.ToType(), [][]uint8{{4, 5, 6}}, []bool{false}), + }, + NewFunctionTestResult(types.T_float64.ToType(), false, []float64{-32}, []bool{false}), + InnerProductArrayViaF32[uint8]) + s, info = tc.Run() + require.True(t, s, fmt.Sprintf("inner_product uint8: %s", info)) +} diff --git a/pkg/sql/plan/function/function_id.go b/pkg/sql/plan/function/function_id.go index b983075b4a36e..4719eca1f569b 100644 --- a/pkg/sql/plan/function/function_id.go +++ b/pkg/sql/plan/function/function_id.go @@ -733,9 +733,24 @@ const ( IS_USED_LOCK = 522 RELEASE_ALL_LOCKS = 523 + // vec{bf16,f16,int8}_from_base64: decode a base64 payload of the narrow type's + // raw bytes into that narrow vector type — the narrow siblings of + // vecf32_from_base64 / vecf64_from_base64. Used by the ivfflat narrow re-rank, + // where the query must be a constant narrow vec literal matching the narrow + // entries (a cast of vecf32_from_base64 does not constant-fold, breaking the + // ORDER BY index pushdown). + // Renumbered to 519-522 on the gpu_plugin_all merge: main added NAME_CONST=518 + // (#24887), which collided with the original VECBF16_FROM_BASE64=518. These + // IDs are referenced by name only (name map + list_builtIn registration), so + // renumbering is safe. + VECBF16_FROM_BASE64 = 524 + VECF16_FROM_BASE64 = 525 + VECINT8_FROM_BASE64 = 526 + VECUINT8_FROM_BASE64 = 527 + // FUNCTION_END_NUMBER is not a function, just a flag to record the max number of function. // TODO: every one should put the new function id in front of this one if you want to make a new function. - FUNCTION_END_NUMBER = 524 + FUNCTION_END_NUMBER = 528 ) // functionIdRegister is what function we have registered already. @@ -1045,6 +1060,10 @@ var functionIdRegister = map[string]int32{ "from_base64": FROM_BASE64, "vecf32_from_base64": VECF32_FROM_BASE64, "vecf64_from_base64": VECF64_FROM_BASE64, + "vecbf16_from_base64": VECBF16_FROM_BASE64, + "vecf16_from_base64": VECF16_FROM_BASE64, + "vecint8_from_base64": VECINT8_FROM_BASE64, + "vecuint8_from_base64": VECUINT8_FROM_BASE64, "serial": SERIAL, "serial_full": SERIAL_FULL, "serial_extract": SERIAL_EXTRACT, diff --git a/pkg/sql/plan/function/function_id_test.go b/pkg/sql/plan/function/function_id_test.go index 66e008523ecf3..aeb8397f28590 100644 --- a/pkg/sql/plan/function/function_id_test.go +++ b/pkg/sql/plan/function/function_id_test.go @@ -577,9 +577,13 @@ var predefinedFunids = map[int]int{ IS_FREE_LOCK: 521, IS_USED_LOCK: 522, RELEASE_ALL_LOCKS: 523, + VECBF16_FROM_BASE64: 524, + VECF16_FROM_BASE64: 525, + VECINT8_FROM_BASE64: 526, + VECUINT8_FROM_BASE64: 527, // FUNCTION_END_NUMBER is not a function, just a flag to record the max number of function. // TODO: every one should put the new function id in front of this one if you want to make a new function. - FUNCTION_END_NUMBER: 524, + FUNCTION_END_NUMBER: 528, } func Test_funids(t *testing.T) { diff --git a/pkg/sql/plan/function/list_builtIn.go b/pkg/sql/plan/function/list_builtIn.go index d503e99acb43b..3fbe22b26df02 100644 --- a/pkg/sql/plan/function/list_builtIn.go +++ b/pkg/sql/plan/function/list_builtIn.go @@ -3068,6 +3068,90 @@ var supportedStringBuiltIns = []FuncNew{ }, }, + // vecbf16_from_base64 + { + functionId: VECBF16_FROM_BASE64, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedTypeMatch, + + Overloads: []overload{ + { + overloadId: 0, + args: []types.T{types.T_varchar}, + retType: func(parameters []types.Type) types.Type { + return types.T_array_bf16.ToType() + }, + newOp: func() executeLogicOfOverload { + return VecFromBase64[types.BF16] + }, + }, + }, + }, + + // vecf16_from_base64 + { + functionId: VECF16_FROM_BASE64, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedTypeMatch, + + Overloads: []overload{ + { + overloadId: 0, + args: []types.T{types.T_varchar}, + retType: func(parameters []types.Type) types.Type { + return types.T_array_float16.ToType() + }, + newOp: func() executeLogicOfOverload { + return VecFromBase64[types.Float16] + }, + }, + }, + }, + + // vecint8_from_base64 + { + functionId: VECINT8_FROM_BASE64, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedTypeMatch, + + Overloads: []overload{ + { + overloadId: 0, + args: []types.T{types.T_varchar}, + retType: func(parameters []types.Type) types.Type { + return types.T_array_int8.ToType() + }, + newOp: func() executeLogicOfOverload { + return VecFromBase64[int8] + }, + }, + }, + }, + + // vecuint8_from_base64 + { + functionId: VECUINT8_FROM_BASE64, + class: plan.Function_STRICT, + layout: STANDARD_FUNCTION, + checkFn: fixedTypeMatch, + + Overloads: []overload{ + { + overloadId: 0, + args: []types.T{types.T_varchar}, + retType: func(parameters []types.Type) types.Type { + return types.T_array_uint8.ToType() + }, + newOp: func() executeLogicOfOverload { + return VecFromBase64[uint8] + }, + }, + }, + }, + // compress { functionId: COMPRESS, @@ -5878,6 +5962,30 @@ var supportedArrayOperations = []FuncNew{ return VectorDimsArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, + newOp: func() executeLogicOfOverload { return VectorDimsArray[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, + newOp: func() executeLogicOfOverload { return VectorDimsArray[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, + newOp: func() executeLogicOfOverload { return VectorDimsArray[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_int64.ToType() }, + newOp: func() executeLogicOfOverload { return VectorDimsArray[uint8] }, + }, }, }, @@ -5909,6 +6017,30 @@ var supportedArrayOperations = []FuncNew{ return InnerProductArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return InnerProductArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return InnerProductArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return InnerProductArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return InnerProductArrayViaF32[uint8] }, + }, }, }, @@ -5940,6 +6072,30 @@ var supportedArrayOperations = []FuncNew{ return CosineSimilarityArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineSimilarityArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineSimilarityArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineSimilarityArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineSimilarityArrayViaF32[uint8] }, + }, }, }, @@ -5971,6 +6127,30 @@ var supportedArrayOperations = []FuncNew{ return L2DistanceArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceArrayViaF32[uint8] }, + }, }, }, @@ -6033,6 +6213,30 @@ var supportedArrayOperations = []FuncNew{ return L2DistanceSqArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceSqArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceSqArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceSqArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return L2DistanceSqArrayViaF32[uint8] }, + }, }, }, @@ -6095,6 +6299,30 @@ var supportedArrayOperations = []FuncNew{ return CosineDistanceArray[float64] }, }, + { + overloadId: 2, + args: []types.T{types.T_array_bf16, types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineDistanceArrayViaF32[types.BF16] }, + }, + { + overloadId: 3, + args: []types.T{types.T_array_float16, types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineDistanceArrayViaF32[types.Float16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_int8, types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineDistanceArrayViaF32[int8] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_uint8, types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return types.T_float64.ToType() }, + newOp: func() executeLogicOfOverload { return CosineDistanceArrayViaF32[uint8] }, + }, }, }, // function `normalize_l2` @@ -6125,6 +6353,30 @@ var supportedArrayOperations = []FuncNew{ return NormalizeL2Array[float64] }, }, + { + overloadId: 3, + args: []types.T{types.T_array_bf16}, + retType: func(parameters []types.Type) types.Type { return parameters[0] }, + newOp: func() executeLogicOfOverload { return NormalizeL2Array[types.BF16] }, + }, + { + overloadId: 4, + args: []types.T{types.T_array_float16}, + retType: func(parameters []types.Type) types.Type { return parameters[0] }, + newOp: func() executeLogicOfOverload { return NormalizeL2Array[types.Float16] }, + }, + { + overloadId: 5, + args: []types.T{types.T_array_int8}, + retType: func(parameters []types.Type) types.Type { return parameters[0] }, + newOp: func() executeLogicOfOverload { return NormalizeL2Array[int8] }, + }, + { + overloadId: 6, + args: []types.T{types.T_array_uint8}, + retType: func(parameters []types.Type) types.Type { return parameters[0] }, + newOp: func() executeLogicOfOverload { return NormalizeL2Array[uint8] }, + }, }, }, // function `subvector` diff --git a/pkg/sql/plan/function/type_check.go b/pkg/sql/plan/function/type_check.go index 3ddb9f32bec5d..a9f074ee2680e 100644 --- a/pkg/sql/plan/function/type_check.go +++ b/pkg/sql/plan/function/type_check.go @@ -1021,6 +1021,10 @@ func initFixed1() { {types.T_varchar, types.T_text, types.T_varchar, types.T_varchar}, {types.T_varchar, types.T_array_float32, types.T_array_float32, types.T_array_float32}, {types.T_varchar, types.T_array_float64, types.T_array_float64, types.T_array_float64}, + {types.T_varchar, types.T_array_bf16, types.T_array_bf16, types.T_array_bf16}, + {types.T_varchar, types.T_array_float16, types.T_array_float16, types.T_array_float16}, + {types.T_varchar, types.T_array_int8, types.T_array_int8, types.T_array_int8}, + {types.T_varchar, types.T_array_uint8, types.T_array_uint8, types.T_array_uint8}, {types.T_json, types.T_any, types.T_json, types.T_json}, {types.T_json, types.T_bool, types.T_bool, types.T_bool}, {types.T_json, types.T_int8, types.T_int8, types.T_int8}, @@ -1160,6 +1164,21 @@ func initFixed1() { {types.T_text, types.T_array_float32, types.T_array_float32, types.T_array_float32}, {types.T_array_float64, types.T_text, types.T_array_float64, types.T_array_float64}, {types.T_text, types.T_array_float64, types.T_array_float64, types.T_array_float64}, + // narrow vector types: string<->narrow for comparison/equality only. + // (No scalar-arithmetic rules below are added for these types, so + - * / + // still fail to resolve — arithmetic requires an explicit CAST to vecf32.) + {types.T_array_bf16, types.T_varchar, types.T_array_bf16, types.T_array_bf16}, + {types.T_array_bf16, types.T_text, types.T_array_bf16, types.T_array_bf16}, + {types.T_text, types.T_array_bf16, types.T_array_bf16, types.T_array_bf16}, + {types.T_array_float16, types.T_varchar, types.T_array_float16, types.T_array_float16}, + {types.T_array_float16, types.T_text, types.T_array_float16, types.T_array_float16}, + {types.T_text, types.T_array_float16, types.T_array_float16, types.T_array_float16}, + {types.T_array_int8, types.T_varchar, types.T_array_int8, types.T_array_int8}, + {types.T_array_int8, types.T_text, types.T_array_int8, types.T_array_int8}, + {types.T_text, types.T_array_int8, types.T_array_int8, types.T_array_int8}, + {types.T_array_uint8, types.T_varchar, types.T_array_uint8, types.T_array_uint8}, + {types.T_array_uint8, types.T_text, types.T_array_uint8, types.T_array_uint8}, + {types.T_text, types.T_array_uint8, types.T_array_uint8, types.T_array_uint8}, /** VEC Scalar => VEC **/ // VECF32 Scalar => VECF32 @@ -1703,6 +1722,10 @@ func initFixed2() { //A {types.T_varchar, types.T_array_float32, types.T_array_float32, types.T_array_float32}, {types.T_varchar, types.T_array_float64, types.T_array_float64, types.T_array_float64}, + {types.T_varchar, types.T_array_bf16, types.T_array_bf16, types.T_array_bf16}, + {types.T_varchar, types.T_array_float16, types.T_array_float16, types.T_array_float16}, + {types.T_varchar, types.T_array_int8, types.T_array_int8, types.T_array_int8}, + {types.T_varchar, types.T_array_uint8, types.T_array_uint8, types.T_array_uint8}, {types.T_binary, types.T_any, types.T_float64, types.T_float64}, {types.T_binary, types.T_int8, types.T_float64, types.T_float64}, {types.T_binary, types.T_int16, types.T_float64, types.T_float64}, @@ -1777,6 +1800,15 @@ func initFixed2() { {types.T_array_float32, types.T_array_float32, types.T_array_float32, types.T_array_float32}, {types.T_array_float64, types.T_varchar, types.T_array_float64, types.T_array_float64}, {types.T_array_float64, types.T_array_float32, types.T_array_float64, types.T_array_float64}, + // narrow vector types: narrow<->string for comparison/equality only. + {types.T_array_bf16, types.T_varchar, types.T_array_bf16, types.T_array_bf16}, + {types.T_array_bf16, types.T_array_bf16, types.T_array_bf16, types.T_array_bf16}, + {types.T_array_float16, types.T_varchar, types.T_array_float16, types.T_array_float16}, + {types.T_array_float16, types.T_array_float16, types.T_array_float16, types.T_array_float16}, + {types.T_array_int8, types.T_varchar, types.T_array_int8, types.T_array_int8}, + {types.T_array_int8, types.T_array_int8, types.T_array_int8, types.T_array_int8}, + {types.T_array_uint8, types.T_varchar, types.T_array_uint8, types.T_array_uint8}, + {types.T_array_uint8, types.T_array_uint8, types.T_array_uint8, types.T_array_uint8}, /** VEC Scalar => VEC **/ // VECF32 Scalar => VECF32 {types.T_array_float32, types.T_int32, types.T_array_float32, types.T_float32}, @@ -2275,6 +2307,10 @@ func initFixed3() { //C {toType: types.T_array_float32, preferLevel: 2}, {toType: types.T_array_float64, preferLevel: 2}, + {toType: types.T_array_bf16, preferLevel: 2}, + {toType: types.T_array_float16, preferLevel: 2}, + {toType: types.T_array_int8, preferLevel: 2}, + {toType: types.T_array_uint8, preferLevel: 2}, }, }, @@ -2393,6 +2429,10 @@ func initFixed3() { {toType: types.T_blob, preferLevel: 2}, {toType: types.T_array_float32, preferLevel: 2}, {toType: types.T_array_float64, preferLevel: 2}, + {toType: types.T_array_bf16, preferLevel: 2}, + {toType: types.T_array_float16, preferLevel: 2}, + {toType: types.T_array_int8, preferLevel: 2}, + {toType: types.T_array_uint8, preferLevel: 2}, }, }, { diff --git a/pkg/sql/plan/make.go b/pkg/sql/plan/make.go index bdce5518d8da8..7d1de52d25546 100644 --- a/pkg/sql/plan/make.go +++ b/pkg/sql/plan/make.go @@ -365,6 +365,62 @@ func makePlan2Vecf64ConstExprWithType(v string, l int32) *plan.Expr { } } +var MakePlan2VecBf16ConstExprWithType = makePlan2VecBf16ConstExprWithType + +// makePlan2VecBf16ConstExprWithType makes a vecbf16 const expr. +func makePlan2VecBf16ConstExprWithType(v string, l int32) *plan.Expr { + return &plan.Expr{ + Expr: makePlan2Vecf32ConstExpr(v), + Typ: plan.Type{ + Id: int32(types.T_array_bf16), + Width: l, + NotNullable: true, + }, + } +} + +var MakePlan2VecF16ConstExprWithType = makePlan2VecF16ConstExprWithType + +// makePlan2VecF16ConstExprWithType makes a vecf16 const expr. +func makePlan2VecF16ConstExprWithType(v string, l int32) *plan.Expr { + return &plan.Expr{ + Expr: makePlan2Vecf32ConstExpr(v), + Typ: plan.Type{ + Id: int32(types.T_array_float16), + Width: l, + NotNullable: true, + }, + } +} + +var MakePlan2VecInt8ConstExprWithType = makePlan2VecInt8ConstExprWithType + +// makePlan2VecInt8ConstExprWithType makes a vecint8 const expr. +func makePlan2VecInt8ConstExprWithType(v string, l int32) *plan.Expr { + return &plan.Expr{ + Expr: makePlan2Vecf32ConstExpr(v), + Typ: plan.Type{ + Id: int32(types.T_array_int8), + Width: l, + NotNullable: true, + }, + } +} + +var MakePlan2VecUint8ConstExprWithType = makePlan2VecUint8ConstExprWithType + +// makePlan2VecUint8ConstExprWithType makes a vecuint8 const expr. +func makePlan2VecUint8ConstExprWithType(v string, l int32) *plan.Expr { + return &plan.Expr{ + Expr: makePlan2Vecf32ConstExpr(v), + Typ: plan.Type{ + Id: int32(types.T_array_uint8), + Width: l, + NotNullable: true, + }, + } +} + var MakePlan2StringVecExprWithType = makePlan2StringVecExprWithType func makePlan2StringVecExprWithType(mp *mpool.MPool, vals ...string) *plan.Expr { diff --git a/pkg/sql/plan/rule/constant_fold.go b/pkg/sql/plan/rule/constant_fold.go index e5ffdc9975d61..a448ce1293488 100644 --- a/pkg/sql/plan/rule/constant_fold.go +++ b/pkg/sql/plan/rule/constant_fold.go @@ -437,7 +437,8 @@ func GetConstantValue(vec *vector.Vector, transAll bool, row uint64) *plan.Liter decimalValue.A = int64(vector.MustFixedColNoTypeCheck[types.Decimal128](vec)[row].B0_63) decimalValue.B = int64(vector.MustFixedColNoTypeCheck[types.Decimal128](vec)[row].B64_127) return &plan.Literal{Value: &plan.Literal_Decimal128Val{Decimal128Val: decimalValue}} - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: data := vec.GetStringAt(int(row)) return &plan.Literal{ Value: &plan.Literal_VecVal{ @@ -574,7 +575,8 @@ func GetConstantValue2(proc *process.Process, expr *plan.Expr, vec *vector.Vecto err = vector.AppendBytes(vec, nil, false, proc.Mp()) return false, err } - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: if val, ok := cExpr.Lit.Value.(*plan.Literal_VecVal); ok { val := val.VecVal err = vector.AppendBytes(vec, []byte(val), false, proc.Mp()) diff --git a/pkg/sql/plan/rule/constant_fold_narrow_test.go b/pkg/sql/plan/rule/constant_fold_narrow_test.go new file mode 100644 index 0000000000000..d70f534aca557 --- /dev/null +++ b/pkg/sql/plan/rule/constant_fold_narrow_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rule + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/stretchr/testify/require" +) + +// Narrow vector constants must fold to a VecVal literal (carrying the raw bytes), +// like float32/float64 — else the ivfflat narrow ORDER BY pushdown can't fold the +// query and the const executor nil-panics on materialization. +func TestGetConstantValueNarrowVec(t *testing.T) { + mp := mpool.MustNewZero() + defer mpool.DeleteMPool(mp) + + cases := []struct { + oid types.T + data []byte + }{ + {types.T_array_float32, types.ArrayToBytes([]float32{1, 2, 3})}, + {types.T_array_float64, types.ArrayToBytes([]float64{1, 2, 3})}, + {types.T_array_bf16, types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2, 3}))}, + {types.T_array_float16, types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2, 3}))}, + {types.T_array_int8, types.ArrayToBytes([]int8{1, 2, 3})}, + {types.T_array_uint8, types.ArrayToBytes([]uint8{1, 2, 3})}, + } + for _, c := range cases { + vec := vector.NewVec(c.oid.ToType()) + require.NoError(t, vector.AppendBytes(vec, c.data, false, mp)) + lit := GetConstantValue(vec, true, 0) + require.NotNilf(t, lit, "%s should fold", c.oid) + vv, ok := lit.Value.(*plan.Literal_VecVal) + require.Truef(t, ok, "%s -> VecVal literal", c.oid) + require.Equalf(t, c.data, []byte(vv.VecVal), "%s bytes preserved", c.oid) + vec.Free(mp) + } +} diff --git a/pkg/vectorindex/brute_force/brute_force.go b/pkg/vectorindex/brute_force/brute_force.go index 2d29aced6e826..5aacf4a2d8730 100644 --- a/pkg/vectorindex/brute_force/brute_force.go +++ b/pkg/vectorindex/brute_force/brute_force.go @@ -41,7 +41,10 @@ type UsearchBruteForceIndex[T types.RealNumbers] struct { deallocator malloc.Deallocator } -type GoBruteForceIndex[T types.RealNumbers] struct { +// GoBruteForceIndex holds vectors of element type T and computes distances in +// result type R (float32 for f32/narrow inputs, float64 for f64). R only ever +// differs from "float32" for f64 input, so the common path stays float32. +type GoBruteForceIndex[T types.ArrayElement, R types.RealNumbers] struct { Dataset [][]T // flattend vector Metric metric.MetricType Dimension uint @@ -49,7 +52,7 @@ type GoBruteForceIndex[T types.RealNumbers] struct { } var _ cache.VectorIndexSearchIf = &UsearchBruteForceIndex[float32]{} -var _ cache.VectorIndexSearchIf = &GoBruteForceIndex[float32]{} +var _ cache.VectorIndexSearchIf = &GoBruteForceIndex[float32, float32]{} func GetUsearchQuantizationFromType(v any) (usearch.Quantization, error) { switch v.(type) { @@ -62,25 +65,57 @@ func GetUsearchQuantizationFromType(v any) (usearch.Quantization, error) { } } -func NewCpuBruteForceIndex[T types.RealNumbers](dataset [][]T, +// NewCpuBruteForceIndex builds a pure-Go brute-force index for any ArrayElement. +// It dispatches by concrete element type and picks the distance result type R: +// float64 only for float64 input, float32 for everything else (f32 + the narrow +// quantizations bf16/f16/int8/uint8 — whose kernels the resolver casts to float32). +func NewCpuBruteForceIndex[T types.ArrayElement](dataset [][]T, dimension uint, m metric.MetricType, elemsz uint) (cache.VectorIndexSearchIf, error) { - return NewGoBruteForceIndex(dataset, dimension, m, elemsz) + // R = element type for f32/f64; float32 for the narrow quantizations. + switch ds := any(dataset).(type) { + case [][]float32: + return newGoBruteForce[float32, float32](ds, dimension, m), nil + case [][]float64: + return newGoBruteForce[float64, float64](ds, dimension, m), nil + case [][]types.BF16: + return newGoBruteForce[types.BF16, float32](ds, dimension, m), nil + case [][]types.Float16: + return newGoBruteForce[types.Float16, float32](ds, dimension, m), nil + case [][]int8: + return newGoBruteForce[int8, float32](ds, dimension, m), nil + case [][]uint8: + return newGoBruteForce[uint8, float32](ds, dimension, m), nil + default: + return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf("brute force: unsupported element type %T", *new(T))) + } +} + +// newGoBruteForce constructs a GoBruteForceIndex with explicit element type T and +// distance result type R. The single constructor for both the public f32/f64 +// entry point and the narrow (R=float32) dispatch. +func newGoBruteForce[T types.ArrayElement, R types.RealNumbers](dataset [][]T, + dimension uint, + m metric.MetricType) cache.VectorIndexSearchIf { + + return &GoBruteForceIndex[T, R]{ + Dataset: dataset, + Metric: m, + Dimension: dimension, + Count: uint(len(dataset)), + } } +// NewGoBruteForceIndex builds an f32/f64 index whose result type equals the +// element type (R=T). Kept as the public one-type-param entry point. func NewGoBruteForceIndex[T types.RealNumbers](dataset [][]T, dimension uint, m metric.MetricType, elemsz uint) (cache.VectorIndexSearchIf, error) { - idx := &GoBruteForceIndex[T]{} - idx.Metric = m - idx.Dimension = dimension - idx.Count = uint(len(dataset)) - idx.Dataset = dataset - return idx, nil + return newGoBruteForce[T, T](dataset, dimension, m), nil } func NewUsearchBruteForceIndex[T types.RealNumbers](dataset [][]T, @@ -278,26 +313,26 @@ func (idx *UsearchBruteForceIndex[T]) Destroy() { } } -func (idx *GoBruteForceIndex[T]) Load(sqlproc *sqlexec.SqlProcess) error { +func (idx *GoBruteForceIndex[T, R]) Load(sqlproc *sqlexec.SqlProcess) error { return nil } -func (idx *GoBruteForceIndex[T]) UpdateConfig(sif cache.VectorIndexSearchIf) error { +func (idx *GoBruteForceIndex[T, R]) UpdateConfig(sif cache.VectorIndexSearchIf) error { return nil } -func (idx *GoBruteForceIndex[T]) Destroy() { +func (idx *GoBruteForceIndex[T, R]) Destroy() { } // SearchFloat32 implements VectorIndexSearchIf — writes results directly into caller-provided // slices, eliminating the intermediate []int64 and []float64 heap allocations of Search. -func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _queries any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { +func (idx *GoBruteForceIndex[T, R]) SearchFloat32(proc *sqlexec.SqlProcess, _queries any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { queries, ok := _queries.([][]T) if !ok { return moerr.NewInternalErrorNoCtx("queries type invalid") } - distfn, err := metric.ResolveDistanceFn[T](idx.Metric) + distfn, err := metric.ResolveDistanceFn[T, R](idx.Metric) if err != nil { return err } @@ -316,10 +351,10 @@ func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _querie nqueries, func(ctx context.Context, thread_id int, start, end int) error { var heapKeysBuf []int64 - var heapDistBuf []T + var heapDistBuf []R if limit > 1 { heapKeysBuf = make([]int64, limit) - heapDistBuf = make([]T, limit) + heapDistBuf = make([]R, limit) } for k := start; k < end; k++ { @@ -329,7 +364,7 @@ func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _querie } if limit == 1 { - minDist := metric.MaxFloat[T]() + minDist := metric.MaxFloat[R]() minIdx := -1 for j := range idx.Dataset { dist, err2 := distfn(q, idx.Dataset[j]) @@ -346,7 +381,7 @@ func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _querie continue } - h := vectorindex.NewFastMaxHeap[T, int64](limit, heapKeysBuf, heapDistBuf) + h := vectorindex.NewFastMaxHeap[R, int64](limit, heapKeysBuf, heapDistBuf) for j := range idx.Dataset { dist, err2 := distfn(q, idx.Dataset[j]) if err2 != nil { @@ -371,13 +406,13 @@ func (idx *GoBruteForceIndex[T]) SearchFloat32(proc *sqlexec.SqlProcess, _querie }) } -func (idx *GoBruteForceIndex[T]) Search(proc *sqlexec.SqlProcess, _queries any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { +func (idx *GoBruteForceIndex[T, R]) Search(proc *sqlexec.SqlProcess, _queries any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { queries, ok := _queries.([][]T) if !ok { return nil, nil, moerr.NewInternalErrorNoCtx("queries type invalid") } - distfn, err := metric.ResolveDistanceFn[T](idx.Metric) + distfn, err := metric.ResolveDistanceFn[T, R](idx.Metric) if err != nil { return nil, nil, err } @@ -401,10 +436,10 @@ func (idx *GoBruteForceIndex[T]) Search(proc *sqlexec.SqlProcess, _queries any, func(ctx context.Context, thread_id int, start, end int) (err2 error) { // Pre-allocate heap buffers for this thread var heapKeysBuf []int64 - var heapDistBuf []T + var heapDistBuf []R if limit > 1 { heapKeysBuf = make([]int64, limit) - heapDistBuf = make([]T, limit) + heapDistBuf = make([]R, limit) } for k := start; k < end; k++ { @@ -414,7 +449,7 @@ func (idx *GoBruteForceIndex[T]) Search(proc *sqlexec.SqlProcess, _queries any, } if limit == 1 { - minDist := metric.MaxFloat[T]() + minDist := metric.MaxFloat[R]() minIdx := -1 for j := range idx.Dataset { dist, err2 := distfn(q, idx.Dataset[j]) @@ -432,7 +467,7 @@ func (idx *GoBruteForceIndex[T]) Search(proc *sqlexec.SqlProcess, _queries any, } // Max-heap logic for K > 1 - h := vectorindex.NewFastMaxHeap[T, int64](limit, heapKeysBuf, heapDistBuf) + h := vectorindex.NewFastMaxHeap[R, int64](limit, heapKeysBuf, heapDistBuf) for j := range idx.Dataset { dist, err2 := distfn(q, idx.Dataset[j]) diff --git a/pkg/vectorindex/brute_force/brute_force_test.go b/pkg/vectorindex/brute_force/brute_force_test.go index a3573f24a0e15..07fff159ed735 100644 --- a/pkg/vectorindex/brute_force/brute_force_test.go +++ b/pkg/vectorindex/brute_force/brute_force_test.go @@ -383,7 +383,7 @@ func TestGoBruteForceLifecycle(t *testing.T) { idx, err := NewGoBruteForceIndex[float32](dataset, 3, metric.Metric_L2sqDistance, 4) require.NoError(t, err) - bf := idx.(*GoBruteForceIndex[float32]) + bf := idx.(*GoBruteForceIndex[float32, float32]) require.NoError(t, bf.Load(nil)) require.NoError(t, bf.UpdateConfig(nil)) bf.Destroy() diff --git a/pkg/vectorindex/brute_force/cpu.go b/pkg/vectorindex/brute_force/cpu.go index c14bb7756765c..a52af753c0576 100644 --- a/pkg/vectorindex/brute_force/cpu.go +++ b/pkg/vectorindex/brute_force/cpu.go @@ -25,7 +25,7 @@ import ( // gpuMode is accepted-but-ignored in non-gpu builds — CPU is the only // option here. The signature matches the gpu.go version so callers // pass the flag uniformly regardless of build tag. -func NewBruteForceIndex[T types.RealNumbers](dataset [][]T, +func NewBruteForceIndex[T types.ArrayElement](dataset [][]T, dimension uint, m metric.MetricType, elemsz uint, diff --git a/pkg/vectorindex/brute_force/gpu.go b/pkg/vectorindex/brute_force/gpu.go index 160a738fb9085..65b80f2c8b17d 100644 --- a/pkg/vectorindex/brute_force/gpu.go +++ b/pkg/vectorindex/brute_force/gpu.go @@ -65,8 +65,8 @@ func NewAdhocBruteForceIndex[T types.RealNumbers](dataset [][]T, switch dset := any(dataset).(type) { case [][]float32: return NewGpuAdhocBruteForceIndex[float32](dset, dimension, m, elemsz) - case [][]uint16: - // Convert [][]uint16 to [][]cuvs.Float16 to pass to NewGpuAdhocBruteForceIndex + case [][]types.Float16: + // types.Float16 (NOT a bare uint16, which could also be BF16) -> cuvs.Float16. f16dset := make([][]cuvs.Float16, len(dset)) for i, v := range dset { f16dset[i] = util.UnsafeSliceCast[cuvs.Float16](v) @@ -224,7 +224,7 @@ func (idx *GpuAdhocBruteForceIndex[T]) Destroy() { } type GpuBruteForceIndex[T cuvs.VectorType] struct { - index *cuvs.GpuBruteForce[T] + index *cuvs.GpuBruteForce[T, T] dimension uint count uint } @@ -248,7 +248,7 @@ func resolveCuvsDistance(m metric.MetricType) cuvs.DistanceType { } } -func NewBruteForceIndex[T types.RealNumbers](dataset [][]T, +func NewBruteForceIndex[T types.ArrayElement](dataset [][]T, dimension uint, m metric.MetricType, elemsz uint, @@ -261,20 +261,20 @@ func NewBruteForceIndex[T types.RealNumbers](dataset [][]T, return NewCpuBruteForceIndex[T](dataset, dimension, m, elemsz) } + // cuVS brute force supports float32 and Float16 only. Switch on the distinct + // Go named type so types.BF16 (also uint16-backed) is never mistaken for f16. switch dset := any(dataset).(type) { - case [][]float64: - return NewCpuBruteForceIndex[T](dataset, dimension, m, elemsz) case [][]float32: return NewGpuBruteForceIndex[float32](dset, dimension, m, elemsz, nthread) - case [][]uint16: - // Convert [][]uint16 to [][]cuvs.Float16 to pass to NewGpuBruteForceIndex + case [][]types.Float16: f16dset := make([][]cuvs.Float16, len(dset)) for i, v := range dset { f16dset[i] = util.UnsafeSliceCast[cuvs.Float16](v) } return NewGpuBruteForceIndex[cuvs.Float16](f16dset, dimension, m, elemsz, nthread) default: - return nil, moerr.NewInternalErrorNoCtx("type not supported for BruteForceIndex") + // float64, bf16, int8, uint8 -> pure-Go CPU brute force. + return NewCpuBruteForceIndex[T](dataset, dimension, m, elemsz) } } @@ -320,7 +320,7 @@ func NewGpuBruteForceIndex[T cuvs.VectorType](dataset [][]T, } deviceID := cuvs.GetNextGpuDeviceId() - km, err := cuvs.NewGpuBruteForce[T](flattened, uint64(len(dataset)), uint32(dimension), resolveCuvsDistance(m), uint32(nthread), deviceID) + km, err := cuvs.NewGpuBruteForce[T, T](flattened, uint64(len(dataset)), uint32(dimension), resolveCuvsDistance(m), uint32(nthread), deviceID) if err != nil { return nil, err } diff --git a/pkg/vectorindex/cagra/build_gpu.go b/pkg/vectorindex/cagra/build_gpu.go index 6c3b4be74cc7e..9109d441eee34 100644 --- a/pkg/vectorindex/cagra/build_gpu.go +++ b/pkg/vectorindex/cagra/build_gpu.go @@ -23,6 +23,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/cuvs" "github.com/matrixorigin/matrixone/pkg/vectorindex" ) @@ -31,17 +32,28 @@ import ( // When the current sub-index reaches IndexCapacity, it is finalized (Build called) and a // new sub-index is created, mirroring the HnswBuild pattern. // +// CagraBuild carries two element types: base/quantizer-source B (the decoded +// source column type — f32 or f16) and storage Q (the cuVS sub-index storage +// type). For a direct index B==Q; for a quantized index (e.g. vecf16 base -> +// int8 storage) B is the base type and Q the 1-byte storage type. +// // CagraBuild is single-threaded; the cagra_create table function runs with IsSingle=true. -type CagraBuild[T cuvs.VectorType] struct { +type CagraBuild[B, Q cuvs.VectorType] struct { uid string idxcfg vectorindex.IndexConfig tblcfg vectorindex.IndexTableConfig - indexes []*CagraModel[T] // completed sub-indexes (Build already called) - current *CagraModel[T] // sub-index currently being filled + indexes []*CagraModel[B, Q] // completed sub-indexes (Build already called) + current *CagraModel[B, Q] // sub-index currently being filled nthread uint32 devices []int count int64 // vectors in current sub-index - idBuf [1]int64 // reusable buffer for AddFloat to avoid per-call heap allocation + idBuf [1]int64 // reusable buffer for AddRow to avoid per-call heap allocation + + // (B, Q) routing tags computed once at construction. bIsHalf: the base + // type is f16. qIsHalf: the storage type is f16 (so a half base goes + // native rather than quantized). + bIsHalf bool + qIsHalf bool // Filter column metadata (INCLUDE columns). Stashed once via SetFilterColumns // and re-applied to every new sub-index allocated by getOrCreateCurrent, so @@ -49,31 +61,33 @@ type CagraBuild[T cuvs.VectorType] struct { filterColMetaJSON string } -// NewCagraBuild creates a new CagraBuild ready for AddFloat calls. -func NewCagraBuild[T cuvs.VectorType]( +// NewCagraBuild creates a new CagraBuild ready for AddRow calls. +func NewCagraBuild[B, Q cuvs.VectorType]( uid string, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, nthread uint32, devices []int, -) (*CagraBuild[T], error) { - return &CagraBuild[T]{ +) (*CagraBuild[B, Q], error) { + return &CagraBuild[B, Q]{ uid: uid, idxcfg: idxcfg, tblcfg: tblcfg, - indexes: make([]*CagraModel[T], 0, 4), + indexes: make([]*CagraModel[B, Q], 0, 4), nthread: nthread, devices: devices, + bIsHalf: cuvs.GetQuantization[B]() == cuvs.F16, + qIsHalf: cuvs.GetQuantization[Q]() == cuvs.F16, }, nil } -func (b *CagraBuild[T]) createKey(n int) string { +func (b *CagraBuild[B, Q]) createKey(n int) string { return fmt.Sprintf("%s:%d", b.uid, n) } // getOrCreateCurrent returns the current sub-index, creating a new one if needed. // When the current sub-index is full it is finalized (Build called) and a new one is started. -func (b *CagraBuild[T]) getOrCreateCurrent() (*CagraModel[T], error) { +func (b *CagraBuild[B, Q]) getOrCreateCurrent() (*CagraModel[B, Q], error) { capacity := b.idxcfg.IndexCapacity if b.current != nil && b.count >= capacity { @@ -88,7 +102,7 @@ func (b *CagraBuild[T]) getOrCreateCurrent() (*CagraModel[T], error) { if b.current == nil { key := b.createKey(len(b.indexes)) - m, err := NewCagraModelForBuild[T](key, b.idxcfg, b.nthread, b.devices) + m, err := NewCagraModelForBuild[B, Q](key, b.idxcfg, b.nthread, b.devices) if err != nil { return nil, err } @@ -111,33 +125,46 @@ func (b *CagraBuild[T]) getOrCreateCurrent() (*CagraModel[T], error) { // SetFilterColumns registers pre-filter (INCLUDE column) metadata. The JSON // is re-applied to each new sub-index allocated during the build. Must be -// called before the first AddFloat. -func (b *CagraBuild[T]) SetFilterColumns(colMetaJSON string) { +// called before the first AddRow. +func (b *CagraBuild[B, Q]) SetFilterColumns(colMetaJSON string) { b.filterColMetaJSON = colMetaJSON } // AddFilterChunk appends nrows raw filter-column bytes to the *current* // sub-index being filled. Call once per filter column per row batch, in the -// same cadence as AddFloat (which drives sub-index rotation). +// same cadence as AddRow (which drives sub-index rotation). // nullBitmap is a packed []uint32 (LSB-first, bit i = 1 means row i IS NULL) // of ceil(nrows/32) entries, or nil when the chunk has no nulls. -func (b *CagraBuild[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (b *CagraBuild[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if b.current == nil { - return moerr.NewInternalErrorNoCtx("CagraBuild.AddFilterChunk: no current sub-index (call AddFloat first)") + return moerr.NewInternalErrorNoCtx("CagraBuild.AddFilterChunk: no current sub-index (call AddRow first)") } return b.current.Index.AddFilterChunk(colIdx, data, nullBitmap, nrows) } -// AddFloat appends one float32 vector with the given int64 id. -// The internal quantization (T) is handled by AddChunkFloat. +// AddRow buffers one source row. vecBytes is the raw little-endian base-type +// bytes of one vector (4*dim for an f32 base, 2*dim for an f16 base) — the +// non-generic cagraBuilder interface can't name the concrete element type B, so +// the bytes are reinterpreted here with UnsafeSliceCast (zero-copy, no per-row +// heap alloc). Routing by (B, Q): +// - f16 base, f16 storage (direct, Q==B): native AddChunk([]Q). +// - otherwise (f32 base, or f16 base -> int8/uint8): AddChunkQuantize([]B), +// which converts B -> Q on device (B==Q copy, or learned/cast quantizer). +// // idBuf is reused across calls to avoid a per-call heap allocation. -func (b *CagraBuild[T]) AddFloat(id int64, vec []float32) error { +func (b *CagraBuild[B, Q]) AddRow(id int64, vecBytes []byte) error { idx, err := b.getOrCreateCurrent() if err != nil { return err } b.idBuf[0] = id - if err = idx.AddChunkFloat(vec, 1, b.idBuf[:]); err != nil { + + if b.bIsHalf && b.qIsHalf { + err = idx.AddChunk(util.UnsafeSliceCast[Q](vecBytes), 1, b.idBuf[:]) + } else { + err = idx.AddChunkQuantize(util.UnsafeSliceCast[B](vecBytes), 1, b.idBuf[:]) + } + if err != nil { return err } b.count++ @@ -146,7 +173,7 @@ func (b *CagraBuild[T]) AddFloat(id int64, vec []float32) error { // ToInsertSql finalizes any in-progress sub-index, serializes all sub-indexes to the // storage table, and returns INSERT SQL statements (storage chunks + single metadata row). -func (b *CagraBuild[T]) ToInsertSql(ts int64) ([]string, error) { +func (b *CagraBuild[B, Q]) ToInsertSql(ts int64) ([]string, error) { // Finalize the current sub-index if it contains vectors. if b.current != nil && b.count > 0 { if err := b.current.Build(); err != nil { @@ -181,7 +208,7 @@ func (b *CagraBuild[T]) ToInsertSql(ts int64) ([]string, error) { } // Destroy frees all GPU memory and removes any temporary files. -func (b *CagraBuild[T]) Destroy() error { +func (b *CagraBuild[B, Q]) Destroy() error { var errs error if b.current != nil { if err := b.current.Destroy(); err != nil { @@ -199,6 +226,6 @@ func (b *CagraBuild[T]) Destroy() error { } // GetIndexes returns the completed sub-indexes (for testing). -func (b *CagraBuild[T]) GetIndexes() []*CagraModel[T] { +func (b *CagraBuild[B, Q]) GetIndexes() []*CagraModel[B, Q] { return b.indexes } diff --git a/pkg/vectorindex/cagra/cdc_load_test.go b/pkg/vectorindex/cagra/cdc_load_test.go index bf9627e221a29..ce49d305b9843 100644 --- a/pkg/vectorindex/cagra/cdc_load_test.go +++ b/pkg/vectorindex/cagra/cdc_load_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -66,7 +67,11 @@ func encodeChunk(t *testing.T, dim, includeBytesPerRow int, ops []cuvscdc.CdcOp, } insIdx++ } - out, err := cuvscdc.EncodeEventRecord(buf, op, pkids[i], v, inc, dim, includeBytesPerRow) + var vb []byte + if v != nil { + vb = util.UnsafeSliceToBytes(v) + } + out, err := cuvscdc.EncodeEventRecord(buf, op, pkids[i], vb, inc, 4*dim, includeBytesPerRow) require.NoError(t, err) buf = out } @@ -95,7 +100,7 @@ func TestLoadCdcEventsFromDB_RoundTrip(t *testing.T) { } defer func() { runSql = orig }() - idx := &CagraModel[float32]{Id: "idx-1"} + idx := &CagraModel[float32, float32]{Id: "idx-1"} got, err := idx.loadCdcEventsFromDB(sqlproc, tblcfg) require.NoError(t, err) require.Len(t, got, 1) @@ -115,7 +120,7 @@ func TestLoadCdcEventsFromDB_Empty(t *testing.T) { } defer func() { runSql = orig }() - idx := &CagraModel[float32]{Id: "idx-1"} + idx := &CagraModel[float32, float32]{Id: "idx-1"} got, err := idx.loadCdcEventsFromDB(sqlproc, testTblcfg()) require.NoError(t, err) require.Empty(t, got) @@ -134,7 +139,7 @@ func TestReplayEventChunks_DeleteInsertDelete(t *testing.T) { ) chunks := []cuvscdc.EventChunk{{ChunkId: 0, Data: chunkBytes}} - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Equal(t, []int64{1}, delPkids) require.Empty(t, ovPkids) @@ -154,7 +159,7 @@ func TestReplayEventChunks_FlattenOverflow(t *testing.T) { ) chunks := []cuvscdc.EventChunk{{ChunkId: 0, Data: chunkBytes}} - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Empty(t, delPkids) require.Equal(t, []int64{10, 20}, ovPkids) @@ -178,7 +183,7 @@ func TestReplayEventChunks_MultiChunkOrder(t *testing.T) { {ChunkId: 1, Data: chunk1}, {ChunkId: 0, Data: chunk0}, } - delPkids, ovPkids, _, _, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, _, _, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Equal(t, []int64{5}, delPkids, "INSERT@chunk0 then DELETE@chunk1 → deleted={5}") @@ -254,7 +259,7 @@ func TestLoadIndex_WithCdcDeltas(t *testing.T) { } defer func() { runSql = origRunSql }() - models, err := LoadMetadata[float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) + models, err := LoadMetadata[float32, float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) require.NoError(t, err) require.Equal(t, 1, len(models)) diff --git a/pkg/vectorindex/cagra/model_gpu.go b/pkg/vectorindex/cagra/model_gpu.go index e8685b889eb85..cdde2343650bd 100644 --- a/pkg/vectorindex/cagra/model_gpu.go +++ b/pkg/vectorindex/cagra/model_gpu.go @@ -29,6 +29,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/cuvs" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -45,9 +46,9 @@ var runSql_streaming = sqlexec.RunStreamingSql // CagraModel wraps a GpuCagra index and handles load/save to the secondary index tables. // The serialized form is a tar file produced by cuvs.Pack / cuvs.Unpack. // T must satisfy cuvs.VectorType (float32 | Float16 | int8 | uint8). -type CagraModel[T cuvs.VectorType] struct { +type CagraModel[B, Q cuvs.VectorType] struct { Id string - Index *cuvs.GpuCagra[T] + Index *cuvs.GpuCagra[B, Q] Path string // local tar file path; empty when index is in GPU memory only FileSize int64 MaxCapacity uint64 @@ -77,7 +78,7 @@ type CagraModel[T cuvs.VectorType] struct { // (quantizer params live in the model tar, not available at CDC write // time). OverflowPkids []int64 - OverflowVecs []float32 // len = len(OverflowPkids) * dim + OverflowVecs []B // len = len(OverflowPkids) * dim (native base type B) // INCLUDE column data carried alongside each overflow row. Layout // matches the EncodeEventRecord INSERT-record include section: @@ -98,8 +99,8 @@ type CagraModel[T cuvs.VectorType] struct { // NewCagraModelForBuild creates a CagraModel ready for bulk-build. // Call InitEmpty once the total vector count is known, then AddChunk, then Build. -func NewCagraModelForBuild[T cuvs.VectorType](id string, cfg vectorindex.IndexConfig, nthread uint32, devices []int) (*CagraModel[T], error) { - return &CagraModel[T]{ +func NewCagraModelForBuild[B, Q cuvs.VectorType](id string, cfg vectorindex.IndexConfig, nthread uint32, devices []int) (*CagraModel[B, Q], error) { + return &CagraModel[B, Q]{ Id: id, Idxcfg: cfg, NThread: nthread, @@ -108,7 +109,7 @@ func NewCagraModelForBuild[T cuvs.VectorType](id string, cfg vectorindex.IndexCo } // cagraConfig returns the cuvs types derived from idx.Idxcfg. -func (idx *CagraModel[T]) cagraConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.CagraBuildParams, mode cuvs.DistributionMode, err error) { +func (idx *CagraModel[B, Q]) cagraConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.CagraBuildParams, mode cuvs.DistributionMode, err error) { cfg := idx.Idxcfg.CuvsCagra var ok bool cuvsMetric, ok = metric.MetricTypeToCuvsMetric[metric.MetricType(cfg.Metric)] @@ -129,7 +130,7 @@ func (idx *CagraModel[T]) cagraConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.C // InitEmpty allocates the GPU buffer for totalCount vectors. // Must be called after NewCagraModelForBuild and before any AddChunk call. -func (idx *CagraModel[T]) InitEmpty(totalCount uint64) error { +func (idx *CagraModel[B, Q]) InitEmpty(totalCount uint64) error { if idx.Index != nil { return moerr.NewInternalErrorNoCtx("CagraModel: index already initialized") } @@ -137,7 +138,7 @@ func (idx *CagraModel[T]) InitEmpty(totalCount uint64) error { if err != nil { return err } - gi, err := cuvs.NewGpuCagraEmpty[T]( + gi, err := cuvs.NewGpuCagraEmpty[B, Q]( totalCount, uint32(idx.Idxcfg.CuvsCagra.Dimensions), cuvsMetric, @@ -159,7 +160,7 @@ func (idx *CagraModel[T]) InitEmpty(totalCount uint64) error { } // AddChunk appends a chunk of typed vectors to the pre-allocated GPU buffer. -func (idx *CagraModel[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) error { +func (idx *CagraModel[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("CagraModel: index not initialized; call InitEmpty first") } @@ -175,17 +176,14 @@ func (idx *CagraModel[T]) AddChunk(chunk []T, chunkCount uint64, ids []int64) er return nil } -// AddChunkFloat appends a chunk of float32 vectors, quantizing on the fly when T is a 1-byte type. -func (idx *CagraModel[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunkQuantize appends a chunk of base-typed (B) vectors, quantizing +// natively to the 1-byte storage type Q (int8/uint8). Used for a vecf16 base +// with QUANTIZATION=int8/uint8 — no f32 detour. +func (idx *CagraModel[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("CagraModel: index not initialized; call InitEmpty first") } - /* - if len(ids) > 0 { - logutil.Infof("[DEBUG] CagraModel.AddChunkFloat: chunkCount=%d, first_id=%d, last_id=%d", chunkCount, ids[0], ids[len(ids)-1]) - } - */ - if err := idx.Index.AddChunkFloat(chunk, chunkCount, ids); err != nil { + if err := idx.Index.AddChunkQuantize(chunk, chunkCount, ids); err != nil { return err } idx.Len += int64(chunkCount) @@ -193,7 +191,7 @@ func (idx *CagraModel[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids } // Build constructs the CAGRA graph from the loaded vectors and starts the worker pool. -func (idx *CagraModel[T]) Build() error { +func (idx *CagraModel[B, Q]) Build() error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("CagraModel: index not initialized") } @@ -205,7 +203,7 @@ func (idx *CagraModel[T]) Build() error { } // Destroy frees GPU memory and removes the local tar file if present. -func (idx *CagraModel[T]) Destroy() error { +func (idx *CagraModel[B, Q]) Destroy() error { if idx.Index != nil { if err := idx.Index.Destroy(); err != nil { return err @@ -224,7 +222,7 @@ func (idx *CagraModel[T]) Destroy() error { // saveToFile serializes the CAGRA index to a local tar file and updates idx.Path / idx.Checksum. // If the index is clean (not dirty) or nil, it is a no-op. // On success the GPU memory is freed and idx.Index is set to nil. -func (idx *CagraModel[T]) saveToFile() error { +func (idx *CagraModel[B, Q]) saveToFile() error { if idx.Index == nil { return nil } @@ -280,7 +278,7 @@ func (idx *CagraModel[T]) saveToFile() error { // ToSql generates INSERT SQL statements to store the model in the secondary index storage table. // Mirrors HnswModel.ToSql — callers are responsible for generating the metadata INSERT. -func (idx *CagraModel[T]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error) { +func (idx *CagraModel[B, Q]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error) { if err := idx.saveToFile(); err != nil { return nil, err } @@ -330,7 +328,7 @@ func (idx *CagraModel[T]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, err } // ToDeleteSql generates DELETE SQL for both the storage and metadata tables. -func (idx *CagraModel[T]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error) { +func (idx *CagraModel[B, Q]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error) { sqls := make([]string, 0, 2) sqls = append(sqls, fmt.Sprintf("DELETE FROM %s WHERE %s = %s", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), catalog.Cagra_TblCol_Storage_Index_Id, sqlquote.String(idx.Id))) @@ -340,17 +338,17 @@ func (idx *CagraModel[T]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]strin } // Empty returns true when no vectors have been added. -func (idx *CagraModel[T]) Empty() bool { +func (idx *CagraModel[B, Q]) Empty() bool { return idx.Len == 0 } // Full returns true when the index has reached its maximum capacity. -func (idx *CagraModel[T]) Full() bool { +func (idx *CagraModel[B, Q]) Full() bool { return idx.MaxCapacity > 0 && uint64(idx.Len) >= idx.MaxCapacity } // Search performs a KNN search and returns external PKs with distances. -func (idx *CagraModel[T]) Search(query []T, limit uint32) (keys []int64, distances []float32, err error) { +func (idx *CagraModel[B, Q]) Search(query []Q, limit uint32) (keys []int64, distances []float32, err error) { if idx.Index == nil { return nil, nil, moerr.NewInternalErrorNoCtx("CagraModel: index not loaded") } @@ -366,7 +364,7 @@ func (idx *CagraModel[T]) Search(query []T, limit uint32) (keys []int64, distanc } // loadChunk reads one streaming result batch and writes each chunk at the correct file offset. -func (idx *CagraModel[T]) loadChunk(ctx context.Context, +func (idx *CagraModel[B, Q]) loadChunk(ctx context.Context, sqlproc *sqlexec.SqlProcess, stream_chan chan executor.Result, error_chan chan error, @@ -414,7 +412,7 @@ func (idx *CagraModel[T]) loadChunk(ctx context.Context, // - tag=0: model tar chunks (streaming, multi-GB) // - tag=1: CDC event log (small KB–MB; replayed once after Unpack to derive // the deleted-pkid set and the brute-force overflow) -func (idx *CagraModel[T]) LoadIndex( +func (idx *CagraModel[B, Q]) LoadIndex( sqlproc *sqlexec.SqlProcess, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, @@ -550,7 +548,7 @@ func (idx *CagraModel[T]) LoadIndex( return err } - gi, err := cuvs.NewGpuCagraEmpty[T]( + gi, err := cuvs.NewGpuCagraEmpty[B, Q]( uint64(idxcfg.IndexCapacity), uint32(idxcfg.CuvsCagra.Dimensions), cuvsMetric, @@ -587,7 +585,7 @@ func (idx *CagraModel[T]) LoadIndex( } includeBytesPerRow = ibpr } - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(eventChunks, dim, includeBytesPerRow) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[B](eventChunks, dim, includeBytesPerRow) if err != nil { gi.Destroy() return err @@ -627,7 +625,7 @@ func (idx *CagraModel[T]) LoadIndex( } // Unload persists dirty state to a local tar file and frees GPU memory. -func (idx *CagraModel[T]) Unload() error { +func (idx *CagraModel[B, Q]) Unload() error { if idx.Index == nil { return nil } @@ -650,7 +648,7 @@ func (idx *CagraModel[T]) Unload() error { // returns one EventChunk per row. The caller (LoadIndex / search) sorts by // chunk_id before replay since record ordering across chunks encodes the // temporal ordering between DELETE and INSERT events for the same pkid. -func (idx *CagraModel[T]) loadCdcEventsFromDB( +func (idx *CagraModel[B, Q]) loadCdcEventsFromDB( sqlproc *sqlexec.SqlProcess, tblcfg vectorindex.IndexTableConfig, ) ([]cuvscdc.EventChunk, error) { @@ -682,16 +680,20 @@ func (idx *CagraModel[T]) loadCdcEventsFromDB( // flattens the (deleted, overflow) replay state into the parallel slices the // CagraModel struct carries (pkids/vecs/include layout that buildOverflow // expects). Pass includeBytesPerRow=0 for indexes without INCLUDE columns. -func replayEventChunks( +func replayEventChunks[B cuvs.VectorType]( chunks []cuvscdc.EventChunk, dim int, includeBytesPerRow int, -) ([]int64, []int64, []float32, []byte, error) { +) ([]int64, []int64, []B, []byte, error) { if len(chunks) == 0 { return nil, nil, nil, nil, nil } cuvscdc.SortChunks(chunks) - state, err := cuvscdc.ReplayEventLog(chunks, dim, includeBytesPerRow) + // The codec stores vectors as opaque bytes; the per-row byte length is + // dim * sizeof(B). Reinterpret each row's bytes back to the native base + // type B for the overflow brute force — no f32 detour. + vecBytesPerRow := dim * int(util.UnsafeSizeOf[B]()) + state, err := cuvscdc.ReplayEventLog(chunks, vecBytesPerRow, includeBytesPerRow) if err != nil { return nil, nil, nil, nil, err } @@ -703,14 +705,15 @@ func replayEventChunks( return deletedPkids, nil, nil, nil, nil } ovPkids := make([]int64, len(state.Overflow)) - ovVecs := make([]float32, len(state.Overflow)*dim) + ovVecs := make([]B, len(state.Overflow)*dim) + ovVecBytes := util.UnsafeSliceToBytes(ovVecs) var ovInc []byte if includeBytesPerRow > 0 { ovInc = make([]byte, len(state.Overflow)*includeBytesPerRow) } for i, e := range state.Overflow { ovPkids[i] = e.Pkid - copy(ovVecs[i*dim:(i+1)*dim], e.Vec) + copy(ovVecBytes[i*vecBytesPerRow:(i+1)*vecBytesPerRow], e.Vec) if includeBytesPerRow > 0 { copy(ovInc[i*includeBytesPerRow:(i+1)*includeBytesPerRow], e.Include) } @@ -720,7 +723,7 @@ func replayEventChunks( // LoadMetadata loads CagraModel descriptors from the metadata table. // Each returned model has Id, Checksum, Timestamp, and FileSize set; Index is nil. -func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*CagraModel[T], error) { +func LoadMetadata[B, Q cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*CagraModel[B, Q], error) { sql := fmt.Sprintf("SELECT * FROM %s ORDER BY timestamp ASC", sqlquote.QualifiedIdent(dbname, metatbl)) res, err := runSql(sqlproc, sql) if err != nil { @@ -733,7 +736,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, total += bat.RowCount() } - indexes := make([]*CagraModel[T], 0, total) + indexes := make([]*CagraModel[B, Q], 0, total) for _, bat := range res.Batches { idVec := bat.Vecs[0] chksumVec := bat.Vecs[1] @@ -744,7 +747,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, chksum := chksumVec.GetStringAt(i) ts := vector.GetFixedAtWithTypeCheck[int64](tsVec, i) fs := vector.GetFixedAtWithTypeCheck[int64](fsVec, i) - idx := &CagraModel[T]{Id: id, Checksum: chksum, Timestamp: ts, FileSize: fs} + idx := &CagraModel[B, Q]{Id: id, Checksum: chksum, Timestamp: ts, FileSize: fs} indexes = append(indexes, idx) } } diff --git a/pkg/vectorindex/cagra/model_test.go b/pkg/vectorindex/cagra/model_test.go index 29ab71065aca1..e2091100df0d9 100644 --- a/pkg/vectorindex/cagra/model_test.go +++ b/pkg/vectorindex/cagra/model_test.go @@ -130,19 +130,19 @@ func makeIndexBatch(proc *process.Process, tarPath string) *batch.Batch { // buildTestModel builds, trains and saves a CagraModel, returning it with Index==nil and // Path/Checksum/FileSize set. The caller is responsible for removing the tar file. -func buildTestModel(t *testing.T, id string, ids []int64) *CagraModel[float32] { +func buildTestModel(t *testing.T, id string, ids []int64) *CagraModel[float32, float32] { t.Helper() idxcfg := testIdxcfg() data := generateTestData(testNVectors, testDim) - m, err := NewCagraModelForBuild[float32](id, idxcfg, 1, []int{0}) + m, err := NewCagraModelForBuild[float32, float32](id, idxcfg, 1, []int{0}) require.NoError(t, err) err = m.InitEmpty(testNVectors) require.NoError(t, err) - err = m.AddChunkFloat(data, testNVectors, ids) + err = m.AddChunkQuantize(data, testNVectors, ids) require.NoError(t, err) err = m.Build() @@ -183,7 +183,7 @@ func TestModelStreamError(t *testing.T) { defer func() { runSql = origRunSql }() // Manually create a model descriptor as if loaded from metadata. - idx := &CagraModel[float32]{ + idx := &CagraModel[float32, float32]{ Id: "test-stream-err", FileSize: 1024, // non-zero triggers DB download Checksum: "fake-checksum", @@ -211,13 +211,13 @@ func TestModelBuildAndLoad(t *testing.T) { } // ---- Build ---- - built, err := NewCagraModelForBuild[float32]("test-build", idxcfg, 1, []int{0}) + built, err := NewCagraModelForBuild[float32, float32]("test-build", idxcfg, 1, []int{0}) require.NoError(t, err) err = built.InitEmpty(testNVectors) require.NoError(t, err) - err = built.AddChunkFloat(data, testNVectors, ids) + err = built.AddChunkQuantize(data, testNVectors, ids) require.NoError(t, err) err = built.Build() @@ -247,7 +247,7 @@ func TestModelBuildAndLoad(t *testing.T) { defer func() { runSql = origRunSql }() // ---- Load from local tar (skips DB download since Path is set) ---- - loader := &CagraModel[float32]{ + loader := &CagraModel[float32, float32]{ Id: "test-build", Path: tarPath, Checksum: checksum, @@ -340,7 +340,7 @@ func TestModelLoadFromDB(t *testing.T) { defer func() { runSql = origRunSql }() // LoadMetadata — creates a model from DB metadata. - models, err := LoadMetadata[float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) + models, err := LoadMetadata[float32, float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) require.NoError(t, err) require.Equal(t, 1, len(models)) @@ -370,7 +370,7 @@ func TestModelNil(t *testing.T) { var tblcfg vectorindex.IndexTableConfig // Zero-value model: no index, no path. - idx := &CagraModel[float32]{} + idx := &CagraModel[float32, float32]{} // InitEmpty fails because Devices is empty. err := idx.InitEmpty(10) @@ -386,7 +386,7 @@ func TestModelNil(t *testing.T) { require.NotNil(t, err) // AddChunkFloat fails because Index is nil. - err = idx.AddChunkFloat([]float32{1, 2}, 1, []int64{1}) + err = idx.AddChunkQuantize([]float32{1, 2}, 1, []int64{1}) require.NotNil(t, err) // Search fails because Index is nil. @@ -394,7 +394,7 @@ func TestModelNil(t *testing.T) { require.NotNil(t, err) // Search with nil query fails. - idx2 := &CagraModel[float32]{} // still nil Index + idx2 := &CagraModel[float32, float32]{} // still nil Index _, _, err = idx2.Search(nil, 1) require.NotNil(t, err) @@ -432,7 +432,7 @@ func TestModelEmptyBuild(t *testing.T) { idxcfg := testIdxcfg() tblcfg := testTblcfg() - built, err := NewCagraModelForBuild[float32]("test-empty", idxcfg, 1, []int{0}) + built, err := NewCagraModelForBuild[float32, float32]("test-empty", idxcfg, 1, []int{0}) require.NoError(t, err) // InitEmpty with 0 would fail in CAGRA, so test saveToFile directly on empty Len. diff --git a/pkg/vectorindex/cagra/plugin/compile/compile.go b/pkg/vectorindex/cagra/plugin/compile/compile.go index 6246f7ec8ff2c..04dd21aeecfa6 100644 --- a/pkg/vectorindex/cagra/plugin/compile/compile.go +++ b/pkg/vectorindex/cagra/plugin/compile/compile.go @@ -234,17 +234,38 @@ func registerIdxcronUpdate( } func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.ReindexParamUpdate) (map[string]string, error) { - return compileplugin.MergeReindexParams(old, alter, "cagra", + // Merge first, then validate the EFFECTIVE quantization via the per-algo + // catalog hook (the single home shared with CREATE). The merged map is the + // index's actual post-reindex config: the value the reindex set, or — when + // the reindex omitted QUANTIZATION (e.g. the idxcron-issued rebuild) — the + // value already stored on the index. Validating the merge (not the raw alter + // delta) means the check is never skipped just because the statement omitted + // quantization, and quantization and op_type come from one consistent source. + merged, err := compileplugin.MergeReindexParams(old, alter, "cagra", catalog.IndexAlgoParamMaxIndexCapacity, catalog.IntermediateGraphDegree, catalog.GraphDegree, catalog.ITopkSize, + catalog.Quantization, ) + if err != nil { + return nil, err + } + if err := (cagraruntime.CatalogHooks{}).ValidQuantization( + merged[catalog.Quantization], merged[catalog.IndexAlgoParamOpType]); err != nil { + return nil, err + } + return merged, nil } -// HandleDropIndex is a no-op: generic hidden-table cleanup is sufficient. func (Hooks) HandleDropIndex(_ compileplugin.CompileContext, defs map[string]*plan.IndexDef) error { logutil.Infof("[plugin] cagra HandleDropIndex: defs=%d", len(defs)) + // Evict the cached search index so its GPU resources are freed NOW, rather + // than lingering until the 5-min VectorIndexCacheTTL housekeeping reaps it. + // Mirrors the create-side cache.Cache.Remove(storageDef.IndexTableName). + if storageDef, ok := defs[catalog.Cagra_TblType_Storage]; ok { + cache.Cache.Remove(storageDef.IndexTableName) + } return nil } diff --git a/pkg/vectorindex/cagra/plugin/compile/compile_test.go b/pkg/vectorindex/cagra/plugin/compile/compile_test.go index 5a0ab619a1b42..c5edc31e70e92 100644 --- a/pkg/vectorindex/cagra/plugin/compile/compile_test.go +++ b/pkg/vectorindex/cagra/plugin/compile/compile_test.go @@ -424,3 +424,34 @@ func TestCagraValidateReindexParams_MergesGraphDegree(t *testing.T) { _, had := old[catalog.GraphDegree] require.False(t, had) } + +// TestCagraValidateReindexParams_Quantization: CAGRA (cuvs) accepts the +// cuvs-supported quantization names and rejects others (e.g. bf16, which the +// cuvs backend does not support even though IVF-FLAT does). +func TestCagraValidateReindexParams_Quantization(t *testing.T) { + got, err := Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "float16"}, + }) + require.NoError(t, err) + require.Equal(t, "float16", got[catalog.Quantization]) + + _, err = Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "bf16"}, + }) + require.Error(t, err) + + // int8/uint8 on a non-L2 (inner-product) index IS rejected at REINDEX via the + // ValidQuantization hook: the merged op_type is inner-product and the + // int8/uint8 affine quantizer only preserves L2 geometry. + _, err = Hooks{}.ValidateReindexParams( + map[string]string{catalog.IndexAlgoParamOpType: "vector_ip_ops"}, + compileplugin.ReindexParamUpdate{Params: map[string]string{catalog.Quantization: "int8"}}) + require.Error(t, err) + + // ...but int8 with L2 (the merged op_type) is accepted. + got, err = Hooks{}.ValidateReindexParams( + map[string]string{catalog.IndexAlgoParamOpType: "vector_l2_ops"}, + compileplugin.ReindexParamUpdate{Params: map[string]string{catalog.Quantization: "int8"}}) + require.NoError(t, err) + require.Equal(t, "int8", got[catalog.Quantization]) +} diff --git a/pkg/vectorindex/cagra/plugin/iscp/iscp.go b/pkg/vectorindex/cagra/plugin/iscp/iscp.go index 15eb40dc81d6c..ab0d4430c0af6 100644 --- a/pkg/vectorindex/cagra/plugin/iscp/iscp.go +++ b/pkg/vectorindex/cagra/plugin/iscp/iscp.go @@ -62,6 +62,6 @@ func (Hooks) Run(c *iscppkg.IndexConsumer, ctx context.Context, errch chan error iscppkg.RunCuvs(c, ctx, errch, r, func(sqlproc *sqlexec.SqlProcess) (iscppkg.CuvsSync, error) { w := c.SqlWriter().(*iscppkg.CuvsCdcWriter) return cagra.NewCagraSync(sqlproc, w.DbName(), w.TblName(), w.IndexName(), - w.IndexDef(), w.Dimension(), w.ColMetaJSON()) + w.IndexDef(), w.Dimension(), w.BaseVectorType(), w.ColMetaJSON()) }) } diff --git a/pkg/vectorindex/cagra/plugin/plan/plan_test.go b/pkg/vectorindex/cagra/plugin/plan/plan_test.go index b4f9661a447e0..968312d5681f1 100644 --- a/pkg/vectorindex/cagra/plugin/plan/plan_test.go +++ b/pkg/vectorindex/cagra/plugin/plan/plan_test.go @@ -168,6 +168,62 @@ func TestBuildSecondaryIndexDefs_OK(t *testing.T) { require.NotNil(t, tblDefs[1].Pkey) } +// indexOnQuant builds a single-column *tree.Index over colName with a +// QUANTIZATION option. +func indexOnQuant(colName, quant string) *tree.Index { + idx := indexOn(colName) + idx.IndexOption = &tree.IndexOption{Quantization: quant} + return idx +} + +// f16ColMap returns a colMap with an int64 pk and a vecf16 base column. +func f16ColMap(pkName, vecName string) map[string]*plan.ColDef { + m := vecColMap(pkName, vecName) + m[vecName].Typ.Id = int32(types.T_array_float16) + return m +} + +// TestBuildSecondaryIndexDefs_F16Base: a vecf16 base column is accepted. +func TestBuildSecondaryIndexDefs_F16Base(t *testing.T) { + idxDefs, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("vec"), f16ColMap("id", "vec"), nil, "id") + require.NoError(t, err) + require.Len(t, idxDefs, 2) +} + +// TestBuildSecondaryIndexDefs_UnsupportedBase: only vecf32 / vecf16 are valid +// base columns; vecf64 / vecbf16 / vecint8 / vecuint8 are rejected. +func TestBuildSecondaryIndexDefs_UnsupportedBase(t *testing.T) { + for _, oid := range []types.T{ + types.T_array_float64, types.T_array_bf16, types.T_array_int8, types.T_array_uint8, + } { + colMap := vecColMap("id", "vec") + colMap["vec"].Typ.Id = int32(oid) + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("vec"), colMap, nil, "id") + require.Error(t, err, "base type %s must be rejected", oid) + } +} + +// TestBuildSecondaryIndexDefs_F16UpcastRejected: vecf16 base + QUANTIZATION +// float32 is an upcast (4 > 2 bytes) and must be rejected by the downcast +// guard. (The accepted downcast path — f16 -> int8/uint8 — is exercised +// end-to-end by the GPU functional BVT, since the full def build past the +// guard needs a richer compiler context than this stub provides.) +func TestBuildSecondaryIndexDefs_F16UpcastRejected(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "float32"), f16ColMap("id", "vec"), nil, "id") + require.Error(t, err) +} + +// TestBuildSecondaryIndexDefs_BF16QuantRejected: QUANTIZATION 'bf16' has no GPU +// bfloat16 storage (cuVS has no bfloat16 index/quantizer), so it must be rejected +// rather than silently falling back to f32 storage — even though it passes the +// downcast width guard (bf16 is 2 bytes). Rejected on both f32 and f16 bases. +func TestBuildSecondaryIndexDefs_BF16QuantRejected(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "bf16"), vecColMap("id", "vec"), nil, "id") + require.Error(t, err, "f32 base + bf16 quant must be rejected") + _, _, err = Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "bf16"), f16ColMap("id", "vec"), nil, "id") + require.Error(t, err, "f16 base + bf16 quant must be rejected") +} + // --- schema.go: BuildFullTextIndexDefs ------------------------------------- func TestBuildFullTextIndexDefs_Unsupported(t *testing.T) { diff --git a/pkg/vectorindex/cagra/plugin/plan/schema.go b/pkg/vectorindex/cagra/plugin/plan/schema.go index fea244c98c1fe..03e04804706b0 100644 --- a/pkg/vectorindex/cagra/plugin/plan/schema.go +++ b/pkg/vectorindex/cagra/plugin/plan/schema.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/sql/util" cagrart "github.com/matrixorigin/matrixone/pkg/vectorindex/cagra/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // cagraCatalogHooks is the shared (stateless) catalog-hooks instance used for @@ -63,7 +64,40 @@ func (Hooks) BuildSecondaryIndexDefs( return nil, nil, moerr.NewInvalidInputf(ctx.GetContext(), "column '%s' is not exist", indexInfo.KeyParts[0].ColName.ColNameOrigin()) } if !catalogplugin.SupportsVectorType(cagraCatalogHooks, types.T(colMap[name].Typ.Id)) { - return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "Cagra only supports VECF32 column types") + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "Cagra only supports VECF32 / VECF16 base column types") + } + // QUANTIZATION is downcast-only: the storage element must be the same width + // or narrower than the base column (f16 base -> int8/uint8 OK; f16 base -> + // float32 is an upcast and rejected). Mirrors ivfflat's guard. + if indexInfo.IndexOption != nil && indexInfo.IndexOption.Quantization != "" { + if qt, ok := quantizer.ToVectorType(indexInfo.IndexOption.Quantization); ok { + // bf16 storage does not exist on the GPU (cuVS/cgo has no bfloat16 + // index or quantizer), so reject it explicitly rather than silently + // falling back to f32 storage. Supported cuvs storage = f16/int8/uint8. + if qt == types.T_array_bf16 { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "Cagra does not support '%s' quantization (no GPU bfloat16 storage); use 'float16', 'int8', or 'uint8'", + indexInfo.IndexOption.Quantization) + } + baseSize := types.Type{Oid: types.T(colMap[name].Typ.Id)}.GetArrayElementSize() + quantSize := types.Type{Oid: qt}.GetArrayElementSize() + if quantSize > baseSize { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "Cagra QUANTIZATION '%s' (%d bytes/element) cannot upcast base column %s (%d bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type", + indexInfo.IndexOption.Quantization, quantSize, + types.T(colMap[name].Typ.Id).String(), baseSize) + } + // int8/uint8 quantization is L2-only (the affine quantizer breaks + // inner-product / cosine geometry). Gated by the per-algo catalog + // hook — the single home shared with REINDEX + // (compile/ValidateReindexParams) — so CREATE and REINDEX cannot + // drift. (bf16 and width/upcast are rejected above with base-column- + // aware messages before reaching here.) + if err := cagraCatalogHooks.ValidQuantization( + indexInfo.IndexOption.Quantization, indexInfo.IndexOption.AlgoParamVectorOpType); err != nil { + return nil, nil, err + } + } } for _, existedIndex := range existedIndexes { if existedIndex.IndexAlgo == catalog.MoIndexCagraAlgo.ToString() && existedIndex.Parts[0] == name { diff --git a/pkg/vectorindex/cagra/plugin/runtime/runtime.go b/pkg/vectorindex/cagra/plugin/runtime/runtime.go index 2a2e91fd778dc..8ed35e2d20a54 100644 --- a/pkg/vectorindex/cagra/plugin/runtime/runtime.go +++ b/pkg/vectorindex/cagra/plugin/runtime/runtime.go @@ -99,12 +99,43 @@ const CagraIndexFlag = "experimental_cagra_index" // ExperimentalFlag: CAGRA DDL is gated by CagraIndexFlag. func (CatalogHooks) ExperimentalFlag() string { return CagraIndexFlag } -// SupportedVectorTypes: CAGRA (cuvs) indexes f32 vectors only. -func (CatalogHooks) SupportedVectorTypes() []types.T { return []types.T{types.T_array_float32} } +// SupportedVectorTypes: CAGRA (cuvs) accepts f32 and f16 base columns. f16 is +// stored natively as half, or downcast-quantized to int8/uint8 via QUANTIZATION. +// int8/uint8 base columns are unsupported (the CDC overflow brute force is f32/f16-only). +func (CatalogHooks) SupportedVectorTypes() []types.T { + return []types.T{types.T_array_float32, types.T_array_float16} +} // SupportedPrimaryKeyTypes: requires an int64 primary key. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return []types.T{types.T_int64} } +// ValidQuantization gates the (quantization, op_type) pair for CAGRA (cuvs): +// the value must be a cuvs storage type (float32/float16/int8/uint8 — bf16 and +// float64 are absent from CuvsQuantizationNameToType), and the 1-byte int8/uint8 +// scalar quantizer is L2-only (its affine map q(x)=scalar*x+offset preserves L2 +// ordering — the offset cancels in a difference — but biases inner-product and +// rotates cosine angles). One home for CREATE (plan/schema) and REINDEX +// (compile/ValidateReindexParams). quant=="" => no quantization (valid); op=="" +// => value rule only. +func (CatalogHooks) ValidQuantization(quant, op string) error { + if quant == "" { + return nil + } + quant = strings.ToLower(quant) + if !metric.ValidQuantization(quant) { + return moerr.NewNotSupportedNoCtxf( + "cagra quantization %q (supported: float32, float16, int8, uint8)", quant) + } + if quant == metric.Quantization_INT8_Str || quant == metric.Quantization_UINT8_Str { + switch strings.ToLower(op) { + case metric.OpType_InnerProduct, metric.OpType_CosineDistance: + return moerr.NewNotSupportedNoCtxf( + "cagra quantization %q is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry", quant) + } + } + return nil +} + // SupportedIncludeColumnTypes: cuvs INCLUDE (pre-filter) columns accept // int32/int64/float32/float64 scalars. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { diff --git a/pkg/vectorindex/cagra/plugin/runtime/runtime_test.go b/pkg/vectorindex/cagra/plugin/runtime/runtime_test.go index f165fcdeb0921..86776a04d48f7 100644 --- a/pkg/vectorindex/cagra/plugin/runtime/runtime_test.go +++ b/pkg/vectorindex/cagra/plugin/runtime/runtime_test.go @@ -181,3 +181,16 @@ func TestCagraJoinIncludeColumns(t *testing.T) { } require.Equal(t, "a,b", joinIncludeColumns(cols)) } + +// TestCagraValidQuantization exercises the per-algo (quant, op) catalog hook +// shared by CREATE and REINDEX. +func TestCagraValidQuantization(t *testing.T) { + h := CatalogHooks{} + require.NoError(t, h.ValidQuantization("", "vector_ip_ops")) // no quantization + require.NoError(t, h.ValidQuantization("float16", "vector_ip_ops")) // f16 fine with any op + require.NoError(t, h.ValidQuantization("int8", "vector_l2_ops")) // int8 + L2 ok + require.Error(t, h.ValidQuantization("int8", "vector_ip_ops")) // int8 + ip rejected + require.Error(t, h.ValidQuantization("uint8", "vector_cosine_ops")) // uint8 + cosine rejected + require.Error(t, h.ValidQuantization("bf16", "vector_l2_ops")) // bad value + require.Error(t, h.ValidQuantization("float64", "vector_l2_ops")) // bad value +} diff --git a/pkg/vectorindex/cagra/search_gpu.go b/pkg/vectorindex/cagra/search_gpu.go index e361b1e37077a..ff994f5d9b2c7 100644 --- a/pkg/vectorindex/cagra/search_gpu.go +++ b/pkg/vectorindex/cagra/search_gpu.go @@ -29,19 +29,19 @@ import ( // CagraSearch implements cache.VectorIndexSearchIf for GPU CAGRA indexes. // Unlike HnswSearch, there is no concurrency gate (Cond/Mutex) because CAGRA // manages GPU thread concurrency internally via its worker pool. -type CagraSearch[T cuvs.VectorType] struct { +type CagraSearch[B, Q cuvs.VectorType] struct { Idxcfg vectorindex.IndexConfig Tblcfg vectorindex.IndexTableConfig - Indexes []*CagraModel[T] - MultiIndex *cuvs.MultiGpuCagra[T] // built once in Load; nil until indexes are loaded - Overflow *cuvs.GpuBruteForce[T] // CDC insert overflow; nil when no overflow records exist + Indexes []*CagraModel[B, Q] + MultiIndex *cuvs.MultiGpuCagra[B, Q] // built once in Load; nil until indexes are loaded + Overflow cuvs.BruteForceOverflow[B] // CDC insert overflow; nil when no overflow records exist Devices []int ThreadsSearch int64 } -func NewCagraSearch[T cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, devices []int) *CagraSearch[T] { +func NewCagraSearch[B, Q cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, devices []int) *CagraSearch[B, Q] { nthread := vectorindex.GetConcurrency(tblcfg.ThreadsSearch) - return &CagraSearch[T]{ + return &CagraSearch[B, Q]{ Idxcfg: idxcfg, Tblcfg: tblcfg, Devices: devices, @@ -50,12 +50,7 @@ func NewCagraSearch[T cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg ve } // Search implements cache.VectorIndexSearchIf. -func (s *CagraSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { - query, ok := anyquery.([]float32) - if !ok { - return nil, nil, moerr.NewInternalErrorNoCtx("CagraSearch: query type mismatch") - } - +func (s *CagraSearch[B, Q]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { limit := rt.Limit if s.MultiIndex == nil { @@ -71,10 +66,18 @@ func (s *CagraSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt ve neighbors64 []int64 dists32 []float32 ) + // Any base (f32 or vecf16) routes its native base-typed (B) query through the + // const-B* search_quantize path — cuVS converts B to storage Q on device (B==Q + // copy for a direct index, learned/cast quantizer for a compressed one). The + // query asserts to []B for both float32 (B==float) and Float16 (B==half) base. + qB, ok := anyquery.([]B) + if !ok { + return nil, nil, moerr.NewInternalErrorNoCtx("CagraSearch: query type mismatch") + } if rt.FilterJSON != "" { - neighbors64, dists32, err = s.MultiIndex.SearchFloat32WithFilter(query, 1, dim, uint32(limit), sp, rt.FilterJSON) + neighbors64, dists32, err = s.MultiIndex.SearchQuantizeWithFilter(qB, 1, dim, uint32(limit), sp, rt.FilterJSON) } else { - neighbors64, dists32, err = s.MultiIndex.SearchFloat32(query, 1, dim, uint32(limit), sp) + neighbors64, dists32, err = s.MultiIndex.SearchQuantize(qB, 1, dim, uint32(limit), sp) } if err != nil { return nil, nil, err @@ -100,7 +103,7 @@ func (s *CagraSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt ve // SearchFloat32 implements cache.VectorIndexSearchIf. // Writes results directly into caller-provided slices to avoid heap allocation. -func (s *CagraSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { +func (s *CagraSearch[B, Q]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { keys, dists, err := s.Search(proc, query, rt) if err != nil { return err @@ -123,8 +126,8 @@ func (s *CagraSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt v // into per-column data + null bitmap and feeds them to the brute-force index // in column order. Mirrors how the build path populates the cuvs main // index's FilterStore. -func addOverflowFilterChunks[T cuvs.VectorType]( - bf *cuvs.GpuBruteForce[T], +func addOverflowFilterChunks[B, OB cuvs.VectorType]( + bf *cuvs.GpuBruteForce[B, OB], colMetaJSON string, includeBytes []byte, nrows uint64, @@ -143,8 +146,8 @@ func addOverflowFilterChunks[T cuvs.VectorType]( } // Load implements cache.VectorIndexSearchIf: loads metadata then index data from the database. -func (s *CagraSearch[T]) Load(sqlproc *sqlexec.SqlProcess) (err error) { - indexes, err := LoadMetadata[T](sqlproc, s.Tblcfg.DbName, s.Tblcfg.MetadataTable) +func (s *CagraSearch[B, Q]) Load(sqlproc *sqlexec.SqlProcess) (err error) { + indexes, err := LoadMetadata[B, Q](sqlproc, s.Tblcfg.DbName, s.Tblcfg.MetadataTable) if err != nil { return err } @@ -187,7 +190,7 @@ func (s *CagraSearch[T]) Load(sqlproc *sqlexec.SqlProcess) (err error) { // index by construction (CDC writer side is fed the same colMetaJSON). If // no sub-index loaded (empty index — never built, or built and dropped), // we have no col-meta and skip; cdc_tail data is moot without a main index. -func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { +func (s *CagraSearch[B, Q]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { var ( includeBytesPerRow int colMetaJSON string @@ -205,7 +208,7 @@ func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } } - stub := &CagraModel[T]{Id: vectorindex.CdcTailId} + stub := &CagraModel[B, Q]{Id: vectorindex.CdcTailId} chunks, err := stub.loadCdcEventsFromDB(sqlproc, s.Tblcfg) if err != nil { return err @@ -229,7 +232,7 @@ func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } dim := int(s.Idxcfg.CuvsCagra.Dimensions) - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, includeBytesPerRow) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[B](chunks, dim, includeBytesPerRow) if err != nil { return err } @@ -245,7 +248,7 @@ func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } } - s.Indexes = append(s.Indexes, &CagraModel[T]{ + s.Indexes = append(s.Indexes, &CagraModel[B, Q]{ Id: vectorindex.CdcTailId, DeletedPkids: delPkids, OverflowPkids: ovPkids, @@ -264,7 +267,7 @@ func (s *CagraSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { // When the underlying index has INCLUDE columns, the brute-force is set up // with the matching FilterStore so a filtered query can prefilter overflow // rows the same way the main cagra index does. -func (s *CagraSearch[T]) buildOverflow() error { +func (s *CagraSearch[B, Q]) buildOverflow() error { total := uint64(0) for _, m := range s.Indexes { total += uint64(len(m.OverflowPkids)) @@ -285,14 +288,43 @@ func (s *CagraSearch[T]) buildOverflow() error { device = s.Devices[0] } - bf, err := cuvs.NewGpuBruteForceEmpty[T]( - total, dim, cuvsMetric, uint32(s.ThreadsSearch), device) + // cuVS brute force can only store float/half. Pick the overflow storage type + // OB from the index storage Q: keep Q when it is float/half, else fall back to + // the base type B (which is always float/half) so the overflow is supported. + // The type-erased BruteForceOverflow[B] interface holds either concrete type. + var ( + ov cuvs.BruteForceOverflow[B] + err error + ) + switch cuvs.GetQuantization[Q]() { + case cuvs.F32, cuvs.F16: + ov, err = buildOverflowBF[B, Q](s.Indexes, total, dim, cuvsMetric, device, uint32(s.ThreadsSearch)) + default: // INT8/UINT8: brute force can't store these → store base B. + ov, err = buildOverflowBF[B, B](s.Indexes, total, dim, cuvsMetric, device, uint32(s.ThreadsSearch)) + } if err != nil { return err } + s.Overflow = ov + return nil +} + +// buildOverflowBF builds a concrete *cuvs.GpuBruteForce[B, OB] from every loaded +// model's CDC insert overflow and returns it behind the type-erased +// BruteForceOverflow[B] interface. OB is the overflow storage type chosen by the +// caller (Q when float/half, else B). Wires the FilterStore when the index has +// INCLUDE columns. +func buildOverflowBF[B, OB cuvs.VectorType, Q cuvs.VectorType]( + indexes []*CagraModel[B, Q], + total uint64, dim uint32, cuvsMetric cuvs.DistanceType, device int, threads uint32, +) (cuvs.BruteForceOverflow[B], error) { + bf, err := cuvs.NewGpuBruteForceEmpty[B, OB](total, dim, cuvsMetric, threads, device) + if err != nil { + return nil, err + } if err = bf.Start(); err != nil { bf.Destroy() - return err + return nil, err } // INCLUDE-column wiring — pull the col-meta JSON from the first loaded @@ -304,7 +336,7 @@ func (s *CagraSearch[T]) buildOverflow() error { colMetaJSON string includeBytesPerRow int ) - for _, m := range s.Indexes { + for _, m := range indexes { if m.Index != nil { colMetaJSON = m.Index.GetFilterColMetaJSON() includeBytesPerRow = m.IncludeBytesPerRow @@ -312,7 +344,7 @@ func (s *CagraSearch[T]) buildOverflow() error { } } if colMetaJSON == "" { - for _, m := range s.Indexes { + for _, m := range indexes { if m.OverflowColMetaJSON != "" { colMetaJSON = m.OverflowColMetaJSON includeBytesPerRow = m.IncludeBytesPerRow @@ -323,32 +355,33 @@ func (s *CagraSearch[T]) buildOverflow() error { if colMetaJSON != "" && includeBytesPerRow > 0 { if err = bf.SetFilterColumns(colMetaJSON, total); err != nil { bf.Destroy() - return err + return nil, err } } - for _, m := range s.Indexes { + for _, m := range indexes { if len(m.OverflowPkids) == 0 { continue } count := uint64(len(m.OverflowPkids)) - if err = bf.AddChunkFloat(m.OverflowVecs, count, m.OverflowPkids); err != nil { + // Overflow vectors are base-typed (B); AddChunkQuantize converts B -> Q + // storage on the C++ side (native store when B==Q, f32->f16 cast otherwise). + if err = bf.AddChunkQuantize(m.OverflowVecs, count, m.OverflowPkids); err != nil { bf.Destroy() - return err + return nil, err } if colMetaJSON != "" && includeBytesPerRow > 0 { if err = addOverflowFilterChunks(bf, colMetaJSON, m.OverflowIncludeBytes, count, includeBytesPerRow); err != nil { bf.Destroy() - return err + return nil, err } } } if err = bf.Build(); err != nil { bf.Destroy() - return err + return nil, err } - s.Overflow = bf - return nil + return bf, nil } // buildMultiIndex assembles a MultiGpuCagra from the loaded indexes. @@ -358,14 +391,14 @@ func (s *CagraSearch[T]) buildOverflow() error { // which returns []int64{}, []float64{} on s.MultiIndex == nil — that's // the load-bearing path for "no main index + no brute-force → empty // result". Any future regression here will fail TestCagraSearchEmpty. -func (s *CagraSearch[T]) buildMultiIndex() (*cuvs.MultiGpuCagra[T], error) { +func (s *CagraSearch[B, Q]) buildMultiIndex() (*cuvs.MultiGpuCagra[B, Q], error) { cuvsMetric, ok := metric.MetricTypeToCuvsMetric[metric.MetricType(s.Idxcfg.CuvsCagra.Metric)] if !ok { // Unsupported metric is a real error — surface it rather than returning a // nil index, which Search would treat as an (empty) success. return nil, moerr.NewInternalErrorNoCtxf("CagraSearch: unsupported metric type %v", s.Idxcfg.CuvsCagra.Metric) } - gpuIndices := make([]*cuvs.GpuCagra[T], 0, len(s.Indexes)) + gpuIndices := make([]*cuvs.GpuCagra[B, Q], 0, len(s.Indexes)) for _, model := range s.Indexes { if model.Index != nil { gpuIndices = append(gpuIndices, model.Index) @@ -382,7 +415,7 @@ func (s *CagraSearch[T]) buildMultiIndex() (*cuvs.MultiGpuCagra[T], error) { // loadIndexes loads each model's index data from the database. // On any error it destroys all partially-loaded indexes and returns the error. -func (s *CagraSearch[T]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*CagraModel[T]) ([]*CagraModel[T], error) { +func (s *CagraSearch[B, Q]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*CagraModel[B, Q]) ([]*CagraModel[B, Q], error) { for _, idx := range indexes { idx.Devices = s.Devices if err := idx.LoadIndex(sqlproc, s.Idxcfg, s.Tblcfg, s.ThreadsSearch, true); err != nil { @@ -396,7 +429,7 @@ func (s *CagraSearch[T]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*Cag } // Destroy implements cache.VectorIndexSearchIf. -func (s *CagraSearch[T]) Destroy() { +func (s *CagraSearch[B, Q]) Destroy() { s.MultiIndex = nil // does not own GPU resources; GpuCagra instances are owned by Indexes if s.Overflow != nil { s.Overflow.Destroy() @@ -409,6 +442,6 @@ func (s *CagraSearch[T]) Destroy() { } // UpdateConfig implements cache.VectorIndexSearchIf. -func (s *CagraSearch[T]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error { +func (s *CagraSearch[B, Q]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error { return nil } diff --git a/pkg/vectorindex/cagra/search_test.go b/pkg/vectorindex/cagra/search_test.go index 35c5018f3c77e..463c7732cf7c0 100644 --- a/pkg/vectorindex/cagra/search_test.go +++ b/pkg/vectorindex/cagra/search_test.go @@ -34,7 +34,7 @@ import ( // loadedModel builds an index, saves it to a tar, then reloads it into GPU // memory from the local file. Returns the model with Index != nil. -func loadedModel(t *testing.T, id string) *CagraModel[float32] { +func loadedModel(t *testing.T, id string) *CagraModel[float32, float32] { t.Helper() built := buildTestModel(t, id, nil) tarPath := built.Path @@ -53,7 +53,7 @@ func loadedModel(t *testing.T, id string) *CagraModel[float32] { } defer func() { runSql = origRunSql }() - loader := &CagraModel[float32]{ + loader := &CagraModel[float32, float32]{ Id: id, Path: tarPath, Checksum: built.Checksum, @@ -72,7 +72,7 @@ func TestCagraSearchEmpty(t *testing.T) { proc := testutil.NewProcessWithMPool(t, "", m) sqlproc := sqlexec.NewSqlProcess(proc) - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) require.Empty(t, s.Indexes) rt := vectorindex.RuntimeConfig{Limit: 4} @@ -98,8 +98,8 @@ func TestCagraSearchTypeMismatch(t *testing.T) { idx := loadedModel(t, "type-mismatch") defer idx.Destroy() - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*CagraModel[float32]{idx} + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*CagraModel[float32, float32]{idx} rt := vectorindex.RuntimeConfig{Limit: 4} @@ -117,8 +117,8 @@ func TestCagraSearchAndSearchFloat32(t *testing.T) { idx := loadedModel(t, "search-single") defer idx.Destroy() - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*CagraModel[float32]{idx} + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*CagraModel[float32, float32]{idx} s.MultiIndex, _ = s.buildMultiIndex() data := generateTestData(testNVectors, testDim) @@ -158,8 +158,8 @@ func TestCagraSearchMultipleIndexes(t *testing.T) { idx1 := loadedModel(t, "multi-1") defer idx1.Destroy() - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*CagraModel[float32]{idx0, idx1} + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*CagraModel[float32, float32]{idx0, idx1} s.MultiIndex, _ = s.buildMultiIndex() data := generateTestData(testNVectors, testDim) @@ -212,7 +212,7 @@ func TestCagraSearchLoad(t *testing.T) { } defer func() { runSql_streaming = origStream }() - s := NewCagraSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) + s := NewCagraSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) err := s.Load(sqlproc) require.NoError(t, err) require.Equal(t, 1, len(s.Indexes)) diff --git a/pkg/vectorindex/cagra/sync.go b/pkg/vectorindex/cagra/sync.go index 4fbbc619fb652..cd1e338f1f869 100644 --- a/pkg/vectorindex/cagra/sync.go +++ b/pkg/vectorindex/cagra/sync.go @@ -50,6 +50,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -74,6 +76,7 @@ type CagraSync struct { activeIndexId string dim int + vecBytesPerRow int // dim * base element size (4*dim for f32, 2*dim for f16) includeBytesPerRow int colMetaJSON string @@ -98,11 +101,19 @@ func NewCagraSync( idxname string, idxdefs []*plan.IndexDef, dimension int32, + baseType types.T, colMetaJSON string, ) (*CagraSync, error) { if dimension <= 0 { return nil, moerr.NewInternalErrorNoCtx("CagraSync: invalid dimension") } + // CDC records carry the vector as raw native base-type bytes: 2*dim for a + // vecf16 base, 4*dim otherwise. Must match the iscp writer's encode width + // and the search-side replayEventChunks[B] read width. + elemSize := 4 + if baseType == types.T_array_float16 { + elemSize = 2 + } var idxtblcfg vectorindex.IndexTableConfig idxtblcfg.DbName = db @@ -134,6 +145,7 @@ func NewCagraSync( tblcfg: idxtblcfg, idxname: idxname, dim: int(dimension), + vecBytesPerRow: int(dimension) * elemSize, includeBytesPerRow: includeBytesPerRow, colMetaJSON: colMetaJSON, activeIndexId: vectorindex.CdcTailId, @@ -226,7 +238,7 @@ func (s *CagraSync) AppendRecords(_ *sqlexec.SqlProcess, recordBytes []byte) err n = 9 // op (1) + pkid (8) case cuvscdc.CdcOpInsert, cuvscdc.CdcOpUpsert: // UPSERT shares INSERT's payload shape; only the op byte differs. - n = 9 + 4*s.dim + s.includeBytesPerRow + n = 9 + s.vecBytesPerRow + s.includeBytesPerRow default: return moerr.NewInternalErrorNoCtx(fmt.Sprintf( "CagraSync.AppendRecords: unknown op %d at offset %d", op, pos)) @@ -263,7 +275,10 @@ func (s *CagraSync) appendRecord(op cuvscdc.CdcOp, pkid int64, vec []float32, in } } before := len(s.pendingRecords) - out, err := cuvscdc.EncodeEventRecord(s.pendingRecords, op, pkid, vec, include, s.dim, s.includeBytesPerRow) + // This synchronous VectorIndexCdc[float32] path is f32-only (vec is + // []float32). vecf16 ongoing ingestion flows through the iscp writer → + // AppendRecords byte path instead, which honors s.vecBytesPerRow. + out, err := cuvscdc.EncodeEventRecord(s.pendingRecords, op, pkid, util.UnsafeSliceToBytes(vec), include, 4*s.dim, s.includeBytesPerRow) if err != nil { return err } diff --git a/pkg/vectorindex/cagra/sync_test.go b/pkg/vectorindex/cagra/sync_test.go index d8d52f70aa09c..7df662fb5f44f 100644 --- a/pkg/vectorindex/cagra/sync_test.go +++ b/pkg/vectorindex/cagra/sync_test.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -143,7 +144,7 @@ func TestCagraSync_Update_AllInsert(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) require.Equal(t, vectorindex.CdcTailId, s.activeIndexId) @@ -164,7 +165,7 @@ func TestCagraSync_Update_AllInsert(t *testing.T) { // Round-trip: replay the persisted chunks and expect 2 overflow rows, no // deletes. - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.Empty(t, state.Deleted) require.Len(t, state.Overflow, 2) @@ -185,7 +186,7 @@ func TestCagraSync_Update_DeleteAndInsert(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -202,7 +203,7 @@ func TestCagraSync_Update_DeleteAndInsert(t *testing.T) { // chunk_id == 7 (nextChunkId mock). require.Contains(t, rec.statements[0], "'cdc_tail', 7,") - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 7), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 7), 16, 0) require.NoError(t, err) require.Equal(t, []int64{42}, state.Deleted) require.Len(t, state.Overflow, 1) @@ -223,7 +224,7 @@ func TestCagraSync_Update_DeleteInsertDelete(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -239,7 +240,7 @@ func TestCagraSync_Update_DeleteInsertDelete(t *testing.T) { require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.Equal(t, []int64{1}, state.Deleted, "final state must have pkid=1 deleted (last event was DELETE)") @@ -260,7 +261,7 @@ func TestCagraSync_Update_DeleteIdempotent(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -275,7 +276,7 @@ func TestCagraSync_Update_DeleteIdempotent(t *testing.T) { require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.ElementsMatch(t, []int64{5, 7}, state.Deleted) } @@ -292,7 +293,7 @@ func TestCagraSync_Update_Upsert(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -306,13 +307,13 @@ func TestCagraSync_Update_Upsert(t *testing.T) { "INSERT + UPSERT → 2 records (UPSERT is a single op, not DELETE+INSERT)") require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.ElementsMatch(t, []int64{100}, state.Deleted, "UPSERT marks pkid in deleted (filters any pre-rebuild main-index entry)") require.Len(t, state.Overflow, 1) require.Equal(t, int64(100), state.Overflow[0].Pkid) - require.Equal(t, []float32{9, 9, 9, 9}, state.Overflow[0].Vec, + require.Equal(t, []float32{9, 9, 9, 9}, util.UnsafeSliceCast[float32](state.Overflow[0].Vec), "UPSERT wrote the latest vec; replay surfaces it") } @@ -324,7 +325,7 @@ func TestCagraSync_Update_DimMismatch(t *testing.T) { sqlproc := sqlexec.NewSqlProcess(proc) s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -355,7 +356,7 @@ func TestCagraSync_Update_WithIncludeBytes(t *testing.T) { require.Equal(t, 9, expectedIBPR) s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, colMetaJSON) + idxdefs("__meta", "__storage"), 4, types.T_array_float32, colMetaJSON) require.NoError(t, err) require.Equal(t, 9, s.includeBytesPerRow) @@ -369,7 +370,7 @@ func TestCagraSync_Update_WithIncludeBytes(t *testing.T) { require.NoError(t, s.Update(sqlproc, cdc)) require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 9) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 9) require.NoError(t, err) require.Len(t, state.Overflow, 1) require.Equal(t, include, state.Overflow[0].Include) @@ -401,7 +402,7 @@ func TestCagraSync_Update_NoOpSaveSkipsSql(t *testing.T) { defer func() { runSql = origRun }() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{} @@ -428,7 +429,7 @@ func TestCagraSync_NewSync_Stateless(t *testing.T) { defer func() { runSql = origRun }() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) require.Equal(t, vectorindex.CdcTailId, s.activeIndexId) require.Equal(t, 0, called) @@ -445,7 +446,7 @@ func TestCagraSync_RunOnce(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -488,7 +489,7 @@ func TestCagraSync_MultiFlush(t *testing.T) { defer rec.install(t)() s, err := NewCagraSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) flush1 := &vectorindex.VectorIndexCdc[float32]{ diff --git a/pkg/vectorindex/cuvs/cdc.go b/pkg/vectorindex/cuvs/cdc.go index 30533f0789adb..ff9481ad8fcb0 100644 --- a/pkg/vectorindex/cuvs/cdc.go +++ b/pkg/vectorindex/cuvs/cdc.go @@ -207,24 +207,29 @@ const ( CdcOpUpsert CdcOp = 2 ) -// CdcEventRecord is the decoded form of one tag=1 record. +// CdcEventRecord is the decoded form of one tag=1 record. Vec holds the raw +// little-endian vector bytes in the index's native base element type (4 bytes +// per element for vecf32, 2 for a vecf16) — the codec is element-type-agnostic; +// the GPU layer reinterprets these bytes to []float32 / []cuvs.Float16. type CdcEventRecord struct { Op CdcOp Pkid int64 - Vec []float32 // populated only for CdcOpInsert - Include []byte // populated only for CdcOpInsert (and only when includeBytesPerRow > 0) + Vec []byte // populated only for CdcOpInsert (vecBytesPerRow bytes) + Include []byte // populated only for CdcOpInsert (and only when includeBytesPerRow > 0) } // EncodeEventRecord appends one record to dst and returns the new slice. // vec is required iff op==CdcOpInsert; include is required iff op==CdcOpInsert -// AND includeBytesPerRow > 0. dim must be the index's dimensionality. +// AND includeBytesPerRow > 0. vec carries the row's vector as raw native +// base-type bytes; vecBytesPerRow is its expected length (dim * element size, +// e.g. 4*dim for f32, 2*dim for f16). func EncodeEventRecord( dst []byte, op CdcOp, pkid int64, - vec []float32, + vec []byte, include []byte, - dim int, + vecBytesPerRow int, includeBytesPerRow int, ) ([]byte, error) { switch op { @@ -243,11 +248,11 @@ func EncodeEventRecord( if op == CdcOpUpsert { opName = "UPSERT" } - if dim <= 0 { - return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s requires positive dim, got %d", opName, dim) + if vecBytesPerRow <= 0 { + return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s requires positive vecBytesPerRow, got %d", opName, vecBytesPerRow) } - if len(vec) != dim { - return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s vec length %d != dim %d", opName, len(vec), dim) + if len(vec) != vecBytesPerRow { + return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s vec length %d != vecBytesPerRow %d", opName, len(vec), vecBytesPerRow) } if includeBytesPerRow > 0 && len(include) != includeBytesPerRow { return nil, moerr.NewInternalErrorNoCtxf("EncodeEventRecord: %s include length %d != includeBytesPerRow %d", @@ -260,11 +265,10 @@ func EncodeEventRecord( var pk [8]byte binary.LittleEndian.PutUint64(pk[:], uint64(pkid)) dst = append(dst, pk[:]...) - var f [4]byte - for _, v := range vec { - binary.LittleEndian.PutUint32(f[:], math.Float32bits(v)) - dst = append(dst, f[:]...) - } + // vec body: raw native base-type bytes, copied verbatim. For f32 this is + // byte-identical to the legacy math.Float32bits + PutUint32 form on + // little-endian targets, so existing f32 CDC streams are unchanged. + dst = append(dst, vec...) if includeBytesPerRow > 0 { dst = append(dst, include...) } @@ -278,12 +282,10 @@ func EncodeEventRecord( // and the number of bytes consumed. Returns ok=false when src cannot start a // valid record (e.g. unknown op byte, or fewer bytes than the record needs) // — the caller treats this as the end of the stream within the chunk. -// -// Vec and Include in the returned record alias into src; copy if you need to -// retain them past the next call. +// vecBytesPerRow is the INSERT-record vector byte length (dim * element size). func DecodeEventRecord( src []byte, - dim int, + vecBytesPerRow int, includeBytesPerRow int, ) (rec CdcEventRecord, n int, ok bool) { if len(src) < 9 { @@ -296,19 +298,18 @@ func DecodeEventRecord( rec.Pkid = int64(binary.LittleEndian.Uint64(src[1:9])) return rec, 9, true case CdcOpInsert, CdcOpUpsert: - need := 9 + 4*dim + includeBytesPerRow - if dim <= 0 || includeBytesPerRow < 0 || len(src) < need { + need := 9 + vecBytesPerRow + includeBytesPerRow + if vecBytesPerRow <= 0 || includeBytesPerRow < 0 || len(src) < need { return rec, 0, false } rec.Op = op rec.Pkid = int64(binary.LittleEndian.Uint64(src[1:9])) - rec.Vec = make([]float32, dim) - for k := 0; k < dim; k++ { - rec.Vec[k] = math.Float32frombits(binary.LittleEndian.Uint32(src[9+k*4:])) - } + // vec body: raw native base-type bytes, copied verbatim. + rec.Vec = make([]byte, vecBytesPerRow) + copy(rec.Vec, src[9:9+vecBytesPerRow]) if includeBytesPerRow > 0 { rec.Include = make([]byte, includeBytesPerRow) - copy(rec.Include, src[9+4*dim:need]) + copy(rec.Include, src[9+vecBytesPerRow:need]) } return rec, need, true default: @@ -454,10 +455,12 @@ type ReplayState struct { ColMetaJSON string } -// OverflowEntry is one row in the brute-force overflow. +// OverflowEntry is one row in the brute-force overflow. Vec holds the raw +// native base-type bytes (no f32 widening for vecf16); the GPU layer +// reinterprets them to the base element type B. type OverflowEntry struct { Pkid int64 - Vec []float32 + Vec []byte Include []byte } @@ -482,16 +485,16 @@ func PeekColMetaJSON(chunks []EventChunk) (string, error) { } // ReplayEventLog walks the chunks (assumed sorted by chunk_id) and applies -// each record in order, returning the final (deleted, overflow) state. dim -// and includeBytesPerRow describe the INSERT record layout. Replay is O(n) -// in event count. +// each record in order, returning the final (deleted, overflow) state. +// vecBytesPerRow (dim * element size) and includeBytesPerRow describe the +// INSERT record layout. Replay is O(n) in event count. func ReplayEventLog( chunks []EventChunk, - dim int, + vecBytesPerRow int, includeBytesPerRow int, ) (ReplayState, error) { - if dim <= 0 { - return ReplayState{}, moerr.NewInternalErrorNoCtxf("ReplayEventLog: invalid dim %d", dim) + if vecBytesPerRow <= 0 { + return ReplayState{}, moerr.NewInternalErrorNoCtxf("ReplayEventLog: invalid vecBytesPerRow %d", vecBytesPerRow) } if includeBytesPerRow < 0 { return ReplayState{}, moerr.NewInternalErrorNoCtxf("ReplayEventLog: negative includeBytesPerRow %d", includeBytesPerRow) @@ -514,14 +517,14 @@ func ReplayEventLog( colMetaJSON = string(header) } for len(data) > 0 { - rec, n, ok := DecodeEventRecord(data, dim, includeBytesPerRow) + rec, n, ok := DecodeEventRecord(data, vecBytesPerRow, includeBytesPerRow) if !ok { // Frame CRC already validated the payload, so any decode // failure here is a record-level bug (encoder/decoder mismatch // on dim or includeBytesPerRow). return ReplayState{}, moerr.NewInternalErrorNoCtxf( - "ReplayEventLog: chunk_id=%d: undecodable record at offset %d (dim=%d includeBytesPerRow=%d)", - ch.ChunkId, len(ch.Data)-cdcFooterSize-len(data), dim, includeBytesPerRow) + "ReplayEventLog: chunk_id=%d: undecodable record at offset %d (vecBytesPerRow=%d includeBytesPerRow=%d)", + ch.ChunkId, len(ch.Data)-cdcFooterSize-len(data), vecBytesPerRow, includeBytesPerRow) } switch rec.Op { case CdcOpDelete: diff --git a/pkg/vectorindex/cuvs/cdc_test.go b/pkg/vectorindex/cuvs/cdc_test.go index a028a7eeb599c..b29b9457e0660 100644 --- a/pkg/vectorindex/cuvs/cdc_test.go +++ b/pkg/vectorindex/cuvs/cdc_test.go @@ -24,6 +24,7 @@ import ( "strings" "testing" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/vectorindex" ) @@ -88,7 +89,11 @@ func encodeBatch( insertIdx++ } before := len(buf) - out, err := EncodeEventRecord(buf, op, pkids[i], v, inc, dim, includeBytesPerRow) + var vb []byte + if v != nil { + vb = util.UnsafeSliceToBytes(v) + } + out, err := EncodeEventRecord(buf, op, pkids[i], vb, inc, 4*dim, includeBytesPerRow) if err != nil { t.Fatalf("EncodeEventRecord(%v, pkid=%d): %v", op, pkids[i], err) } @@ -101,14 +106,14 @@ func encodeBatch( // TestEncodeDecodeEventRecord_Delete: round-trip a DELETE record. func TestEncodeDecodeEventRecord_Delete(t *testing.T) { for _, pkid := range []int64{1, -7, math.MaxInt64, math.MinInt64, 0} { - buf, err := EncodeEventRecord(nil, CdcOpDelete, pkid, nil, nil, 4, 0) + buf, err := EncodeEventRecord(nil, CdcOpDelete, pkid, nil, nil, 16, 0) if err != nil { t.Fatalf("encode pkid=%d: %v", pkid, err) } if len(buf) != 9 { t.Fatalf("DELETE record should be 9 bytes, got %d", len(buf)) } - rec, n, ok := DecodeEventRecord(buf, 4, 0) + rec, n, ok := DecodeEventRecord(buf, 16, 0) if !ok || n != 9 { t.Fatalf("decode failed: ok=%v n=%d", ok, n) } @@ -123,7 +128,7 @@ func TestEncodeDecodeEventRecord_Insert(t *testing.T) { dim := 3 pkid := int64(42) vec := []float32{1.5, -2.25, math.MaxFloat32} - buf, err := EncodeEventRecord(nil, CdcOpInsert, pkid, vec, nil, dim, 0) + buf, err := EncodeEventRecord(nil, CdcOpInsert, pkid, util.UnsafeSliceToBytes(vec), nil, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -131,16 +136,17 @@ func TestEncodeDecodeEventRecord_Insert(t *testing.T) { if len(buf) != want { t.Fatalf("INSERT record len %d, want %d", len(buf), want) } - rec, n, ok := DecodeEventRecord(buf, dim, 0) + rec, n, ok := DecodeEventRecord(buf, 4*dim, 0) if !ok || n != want { t.Fatalf("decode: ok=%v n=%d", ok, n) } if rec.Op != CdcOpInsert || rec.Pkid != pkid { t.Fatalf("op/pkid mismatch") } + gotVec := util.UnsafeSliceCast[float32](rec.Vec) for i, v := range vec { - if math.Float32bits(rec.Vec[i]) != math.Float32bits(v) { - t.Fatalf("vec[%d]: got %v want %v", i, rec.Vec[i], v) + if math.Float32bits(gotVec[i]) != math.Float32bits(v) { + t.Fatalf("vec[%d]: got %v want %v", i, gotVec[i], v) } } if len(rec.Include) != 0 { @@ -156,7 +162,7 @@ func TestEncodeDecodeEventRecord_InsertWithInclude(t *testing.T) { binary.LittleEndian.PutUint32(include[0:4], 0xdeadbeef) binary.LittleEndian.PutUint64(include[4:12], 0x1122334455667788) include[12] = 0x02 - buf, err := EncodeEventRecord(nil, CdcOpInsert, 1, []float32{0.1, 0.2}, include, dim, includeBytesPerRow) + buf, err := EncodeEventRecord(nil, CdcOpInsert, 1, util.UnsafeSliceToBytes([]float32{0.1, 0.2}), include, 4*dim, includeBytesPerRow) if err != nil { t.Fatal(err) } @@ -164,7 +170,7 @@ func TestEncodeDecodeEventRecord_InsertWithInclude(t *testing.T) { if len(buf) != want { t.Fatalf("len %d, want %d", len(buf), want) } - rec, n, ok := DecodeEventRecord(buf, dim, includeBytesPerRow) + rec, n, ok := DecodeEventRecord(buf, 4*dim, includeBytesPerRow) if !ok || n != want { t.Fatalf("decode: ok=%v n=%d", ok, n) } @@ -181,19 +187,19 @@ func TestEncodeDecodeEventRecord_InsertWithInclude(t *testing.T) { // TestEncodeEventRecord_Rejects: encoder rejects malformed inputs. func TestEncodeEventRecord_Rejects(t *testing.T) { // DELETE with vec. - if _, err := EncodeEventRecord(nil, CdcOpDelete, 1, []float32{1}, nil, 1, 0); err == nil { + if _, err := EncodeEventRecord(nil, CdcOpDelete, 1, util.UnsafeSliceToBytes([]float32{1}), nil, 4, 0); err == nil { t.Fatal("expected error on DELETE with vec") } // INSERT with wrong dim. - if _, err := EncodeEventRecord(nil, CdcOpInsert, 1, []float32{1, 2}, nil, 4, 0); err == nil { + if _, err := EncodeEventRecord(nil, CdcOpInsert, 1, util.UnsafeSliceToBytes([]float32{1, 2}), nil, 4, 0); err == nil { t.Fatal("expected error on dim mismatch") } // INSERT with include but includeBytesPerRow=0. - if _, err := EncodeEventRecord(nil, CdcOpInsert, 1, []float32{1}, []byte{0xff}, 1, 0); err == nil { + if _, err := EncodeEventRecord(nil, CdcOpInsert, 1, util.UnsafeSliceToBytes([]float32{1}), []byte{0xff}, 4, 0); err == nil { t.Fatal("expected error on extraneous include bytes") } // Unknown op. - if _, err := EncodeEventRecord(nil, CdcOp(99), 1, nil, nil, 1, 0); err == nil { + if _, err := EncodeEventRecord(nil, CdcOp(99), 1, nil, nil, 4, 0); err == nil { t.Fatal("expected error on unknown op") } } @@ -203,21 +209,21 @@ func TestEncodeEventRecord_Rejects(t *testing.T) { // no-bytes-left case and an unknown op byte at the boundary). func TestDecodeEventRecord_StopsAtPad(t *testing.T) { // Empty buffer: decoder reports not-ok. - if _, _, ok := DecodeEventRecord(nil, 4, 0); ok { + if _, _, ok := DecodeEventRecord(nil, 16, 0); ok { t.Fatal("decoder should not accept empty input") } // 7 bytes: not enough for any record. - if _, _, ok := DecodeEventRecord(make([]byte, 7), 4, 0); ok { + if _, _, ok := DecodeEventRecord(make([]byte, 7), 16, 0); ok { t.Fatal("decoder should not accept 7 bytes") } // Bogus op byte. bogus := []byte{42, 0, 0, 0, 0, 0, 0, 0, 0} - if _, _, ok := DecodeEventRecord(bogus, 4, 0); ok { + if _, _, ok := DecodeEventRecord(bogus, 16, 0); ok { t.Fatal("decoder should reject unknown op byte") } // INSERT op but truncated payload. short := []byte{byte(CdcOpInsert), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} // missing vec bytes - if _, _, ok := DecodeEventRecord(short, 4, 0); ok { + if _, _, ok := DecodeEventRecord(short, 16, 0); ok { t.Fatal("decoder should reject truncated INSERT") } } @@ -259,7 +265,7 @@ func TestCdcAppendEventsSql_DeleteOnly(t *testing.T) { } // Round-trip via the loader path. chunks := []EventChunk{{ChunkId: 0, Data: blobs[0]}} - state, err := ReplayEventLog(chunks, 4, 0) + state, err := ReplayEventLog(chunks, 16, 0) if err != nil { t.Fatal(err) } @@ -290,7 +296,7 @@ func TestCdcAppendEventsSql_InsertOnly(t *testing.T) { } blobs := extractUnhexBlobs(t, sqls[0]) chunks := []EventChunk{{ChunkId: 0, Data: blobs[0]}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -301,9 +307,10 @@ func TestCdcAppendEventsSql_InsertOnly(t *testing.T) { if e.Pkid != pkids[i] { t.Fatalf("overflow[%d].Pkid: got %d want %d", i, e.Pkid, pkids[i]) } + gotVec := util.UnsafeSliceCast[float32](e.Vec) for k, v := range vecs[i] { - if math.Float32bits(e.Vec[k]) != math.Float32bits(v) { - t.Fatalf("overflow[%d].Vec[%d]: got %v want %v", i, k, e.Vec[k], v) + if math.Float32bits(gotVec[k]) != math.Float32bits(v) { + t.Fatalf("overflow[%d].Vec[%d]: got %v want %v", i, k, gotVec[k], v) } } } @@ -327,7 +334,7 @@ func TestCdcAppendEventsSql_Mixed(t *testing.T) { } blobs := extractUnhexBlobs(t, sqls[0]) chunks := []EventChunk{{ChunkId: 0, Data: blobs[0]}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -385,7 +392,7 @@ func TestCdcAppendEventsSql_ChunkPacking(t *testing.T) { {ChunkId: 5, Data: blobs[0]}, {ChunkId: 6, Data: blobs[1]}, } - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -408,7 +415,7 @@ func TestReplayEventLog_DeleteInsertDelete(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, ops, pkids, vecs, nil) chunks := []EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -440,7 +447,7 @@ func TestReplayEventLog_InsertDeleteInsert(t *testing.T) { vecs := [][]float32{{1, 1}, {9, 9}} buf, _ := encodeBatch(t, dim, 0, ops, pkids, vecs, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -451,8 +458,9 @@ func TestReplayEventLog_InsertDeleteInsert(t *testing.T) { t.Fatalf("overflow: got %v want one entry pkid=7", state.Overflow) } // Last INSERT's vec wins for the overflow entry. - if state.Overflow[0].Vec[0] != 9 || state.Overflow[0].Vec[1] != 9 { - t.Fatalf("vec: got %v want [9 9]", state.Overflow[0].Vec) + gotVec := util.UnsafeSliceCast[float32](state.Overflow[0].Vec) + if gotVec[0] != 9 || gotVec[1] != 9 { + t.Fatalf("vec: got %v want [9 9]", gotVec) } } @@ -465,7 +473,7 @@ func TestReplayEventLog_UpsertSingle(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpUpsert}, []int64{7}, [][]float32{{1, 1}}, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -475,8 +483,9 @@ func TestReplayEventLog_UpsertSingle(t *testing.T) { if len(state.Overflow) != 1 || state.Overflow[0].Pkid != 7 { t.Fatalf("overflow: got %v want one entry pkid=7 (UPSERT writes new vec to brute-force overflow)", state.Overflow) } - if state.Overflow[0].Vec[0] != 1 || state.Overflow[0].Vec[1] != 1 { - t.Fatalf("vec: got %v want [1 1]", state.Overflow[0].Vec) + gotVec := util.UnsafeSliceCast[float32](state.Overflow[0].Vec) + if gotVec[0] != 1 || gotVec[1] != 1 { + t.Fatalf("vec: got %v want [1 1]", gotVec) } } @@ -489,7 +498,7 @@ func TestReplayEventLog_UpsertThenDelete(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpUpsert, CdcOpDelete}, []int64{7, 7}, [][]float32{{1, 1}}, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -512,7 +521,7 @@ func TestReplayEventLog_UpsertReplayIdempotent(t *testing.T) { []int64{7, 7, 7}, [][]float32{{1, 1}, {1, 1}, {1, 1}}, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -522,8 +531,9 @@ func TestReplayEventLog_UpsertReplayIdempotent(t *testing.T) { if len(state.Overflow) != 1 || state.Overflow[0].Pkid != 7 { t.Fatalf("overflow: got %v want one entry pkid=7", state.Overflow) } - if state.Overflow[0].Vec[0] != 1 || state.Overflow[0].Vec[1] != 1 { - t.Fatalf("vec: got %v want [1 1]", state.Overflow[0].Vec) + gotVec := util.UnsafeSliceCast[float32](state.Overflow[0].Vec) + if gotVec[0] != 1 || gotVec[1] != 1 { + t.Fatalf("vec: got %v want [1 1]", gotVec) } } @@ -537,7 +547,7 @@ func TestReplayEventLog_InsertAfterDeleteDoesNotUnfilter(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpDelete, CdcOpInsert}, []int64{7, 7}, [][]float32{{9, 9}}, nil) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -547,8 +557,9 @@ func TestReplayEventLog_InsertAfterDeleteDoesNotUnfilter(t *testing.T) { if len(state.Overflow) != 1 || state.Overflow[0].Pkid != 7 { t.Fatalf("overflow: got %v want one entry pkid=7", state.Overflow) } - if state.Overflow[0].Vec[0] != 9 || state.Overflow[0].Vec[1] != 9 { - t.Fatalf("vec: got %v want [9 9]", state.Overflow[0].Vec) + gotVec := util.UnsafeSliceCast[float32](state.Overflow[0].Vec) + if gotVec[0] != 9 || gotVec[1] != 9 { + t.Fatalf("vec: got %v want [9 9]", gotVec) } } @@ -570,7 +581,7 @@ func TestReplayEventLog_MultiChunk(t *testing.T) { {ChunkId: 0, Data: FrameCdcChunk(buf0, nil, 0, 0, 0)}, } SortChunks(chunks) - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -596,7 +607,7 @@ func TestReplayEventLog_WithInclude(t *testing.T) { [][]float32{{1, 2}}, [][]byte{include}, ) - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, dim, includeBytesPerRow) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}}, 4*dim, includeBytesPerRow) if err != nil { t.Fatal(err) } @@ -618,7 +629,7 @@ func TestReplayEventLog_CapturesColMetaJSON(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpInsert}, []int64{1}, [][]float32{{1, 2}}, nil) chunks := []EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, []byte(colMetaJSON), 0, 0, 0)}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -637,7 +648,7 @@ func TestReplayEventLog_NoColMetaJSON(t *testing.T) { buf, _ := encodeBatch(t, dim, 0, []CdcOp{CdcOpInsert}, []int64{1}, [][]float32{{1, 2}}, nil) chunks := []EventChunk{{ChunkId: 0, Data: FrameCdcChunk(buf, nil, 0, 0, 0)}} - state, err := ReplayEventLog(chunks, dim, 0) + state, err := ReplayEventLog(chunks, 4*dim, 0) if err != nil { t.Fatal(err) } @@ -682,14 +693,14 @@ func TestReplayEventLog_RejectsCorruptFrame(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - _, err := ReplayEventLog([]EventChunk{{ChunkId: 7, Data: tc.mut(good)}}, dim, 0) + _, err := ReplayEventLog([]EventChunk{{ChunkId: 7, Data: tc.mut(good)}}, 4*dim, 0) if err == nil { t.Fatalf("expected error for %s, got nil", tc.name) } }) } // Sanity: the unmodified frame round-trips. - state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: good}}, dim, 0) + state, err := ReplayEventLog([]EventChunk{{ChunkId: 0, Data: good}}, 4*dim, 0) if err != nil { t.Fatal(err) } diff --git a/pkg/vectorindex/cuvs/small_tail.go b/pkg/vectorindex/cuvs/small_tail.go index 160d1acdc4111..9c04396e0c486 100644 --- a/pkg/vectorindex/cuvs/small_tail.go +++ b/pkg/vectorindex/cuvs/small_tail.go @@ -28,7 +28,7 @@ import ( // columns. type PendingRecord struct { Pkid int64 - Vec []float32 + Vec []byte // raw native base-type bytes (vecBytesPerRow) Include []byte } @@ -55,7 +55,7 @@ type PendingRecord struct { func SaveSmallTailAsCdc( tblcfg vectorindex.IndexTableConfig, rows []PendingRecord, - dim int, + vecBytesPerRow int, includeBytesPerRow int, colMetaJSON string, ) ([]string, error) { @@ -63,16 +63,16 @@ func SaveSmallTailAsCdc( return nil, nil } - // Pre-size the buffer: 9 (op + pkid) + 4*dim + ibpr bytes per + // Pre-size the buffer: 9 (op + pkid) + vecBytesPerRow + ibpr bytes per // INSERT record. Avoids ~len(rows) reallocs in EncodeEventRecord. - perRow := 9 + 4*dim + includeBytesPerRow + perRow := 9 + vecBytesPerRow + includeBytesPerRow records := make([]byte, 0, perRow*len(rows)) sizes := make([]int, 0, len(rows)) for _, r := range rows { before := len(records) out, err := EncodeEventRecord(records, CdcOpInsert, - r.Pkid, r.Vec, r.Include, dim, includeBytesPerRow) + r.Pkid, r.Vec, r.Include, vecBytesPerRow, includeBytesPerRow) if err != nil { return nil, err } diff --git a/pkg/vectorindex/cuvs/small_tail_test.go b/pkg/vectorindex/cuvs/small_tail_test.go index 6b96b3eefe7bd..a05454c4ba21a 100644 --- a/pkg/vectorindex/cuvs/small_tail_test.go +++ b/pkg/vectorindex/cuvs/small_tail_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/vectorindex" ) @@ -46,12 +47,12 @@ func TestSaveSmallTailAsCdc_Empty(t *testing.T) { func TestSaveSmallTailAsCdc_NoInclude(t *testing.T) { const dim = 3 rows := []PendingRecord{ - {Pkid: 1, Vec: []float32{1, 2, 3}}, - {Pkid: 2, Vec: []float32{4, 5, 6}}, - {Pkid: -3, Vec: []float32{math.MaxFloat32, 0, -1}}, + {Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{1, 2, 3})}, + {Pkid: 2, Vec: util.UnsafeSliceToBytes([]float32{4, 5, 6})}, + {Pkid: -3, Vec: util.UnsafeSliceToBytes([]float32{math.MaxFloat32, 0, -1})}, } - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, dim, 0, "") + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4*dim, 0, "") require.NoError(t, err) require.NotEmpty(t, sqls) @@ -69,7 +70,7 @@ func TestSaveSmallTailAsCdc_NoInclude(t *testing.T) { require.NoError(t, err) pos := 0 for pos < len(records) { - rec, n, ok := DecodeEventRecord(records[pos:], dim, 0) + rec, n, ok := DecodeEventRecord(records[pos:], 4*dim, 0) require.True(t, ok) got = append(got, rec) pos += n @@ -80,9 +81,11 @@ func TestSaveSmallTailAsCdc_NoInclude(t *testing.T) { for i, in := range rows { require.Equal(t, CdcOpInsert, got[i].Op) require.Equal(t, in.Pkid, got[i].Pkid) - require.Len(t, got[i].Vec, dim) - for j, v := range in.Vec { - require.Equal(t, math.Float32bits(v), math.Float32bits(got[i].Vec[j]), + require.Len(t, got[i].Vec, 4*dim) + inVec := util.UnsafeSliceCast[float32](in.Vec) + gotVec := util.UnsafeSliceCast[float32](got[i].Vec) + for j, v := range inVec { + require.Equal(t, math.Float32bits(v), math.Float32bits(gotVec[j]), "row %d vec[%d] mismatch", i, j) } } @@ -94,13 +97,13 @@ func TestSaveSmallTailAsCdc_WithInclude(t *testing.T) { const dim = 2 const ibpr = 8 // one int64-shaped INCLUDE col + zero-mask byte rounded rows := []PendingRecord{ - {Pkid: 10, Vec: []float32{0.1, 0.2}, Include: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, - {Pkid: 11, Vec: []float32{0.3, 0.4}, Include: []byte{9, 10, 11, 12, 13, 14, 15, 16}}, + {Pkid: 10, Vec: util.UnsafeSliceToBytes([]float32{0.1, 0.2}), Include: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, + {Pkid: 11, Vec: util.UnsafeSliceToBytes([]float32{0.3, 0.4}), Include: []byte{9, 10, 11, 12, 13, 14, 15, 16}}, } // Empty colMetaJSON to keep this test focused on tag=1 INSERT // round-trip; the header-emission case has its own test below. - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, dim, ibpr, "") + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4*dim, ibpr, "") require.NoError(t, err) require.NotEmpty(t, sqls) @@ -116,7 +119,7 @@ func TestSaveSmallTailAsCdc_WithInclude(t *testing.T) { require.NoError(t, err) pos := 0 for pos < len(records) { - rec, n, ok := DecodeEventRecord(records[pos:], dim, ibpr) + rec, n, ok := DecodeEventRecord(records[pos:], 4*dim, ibpr) require.True(t, ok) got = append(got, rec) pos += n @@ -137,9 +140,9 @@ func TestSaveSmallTailAsCdc_IncludeMismatchErrors(t *testing.T) { const dim = 2 const ibpr = 8 rows := []PendingRecord{ - {Pkid: 1, Vec: []float32{0.1, 0.2}, Include: []byte{1, 2, 3}}, // wrong length + {Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{0.1, 0.2}), Include: []byte{1, 2, 3}}, // wrong length } - _, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, dim, ibpr, "") + _, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4*dim, ibpr, "") require.Error(t, err) } @@ -147,8 +150,8 @@ func TestSaveSmallTailAsCdc_IncludeMismatchErrors(t *testing.T) { // must be the well-known CdcTailId sentinel so the search-side // replay finds it. func TestSaveSmallTailAsCdc_UsesCdcTailId(t *testing.T) { - rows := []PendingRecord{{Pkid: 1, Vec: []float32{1, 2, 3, 4}}} - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4, 0, "") + rows := []PendingRecord{{Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{1, 2, 3, 4})}} + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 16, 0, "") require.NoError(t, err) require.NotEmpty(t, sqls) require.Contains(t, sqls[0], "'"+vectorindex.CdcTailId+"'") @@ -163,10 +166,10 @@ func TestSaveSmallTailAsCdc_EmbedsColMetaInEveryChunk(t *testing.T) { const ibpr = 8 colMetaJSON := `[{"name":"a","type":1}]` rows := []PendingRecord{ - {Pkid: 1, Vec: []float32{0.1, 0.2}, Include: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, - {Pkid: 2, Vec: []float32{0.3, 0.4}, Include: []byte{9, 10, 11, 12, 13, 14, 15, 16}}, + {Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{0.1, 0.2}), Include: []byte{1, 2, 3, 4, 5, 6, 7, 8}}, + {Pkid: 2, Vec: util.UnsafeSliceToBytes([]float32{0.3, 0.4}), Include: []byte{9, 10, 11, 12, 13, 14, 15, 16}}, } - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, dim, ibpr, colMetaJSON) + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4*dim, ibpr, colMetaJSON) require.NoError(t, err) require.NotEmpty(t, sqls) @@ -183,7 +186,7 @@ func TestSaveSmallTailAsCdc_EmbedsColMetaInEveryChunk(t *testing.T) { "every chunk's frame header section must carry colMetaJSON") // Records are still pure Delete/Insert event ops (no special // header record in the records section). - rec, _, ok := DecodeEventRecord(records, dim, ibpr) + rec, _, ok := DecodeEventRecord(records, 4*dim, ibpr) require.True(t, ok) require.Equal(t, CdcOpInsert, rec.Op) } @@ -193,11 +196,11 @@ func TestSaveSmallTailAsCdc_EmbedsColMetaInEveryChunk(t *testing.T) { // recovers the colMetaJSON from the chunk frame header. func TestPeekColMetaJSON_RoundTrip(t *testing.T) { colMetaJSON := `[{"name":"a","type":1},{"name":"b","type":2}]` - rows := []PendingRecord{{Pkid: 1, Vec: []float32{1, 2}}} + rows := []PendingRecord{{Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{1, 2})}} // Note: ibpr=0 here because rows[0].Include is empty; the embedded // colMetaJSON is for the search side's INCLUDE-column wiring, not // the encode-time layout in this contrived test. - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 2, 0, colMetaJSON) + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 8, 0, colMetaJSON) require.NoError(t, err) require.NotEmpty(t, sqls) @@ -215,8 +218,8 @@ func TestPeekColMetaJSON_RoundTrip(t *testing.T) { // TestPeekColMetaJSON_NoHeader: when colMetaJSON is empty the chunk's // frame header section is empty too — PeekColMetaJSON returns "". func TestPeekColMetaJSON_NoHeader(t *testing.T) { - rows := []PendingRecord{{Pkid: 1, Vec: []float32{1, 2, 3, 4}}} - sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 4, 0, "") + rows := []PendingRecord{{Pkid: 1, Vec: util.UnsafeSliceToBytes([]float32{1, 2, 3, 4})}} + sqls, err := SaveSmallTailAsCdc(smallTailTblcfg(), rows, 16, 0, "") require.NoError(t, err) re := regexp.MustCompile(`unhex\('([0-9a-fA-F]*)'\)`) diff --git a/pkg/vectorindex/hnsw/plugin/compile/compile.go b/pkg/vectorindex/hnsw/plugin/compile/compile.go index 37e6fe4979305..d28f6685dd160 100644 --- a/pkg/vectorindex/hnsw/plugin/compile/compile.go +++ b/pkg/vectorindex/hnsw/plugin/compile/compile.go @@ -200,6 +200,12 @@ func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.Re // not covered there. func (Hooks) HandleDropIndex(_ compileplugin.CompileContext, defs map[string]*plan.IndexDef) error { logutil.Infof("[plugin] hnsw HandleDropIndex: defs=%d", len(defs)) + // Evict the cached search index so its resources are freed NOW, rather than + // lingering until the 5-min VectorIndexCacheTTL. Mirrors the create-side + // cache.Cache.Remove(storageDef.IndexTableName). + if storageDef, ok := defs[catalog.Hnsw_TblType_Storage]; ok { + cache.Cache.Remove(storageDef.IndexTableName) + } return nil } diff --git a/pkg/vectorindex/hnsw/plugin/runtime/runtime.go b/pkg/vectorindex/hnsw/plugin/runtime/runtime.go index f2dff8120d67d..93995bc2e7730 100644 --- a/pkg/vectorindex/hnsw/plugin/runtime/runtime.go +++ b/pkg/vectorindex/hnsw/plugin/runtime/runtime.go @@ -93,6 +93,11 @@ func (CatalogHooks) SupportedVectorTypes() []types.T { // SupportedPrimaryKeyTypes: requires an int64 primary key. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return []types.T{types.T_int64} } +// ValidQuantization: HNSW (usearch) validates quantization on its own +// param-build path (ParamsFromTree), and REINDEX does not accept a quantization +// change, so there is nothing to gate here. +func (CatalogHooks) ValidQuantization(_, _ string) error { return nil } + // SupportedIncludeColumnTypes: this index has no INCLUDE-column support. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } diff --git a/pkg/vectorindex/idxcron/executor_test.go b/pkg/vectorindex/idxcron/executor_test.go index 8c4f3e40d0a1c..5b3539527b912 100644 --- a/pkg/vectorindex/idxcron/executor_test.go +++ b/pkg/vectorindex/idxcron/executor_test.go @@ -265,6 +265,7 @@ func (m mockCatalogHooks) SupportedOpTypes() map[string]string func (m mockCatalogHooks) SupportedVectorTypes() []types.T { return nil } func (m mockCatalogHooks) SupportedPrimaryKeyTypes() []types.T { return nil } func (m mockCatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } +func (m mockCatalogHooks) ValidQuantization(_, _ string) error { return nil } func (m mockCatalogHooks) ExperimentalFlag() string { return "" } func (m mockCatalogHooks) AlterTableCloneBehavior() catalogplugin.AlterTableCloneBehavior { return catalogplugin.AlterTableCloneBehavior{} diff --git a/pkg/vectorindex/ivfflat/kmeans/device/gpu.go b/pkg/vectorindex/ivfflat/kmeans/device/gpu.go index 7bf0eafbef676..2d81157f7e5f0 100644 --- a/pkg/vectorindex/ivfflat/kmeans/device/gpu.go +++ b/pkg/vectorindex/ivfflat/kmeans/device/gpu.go @@ -75,23 +75,6 @@ func (c *GpuClusterer[T]) Close() error { return nil } -func resolveCuvsDistanceForDense(distance metric.MetricType) cuvs.DistanceType { - switch distance { - case metric.Metric_L2sqDistance: - return cuvs.L2Expanded - case metric.Metric_L2Distance: - return cuvs.L2Expanded - case metric.Metric_InnerProduct: - return cuvs.InnerProduct - case metric.Metric_CosineDistance: - return cuvs.CosineSimilarity - case metric.Metric_L1Distance: - return cuvs.L1 - default: - return cuvs.L2Expanded - } -} - func NewKMeans[T types.RealNumbers](vectors [][]T, clusterCnt, maxIterations int, deltaThreshold float64, distanceType metric.MetricType, _ kmeans.InitType, @@ -122,7 +105,13 @@ func NewKMeans[T types.RealNumbers](vectors [][]T, clusterCnt, deviceID := 0 nthread := uint32(1) - km, err := cuvs.NewGpuKMeans[float32](uint32(clusterCnt), uint32(dim), resolveCuvsDistanceForDense(distanceType), maxIterations, deviceID, nthread) + // Dense centroid clustering always uses L2, independent of the index's + // search metric (matches the CPU path, ResolveKmeansDistanceFnForDense, + // which forces L2 for every metric). cuVS kmeans_balanced only supports + // L2/InnerProduct — forwarding the search metric (e.g. L1, cosine) makes + // cuVS abort with "distance metric not supported". The search metric is + // applied later at query time (centroid scan + re-rank), not in clustering. + km, err := cuvs.NewGpuKMeans[float32](uint32(clusterCnt), uint32(dim), cuvs.L2Expanded, maxIterations, deviceID, nthread) if err != nil { return nil, err } diff --git a/pkg/vectorindex/ivfflat/kmeans/device/issue_test.go b/pkg/vectorindex/ivfflat/kmeans/device/issue_test.go index 8202874c783f0..c7b7ab92db133 100644 --- a/pkg/vectorindex/ivfflat/kmeans/device/issue_test.go +++ b/pkg/vectorindex/ivfflat/kmeans/device/issue_test.go @@ -85,7 +85,7 @@ func Search(datasetvec [][]float32, queriesvec [][]float32, limit uint, distance deviceID := 0 nthread := uint32(1) - bf, err := cuvs.NewGpuBruteForce[float32](flattenedDataset, uint64(len(datasetvec)), uint32(dim), distanceType, nthread, deviceID) + bf, err := cuvs.NewGpuBruteForce[float32, float32](flattenedDataset, uint64(len(datasetvec)), uint32(dim), distanceType, nthread, deviceID) if err != nil { return nil, nil, err } diff --git a/pkg/vectorindex/ivfflat/plugin/compile/compile.go b/pkg/vectorindex/ivfflat/plugin/compile/compile.go index 81b3f240a3504..0d3849b3fe598 100644 --- a/pkg/vectorindex/ivfflat/plugin/compile/compile.go +++ b/pkg/vectorindex/ivfflat/plugin/compile/compile.go @@ -34,6 +34,7 @@ import ( "github.com/bytedance/sonic" "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -41,6 +42,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/util/executor" "github.com/matrixorigin/matrixone/pkg/vectorindex" "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" + ivfflatruntime "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // actionIvfflatReindex mirrors idxcron.Action_Ivfflat_Reindex. Inlined @@ -87,11 +90,24 @@ func (Hooks) RestoreInitSQL(ctx compileplugin.CompileContext, indexDefs map[stri // inline — that persistence stays at the SQL-layer call site, so this // hook only performs the map merge. func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.ReindexParamUpdate) (map[string]string, error) { - return compileplugin.MergeReindexParams(old, alter, "ivfflat", + // Merge first, then validate the EFFECTIVE quantization via the per-algo + // catalog hook (the single home shared with CREATE; the value the reindex + // set, or the index's stored value when the statement omitted it — e.g. the + // idxcron-issued rebuild). + merged, err := compileplugin.MergeReindexParams(old, alter, "ivfflat", catalog.IndexAlgoParamLists, catalog.IndexAlgoParamKmeansTrainPercent, catalog.IndexAlgoParamKmeansMaxIteration, + catalog.Quantization, ) + if err != nil { + return nil, err + } + if err := (ivfflatruntime.CatalogHooks{}).ValidQuantization( + merged[catalog.Quantization], merged[catalog.IndexAlgoParamOpType]); err != nil { + return nil, err + } + return merged, nil } // HandleDropIndex: IVF-FLAT generic hidden-table deletion is performed @@ -100,6 +116,12 @@ func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.Re // (pkg/sql/compile/ddl.go DropIndex path). No additional cleanup here. func (Hooks) HandleDropIndex(_ compileplugin.CompileContext, defs map[string]*plan.IndexDef) error { logutil.Infof("[plugin] ivfflat HandleDropIndex: defs=%d", len(defs)) + // Evict the cached search index immediately rather than waiting for the + // 5-min VectorIndexCacheTTL. Mirrors the create-side + // cache.Cache.Remove(fmt.Sprintf("%s:0", centroidsDef.IndexTableName)). + if centroidsDef, ok := defs[catalog.SystemSI_IVFFLAT_TblType_Centroids]; ok { + cache.Cache.Remove(fmt.Sprintf("%s:0", centroidsDef.IndexTableName)) + } return nil } @@ -195,18 +217,33 @@ func runCreateOrReindex(ctx compileplugin.CompileContext, indexDefs map[string]* return err } - // 4.b populate centroids table - if err = ivfIndexCentroidsTable(ctx, centroidsDef, qryDatabase, originalTableDef, - totalCnt, metaDef.IndexTableName, forceSync); err != nil { - return err - } - - if !async || forceSync { - // 4.c populate entries table - if err = ivfIndexEntriesTable(ctx, entriesDef, qryDatabase, originalTableDef, - metaDef.IndexTableName, centroidsDef.IndexTableName); err != nil { + // 4.b + 4.c: build the index. Both kmeans (4.b) and entry assignment (4.c) + // scan the source table, but queries never re-read it (re-rank fetches only a + // handful of rows), so run the build's reads with SkipMemoryCacheWrites — this + // one-shot source scan must not evict the index-entry working set the queries + // actually hit from the fileservice cache. The optional-interface keeps the + // CompileContext interface (and its plugin mocks) untouched; non-supporting + // contexts just build directly. + buildIndex := func() error { + if err := ivfIndexCentroidsTable(ctx, centroidsDef, qryDatabase, originalTableDef, + totalCnt, metaDef.IndexTableName, forceSync); err != nil { return err } + if !async || forceSync { + if err := ivfIndexEntriesTable(ctx, entriesDef, qryDatabase, originalTableDef, + metaDef.IndexTableName, centroidsDef.IndexTableName); err != nil { + return err + } + } + return nil + } + if r, ok := ctx.(interface{ RunWithSourceReadCacheSkip(func() error) error }); ok { + err = r.RunWithSourceReadCacheSkip(buildIndex) + } else { + err = buildIndex() + } + if err != nil { + return err } // 4.d delete older entries in index table. @@ -245,6 +282,33 @@ func indexColCount(ctx compileplugin.CompileContext, indexDef *plan.IndexDef, return n, nil } +// readQuantizeBound reads a scalar DOUBLE metadata value (e.g. quantize_min / +// quantize_max) by key. found=false when the row is absent (e.g. a pre-quantizer +// index), in which case the caller falls back to a raw cast. +func readQuantizeBound(ctx compileplugin.CompileContext, qryDatabase, metaTbl, key string) (val float64, found bool, err error) { + sql := fmt.Sprintf("SELECT CAST(`%s` AS DOUBLE) FROM `%s`.`%s` WHERE `%s` = '%s'", + catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, qryDatabase, metaTbl, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, key) + rs, err := ctx.RunSqlWithResult(sql) + if err != nil { + return 0, false, err + } + defer rs.Close() + rs.ReadRows(func(_ int, cols []*vector.Vector) bool { + if len(cols) == 0 { + return false + } + rows := executor.GetFixedRows[float64](cols[0]) + if len(rows) == 0 { + return false + } + val = rows[0] + found = true + return false + }) + return val, found, nil +} + // ivfIndexMetaTable is lifted from Scope.handleIvfIndexMetaTable // (pkg/sql/compile/ddl_index_algo.go:221). func ivfIndexMetaTable(ctx compileplugin.CompileContext, indexDef *plan.IndexDef, qryDatabase string) error { @@ -395,6 +459,51 @@ func ivfIndexEntriesTable( return err } + // QUANTIZATION: if set, entries are stored as the quantization type (the entry + // column was created with that type in schema.go), so the SELECT casts the + // base vectors to it. The CENTROIDX assignment still uses the f32 base column. + indexColName := indexDef.Parts[0] + entrySelectExpr := fmt.Sprintf("`%s`", indexColName) + if qv, qerr := sonic.Get([]byte(indexDef.IndexAlgoParams), catalog.Quantization); qerr == nil { + if qstr, serr := qv.String(); serr == nil && qstr != "" { + if qt, ok := quantizer.ToVectorType(qstr); ok { + var dim int32 + for _, c := range originalTableDef.Cols { + if c.Name == indexColName { + dim = c.Typ.Width + break + } + } + if qt == types.T_array_int8 || qt == types.T_array_uint8 { + // cuVS-style asymmetric scalar quantizer: map the trained + // [min,max] (stored in metadata by ivf_create) onto the full int8 + // range [-128,127] (or uint8 [0,255]) via q(x)=round(x*mul+add). + // float16 needs no scale. + qmin, ok1, err := readQuantizeBound(ctx, qryDatabase, metadataTableName, catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin) + if err != nil { + return err + } + qmax, ok2, err := readQuantizeBound(ctx, qryDatabase, metadataTableName, catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax) + if err != nil { + return err + } + col := fmt.Sprintf("`%s`", indexColName) + if ok1 && ok2 && qt == types.T_array_int8 { + mul, add := quantizer.Int8Params(qmin, qmax) + entrySelectExpr = quantizer.Int8EntrySQL(col, mul, add, dim) + } else if ok1 && ok2 { + mul, add := quantizer.Uint8Params(qmin, qmax) + entrySelectExpr = quantizer.Uint8EntrySQL(col, mul, add, dim) + } else { + entrySelectExpr = quantizer.CastSQL(col, qt, dim) + } + } else { + entrySelectExpr = quantizer.CastSQL(fmt.Sprintf("`%s`", indexColName), qt, dim) + } + } + } + } + var originalTblPkColsCommaSeparated, originalTblPkColMaySerial string if originalTableDef.Pkey.PkeyColName == catalog.CPrimaryKeyColName { for i, part := range originalTableDef.Pkey.Names { @@ -433,14 +542,14 @@ func ivfIndexEntriesTable( indexColumnName := indexDef.Parts[0] centroidsCrossL2JoinTbl := fmt.Sprintf("%s "+ - "SELECT `%s`, `%s`, %s, `%s`"+ + "SELECT `%s`, `%s`, %s, %s"+ " FROM `%s`.`%s` CENTROIDX ('%s') join %s "+ " using (`%s`, `%s`) ", insertSQL, catalog.SystemSI_IVFFLAT_TblCol_Centroids_version, catalog.SystemSI_IVFFLAT_TblCol_Centroids_id, originalTblPkColMaySerial, - indexColumnName, + entrySelectExpr, // base column, or cast(base as ) under QUANTIZATION qryDatabase, originalTableDef.Name, optype, diff --git a/pkg/vectorindex/ivfflat/plugin/compile/compile_smoke_test.go b/pkg/vectorindex/ivfflat/plugin/compile/compile_smoke_test.go index fe543bde83de2..35a4a6105b608 100644 --- a/pkg/vectorindex/ivfflat/plugin/compile/compile_smoke_test.go +++ b/pkg/vectorindex/ivfflat/plugin/compile/compile_smoke_test.go @@ -17,6 +17,7 @@ package compile import ( "testing" + "github.com/matrixorigin/matrixone/pkg/catalog" compileplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/compile" "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -89,6 +90,21 @@ func TestIvfflatValidateReindexParams_Passthrough(t *testing.T) { require.Equal(t, old, got) } +// TestIvfflatValidateReindexParams_Quantization: IVF-FLAT honors a narrow-type +// quantization on reindex (same set as CREATE) and rejects unknown values. +func TestIvfflatValidateReindexParams_Quantization(t *testing.T) { + got, err := Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "int8"}, + }) + require.NoError(t, err) + require.Equal(t, "int8", got[catalog.Quantization]) + + _, err = Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "garbage"}, + }) + require.Error(t, err) +} + // TestIvfflatIdxcronMetadata_BackgroundLog covers the entry log line // of IdxcronMetadata via the isFrontend=false path (which short- // circuits through BuildIdxcronMetadata's IsFrontend guard). diff --git a/pkg/vectorindex/ivfflat/plugin/plan/schema.go b/pkg/vectorindex/ivfflat/plugin/plan/schema.go index 5207c442e44d2..7d147dd89586a 100644 --- a/pkg/vectorindex/ivfflat/plugin/plan/schema.go +++ b/pkg/vectorindex/ivfflat/plugin/plan/schema.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/sql/util" ivfflatrt "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfflat/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // ivfflatCatalogHooks is the shared (stateless) catalog-hooks instance used for @@ -146,14 +147,31 @@ func (Hooks) BuildSecondaryIndexDefs( Typ: plan.Type{Id: int32(types.T_int64)}, Default: &plan.Default{NullAbility: false, Expr: nil, OriginString: ""}, } + // Centroid type is decoupled from the entry type. Centroids are f32 whenever + // the entries are NOT a plain f32/f64 column: i.e. for a narrow base + // (bf16/f16/int8) or for ANY base under QUANTIZATION (incl. f64). f32 gives + // accurate assignment, fast f32 search, and tiny RAM for the few centroids; + // the entries carry the memory win. A plain f32/f64 column keeps its type. + centroidTyp := plan.Type{ + Id: colMap[colName].Typ.Id, + Width: colMap[colName].Typ.Width, + Scale: colMap[colName].Typ.Scale, + } + quantized := indexInfo.IndexOption != nil && indexInfo.IndexOption.Quantization != "" + switch types.T(centroidTyp.Id) { + case types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: + centroidTyp.Id = int32(types.T_array_float32) + centroidTyp.Scale = 0 + default: + if quantized { + centroidTyp.Id = int32(types.T_array_float32) + centroidTyp.Scale = 0 + } + } tableDefs[1].Cols[2] = &plan.ColDef{ - Name: catalog.SystemSI_IVFFLAT_TblCol_Centroids_centroid, - Alg: plan.CompressType_Lz4, - Typ: plan.Type{ - Id: colMap[colName].Typ.Id, - Width: colMap[colName].Typ.Width, - Scale: colMap[colName].Typ.Scale, - }, + Name: catalog.SystemSI_IVFFLAT_TblCol_Centroids_centroid, + Alg: plan.CompressType_Lz4, + Typ: centroidTyp, Default: &plan.Default{NullAbility: true, Expr: nil, OriginString: ""}, } tableDefs[1].Cols[3] = planplugin.MakeHiddenColDefByName(catalog.CPrimaryKeyColName) @@ -218,14 +236,38 @@ func (Hooks) BuildSecondaryIndexDefs( }, Default: &plan.Default{NullAbility: false, Expr: nil, OriginString: ""}, } + // Entry type follows the QUANTIZATION option: CREATE INDEX ... USING + // ivfflat ... QUANTIZATION='int8' stores entries as vecint8 (quantized from + // the base vectors), while the base column and the f32 centroids are + // unchanged. Without QUANTIZATION the entries keep the base column type. + entryTyp := plan.Type{ + Id: colMap[colName].Typ.Id, + Width: colMap[colName].Typ.Width, + Scale: colMap[colName].Typ.Scale, + } + if indexInfo.IndexOption != nil && indexInfo.IndexOption.Quantization != "" { + if qt, ok := quantizer.ToVectorType(indexInfo.IndexOption.Quantization); ok { + // QUANTIZATION is downcast-only: the quantized entry element must be the + // same width or narrower than the base column. Upcasting (e.g. a bf16 or + // int8 base with QUANTIZATION='float32') is unsupported — it costs 2-4x the + // entry storage for no precision gain and forces the f32 distance kernel + // over narrow entries. Omit QUANTIZATION to keep the base-width entries. + baseSize := types.Type{Oid: types.T(colMap[colName].Typ.Id)}.GetArrayElementSize() + quantSize := types.Type{Oid: qt}.GetArrayElementSize() + if quantSize > baseSize { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "ivfflat QUANTIZATION '%s' (%d bytes/element) cannot upcast base column %s (%d bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type", + indexInfo.IndexOption.Quantization, quantSize, + types.T(colMap[colName].Typ.Id).String(), baseSize) + } + entryTyp.Id = int32(qt) + entryTyp.Scale = 0 + } + } tableDefs[2].Cols[3] = &plan.ColDef{ - Name: catalog.SystemSI_IVFFLAT_TblCol_Entries_entry, - Alg: plan.CompressType_Lz4, - Typ: plan.Type{ - Id: colMap[colName].Typ.Id, - Width: colMap[colName].Typ.Width, - Scale: colMap[colName].Typ.Scale, - }, + Name: catalog.SystemSI_IVFFLAT_TblCol_Entries_entry, + Alg: plan.CompressType_Lz4, + Typ: entryTyp, Default: &plan.Default{NullAbility: true, Expr: nil, OriginString: ""}, } tableDefs[2].Cols[4] = planplugin.MakeHiddenColDefByName(catalog.CPrimaryKeyColName) diff --git a/pkg/vectorindex/ivfflat/plugin/runtime/runtime.go b/pkg/vectorindex/ivfflat/plugin/runtime/runtime.go index a64904ee68cbd..694f7ad57026b 100644 --- a/pkg/vectorindex/ivfflat/plugin/runtime/runtime.go +++ b/pkg/vectorindex/ivfflat/plugin/runtime/runtime.go @@ -30,6 +30,7 @@ import ( catalogplugin "github.com/matrixorigin/matrixone/pkg/indexplugin/catalog" "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // actionIvfflatReindex mirrors idxcron.Action_Ivfflat_Reindex. Inlined @@ -127,15 +128,36 @@ func (CatalogHooks) ExperimentalFlag() string { return "" } // SupportedOpTypes returns IVF-FLAT's metric registry. IVF uses a // distinct metric table from HNSW/USearch (OpTypeToIvfMetric). -// SupportedVectorTypes: IVF-FLAT indexes f32 or f64 vectors. +// SupportedVectorTypes: IVF-FLAT indexes all vector element types. Entries are +// stored in their own (narrow) type; centroids are f32 (decoupled). kmeans runs +// in f32, narrow distances go through the float32 bridge / narrow kernels. func (CatalogHooks) SupportedVectorTypes() []types.T { - return []types.T{types.T_array_float32, types.T_array_float64} + return []types.T{ + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, + } } // SupportedPrimaryKeyTypes: IVF-FLAT imposes no PK-type constraint — the // primary key may be any type. nil = "no constraint". func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return nil } +// ValidQuantization gates the quantization value for IVF-FLAT: it must name a +// narrow vector type IVF-FLAT supports (float32/float16/bf16/int8/uint8, via +// quantizer.ToVectorType). IVF-FLAT re-ranks on the CPU from the stored entries, +// so unlike the cuvs backends it imposes no op_type restriction; op is unused. +// One home for CREATE (plan/schema) and REINDEX (compile/ValidateReindexParams). +func (CatalogHooks) ValidQuantization(quant, _ string) error { + if quant == "" { + return nil + } + if _, ok := quantizer.ToVectorType(quant); !ok { + return moerr.NewNotSupportedNoCtxf( + "ivfflat quantization %q (supported: float32, float16, bf16, int8, uint8)", quant) + } + return nil +} + // SupportedIncludeColumnTypes: this index has no INCLUDE-column support. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { return nil } @@ -222,5 +244,17 @@ func (CatalogHooks) ParamsFromTree(idx *tree.Index) (map[string]string, error) { if idx.IndexOption.KmeansMaxIteration > 0 { res[catalog.IndexAlgoParamKmeansMaxIteration] = strconv.FormatInt(idx.IndexOption.KmeansMaxIteration, 10) } + + // QUANTIZATION stores the ivfflat ENTRIES in a narrow type (float16/int8); + // the base column and f32 centroids are unchanged. Persist it in algo_params + // so the entries build (compile) and the search can read it back. Only the + // predefined names that map to a MO narrow vector type are accepted. + if q := idx.IndexOption.Quantization; q != "" { + if _, ok := quantizer.ToVectorType(q); !ok { + return nil, moerr.NewInternalErrorNoCtx(fmt.Sprintf( + "ivfflat: unsupported quantization '%s' (supported: 'float32', 'float16', 'bf16', 'int8', 'uint8')", q)) + } + res[catalog.Quantization] = catalog.ToLower(q) + } return res, nil } diff --git a/pkg/vectorindex/ivfflat/plugin/runtime/runtime_test.go b/pkg/vectorindex/ivfflat/plugin/runtime/runtime_test.go index 372d5ac2d1f62..723dd1765c113 100644 --- a/pkg/vectorindex/ivfflat/plugin/runtime/runtime_test.go +++ b/pkg/vectorindex/ivfflat/plugin/runtime/runtime_test.go @@ -137,3 +137,27 @@ func TestIvfflatParamsFromTree_InvalidOpType(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "invalid op_type") } + +func TestIvfflatParamsFromTree_Quantization(t *testing.T) { + for _, q := range []string{"int8", "uint8", "float16", "bf16", "float32", "INT8", "Bf16", "UINT8"} { + idx := &tree.Index{IndexOption: &tree.IndexOption{Quantization: q}} + got, err := CatalogHooks{}.ParamsFromTree(idx) + require.NoErrorf(t, err, "quantization %q", q) + require.Equalf(t, catalog.ToLower(q), got[catalog.Quantization], "stored quantization %q", q) + } + // omitted -> not present in params + idx := &tree.Index{IndexOption: &tree.IndexOption{}} + got, err := CatalogHooks{}.ParamsFromTree(idx) + require.NoError(t, err) + _, present := got[catalog.Quantization] + require.False(t, present) +} + +func TestIvfflatParamsFromTree_InvalidQuantization(t *testing.T) { + for _, q := range []string{"float64", "f16", "garbage"} { + idx := &tree.Index{IndexOption: &tree.IndexOption{Quantization: q}} + _, err := CatalogHooks{}.ParamsFromTree(idx) + require.Errorf(t, err, "quantization %q should be rejected", q) + require.Contains(t, err.Error(), "unsupported quantization") + } +} diff --git a/pkg/vectorindex/ivfflat/search.go b/pkg/vectorindex/ivfflat/search.go index fd7ef78dc6af6..8329bced52e79 100644 --- a/pkg/vectorindex/ivfflat/search.go +++ b/pkg/vectorindex/ivfflat/search.go @@ -30,6 +30,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/vectorindex/brute_force" "github.com/matrixorigin/matrixone/pkg/vectorindex/cache" "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" "github.com/matrixorigin/matrixone/pkg/vectorindex/sqlexec" ) @@ -47,6 +48,11 @@ var runSql = sqlexec.RunSql type IvfflatSearchIndex[T types.RealNumbers] struct { Version int64 Centroids cache.VectorIndexSearchIf + // QuantMul/QuantAdd are the int8 scalar-quantizer params (q(x)=round(x*mul+add)) + // derived from the trained [min,max] in metadata; the query uses the same + // transform as the entries. Defaults (1,0) = identity when not int8-quantized. + QuantMul float64 + QuantAdd float64 } // This is the Ivf search implementation that implement VectorIndexSearchIf interface @@ -127,16 +133,59 @@ func (idx *IvfflatSearchIndex[T]) LoadCentroids(proc *sqlexec.SqlProcess, idxcfg func (idx *IvfflatSearchIndex[T]) LoadIndex(proc *sqlexec.SqlProcess, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, nthread int64) (err error) { idx.Version = idxcfg.Ivfflat.Version + idx.QuantMul = 1.0 + idx.QuantAdd = 0.0 err = idx.LoadCentroids(proc, idxcfg, tblcfg, nthread) if err != nil { return err } + // int8/uint8 QUANTIZATION: load the trained [min,max] and derive the same + // transform the entries were quantized with, so the query maps identically. + if vt := types.T(idxcfg.Ivfflat.VectorType); vt == types.T_array_int8 || vt == types.T_array_uint8 { + if err = idx.loadQuantizeBounds(proc, tblcfg, vt); err != nil { + return err + } + } + + return nil +} + +func (idx *IvfflatSearchIndex[T]) loadQuantizeBounds(proc *sqlexec.SqlProcess, tblcfg vectorindex.IndexTableConfig, vt types.T) error { + read := func(key string) (float64, bool, error) { + sql := fmt.Sprintf("SELECT CAST(`%s` AS DOUBLE) FROM `%s`.`%s` WHERE `%s` = '%s'", + catalog.SystemSI_IVFFLAT_TblCol_Metadata_val, tblcfg.DbName, tblcfg.MetadataTable, + catalog.SystemSI_IVFFLAT_TblCol_Metadata_key, key) + res, err := runSql(proc, sql) + if err != nil { + return 0, false, err + } + defer res.Close() + if len(res.Batches) == 0 || res.Batches[0].RowCount() == 0 { + return 0, false, nil + } + return vector.GetFixedAtNoTypeCheck[float64](res.Batches[0].Vecs[0], 0), true, nil + } + qmin, ok1, err := read(catalog.SystemSI_IVFFLAT_Metadata_QuantizeMin) + if err != nil { + return err + } + qmax, ok2, err := read(catalog.SystemSI_IVFFLAT_Metadata_QuantizeMax) + if err != nil { + return err + } + if ok1 && ok2 { + if vt == types.T_array_uint8 { + idx.QuantMul, idx.QuantAdd = quantizer.Uint8Params(qmin, qmax) + } else { + idx.QuantMul, idx.QuantAdd = quantizer.Int8Params(qmin, qmax) + } + } return nil } -func (idx *IvfflatSearchIndex[T]) findCentroids(sqlproc *sqlexec.SqlProcess, query []T, distfn metric.DistanceFunction[T], idxcfg vectorindex.IndexConfig, probe uint, _ int64) ([]int64, error) { +func (idx *IvfflatSearchIndex[T]) findCentroids(sqlproc *sqlexec.SqlProcess, query []T, idxcfg vectorindex.IndexConfig, probe uint, _ int64) ([]int64, error) { if idx.Centroids == nil { // empty index has id = 1 @@ -244,12 +293,7 @@ func (idx *IvfflatSearchIndex[T]) Search( nthread int64, ) (keys any, distances []float64, err error) { - distfn, err := metric.ResolveDistanceFn[T](metric.MetricType(idxcfg.Ivfflat.Metric)) - if err != nil { - return - } - - centroids_ids, err := idx.findCentroids(sqlproc, query, distfn, idxcfg, rt.Probe, nthread) + centroids_ids, err := idx.findCentroids(sqlproc, query, idxcfg, rt.Probe, nthread) if err != nil { return } @@ -280,6 +324,36 @@ func (idx *IvfflatSearchIndex[T]) Search( vecFromB64Fn = "vecf64_from_base64" } + // Re-rank distance. The ENTRY must stay a plain column so the ORDER BY + // index-param pushdown (readutil.SetIndexParam) can identify it — wrapping it + // in a CAST makes Args[0] a function and panics. The query must be a CONSTANT + // vec literal of the SAME (narrow) type as the entries, or the pushdown can't + // fold it and the pushed top-limit stays 0 ("top limit must be positive"). A + // cast of vecf32_from_base64(...) does NOT fold (vector casts aren't constant- + // folded), so for narrow entries quantize the f32 query to the entry type here + // and pass it via vec{bf16,f16,int8}_from_base64 — a STRICT decode that folds + // to a narrow literal, the narrow sibling of vecf32_from_base64. f32/f64 use + // the plain f32 base64 decode. + entryCol := fmt.Sprintf("`%s`", catalog.SystemSI_IVFFLAT_TblCol_Entries_entry) + queryExpr := fmt.Sprintf("%s('%s')", vecFromB64Fn, queryB64) + if qf32, ok := any(query).([]float32); ok { + switch types.T(idxcfg.Ivfflat.VectorType) { + case types.T_array_bf16: + queryExpr = fmt.Sprintf("vecbf16_from_base64('%s')", types.ArrayToBase64(types.Float32ToBF16Slice(qf32))) + case types.T_array_float16: + queryExpr = fmt.Sprintf("vecf16_from_base64('%s')", types.ArrayToBase64(types.Float32ToFloat16Slice(qf32))) + case types.T_array_int8: + // apply the same q(x)=x*mul+add transform as the entries, then round+clamp + // to int8. (mul,add)=(1,0) falls back to the raw cast (no quantizer). + sq := quantizer.ApplyInt8(qf32, idx.QuantMul, idx.QuantAdd) + queryExpr = fmt.Sprintf("vecint8_from_base64('%s')", types.ArrayToBase64(sq)) + case types.T_array_uint8: + // same transform as int8, narrowed to the unsigned [0,255] range. + sq := quantizer.ApplyUint8(qf32, idx.QuantMul, idx.QuantAdd) + queryExpr = fmt.Sprintf("vecuint8_from_base64('%s')", types.ArrayToBase64(sq)) + } + } + if sqlproc != nil && sqlproc.ExactPkFilter != "" { // Exact PK path: WaitUniqueJoinKeys converted small key set into ExactPkFilter. // Query entries directly by pk list, skip centroid-based filtering. @@ -294,12 +368,11 @@ func (idx *IvfflatSearchIndex[T]) Search( // a plain filtered read that returns the full candidate set; the downstream // Node_SORT + LIMIT k does the ranking and truncation. sql = fmt.Sprintf( - "SELECT `%s`, %s(`%s`, %s('%s')) as vec_dist FROM `%s`.`%s` WHERE `%s` = %d AND `%s` IN (%s)", + "SELECT `%s`, %s(%s, %s) as vec_dist FROM `%s`.`%s` WHERE `%s` = %d AND `%s` IN (%s)", catalog.SystemSI_IVFFLAT_TblCol_Entries_pk, metric.MetricTypeToDistFuncName[metric.MetricType(idxcfg.Ivfflat.Metric)], - catalog.SystemSI_IVFFLAT_TblCol_Entries_entry, - vecFromB64Fn, - queryB64, + entryCol, + queryExpr, tblcfg.DbName, tblcfg.EntriesTable, catalog.SystemSI_IVFFLAT_TblCol_Entries_version, idx.Version, @@ -309,12 +382,11 @@ func (idx *IvfflatSearchIndex[T]) Search( } else { // Standard centroid-based path with optional CBloomFilter pre-filtering. sql = fmt.Sprintf( - "SELECT `%s`, %s(`%s`, %s('%s')) as vec_dist FROM `%s`.`%s` WHERE `%s` = %d AND `%s` IN (%s) ORDER BY vec_dist LIMIT %d", + "SELECT `%s`, %s(%s, %s) as vec_dist FROM `%s`.`%s` WHERE `%s` = %d AND `%s` IN (%s) ORDER BY vec_dist LIMIT %d", catalog.SystemSI_IVFFLAT_TblCol_Entries_pk, metric.MetricTypeToDistFuncName[metric.MetricType(idxcfg.Ivfflat.Metric)], - catalog.SystemSI_IVFFLAT_TblCol_Entries_entry, - vecFromB64Fn, - queryB64, + entryCol, + queryExpr, tblcfg.DbName, tblcfg.EntriesTable, catalog.SystemSI_IVFFLAT_TblCol_Entries_version, idx.Version, @@ -325,8 +397,6 @@ func (idx *IvfflatSearchIndex[T]) Search( } //fmt.Println("IVFFlat SQL: ", sql) - //os.Stderr.WriteString(sql) - //os.Stderr.WriteString("\n") res, err := runSql(sqlproc, sql) if err != nil { diff --git a/pkg/vectorindex/ivfpq/build_gpu.go b/pkg/vectorindex/ivfpq/build_gpu.go index 1dbdbd2dac5a5..cd67f4b8066f1 100644 --- a/pkg/vectorindex/ivfpq/build_gpu.go +++ b/pkg/vectorindex/ivfpq/build_gpu.go @@ -23,6 +23,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/cuvs" "github.com/matrixorigin/matrixone/pkg/vectorindex" ) @@ -31,44 +32,57 @@ import ( // When the current sub-index reaches IndexCapacity, it is finalized (Build called) and a // new sub-index is created, mirroring the CagraBuild pattern. // +// IvfpqBuild carries two element types: base/quantizer-source B (the decoded +// source column type — f32 or f16) and storage Q (the cuVS sub-index storage +// type). For a direct index B==Q; for a quantized index (e.g. vecf16 base -> +// int8 storage) B is the base type and Q the 1-byte storage type. +// // IvfpqBuild is single-threaded; the ivfpq_create table function runs with IsSingle=true. -type IvfpqBuild[T cuvs.VectorType] struct { +type IvfpqBuild[B, Q cuvs.VectorType] struct { uid string idxcfg vectorindex.IndexConfig tblcfg vectorindex.IndexTableConfig - indexes []*IvfpqModel[T] - current *IvfpqModel[T] + indexes []*IvfpqModel[B, Q] + current *IvfpqModel[B, Q] nthread uint32 devices []int count int64 idBuf [1]int64 + // (B, Q) routing tags computed once at construction. bIsHalf: the base + // type is f16. qIsHalf: the storage type is f16 (so a half base goes + // native rather than quantized). + bIsHalf bool + qIsHalf bool + // Filter column metadata (INCLUDE columns) — see CagraBuild.filterColMetaJSON. filterColMetaJSON string } -func NewIvfpqBuild[T cuvs.VectorType]( +func NewIvfpqBuild[B, Q cuvs.VectorType]( uid string, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, nthread uint32, devices []int, -) (*IvfpqBuild[T], error) { - return &IvfpqBuild[T]{ +) (*IvfpqBuild[B, Q], error) { + return &IvfpqBuild[B, Q]{ uid: uid, idxcfg: idxcfg, tblcfg: tblcfg, - indexes: make([]*IvfpqModel[T], 0, 4), + indexes: make([]*IvfpqModel[B, Q], 0, 4), nthread: nthread, devices: devices, + bIsHalf: cuvs.GetQuantization[B]() == cuvs.F16, + qIsHalf: cuvs.GetQuantization[Q]() == cuvs.F16, }, nil } -func (b *IvfpqBuild[T]) createKey(n int) string { +func (b *IvfpqBuild[B, Q]) createKey(n int) string { return fmt.Sprintf("%s:%d", b.uid, n) } -func (b *IvfpqBuild[T]) getOrCreateCurrent() (*IvfpqModel[T], error) { +func (b *IvfpqBuild[B, Q]) getOrCreateCurrent() (*IvfpqModel[B, Q], error) { capacity := b.idxcfg.IndexCapacity if b.current != nil && b.count >= capacity { @@ -82,7 +96,7 @@ func (b *IvfpqBuild[T]) getOrCreateCurrent() (*IvfpqModel[T], error) { if b.current == nil { key := b.createKey(len(b.indexes)) - m, err := NewIvfpqModelForBuild[T](key, b.idxcfg, b.nthread, b.devices) + m, err := NewIvfpqModelForBuild[B, Q](key, b.idxcfg, b.nthread, b.devices) if err != nil { return nil, err } @@ -104,32 +118,46 @@ func (b *IvfpqBuild[T]) getOrCreateCurrent() (*IvfpqModel[T], error) { } // SetFilterColumns — see cagra.CagraBuild.SetFilterColumns. -func (b *IvfpqBuild[T]) SetFilterColumns(colMetaJSON string) { +func (b *IvfpqBuild[B, Q]) SetFilterColumns(colMetaJSON string) { b.filterColMetaJSON = colMetaJSON } // AddFilterChunk — see cagra.CagraBuild.AddFilterChunk. -func (b *IvfpqBuild[T]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { +func (b *IvfpqBuild[B, Q]) AddFilterChunk(colIdx uint32, data []byte, nullBitmap []uint32, nrows uint64) error { if b.current == nil { - return moerr.NewInternalErrorNoCtx("IvfpqBuild.AddFilterChunk: no current sub-index (call AddFloat first)") + return moerr.NewInternalErrorNoCtx("IvfpqBuild.AddFilterChunk: no current sub-index (call AddRow first)") } return b.current.Index.AddFilterChunk(colIdx, data, nullBitmap, nrows) } -func (b *IvfpqBuild[T]) AddFloat(id int64, vec []float32) error { +// AddRow buffers one source row. vecBytes is the raw little-endian base-type +// bytes of one vector (4*dim for an f32 base, 2*dim for an f16 base) — the +// non-generic ivfpqBuilder interface can't name the concrete element type B, so +// the bytes are reinterpreted here with UnsafeSliceCast (zero-copy, no per-row +// heap alloc). Routing by (B, Q): +// - f16 base, f16 storage (direct, Q==B): native AddChunk([]Q). +// - otherwise (f32 base, or f16 base -> int8/uint8): AddChunkQuantize([]B), +// which converts B -> Q on device (B==Q copy, or learned/cast quantizer). +func (b *IvfpqBuild[B, Q]) AddRow(id int64, vecBytes []byte) error { idx, err := b.getOrCreateCurrent() if err != nil { return err } b.idBuf[0] = id - if err = idx.AddChunkFloat(vec, 1, b.idBuf[:]); err != nil { + + if b.bIsHalf && b.qIsHalf { + err = idx.AddChunk(util.UnsafeSliceCast[Q](vecBytes), 1, b.idBuf[:]) + } else { + err = idx.AddChunkQuantize(util.UnsafeSliceCast[B](vecBytes), 1, b.idBuf[:]) + } + if err != nil { return err } b.count++ return nil } -func (b *IvfpqBuild[T]) ToInsertSql(ts int64) ([]string, error) { +func (b *IvfpqBuild[B, Q]) ToInsertSql(ts int64) ([]string, error) { if b.current != nil && b.count > 0 { if err := b.current.Build(); err != nil { return nil, err @@ -160,7 +188,7 @@ func (b *IvfpqBuild[T]) ToInsertSql(ts int64) ([]string, error) { return sqls, nil } -func (b *IvfpqBuild[T]) Destroy() error { +func (b *IvfpqBuild[B, Q]) Destroy() error { var errs error if b.current != nil { if err := b.current.Destroy(); err != nil { @@ -177,6 +205,6 @@ func (b *IvfpqBuild[T]) Destroy() error { return errs } -func (b *IvfpqBuild[T]) GetIndexes() []*IvfpqModel[T] { +func (b *IvfpqBuild[B, Q]) GetIndexes() []*IvfpqModel[B, Q] { return b.indexes } diff --git a/pkg/vectorindex/ivfpq/cdc_load_test.go b/pkg/vectorindex/ivfpq/cdc_load_test.go index 252ae3985e330..f1810c027088f 100644 --- a/pkg/vectorindex/ivfpq/cdc_load_test.go +++ b/pkg/vectorindex/ivfpq/cdc_load_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -60,7 +61,11 @@ func encodeChunk(t *testing.T, dim, includeBytesPerRow int, ops []cuvscdc.CdcOp, } insIdx++ } - out, err := cuvscdc.EncodeEventRecord(buf, op, pkids[i], v, inc, dim, includeBytesPerRow) + var vb []byte + if v != nil { + vb = util.UnsafeSliceToBytes(v) + } + out, err := cuvscdc.EncodeEventRecord(buf, op, pkids[i], vb, inc, 4*dim, includeBytesPerRow) require.NoError(t, err) buf = out } @@ -86,7 +91,7 @@ func TestLoadCdcEventsFromDB_RoundTrip(t *testing.T) { } defer func() { runSql = orig }() - idx := &IvfpqModel[float32]{Id: "idx-1"} + idx := &IvfpqModel[float32, float32]{Id: "idx-1"} got, err := idx.loadCdcEventsFromDB(sqlproc, tblcfg) require.NoError(t, err) require.Len(t, got, 1) @@ -105,7 +110,7 @@ func TestLoadCdcEventsFromDB_Empty(t *testing.T) { } defer func() { runSql = orig }() - idx := &IvfpqModel[float32]{Id: "idx-1"} + idx := &IvfpqModel[float32, float32]{Id: "idx-1"} got, err := idx.loadCdcEventsFromDB(sqlproc, testTblcfg()) require.NoError(t, err) require.Empty(t, got) @@ -121,7 +126,7 @@ func TestReplayEventChunks_DeleteInsertDelete(t *testing.T) { ) chunks := []cuvscdc.EventChunk{{ChunkId: 0, Data: chunkBytes}} - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Equal(t, []int64{1}, delPkids) require.Empty(t, ovPkids) @@ -139,7 +144,7 @@ func TestReplayEventChunks_FlattenOverflow(t *testing.T) { ) chunks := []cuvscdc.EventChunk{{ChunkId: 0, Data: chunkBytes}} - delPkids, ovPkids, ovVecs, _, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, ovVecs, _, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Empty(t, delPkids) require.Equal(t, []int64{10, 20}, ovPkids) @@ -158,7 +163,7 @@ func TestReplayEventChunks_MultiChunkOrder(t *testing.T) { {ChunkId: 1, Data: chunk1}, {ChunkId: 0, Data: chunk0}, } - delPkids, ovPkids, _, _, err := replayEventChunks(chunks, dim, 0) + delPkids, ovPkids, _, _, err := replayEventChunks[float32](chunks, dim, 0) require.NoError(t, err) require.Equal(t, []int64{5}, delPkids) require.Empty(t, ovPkids) @@ -225,7 +230,7 @@ func TestLoadIndex_WithCdcDeltas(t *testing.T) { } defer func() { runSql = origRunSql }() - models, err := LoadMetadata[float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) + models, err := LoadMetadata[float32, float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) require.NoError(t, err) require.Equal(t, 1, len(models)) @@ -245,7 +250,7 @@ func TestLoadIndex_WithCdcDeltas(t *testing.T) { // prefilter should drop it from the result set. data := generateTestData(testNVectors, testDim) query := data[:testDim] - keys, _, err := idx.SearchF32(query, 1, 0) + keys, _, err := idx.SearchQuantize(query, 1, 0) require.NoError(t, err) require.Equal(t, 1, len(keys)) require.NotEqual(t, ids[0], keys[0], diff --git a/pkg/vectorindex/ivfpq/model_gpu.go b/pkg/vectorindex/ivfpq/model_gpu.go index cd1ae161f8449..7e3a04ee41552 100644 --- a/pkg/vectorindex/ivfpq/model_gpu.go +++ b/pkg/vectorindex/ivfpq/model_gpu.go @@ -28,6 +28,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/sqlquote" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/cuvs" "github.com/matrixorigin/matrixone/pkg/logutil" @@ -42,9 +43,9 @@ var runSql = sqlexec.RunSql var runSql_streaming = sqlexec.RunStreamingSql // IvfpqModel wraps a GpuIvfPq index and handles load/save to secondary index tables. -type IvfpqModel[T cuvs.VectorType] struct { +type IvfpqModel[B, Q cuvs.VectorType] struct { Id string - Index *cuvs.GpuIvfPq[T] + Index *cuvs.GpuIvfPq[B, Q] Path string FileSize int64 MaxCapacity uint64 @@ -67,10 +68,10 @@ type IvfpqModel[T cuvs.VectorType] struct { // CDC insert overflow — pkids that the replay left in the brute-force // overflow (INSERT record with no later DELETE). Brute-force searched at - // query time and merged with main-index results. Always F32 regardless - // of T. + // query time and merged with main-index results. Stored in the native + // base type B (f32 or f16), matching the base-typed overflow brute force. OverflowPkids []int64 - OverflowVecs []float32 // len = len(OverflowPkids) * dim + OverflowVecs []B // len = len(OverflowPkids) * dim // INCLUDE column data carried alongside each overflow row. Layout // matches the EncodeEventRecord INSERT-record include section: @@ -89,8 +90,8 @@ type IvfpqModel[T cuvs.VectorType] struct { OverflowColMetaJSON string } -func NewIvfpqModelForBuild[T cuvs.VectorType](id string, cfg vectorindex.IndexConfig, nthread uint32, devices []int) (*IvfpqModel[T], error) { - return &IvfpqModel[T]{ +func NewIvfpqModelForBuild[B, Q cuvs.VectorType](id string, cfg vectorindex.IndexConfig, nthread uint32, devices []int) (*IvfpqModel[B, Q], error) { + return &IvfpqModel[B, Q]{ Id: id, Idxcfg: cfg, NThread: nthread, @@ -98,7 +99,7 @@ func NewIvfpqModelForBuild[T cuvs.VectorType](id string, cfg vectorindex.IndexCo }, nil } -func (idx *IvfpqModel[T]) ivfpqConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.IvfPqBuildParams, mode cuvs.DistributionMode, err error) { +func (idx *IvfpqModel[B, Q]) ivfpqConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.IvfPqBuildParams, mode cuvs.DistributionMode, err error) { cfg := idx.Idxcfg.CuvsIvfpq var ok bool cuvsMetric, ok = metric.MetricTypeToCuvsMetric[metric.MetricType(cfg.Metric)] @@ -124,7 +125,7 @@ func (idx *IvfpqModel[T]) ivfpqConfig() (cuvsMetric cuvs.DistanceType, bp cuvs.I } // InitEmpty allocates the GPU buffer for totalCount vectors. -func (idx *IvfpqModel[T]) InitEmpty(totalCount uint64) error { +func (idx *IvfpqModel[B, Q]) InitEmpty(totalCount uint64) error { if idx.Index != nil { return moerr.NewInternalErrorNoCtx("IvfpqModel: index already initialized") } @@ -138,7 +139,7 @@ func (idx *IvfpqModel[T]) InitEmpty(totalCount uint64) error { if buildMode == cuvs.Replicated { buildMode = cuvs.SingleGpu } - gi, err := cuvs.NewGpuIvfPqEmpty[T]( + gi, err := cuvs.NewGpuIvfPqEmpty[B, Q]( totalCount, uint32(idx.Idxcfg.CuvsIvfpq.Dimensions), cuvsMetric, @@ -159,18 +160,35 @@ func (idx *IvfpqModel[T]) InitEmpty(totalCount uint64) error { return nil } -func (idx *IvfpqModel[T]) AddChunkFloat(chunk []float32, chunkCount uint64, ids []int64) error { +// AddChunk appends a chunk of native storage-type (T) vectors with no +// quantization — used when the base column type equals the storage type +// (e.g. a vecf16 base stored as half). Mirrors AddChunkFloat but raw. +func (idx *IvfpqModel[B, Q]) AddChunk(chunk []Q, chunkCount uint64, ids []int64) error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("IvfpqModel: index not initialized; call InitEmpty first") } - if err := idx.Index.AddChunkFloat(chunk, chunkCount, ids); err != nil { + if err := idx.Index.AddChunk(chunk, chunkCount, ids); err != nil { return err } idx.Len += int64(chunkCount) return nil } -func (idx *IvfpqModel[T]) Build() error { +// AddChunkQuantize appends a chunk of base-typed (B) vectors, quantizing +// natively to the 1-byte storage type Q (int8/uint8). Used for a vecf16 base +// with QUANTIZATION=int8/uint8 — no f32 detour. +func (idx *IvfpqModel[B, Q]) AddChunkQuantize(chunk []B, chunkCount uint64, ids []int64) error { + if idx.Index == nil { + return moerr.NewInternalErrorNoCtx("IvfpqModel: index not initialized; call InitEmpty first") + } + if err := idx.Index.AddChunkQuantize(chunk, chunkCount, ids); err != nil { + return err + } + idx.Len += int64(chunkCount) + return nil +} + +func (idx *IvfpqModel[B, Q]) Build() error { if idx.Index == nil { return moerr.NewInternalErrorNoCtx("IvfpqModel: index not initialized") } @@ -181,7 +199,7 @@ func (idx *IvfpqModel[T]) Build() error { return nil } -func (idx *IvfpqModel[T]) Destroy() error { +func (idx *IvfpqModel[B, Q]) Destroy() error { if idx.Index != nil { if err := idx.Index.Destroy(); err != nil { return err @@ -195,7 +213,7 @@ func (idx *IvfpqModel[T]) Destroy() error { return nil } -func (idx *IvfpqModel[T]) saveToFile() error { +func (idx *IvfpqModel[B, Q]) saveToFile() error { if idx.Index == nil { return nil } @@ -246,7 +264,7 @@ func (idx *IvfpqModel[T]) saveToFile() error { return nil } -func (idx *IvfpqModel[T]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error) { +func (idx *IvfpqModel[B, Q]) ToSql(cfg vectorindex.IndexTableConfig) ([]string, error) { if err := idx.saveToFile(); err != nil { return nil, err } @@ -306,16 +324,17 @@ func joinStrings(ss []string, sep string) string { return result } -func (idx *IvfpqModel[T]) Empty() bool { +func (idx *IvfpqModel[B, Q]) Empty() bool { return idx.Len == 0 } -func (idx *IvfpqModel[T]) Full() bool { +func (idx *IvfpqModel[B, Q]) Full() bool { return idx.MaxCapacity > 0 && uint64(idx.Len) >= idx.MaxCapacity } -// SearchF32 performs a KNN search using a float32 query vector. -func (idx *IvfpqModel[T]) SearchF32(query []float32, limit uint32, nprobes uint32) (keys []int64, distances []float32, err error) { +// SearchQuantize performs a KNN search using a base-typed (B) query vector; the +// index converts B -> its storage type Q on device (was SearchF32, f32-only). +func (idx *IvfpqModel[B, Q]) SearchQuantize(query []B, limit uint32, nprobes uint32) (keys []int64, distances []float32, err error) { if idx.Index == nil { return nil, nil, moerr.NewInternalErrorNoCtx("IvfpqModel: index not loaded") } @@ -326,14 +345,14 @@ func (idx *IvfpqModel[T]) SearchF32(query []float32, limit uint32, nprobes uint3 if sp.NProbes == 0 { sp = cuvs.DefaultIvfPqSearchParams() } - res, err := idx.Index.SearchFloat(query, 1, uint32(idx.Idxcfg.CuvsIvfpq.Dimensions), limit, sp) + res, err := idx.Index.SearchQuantize(query, 1, uint32(idx.Idxcfg.CuvsIvfpq.Dimensions), limit, sp) if err != nil { return nil, nil, err } return res.Neighbors, res.Distances, nil } -func (idx *IvfpqModel[T]) Search(query []T, limit uint32, nprobes uint32) (keys []int64, distances []float32, err error) { +func (idx *IvfpqModel[B, Q]) Search(query []Q, limit uint32, nprobes uint32) (keys []int64, distances []float32, err error) { if idx.Index == nil { return nil, nil, moerr.NewInternalErrorNoCtx("IvfpqModel: index not loaded") } @@ -351,7 +370,7 @@ func (idx *IvfpqModel[T]) Search(query []T, limit uint32, nprobes uint32) (keys return res.Neighbors, res.Distances, nil } -func (idx *IvfpqModel[T]) loadChunk(ctx context.Context, +func (idx *IvfpqModel[B, Q]) loadChunk(ctx context.Context, sqlproc *sqlexec.SqlProcess, stream_chan chan executor.Result, error_chan chan error, @@ -395,7 +414,7 @@ func (idx *IvfpqModel[T]) loadChunk(ctx context.Context, // the storage table in parallel, then unpacks the tar onto the GPU, replays // the event log to derive (deleted, overflow), and applies the deletes via // Index.DeleteIds. -func (idx *IvfpqModel[T]) LoadIndex( +func (idx *IvfpqModel[B, Q]) LoadIndex( sqlproc *sqlexec.SqlProcess, idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, @@ -529,7 +548,7 @@ func (idx *IvfpqModel[T]) LoadIndex( return err } - gi, err := cuvs.NewGpuIvfPqEmpty[T]( + gi, err := cuvs.NewGpuIvfPqEmpty[B, Q]( uint64(idxcfg.IndexCapacity), uint32(idxcfg.CuvsIvfpq.Dimensions), cuvsMetric, @@ -565,7 +584,7 @@ func (idx *IvfpqModel[T]) LoadIndex( } includeBytesPerRow = ibpr } - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(eventChunks, dim, includeBytesPerRow) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[B](eventChunks, dim, includeBytesPerRow) if err != nil { gi.Destroy() return err @@ -602,7 +621,7 @@ func (idx *IvfpqModel[T]) LoadIndex( // loadCdcEventsFromDB reads the tag=1 event-log rows for this index. See // pkg/vectorindex/cagra/model_gpu.go for design notes. -func (idx *IvfpqModel[T]) loadCdcEventsFromDB( +func (idx *IvfpqModel[B, Q]) loadCdcEventsFromDB( sqlproc *sqlexec.SqlProcess, tblcfg vectorindex.IndexTableConfig, ) ([]cuvscdc.EventChunk, error) { @@ -633,16 +652,20 @@ func (idx *IvfpqModel[T]) loadCdcEventsFromDB( // replayEventChunks sorts the chunks by chunk_id, replays the records, and // flattens (deleted, overflow) into the parallel slices the IvfpqModel // struct carries (the layout buildOverflow consumes). -func replayEventChunks( +func replayEventChunks[B cuvs.VectorType]( chunks []cuvscdc.EventChunk, dim int, includeBytesPerRow int, -) ([]int64, []int64, []float32, []byte, error) { +) ([]int64, []int64, []B, []byte, error) { if len(chunks) == 0 { return nil, nil, nil, nil, nil } cuvscdc.SortChunks(chunks) - state, err := cuvscdc.ReplayEventLog(chunks, dim, includeBytesPerRow) + // The codec stores vectors as opaque bytes; the per-row byte length is + // dim * sizeof(B). Reinterpret each row's bytes back to the native base + // type B for the overflow brute force — no f32 detour. + vecBytesPerRow := dim * int(util.UnsafeSizeOf[B]()) + state, err := cuvscdc.ReplayEventLog(chunks, vecBytesPerRow, includeBytesPerRow) if err != nil { return nil, nil, nil, nil, err } @@ -654,14 +677,15 @@ func replayEventChunks( return deletedPkids, nil, nil, nil, nil } ovPkids := make([]int64, len(state.Overflow)) - ovVecs := make([]float32, len(state.Overflow)*dim) + ovVecs := make([]B, len(state.Overflow)*dim) + ovVecBytes := util.UnsafeSliceToBytes(ovVecs) var ovInc []byte if includeBytesPerRow > 0 { ovInc = make([]byte, len(state.Overflow)*includeBytesPerRow) } for i, e := range state.Overflow { ovPkids[i] = e.Pkid - copy(ovVecs[i*dim:(i+1)*dim], e.Vec) + copy(ovVecBytes[i*vecBytesPerRow:(i+1)*vecBytesPerRow], e.Vec) if includeBytesPerRow > 0 { copy(ovInc[i*includeBytesPerRow:(i+1)*includeBytesPerRow], e.Include) } @@ -669,7 +693,7 @@ func replayEventChunks( return deletedPkids, ovPkids, ovVecs, ovInc, nil } -func (idx *IvfpqModel[T]) Unload() error { +func (idx *IvfpqModel[B, Q]) Unload() error { if idx.Index == nil { return nil } @@ -688,7 +712,7 @@ func (idx *IvfpqModel[T]) Unload() error { } // LoadMetadata loads IvfpqModel descriptors from the metadata table. -func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*IvfpqModel[T], error) { +func LoadMetadata[B, Q cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, metatbl string) ([]*IvfpqModel[B, Q], error) { sql := fmt.Sprintf("SELECT * FROM %s ORDER BY timestamp ASC", sqlquote.QualifiedIdent(dbname, metatbl)) res, err := runSql(sqlproc, sql) if err != nil { @@ -701,7 +725,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, total += bat.RowCount() } - indexes := make([]*IvfpqModel[T], 0, total) + indexes := make([]*IvfpqModel[B, Q], 0, total) for _, bat := range res.Batches { idVec := bat.Vecs[0] chksumVec := bat.Vecs[1] @@ -712,7 +736,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, chksum := chksumVec.GetStringAt(i) ts := vector.GetFixedAtWithTypeCheck[int64](tsVec, i) fs := vector.GetFixedAtWithTypeCheck[int64](fsVec, i) - idx := &IvfpqModel[T]{Id: id, Checksum: chksum, Timestamp: ts, FileSize: fs} + idx := &IvfpqModel[B, Q]{Id: id, Checksum: chksum, Timestamp: ts, FileSize: fs} indexes = append(indexes, idx) } } @@ -720,7 +744,7 @@ func LoadMetadata[T cuvs.VectorType](sqlproc *sqlexec.SqlProcess, dbname string, } // ToDeleteSql generates DELETE SQL for storage and metadata tables. -func (idx *IvfpqModel[T]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error) { +func (idx *IvfpqModel[B, Q]) ToDeleteSql(cfg vectorindex.IndexTableConfig) ([]string, error) { sqls := make([]string, 0, 2) sqls = append(sqls, fmt.Sprintf("DELETE FROM %s WHERE %s = %s", sqlquote.QualifiedIdent(cfg.DbName, cfg.IndexTable), catalog.Ivfpq_TblCol_Storage_Index_Id, sqlquote.String(idx.Id))) diff --git a/pkg/vectorindex/ivfpq/model_test.go b/pkg/vectorindex/ivfpq/model_test.go index 8b41fb7e8251b..f2b9bd66859fb 100644 --- a/pkg/vectorindex/ivfpq/model_test.go +++ b/pkg/vectorindex/ivfpq/model_test.go @@ -128,7 +128,7 @@ func makeIndexBatch(proc *process.Process, tarPath string) *batch.Batch { // buildTestModel builds an IvfpqModel, calls Build, and saves via ToSql. // Index is nil after ToSql (GPU memory freed). Path/Checksum/FileSize are set. -func buildTestModel(t *testing.T, id string, ids []int64) *IvfpqModel[float32] { +func buildTestModel(t *testing.T, id string, ids []int64) *IvfpqModel[float32, float32] { t.Helper() idxcfg := testIdxcfg() @@ -141,13 +141,13 @@ func buildTestModel(t *testing.T, id string, ids []int64) *IvfpqModel[float32] { } } - m, err := NewIvfpqModelForBuild[float32](id, idxcfg, 1, []int{0}) + m, err := NewIvfpqModelForBuild[float32, float32](id, idxcfg, 1, []int{0}) require.NoError(t, err) err = m.InitEmpty(testNVectors) require.NoError(t, err) - err = m.AddChunkFloat(data, testNVectors, ids) + err = m.AddChunkQuantize(data, testNVectors, ids) require.NoError(t, err) err = m.Build() @@ -185,7 +185,7 @@ func TestModelStreamError(t *testing.T) { } defer func() { runSql = origRunSql }() - idx := &IvfpqModel[float32]{ + idx := &IvfpqModel[float32, float32]{ Id: "test-stream-err", FileSize: 1024, Checksum: "fake-checksum", @@ -212,13 +212,13 @@ func TestModelBuildAndLoad(t *testing.T) { } // ---- Build ---- - built, err := NewIvfpqModelForBuild[float32]("test-build", idxcfg, 1, []int{0}) + built, err := NewIvfpqModelForBuild[float32, float32]("test-build", idxcfg, 1, []int{0}) require.NoError(t, err) err = built.InitEmpty(testNVectors) require.NoError(t, err) - err = built.AddChunkFloat(data, testNVectors, ids) + err = built.AddChunkQuantize(data, testNVectors, ids) require.NoError(t, err) err = built.Build() @@ -248,7 +248,7 @@ func TestModelBuildAndLoad(t *testing.T) { defer func() { runSql = origRunSql }() // ---- Load from local tar ---- - loader := &IvfpqModel[float32]{ + loader := &IvfpqModel[float32, float32]{ Id: "test-build", Path: tarPath, Checksum: checksum, @@ -267,7 +267,7 @@ func TestModelBuildAndLoad(t *testing.T) { // ---- Search ---- query := data[:testDim] - keys, dists, err := loader.SearchF32(query, 1, 0) + keys, dists, err := loader.SearchQuantize(query, 1, 0) require.NoError(t, err) require.Equal(t, 1, len(keys)) require.Equal(t, 1, len(dists)) @@ -335,7 +335,7 @@ func TestModelLoadFromDB(t *testing.T) { } defer func() { runSql = origRunSql }() - models, err := LoadMetadata[float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) + models, err := LoadMetadata[float32, float32](sqlproc, tblcfg.DbName, tblcfg.MetadataTable) require.NoError(t, err) require.Equal(t, 1, len(models)) @@ -350,7 +350,7 @@ func TestModelLoadFromDB(t *testing.T) { data := generateTestData(testNVectors, testDim) query := data[:testDim] - keys, dists, err := idx.SearchF32(query, 1, 0) + keys, dists, err := idx.SearchQuantize(query, 1, 0) require.NoError(t, err) require.Equal(t, 1, len(keys)) fmt.Printf("LoadFromDB SearchF32: keys=%v dists=%v\n", keys, dists) @@ -360,7 +360,7 @@ func TestModelLoadFromDB(t *testing.T) { func TestModelNil(t *testing.T) { var tblcfg vectorindex.IndexTableConfig - idx := &IvfpqModel[float32]{} + idx := &IvfpqModel[float32, float32]{} // InitEmpty fails because Devices is empty. err := idx.InitEmpty(10) @@ -372,16 +372,16 @@ func TestModelNil(t *testing.T) { require.NotNil(t, err) // AddChunkFloat fails because Index is nil. - err = idx.AddChunkFloat([]float32{1, 2, 3, 4}, 1, []int64{1}) + err = idx.AddChunkQuantize([]float32{1, 2, 3, 4}, 1, []int64{1}) require.NotNil(t, err) // SearchF32 fails because Index is nil. - _, _, err = idx.SearchF32([]float32{0, 0, 0, 0}, 1, 0) + _, _, err = idx.SearchQuantize([]float32{0, 0, 0, 0}, 1, 0) require.NotNil(t, err) // SearchF32 with nil query fails. - idx2 := &IvfpqModel[float32]{} - _, _, err = idx2.SearchF32(nil, 1, 0) + idx2 := &IvfpqModel[float32, float32]{} + _, _, err = idx2.SearchQuantize(nil, 1, 0) require.NotNil(t, err) // ToSql on a never-built model returns empty. @@ -411,7 +411,7 @@ func TestModelEmptyBuild(t *testing.T) { idxcfg := testIdxcfg() tblcfg := testTblcfg() - built, err := NewIvfpqModelForBuild[float32]("test-empty", idxcfg, 1, []int{0}) + built, err := NewIvfpqModelForBuild[float32, float32]("test-empty", idxcfg, 1, []int{0}) require.NoError(t, err) // Not dirty → ToSql returns empty slice. diff --git a/pkg/vectorindex/ivfpq/plugin/compile/compile.go b/pkg/vectorindex/ivfpq/plugin/compile/compile.go index a8695affe0e81..2094983975da2 100644 --- a/pkg/vectorindex/ivfpq/plugin/compile/compile.go +++ b/pkg/vectorindex/ivfpq/plugin/compile/compile.go @@ -279,14 +279,30 @@ func registerIdxcronUpdate( // IVF-PQ supports updating `lists` at REINDEX time — mirrors IVF-FLAT // since both algorithms key on the inverted-list count for their build. func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.ReindexParamUpdate) (map[string]string, error) { - return compileplugin.MergeReindexParams(old, alter, "ivfpq", + // Merge first, then validate the EFFECTIVE quantization via the per-algo + // catalog hook (the single home shared with CREATE). The merged map is the + // index's actual post-reindex config: the value the reindex set, or — when + // the reindex omitted QUANTIZATION (e.g. the idxcron-issued rebuild) — the + // value already stored on the index. Validating the merge (not the raw alter + // delta) means the check is never skipped just because the statement omitted + // quantization, and quantization and op_type come from one consistent source. + merged, err := compileplugin.MergeReindexParams(old, alter, "ivfpq", catalog.IndexAlgoParamLists, catalog.IndexAlgoParamKmeansTrainPercent, catalog.IndexAlgoParamKmeansMaxIteration, catalog.IndexAlgoParamMaxIndexCapacity, catalog.HnswM, catalog.BitsPerCode, + catalog.Quantization, ) + if err != nil { + return nil, err + } + if err := (ivfpqruntime.CatalogHooks{}).ValidQuantization( + merged[catalog.Quantization], merged[catalog.IndexAlgoParamOpType]); err != nil { + return nil, err + } + return merged, nil } // HandleDropIndex runs algorithm-specific cleanup beyond the generic @@ -299,6 +315,12 @@ func (Hooks) ValidateReindexParams(old map[string]string, alter compileplugin.Re // so this is a no-op. Compare HNSW, which does maintain CDC tasks. func (Hooks) HandleDropIndex(_ compileplugin.CompileContext, defs map[string]*plan.IndexDef) error { logutil.Infof("[plugin] ivfpq HandleDropIndex: defs=%d", len(defs)) + // Evict the cached search index so its GPU resources are freed NOW, rather + // than lingering until the 5-min VectorIndexCacheTTL housekeeping reaps it. + // Mirrors the create-side cache.Cache.Remove(storageDef.IndexTableName). + if storageDef, ok := defs[catalog.Ivfpq_TblType_Storage]; ok { + cache.Cache.Remove(storageDef.IndexTableName) + } return nil } diff --git a/pkg/vectorindex/ivfpq/plugin/compile/compile_test.go b/pkg/vectorindex/ivfpq/plugin/compile/compile_test.go index 94bad4e94640f..e1e4c3b74b69a 100644 --- a/pkg/vectorindex/ivfpq/plugin/compile/compile_test.go +++ b/pkg/vectorindex/ivfpq/plugin/compile/compile_test.go @@ -254,6 +254,36 @@ func TestIvfpqValidateReindexParams_Unsupported(t *testing.T) { require.Contains(t, err.Error(), catalog.HnswEfConstruction) } +// TestIvfpqValidateReindexParams_Quantization: IVF-PQ (cuvs) accepts the +// cuvs-supported quantization names and rejects others (e.g. bf16). +func TestIvfpqValidateReindexParams_Quantization(t *testing.T) { + got, err := Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "int8"}, + }) + require.NoError(t, err) + require.Equal(t, "int8", got[catalog.Quantization]) + + _, err = Hooks{}.ValidateReindexParams(nil, compileplugin.ReindexParamUpdate{ + Params: map[string]string{catalog.Quantization: "bf16"}, + }) + require.Error(t, err) + + // int8/uint8 on a non-L2 (inner-product) index IS rejected at REINDEX via the + // ValidQuantization hook: the merged op_type is inner-product and the + // int8/uint8 affine quantizer only preserves L2 geometry. + _, err = Hooks{}.ValidateReindexParams( + map[string]string{catalog.IndexAlgoParamOpType: "vector_ip_ops"}, + compileplugin.ReindexParamUpdate{Params: map[string]string{catalog.Quantization: "uint8"}}) + require.Error(t, err) + + // ...but uint8 with L2 (the merged op_type) is accepted. + got, err = Hooks{}.ValidateReindexParams( + map[string]string{catalog.IndexAlgoParamOpType: "vector_l2_ops"}, + compileplugin.ReindexParamUpdate{Params: map[string]string{catalog.Quantization: "uint8"}}) + require.NoError(t, err) + require.Equal(t, "uint8", got[catalog.Quantization]) +} + func TestIvfpqHandleDropIndex(t *testing.T) { require.NoError(t, Hooks{}.HandleDropIndex(nil, nil)) } diff --git a/pkg/vectorindex/ivfpq/plugin/iscp/iscp.go b/pkg/vectorindex/ivfpq/plugin/iscp/iscp.go index 24adcae0a1103..c444f71779131 100644 --- a/pkg/vectorindex/ivfpq/plugin/iscp/iscp.go +++ b/pkg/vectorindex/ivfpq/plugin/iscp/iscp.go @@ -49,6 +49,6 @@ func (Hooks) Run(c *iscppkg.IndexConsumer, ctx context.Context, errch chan error iscppkg.RunCuvs(c, ctx, errch, r, func(sqlproc *sqlexec.SqlProcess) (iscppkg.CuvsSync, error) { w := c.SqlWriter().(*iscppkg.CuvsCdcWriter) return ivfpq.NewIvfpqSync(sqlproc, w.DbName(), w.TblName(), w.IndexName(), - w.IndexDef(), w.Dimension(), w.ColMetaJSON()) + w.IndexDef(), w.Dimension(), w.BaseVectorType(), w.ColMetaJSON()) }) } diff --git a/pkg/vectorindex/ivfpq/plugin/plan/plan_test.go b/pkg/vectorindex/ivfpq/plugin/plan/plan_test.go index 9eabac4a943e4..45e2f7b9bd268 100644 --- a/pkg/vectorindex/ivfpq/plugin/plan/plan_test.go +++ b/pkg/vectorindex/ivfpq/plugin/plan/plan_test.go @@ -168,6 +168,62 @@ func TestBuildSecondaryIndexDefs_OK(t *testing.T) { require.NotNil(t, tblDefs[1].Pkey) } +// indexOnQuant builds a single-column *tree.Index over colName with a +// QUANTIZATION option. +func indexOnQuant(colName, quant string) *tree.Index { + idx := indexOn(colName) + idx.IndexOption = &tree.IndexOption{Quantization: quant} + return idx +} + +// f16ColMap returns a colMap with an int64 pk and a vecf16 base column. +func f16ColMap(pkName, vecName string) map[string]*plan.ColDef { + m := vecColMap(pkName, vecName) + m[vecName].Typ.Id = int32(types.T_array_float16) + return m +} + +// TestBuildSecondaryIndexDefs_F16Base: a vecf16 base column is accepted. +func TestBuildSecondaryIndexDefs_F16Base(t *testing.T) { + idxDefs, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("vec"), f16ColMap("id", "vec"), nil, "id") + require.NoError(t, err) + require.Len(t, idxDefs, 2) +} + +// TestBuildSecondaryIndexDefs_UnsupportedBase: only vecf32 / vecf16 are valid +// base columns; vecf64 / vecbf16 / vecint8 / vecuint8 are rejected. +func TestBuildSecondaryIndexDefs_UnsupportedBase(t *testing.T) { + for _, oid := range []types.T{ + types.T_array_float64, types.T_array_bf16, types.T_array_int8, types.T_array_uint8, + } { + colMap := vecColMap("id", "vec") + colMap["vec"].Typ.Id = int32(oid) + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOn("vec"), colMap, nil, "id") + require.Error(t, err, "base type %s must be rejected", oid) + } +} + +// TestBuildSecondaryIndexDefs_F16UpcastRejected: vecf16 base + QUANTIZATION +// float32 is an upcast (4 > 2 bytes) and must be rejected by the downcast +// guard. (The accepted downcast path — f16 -> int8/uint8 — is exercised +// end-to-end by the GPU functional BVT, since the full def build past the +// guard needs a richer compiler context than this stub provides.) +func TestBuildSecondaryIndexDefs_F16UpcastRejected(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "float32"), f16ColMap("id", "vec"), nil, "id") + require.Error(t, err) +} + +// TestBuildSecondaryIndexDefs_BF16QuantRejected: QUANTIZATION 'bf16' has no GPU +// bfloat16 storage (cuVS has no bfloat16 index/quantizer), so it must be rejected +// rather than silently falling back to f32 storage — even though it passes the +// downcast width guard (bf16 is 2 bytes). Rejected on both f32 and f16 bases. +func TestBuildSecondaryIndexDefs_BF16QuantRejected(t *testing.T) { + _, _, err := Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "bf16"), vecColMap("id", "vec"), nil, "id") + require.Error(t, err, "f32 base + bf16 quant must be rejected") + _, _, err = Hooks{}.BuildSecondaryIndexDefs(newStubCompilerContext(), indexOnQuant("vec", "bf16"), f16ColMap("id", "vec"), nil, "id") + require.Error(t, err, "f16 base + bf16 quant must be rejected") +} + // --- schema.go: BuildFullTextIndexDefs ------------------------------------- func TestBuildFullTextIndexDefs_Unsupported(t *testing.T) { diff --git a/pkg/vectorindex/ivfpq/plugin/plan/schema.go b/pkg/vectorindex/ivfpq/plugin/plan/schema.go index f4b291ba03264..4c24e61b4c4b7 100644 --- a/pkg/vectorindex/ivfpq/plugin/plan/schema.go +++ b/pkg/vectorindex/ivfpq/plugin/plan/schema.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" "github.com/matrixorigin/matrixone/pkg/sql/util" ivfpqrt "github.com/matrixorigin/matrixone/pkg/vectorindex/ivfpq/plugin/runtime" + "github.com/matrixorigin/matrixone/pkg/vectorindex/quantizer" ) // ivfpqCatalogHooks is the shared (stateless) catalog-hooks instance used for @@ -88,7 +89,40 @@ func (Hooks) BuildSecondaryIndexDefs( return nil, nil, moerr.NewInvalidInputf(ctx.GetContext(), "column '%s' is not exist", indexInfo.KeyParts[0].ColName.ColNameOrigin()) } if !catalogplugin.SupportsVectorType(ivfpqCatalogHooks, types.T(colMap[name].Typ.Id)) { - return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "IvfPQ only supports VECF32 column types") + return nil, nil, moerr.NewNotSupported(ctx.GetContext(), "IvfPQ only supports VECF32 / VECF16 base column types") + } + // QUANTIZATION is downcast-only: the storage element must be the same width + // or narrower than the base column (f16 base -> int8/uint8 OK; f16 base -> + // float32 is an upcast and rejected). Mirrors ivfflat's guard. + if indexInfo.IndexOption != nil && indexInfo.IndexOption.Quantization != "" { + if qt, ok := quantizer.ToVectorType(indexInfo.IndexOption.Quantization); ok { + // bf16 storage does not exist on the GPU (cuVS/cgo has no bfloat16 + // index or quantizer), so reject it explicitly rather than silently + // falling back to f32 storage. Supported cuvs storage = f16/int8/uint8. + if qt == types.T_array_bf16 { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "IvfPQ does not support '%s' quantization (no GPU bfloat16 storage); use 'float16', 'int8', or 'uint8'", + indexInfo.IndexOption.Quantization) + } + baseSize := types.Type{Oid: types.T(colMap[name].Typ.Id)}.GetArrayElementSize() + quantSize := types.Type{Oid: qt}.GetArrayElementSize() + if quantSize > baseSize { + return nil, nil, moerr.NewNotSupportedf(ctx.GetContext(), + "IvfPQ QUANTIZATION '%s' (%d bytes/element) cannot upcast base column %s (%d bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type", + indexInfo.IndexOption.Quantization, quantSize, + types.T(colMap[name].Typ.Id).String(), baseSize) + } + // int8/uint8 quantization is L2-only (the affine quantizer breaks + // inner-product / cosine geometry). Gated by the per-algo catalog + // hook — the single home shared with REINDEX + // (compile/ValidateReindexParams) — so CREATE and REINDEX cannot + // drift. (bf16 and width/upcast are rejected above with base-column- + // aware messages before reaching here.) + if err := ivfpqCatalogHooks.ValidQuantization( + indexInfo.IndexOption.Quantization, indexInfo.IndexOption.AlgoParamVectorOpType); err != nil { + return nil, nil, err + } + } } for _, existedIndex := range existedIndexes { if existedIndex.IndexAlgo == catalog.MoIndexIvfpqAlgo.ToString() && existedIndex.Parts[0] == name { diff --git a/pkg/vectorindex/ivfpq/plugin/runtime/runtime.go b/pkg/vectorindex/ivfpq/plugin/runtime/runtime.go index ea8b5144ff34f..d0fb8190b43f2 100644 --- a/pkg/vectorindex/ivfpq/plugin/runtime/runtime.go +++ b/pkg/vectorindex/ivfpq/plugin/runtime/runtime.go @@ -126,12 +126,43 @@ func (CatalogHooks) ExperimentalFlag() string { return IvfpqIndexFlag } // "vector_l2_ops") to a stable internal identifier. Used by plan-side // op_type validation when matching an ORDER BY distance function against // the index's declared op_type. -// SupportedVectorTypes: IVF-PQ (cuvs) indexes f32 vectors only. -func (CatalogHooks) SupportedVectorTypes() []types.T { return []types.T{types.T_array_float32} } +// SupportedVectorTypes: IVF-PQ (cuvs) accepts f32 and f16 base columns. f16 is +// stored natively as half, or downcast-quantized to int8/uint8 via QUANTIZATION. +// int8/uint8 base columns are unsupported (the CDC overflow brute force is f32/f16-only). +func (CatalogHooks) SupportedVectorTypes() []types.T { + return []types.T{types.T_array_float32, types.T_array_float16} +} // SupportedPrimaryKeyTypes: requires an int64 primary key. func (CatalogHooks) SupportedPrimaryKeyTypes() []types.T { return []types.T{types.T_int64} } +// ValidQuantization gates the (quantization, op_type) pair for IVF-PQ (cuvs): +// the value must be a cuvs storage type (float32/float16/int8/uint8 — bf16 and +// float64 are absent from CuvsQuantizationNameToType), and the 1-byte int8/uint8 +// scalar quantizer is L2-only (its affine map q(x)=scalar*x+offset preserves L2 +// ordering — the offset cancels in a difference — but biases inner-product and +// rotates cosine angles). One home for CREATE (plan/schema) and REINDEX +// (compile/ValidateReindexParams). quant=="" => no quantization (valid); op=="" +// => value rule only. +func (CatalogHooks) ValidQuantization(quant, op string) error { + if quant == "" { + return nil + } + quant = strings.ToLower(quant) + if !metric.ValidQuantization(quant) { + return moerr.NewNotSupportedNoCtxf( + "ivfpq quantization %q (supported: float32, float16, int8, uint8)", quant) + } + if quant == metric.Quantization_INT8_Str || quant == metric.Quantization_UINT8_Str { + switch strings.ToLower(op) { + case metric.OpType_InnerProduct, metric.OpType_CosineDistance: + return moerr.NewNotSupportedNoCtxf( + "ivfpq quantization %q is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry", quant) + } + } + return nil +} + // SupportedIncludeColumnTypes: cuvs INCLUDE (pre-filter) columns accept // int32/int64/float32/float64 scalars. func (CatalogHooks) SupportedIncludeColumnTypes() []types.T { diff --git a/pkg/vectorindex/ivfpq/plugin/runtime/runtime_test.go b/pkg/vectorindex/ivfpq/plugin/runtime/runtime_test.go index 7aef203e7c144..20adf3852562f 100644 --- a/pkg/vectorindex/ivfpq/plugin/runtime/runtime_test.go +++ b/pkg/vectorindex/ivfpq/plugin/runtime/runtime_test.go @@ -189,3 +189,15 @@ func TestIvfpqJoinIncludeColumns(t *testing.T) { } require.Equal(t, "a,b", joinIncludeColumns(cols)) } + +// TestIvfpqValidQuantization exercises the per-algo (quant, op) catalog hook +// shared by CREATE and REINDEX. +func TestIvfpqValidQuantization(t *testing.T) { + h := CatalogHooks{} + require.NoError(t, h.ValidQuantization("", "vector_ip_ops")) + require.NoError(t, h.ValidQuantization("float16", "vector_ip_ops")) + require.NoError(t, h.ValidQuantization("uint8", "vector_l2_ops")) + require.Error(t, h.ValidQuantization("int8", "vector_ip_ops")) + require.Error(t, h.ValidQuantization("uint8", "vector_cosine_ops")) + require.Error(t, h.ValidQuantization("bf16", "vector_l2_ops")) +} diff --git a/pkg/vectorindex/ivfpq/search_gpu.go b/pkg/vectorindex/ivfpq/search_gpu.go index 01aecf993f086..078b89ea4aa50 100644 --- a/pkg/vectorindex/ivfpq/search_gpu.go +++ b/pkg/vectorindex/ivfpq/search_gpu.go @@ -27,19 +27,19 @@ import ( ) // IvfpqSearch implements cache.VectorIndexSearchIf for GPU IVF-PQ indexes. -type IvfpqSearch[T cuvs.VectorType] struct { +type IvfpqSearch[B, Q cuvs.VectorType] struct { Idxcfg vectorindex.IndexConfig Tblcfg vectorindex.IndexTableConfig - Indexes []*IvfpqModel[T] - MultiIndex *cuvs.MultiGpuIvfPq[T] - Overflow *cuvs.GpuBruteForce[T] // CDC insert overflow; nil when no overflow records exist + Indexes []*IvfpqModel[B, Q] + MultiIndex *cuvs.MultiGpuIvfPq[B, Q] + Overflow cuvs.BruteForceOverflow[B] // CDC insert overflow; nil when no overflow records exist Devices []int ThreadsSearch int64 } -func NewIvfpqSearch[T cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, devices []int) *IvfpqSearch[T] { +func NewIvfpqSearch[B, Q cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg vectorindex.IndexTableConfig, devices []int) *IvfpqSearch[B, Q] { nthread := vectorindex.GetConcurrency(tblcfg.ThreadsSearch) - return &IvfpqSearch[T]{ + return &IvfpqSearch[B, Q]{ Idxcfg: idxcfg, Tblcfg: tblcfg, Devices: devices, @@ -48,12 +48,7 @@ func NewIvfpqSearch[T cuvs.VectorType](idxcfg vectorindex.IndexConfig, tblcfg ve } // Search implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { - query, ok := anyquery.([]float32) - if !ok { - return nil, nil, moerr.NewInternalErrorNoCtx("IvfpqSearch: query type mismatch") - } - +func (s *IvfpqSearch[B, Q]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt vectorindex.RuntimeConfig) (keys any, distances []float64, err error) { limit := rt.Limit if s.MultiIndex == nil { @@ -77,10 +72,18 @@ func (s *IvfpqSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt ve neighbors64 []int64 dists32 []float32 ) + // Any base (f32 or vecf16) routes its native base-typed (B) query through the + // const-B* search_quantize path — cuVS converts B to storage Q on device (B==Q + // copy for a direct index, learned/cast quantizer for a compressed one). The + // query asserts to []B for both float32 (B==float) and Float16 (B==half) base. + qB, ok := anyquery.([]B) + if !ok { + return nil, nil, moerr.NewInternalErrorNoCtx("IvfpqSearch: query type mismatch") + } if rt.FilterJSON != "" { - neighbors64, dists32, err = s.MultiIndex.SearchFloat32WithFilter(query, 1, dim, uint32(limit), sp, rt.FilterJSON) + neighbors64, dists32, err = s.MultiIndex.SearchQuantizeWithFilter(qB, 1, dim, uint32(limit), sp, rt.FilterJSON) } else { - neighbors64, dists32, err = s.MultiIndex.SearchFloat32(query, 1, dim, uint32(limit), sp) + neighbors64, dists32, err = s.MultiIndex.SearchQuantize(qB, 1, dim, uint32(limit), sp) } if err != nil { return nil, nil, err @@ -104,7 +107,7 @@ func (s *IvfpqSearch[T]) Search(sqlproc *sqlexec.SqlProcess, anyquery any, rt ve } // SearchFloat32 implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { +func (s *IvfpqSearch[B, Q]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt vectorindex.RuntimeConfig, outKeys []int64, outDists []float32) error { keys, dists, err := s.Search(proc, query, rt) if err != nil { return err @@ -124,8 +127,8 @@ func (s *IvfpqSearch[T]) SearchFloat32(proc *sqlexec.SqlProcess, query any, rt v } // Load implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) Load(sqlproc *sqlexec.SqlProcess) (err error) { - indexes, err := LoadMetadata[T](sqlproc, s.Tblcfg.DbName, s.Tblcfg.MetadataTable) +func (s *IvfpqSearch[B, Q]) Load(sqlproc *sqlexec.SqlProcess) (err error) { + indexes, err := LoadMetadata[B, Q](sqlproc, s.Tblcfg.DbName, s.Tblcfg.MetadataTable) if err != nil { return err } @@ -158,7 +161,7 @@ func (s *IvfpqSearch[T]) Load(sqlproc *sqlexec.SqlProcess) (err error) { // loadCdcTail mirrors cagra.CagraSearch.loadCdcTail — see that for the // architectural commentary. Differs only in the IndexConfig type slot and // the GpuIvfPq element type. -func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { +func (s *IvfpqSearch[B, Q]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { var ( includeBytesPerRow int colMetaJSON string @@ -175,7 +178,7 @@ func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } } - stub := &IvfpqModel[T]{Id: vectorindex.CdcTailId} + stub := &IvfpqModel[B, Q]{Id: vectorindex.CdcTailId} chunks, err := stub.loadCdcEventsFromDB(sqlproc, s.Tblcfg) if err != nil { return err @@ -199,7 +202,7 @@ func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } dim := int(s.Idxcfg.CuvsIvfpq.Dimensions) - delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks(chunks, dim, includeBytesPerRow) + delPkids, ovPkids, ovVecs, ovInc, err := replayEventChunks[B](chunks, dim, includeBytesPerRow) if err != nil { return err } @@ -215,7 +218,7 @@ func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } } - s.Indexes = append(s.Indexes, &IvfpqModel[T]{ + s.Indexes = append(s.Indexes, &IvfpqModel[B, Q]{ Id: vectorindex.CdcTailId, DeletedPkids: delPkids, OverflowPkids: ovPkids, @@ -228,8 +231,8 @@ func (s *IvfpqSearch[T]) loadCdcTail(sqlproc *sqlexec.SqlProcess) error { } // addOverflowFilterChunks — see cagra/search_gpu.go for docs. -func addOverflowFilterChunks[T cuvs.VectorType]( - bf *cuvs.GpuBruteForce[T], +func addOverflowFilterChunks[B, OB cuvs.VectorType]( + bf *cuvs.GpuBruteForce[B, OB], colMetaJSON string, includeBytes []byte, nrows uint64, @@ -251,7 +254,7 @@ func addOverflowFilterChunks[T cuvs.VectorType]( // loaded model's CDC insert overflow. When the underlying index has INCLUDE // columns, the brute-force is set up with the matching FilterStore so a // filtered query can prefilter overflow rows. -func (s *IvfpqSearch[T]) buildOverflow() error { +func (s *IvfpqSearch[B, Q]) buildOverflow() error { total := uint64(0) for _, m := range s.Indexes { total += uint64(len(m.OverflowPkids)) @@ -272,25 +275,46 @@ func (s *IvfpqSearch[T]) buildOverflow() error { device = s.Devices[0] } - bf, err := cuvs.NewGpuBruteForceEmpty[T]( - total, dim, cuvsMetric, uint32(s.ThreadsSearch), device) + // cuVS brute force can only store float/half. Pick the overflow storage type + // OB from the index storage Q: keep Q when float/half, else fall back to base + // B (which is always float/half). The type-erased BruteForceOverflow[B] holds + // either concrete *GpuBruteForce[B, OB]. + var ( + ov cuvs.BruteForceOverflow[B] + err error + ) + switch cuvs.GetQuantization[Q]() { + case cuvs.F32, cuvs.F16: + ov, err = buildOverflowBF[B, Q](s.Indexes, total, dim, cuvsMetric, device, uint32(s.ThreadsSearch)) + default: // INT8/UINT8: brute force can't store these → store base B. + ov, err = buildOverflowBF[B, B](s.Indexes, total, dim, cuvsMetric, device, uint32(s.ThreadsSearch)) + } if err != nil { return err } + s.Overflow = ov + return nil +} + +// buildOverflowBF — see cagra/search_gpu.go. +func buildOverflowBF[B, OB cuvs.VectorType, Q cuvs.VectorType]( + indexes []*IvfpqModel[B, Q], + total uint64, dim uint32, cuvsMetric cuvs.DistanceType, device int, threads uint32, +) (cuvs.BruteForceOverflow[B], error) { + bf, err := cuvs.NewGpuBruteForceEmpty[B, OB](total, dim, cuvsMetric, threads, device) + if err != nil { + return nil, err + } if err = bf.Start(); err != nil { bf.Destroy() - return err + return nil, err } - // INCLUDE-column wiring — pull the col-meta JSON from the first loaded - // model (every shard agrees by construction). For small-data-only - // indexes (no tag=0 sub-index ever built) the synthetic CDC-tail model - // carries the colMetaJSON recovered from the CdcOpHeader record. var ( colMetaJSON string includeBytesPerRow int ) - for _, m := range s.Indexes { + for _, m := range indexes { if m.Index != nil { colMetaJSON = m.Index.GetFilterColMetaJSON() includeBytesPerRow = m.IncludeBytesPerRow @@ -298,7 +322,7 @@ func (s *IvfpqSearch[T]) buildOverflow() error { } } if colMetaJSON == "" { - for _, m := range s.Indexes { + for _, m := range indexes { if m.OverflowColMetaJSON != "" { colMetaJSON = m.OverflowColMetaJSON includeBytesPerRow = m.IncludeBytesPerRow @@ -309,32 +333,33 @@ func (s *IvfpqSearch[T]) buildOverflow() error { if colMetaJSON != "" && includeBytesPerRow > 0 { if err = bf.SetFilterColumns(colMetaJSON, total); err != nil { bf.Destroy() - return err + return nil, err } } - for _, m := range s.Indexes { + for _, m := range indexes { if len(m.OverflowPkids) == 0 { continue } count := uint64(len(m.OverflowPkids)) - if err = bf.AddChunkFloat(m.OverflowVecs, count, m.OverflowPkids); err != nil { + // Base-typed (B) overflow vectors; AddChunkQuantize converts B -> Q storage + // on the C++ side (native store when B==Q, f32->f16 cast otherwise). + if err = bf.AddChunkQuantize(m.OverflowVecs, count, m.OverflowPkids); err != nil { bf.Destroy() - return err + return nil, err } if colMetaJSON != "" && includeBytesPerRow > 0 { if err = addOverflowFilterChunks(bf, colMetaJSON, m.OverflowIncludeBytes, count, includeBytesPerRow); err != nil { bf.Destroy() - return err + return nil, err } } } if err = bf.Build(); err != nil { bf.Destroy() - return err + return nil, err } - s.Overflow = bf - return nil + return bf, nil } // buildMultiIndex assembles a MultiGpuIvfPq from the loaded indexes. @@ -344,14 +369,14 @@ func (s *IvfpqSearch[T]) buildOverflow() error { // s.MultiIndex == nil — that's the load-bearing path for "no main // index + no brute-force → empty result". Any future regression here // will fail TestIvfpqSearchEmpty. -func (s *IvfpqSearch[T]) buildMultiIndex() (*cuvs.MultiGpuIvfPq[T], error) { +func (s *IvfpqSearch[B, Q]) buildMultiIndex() (*cuvs.MultiGpuIvfPq[B, Q], error) { cuvsMetric, ok := metric.MetricTypeToCuvsMetric[metric.MetricType(s.Idxcfg.CuvsIvfpq.Metric)] if !ok { // Unsupported metric is a real error — surface it rather than returning a // nil index, which Search would treat as an (empty) success. return nil, moerr.NewInternalErrorNoCtxf("IvfpqSearch: unsupported metric type %v", s.Idxcfg.CuvsIvfpq.Metric) } - gpuIndices := make([]*cuvs.GpuIvfPq[T], 0, len(s.Indexes)) + gpuIndices := make([]*cuvs.GpuIvfPq[B, Q], 0, len(s.Indexes)) for _, model := range s.Indexes { if model.Index != nil { gpuIndices = append(gpuIndices, model.Index) @@ -367,7 +392,7 @@ func (s *IvfpqSearch[T]) buildMultiIndex() (*cuvs.MultiGpuIvfPq[T], error) { } // loadIndexes loads each model's index data from the database. -func (s *IvfpqSearch[T]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*IvfpqModel[T]) ([]*IvfpqModel[T], error) { +func (s *IvfpqSearch[B, Q]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*IvfpqModel[B, Q]) ([]*IvfpqModel[B, Q], error) { for _, idx := range indexes { idx.Devices = s.Devices if err := idx.LoadIndex(sqlproc, s.Idxcfg, s.Tblcfg, s.ThreadsSearch, true); err != nil { @@ -381,7 +406,7 @@ func (s *IvfpqSearch[T]) loadIndexes(sqlproc *sqlexec.SqlProcess, indexes []*Ivf } // Destroy implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) Destroy() { +func (s *IvfpqSearch[B, Q]) Destroy() { s.MultiIndex = nil if s.Overflow != nil { s.Overflow.Destroy() @@ -394,6 +419,6 @@ func (s *IvfpqSearch[T]) Destroy() { } // UpdateConfig implements cache.VectorIndexSearchIf. -func (s *IvfpqSearch[T]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error { +func (s *IvfpqSearch[B, Q]) UpdateConfig(newalgo cache.VectorIndexSearchIf) error { return nil } diff --git a/pkg/vectorindex/ivfpq/search_test.go b/pkg/vectorindex/ivfpq/search_test.go index 28532a76597aa..222d636fb07d8 100644 --- a/pkg/vectorindex/ivfpq/search_test.go +++ b/pkg/vectorindex/ivfpq/search_test.go @@ -34,7 +34,7 @@ import ( // loadedModel builds an index, saves it, then reloads it into GPU memory from // the local tar file. Returns the model with Index != nil. -func loadedModel(t *testing.T, id string) *IvfpqModel[float32] { +func loadedModel(t *testing.T, id string) *IvfpqModel[float32, float32] { t.Helper() built := buildTestModel(t, id, nil) tarPath := built.Path @@ -53,7 +53,7 @@ func loadedModel(t *testing.T, id string) *IvfpqModel[float32] { } defer func() { runSql = origRunSql }() - loader := &IvfpqModel[float32]{ + loader := &IvfpqModel[float32, float32]{ Id: id, Path: tarPath, Checksum: built.Checksum, @@ -72,7 +72,7 @@ func TestIvfpqSearchEmpty(t *testing.T) { proc := testutil.NewProcessWithMPool(t, "", m) sqlproc := sqlexec.NewSqlProcess(proc) - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) require.Empty(t, s.Indexes) rt := vectorindex.RuntimeConfig{Limit: 4} @@ -98,8 +98,8 @@ func TestIvfpqSearchTypeMismatch(t *testing.T) { idx := loadedModel(t, "type-mismatch") defer idx.Destroy() - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*IvfpqModel[float32]{idx} + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*IvfpqModel[float32, float32]{idx} s.MultiIndex, _ = s.buildMultiIndex() rt := vectorindex.RuntimeConfig{Limit: 4} @@ -118,8 +118,8 @@ func TestIvfpqSearchAndSearchFloat32(t *testing.T) { idx := loadedModel(t, "search-single") defer idx.Destroy() - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*IvfpqModel[float32]{idx} + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*IvfpqModel[float32, float32]{idx} s.MultiIndex, _ = s.buildMultiIndex() data := generateTestData(testNVectors, testDim) @@ -156,8 +156,8 @@ func TestIvfpqSearchMultipleIndexes(t *testing.T) { idx1 := loadedModel(t, "multi-1") defer idx1.Destroy() - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) - s.Indexes = []*IvfpqModel[float32]{idx0, idx1} + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) + s.Indexes = []*IvfpqModel[float32, float32]{idx0, idx1} s.MultiIndex, _ = s.buildMultiIndex() data := generateTestData(testNVectors, testDim) @@ -206,7 +206,7 @@ func TestIvfpqSearchLoad(t *testing.T) { } defer func() { runSql_streaming = origStream }() - s := NewIvfpqSearch[float32](testIdxcfg(), testTblcfg(), []int{0}) + s := NewIvfpqSearch[float32, float32](testIdxcfg(), testTblcfg(), []int{0}) err := s.Load(sqlproc) require.NoError(t, err) require.Equal(t, 1, len(s.Indexes)) diff --git a/pkg/vectorindex/ivfpq/sync.go b/pkg/vectorindex/ivfpq/sync.go index 54144bd66f418..1c20aaff04803 100644 --- a/pkg/vectorindex/ivfpq/sync.go +++ b/pkg/vectorindex/ivfpq/sync.go @@ -28,6 +28,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/util" + "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/pb/plan" @@ -48,6 +50,7 @@ type IvfpqSync struct { activeIndexId string dim int + vecBytesPerRow int // dim * base element size (4*dim for f32, 2*dim for f16) includeBytesPerRow int colMetaJSON string @@ -66,11 +69,19 @@ func NewIvfpqSync( idxname string, idxdefs []*plan.IndexDef, dimension int32, + baseType types.T, colMetaJSON string, ) (*IvfpqSync, error) { if dimension <= 0 { return nil, moerr.NewInternalErrorNoCtx("IvfpqSync: invalid dimension") } + // CDC records carry the vector as raw native base-type bytes: 2*dim for a + // vecf16 base, 4*dim otherwise. Must match the iscp writer's encode width + // and the search-side replayEventChunks[B] read width. + elemSize := 4 + if baseType == types.T_array_float16 { + elemSize = 2 + } var idxtblcfg vectorindex.IndexTableConfig idxtblcfg.DbName = db @@ -102,6 +113,7 @@ func NewIvfpqSync( tblcfg: idxtblcfg, idxname: idxname, dim: int(dimension), + vecBytesPerRow: int(dimension) * elemSize, includeBytesPerRow: includeBytesPerRow, colMetaJSON: colMetaJSON, activeIndexId: vectorindex.CdcTailId, @@ -176,7 +188,7 @@ func (s *IvfpqSync) AppendRecords(_ *sqlexec.SqlProcess, recordBytes []byte) err n = 9 // op (1) + pkid (8) case cuvscdc.CdcOpInsert, cuvscdc.CdcOpUpsert: // UPSERT shares INSERT's payload shape; only the op byte differs. - n = 9 + 4*s.dim + s.includeBytesPerRow + n = 9 + s.vecBytesPerRow + s.includeBytesPerRow default: return moerr.NewInternalErrorNoCtx(fmt.Sprintf( "IvfpqSync.AppendRecords: unknown op %d at offset %d", op, pos)) @@ -212,7 +224,10 @@ func (s *IvfpqSync) appendRecord(op cuvscdc.CdcOp, pkid int64, vec []float32, in } } before := len(s.pendingRecords) - out, err := cuvscdc.EncodeEventRecord(s.pendingRecords, op, pkid, vec, include, s.dim, s.includeBytesPerRow) + // This synchronous VectorIndexCdc[float32] path is f32-only (vec is + // []float32). vecf16 ongoing ingestion flows through the iscp writer → + // AppendRecords byte path instead, which honors s.vecBytesPerRow. + out, err := cuvscdc.EncodeEventRecord(s.pendingRecords, op, pkid, util.UnsafeSliceToBytes(vec), include, 4*s.dim, s.includeBytesPerRow) if err != nil { return err } diff --git a/pkg/vectorindex/ivfpq/sync_test.go b/pkg/vectorindex/ivfpq/sync_test.go index eda8956df5219..5f72650799d89 100644 --- a/pkg/vectorindex/ivfpq/sync_test.go +++ b/pkg/vectorindex/ivfpq/sync_test.go @@ -24,6 +24,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/catalog" "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/batch" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" @@ -112,7 +113,7 @@ func TestIvfpqSync_Update_AllInsert(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) require.Equal(t, vectorindex.CdcTailId, s.activeIndexId) @@ -129,7 +130,7 @@ func TestIvfpqSync_Update_AllInsert(t *testing.T) { require.Len(t, rec.statements, 1) require.Contains(t, rec.statements[0], "'cdc_tail', 0,") - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.Empty(t, state.Deleted) require.Len(t, state.Overflow, 2) @@ -145,7 +146,7 @@ func TestIvfpqSync_Update_DeleteAndInsert(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -159,7 +160,7 @@ func TestIvfpqSync_Update_DeleteAndInsert(t *testing.T) { require.Len(t, rec.statements, 1) require.Contains(t, rec.statements[0], "'cdc_tail', 7,") - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 7), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 7), 16, 0) require.NoError(t, err) require.Equal(t, []int64{42}, state.Deleted) require.Len(t, state.Overflow, 1) @@ -176,7 +177,7 @@ func TestIvfpqSync_Update_DeleteInsertDelete(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -191,7 +192,7 @@ func TestIvfpqSync_Update_DeleteInsertDelete(t *testing.T) { require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.Equal(t, []int64{1}, state.Deleted) require.Empty(t, state.Overflow) @@ -207,7 +208,7 @@ func TestIvfpqSync_Update_DeleteIdempotent(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -220,7 +221,7 @@ func TestIvfpqSync_Update_DeleteIdempotent(t *testing.T) { require.NoError(t, s.Update(sqlproc, cdc)) require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.ElementsMatch(t, []int64{5, 7}, state.Deleted) } @@ -235,7 +236,7 @@ func TestIvfpqSync_Update_Upsert(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -248,11 +249,11 @@ func TestIvfpqSync_Update_Upsert(t *testing.T) { require.Len(t, s.pendingSizes, 2) require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 0) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 0) require.NoError(t, err) require.ElementsMatch(t, []int64{100}, state.Deleted) require.Len(t, state.Overflow, 1) - require.Equal(t, []float32{9, 9, 9, 9}, state.Overflow[0].Vec) + require.Equal(t, []float32{9, 9, 9, 9}, util.UnsafeSliceCast[float32](state.Overflow[0].Vec)) } func TestIvfpqSync_Update_DimMismatch(t *testing.T) { @@ -261,7 +262,7 @@ func TestIvfpqSync_Update_DimMismatch(t *testing.T) { sqlproc := sqlexec.NewSqlProcess(proc) s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ @@ -289,7 +290,7 @@ func TestIvfpqSync_Update_WithIncludeBytes(t *testing.T) { require.Equal(t, 9, expectedIBPR) s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, colMetaJSON) + idxdefs("__meta", "__storage"), 4, types.T_array_float32, colMetaJSON) require.NoError(t, err) require.Equal(t, 9, s.includeBytesPerRow) @@ -303,7 +304,7 @@ func TestIvfpqSync_Update_WithIncludeBytes(t *testing.T) { require.NoError(t, s.Update(sqlproc, cdc)) require.NoError(t, s.Save(sqlproc)) - state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 4, 9) + state, err := cuvscdc.ReplayEventLog(chunksFromSql(t, rec.statements, 0), 16, 9) require.NoError(t, err) require.Len(t, state.Overflow, 1) require.Equal(t, include, state.Overflow[0].Include) @@ -330,7 +331,7 @@ func TestIvfpqSync_Update_NoOpSaveSkipsSql(t *testing.T) { defer func() { runSql = origRun }() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{} @@ -354,7 +355,7 @@ func TestIvfpqSync_NewSync_Stateless(t *testing.T) { defer func() { runSql = origRun }() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) require.Equal(t, vectorindex.CdcTailId, s.activeIndexId) require.Equal(t, 0, called) @@ -370,7 +371,7 @@ func TestIvfpqSync_RunOnce(t *testing.T) { defer rec.install(t)() s, err := NewIvfpqSync(sqlproc, "db", "src", "idxname", - idxdefs("__meta", "__storage"), 4, "") + idxdefs("__meta", "__storage"), 4, types.T_array_float32, "") require.NoError(t, err) cdc := &vectorindex.VectorIndexCdc[float32]{ diff --git a/pkg/vectorindex/metric/cpu.go b/pkg/vectorindex/metric/cpu.go index 605d10c408fe2..3ca9222647712 100644 --- a/pkg/vectorindex/metric/cpu.go +++ b/pkg/vectorindex/metric/cpu.go @@ -29,7 +29,7 @@ const GPUThresholdSQL = GPUThresholdSync / 4 // gpuMode is accepted-but-ignored in non-gpu builds — CPU is the only // option here. The signature matches the gpu.go variant so callers // pass the flag uniformly. -func PairWiseDistance[T types.RealNumbers]( +func PairWiseDistance[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, @@ -38,7 +38,7 @@ func PairWiseDistance[T types.RealNumbers]( return GoPairWiseDistance(x, y, metric) } -func PairwiseDistanceLaunch[T types.RealNumbers]( +func PairwiseDistanceLaunch[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, diff --git a/pkg/vectorindex/metric/distance_func.go b/pkg/vectorindex/metric/distance_func.go index ba91d5f5dad48..ce4748adb2162 100644 --- a/pkg/vectorindex/metric/distance_func.go +++ b/pkg/vectorindex/metric/distance_func.go @@ -1,3 +1,5 @@ +//go:build !(amd64 && go1.26 && goexperiment.simd) + // Copyright 2023 Matrix Origin // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -438,122 +440,3 @@ func ScaleInPlace[T types.RealNumbers](v []T, scale T) { v[i] *= scale } } - -// IMPORTANT: Elkans Kmeans always use L2Distance for dense vector or images. After getting the centroids, we can use other distance function -// specified by user to assign vector to corresponding centroids (CENTROIDX JOIN / ProductL2). - -func ResolveKmeansDistanceFn[T types.RealNumbers](metric MetricType, spherical bool) (DistanceFunction[T], bool, error) { - if spherical { - return ResolveKmeansDistanceFnForSparse[T](metric) - } - return ResolveKmeansDistanceFnForDense[T](metric) -} - -func ResolveKmeansDistanceFnForDense[T types.RealNumbers](metric MetricType) (DistanceFunction[T], bool, error) { - var distanceFunction DistanceFunction[T] - normalize := false - switch metric { - case Metric_L2Distance: - distanceFunction = L2Distance[T] - normalize = false - case Metric_L2sqDistance: - // Elkans Kmeans always uses true L2Distance regardless of user metric. - distanceFunction = L2Distance[T] - normalize = false - case Metric_InnerProduct: - distanceFunction = L2Distance[T] - normalize = false - case Metric_CosineDistance: - distanceFunction = L2Distance[T] - normalize = false - case Metric_L1Distance: - distanceFunction = L2Distance[T] - normalize = false - default: - return nil, normalize, moerr.NewInternalErrorNoCtx("invalid distance type") - } - return distanceFunction, normalize, nil -} - -// IMPORTANT: Spherical Kmeans always use Spherical Distance / Cosine Similarity for Sparse vector or text embedding (TD-IDF). -// After getting the centroids, we can use other distance function -// specified by user to assign vector to corresponding centroids (CENTROIDX JOIN / ProductL2). -func ResolveKmeansDistanceFnForSparse[T types.RealNumbers](metric MetricType) (DistanceFunction[T], bool, error) { - var distanceFunction DistanceFunction[T] - normalize := false - switch metric { - case Metric_L2Distance: - distanceFunction = L2Distance[T] - normalize = false - case Metric_L2sqDistance: - distanceFunction = L2Distance[T] - normalize = false - case Metric_InnerProduct: - distanceFunction = SphericalDistance[T] - normalize = true - case Metric_CosineDistance: - distanceFunction = SphericalDistance[T] - normalize = true - case Metric_L1Distance: - distanceFunction = L2Distance[T] - normalize = false - default: - return nil, normalize, moerr.NewInternalErrorNoCtx("invalid distance type") - } - return distanceFunction, normalize, nil -} - -// ResolveDistanceFn is used for similarity score for search and assign vector to centroids (CENTROIDX JOIN / ProductL2). -// IMPORTANT: Don't use it for Elkans Kmeans. -// NOTE: Metric_L2Distance returns L2DistanceSq (squared distance). Callers that need true L2 -// must apply sqrt to each result afterwards (as GoPairWiseDistance does). -func ResolveDistanceFn[T types.RealNumbers](metric MetricType) (DistanceFunction[T], error) { - var distanceFunction DistanceFunction[T] - switch metric { - case Metric_L2Distance: - distanceFunction = L2DistanceSq[T] // caller must sqrt; see function doc above - case Metric_L2sqDistance: - distanceFunction = L2DistanceSq[T] - case Metric_InnerProduct: - distanceFunction = InnerProduct[T] - case Metric_CosineDistance: - distanceFunction = CosineDistance[T] - case Metric_L1Distance: - distanceFunction = L1Distance[T] - default: - return nil, moerr.NewInternalErrorNoCtx("invalid distance type") - } - return distanceFunction, nil -} - -func GoPairWiseDistance[T types.RealNumbers]( - x [][]T, - y [][]T, - metric MetricType, -) ([]float32, error) { - distFn, err := ResolveDistanceFn[T](metric) - if err != nil { - return nil, err - } - - nX := len(x) - nY := len(y) - res := make([]float32, nX*nY) - for i := 0; i < nX; i++ { - for j := 0; j < nY; j++ { - d, err := distFn(x[i], y[j]) - if err != nil { - return nil, err - } - res[i*nY+j] = float32(d) - } - } - - if metric == Metric_L2Distance { - for i := range res { - res[i] = float32(math.Sqrt(float64(res[i]))) - } - } - - return res, nil -} diff --git a/pkg/vectorindex/metric/distance_func_amd64.go b/pkg/vectorindex/metric/distance_func_amd64.go new file mode 100644 index 0000000000000..07cf2ae3438f6 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_amd64.go @@ -0,0 +1,655 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "math" + "os" + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// hasAVX512 gates the top kernel tier. TESTING-ONLY override: set +// MO_METRIC_NO_AVX512=1 to force AVX512-capable CPUs down to the AVX2 (and +// then scalar) path, so the lower tiers get real coverage on this hardware. +// Package-level var initializers run before every init() selector, so the +// override is seen by all kernel-selection init() funcs in this package. +var ( + hasAVX512 = archsimd.X86.AVX512() && os.Getenv("MO_METRIC_NO_AVX512") == "" +) + +// Reduction Helpers - Simple Store and Tree Sum for maximum throughput +func sumF32x16(v archsimd.Float32x16) float32 { + var a [16]float32 + v.Store(&a) + s0 := (a[0] + a[1]) + (a[2] + a[3]) + s1 := (a[4] + a[5]) + (a[6] + a[7]) + s2 := (a[8] + a[9]) + (a[10] + a[11]) + s3 := (a[12] + a[13]) + (a[14] + a[15]) + return (s0 + s1) + (s2 + s3) +} + +func sumF64x8(v archsimd.Float64x8) float64 { + var a [8]float64 + v.Store(&a) + return (a[0] + a[1] + a[2] + a[3]) + (a[4] + a[5] + a[6] + a[7]) +} + +// L2 Distance Squared kernels +func L2DistanceSqFloat32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + + var sum float32 + i := 0 + + if hasAVX512 && n >= 64 { + acc0, acc1, acc2, acc3 := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-64 { + as, bs := a[i:i+64:i+64], b[i:i+64:i+64] + d0 := archsimd.LoadFloat32x16Slice(as[0:16]).Sub(archsimd.LoadFloat32x16Slice(bs[0:16])) + d1 := archsimd.LoadFloat32x16Slice(as[16:32]).Sub(archsimd.LoadFloat32x16Slice(bs[16:32])) + d2 := archsimd.LoadFloat32x16Slice(as[32:48]).Sub(archsimd.LoadFloat32x16Slice(bs[32:48])) + d3 := archsimd.LoadFloat32x16Slice(as[48:64]).Sub(archsimd.LoadFloat32x16Slice(bs[48:64])) + + acc0 = d0.MulAdd(d0, acc0) + acc1 = d1.MulAdd(d1, acc1) + acc2 = d2.MulAdd(d2, acc2) + acc3 = d3.MulAdd(d3, acc3) + i += 64 + } + sum += sumF32x16(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + d0 := as[0] - bs[0] + d1 := as[1] - bs[1] + d2 := as[2] - bs[2] + d3 := as[3] - bs[3] + d4 := as[4] - bs[4] + d5 := as[5] - bs[5] + d6 := as[6] - bs[6] + d7 := as[7] - bs[7] + sum += (d0*d0 + d1*d1) + (d2*d2 + d3*d3) + (d4*d4 + d5*d5) + (d6*d6 + d7*d7) + i += 8 + } + + for ; i < n; i++ { + diff := a[i] - b[i] + sum += diff * diff + } + return sum, nil +} + +func InnerProductFloat32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + + var total float32 + i := 0 + + if hasAVX512 && n >= 64 { + acc0, acc1, acc2, acc3 := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-64 { + as, bs := a[i:i+64:i+64], b[i:i+64:i+64] + acc0 = archsimd.LoadFloat32x16Slice(as[0:16]).MulAdd(archsimd.LoadFloat32x16Slice(bs[0:16]), acc0) + acc1 = archsimd.LoadFloat32x16Slice(as[16:32]).MulAdd(archsimd.LoadFloat32x16Slice(bs[16:32]), acc1) + acc2 = archsimd.LoadFloat32x16Slice(as[32:48]).MulAdd(archsimd.LoadFloat32x16Slice(bs[32:48]), acc2) + acc3 = archsimd.LoadFloat32x16Slice(as[48:64]).MulAdd(archsimd.LoadFloat32x16Slice(bs[48:64]), acc3) + i += 64 + } + total += sumF32x16(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + total += as[0]*bs[0] + as[1]*bs[1] + as[2]*bs[2] + as[3]*bs[3] + + as[4]*bs[4] + as[5]*bs[5] + as[6]*bs[6] + as[7]*bs[7] + i += 8 + } + + for ; i < n; i++ { + total += a[i] * b[i] + } + return -total, nil +} + +func L2Distance[T types.RealNumbers](v1, v2 []T) (T, error) { + if pf32, ok := any(v1).([]float32); ok { + dist, err := L2DistanceSqFloat32(pf32, any(v2).([]float32)) + if err != nil { + return 0, err + } + return T(math.Sqrt(float64(dist))), nil + } + if pf64, ok := any(v1).([]float64); ok { + dist, err := L2DistanceSqFloat64(pf64, any(v2).([]float64)) + if err != nil { + return 0, err + } + return T(math.Sqrt(dist)), nil + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func L2DistanceSqFloat64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float64 + i := 0 + if hasAVX512 && n >= 32 { + acc0, acc1, acc2, acc3 := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-32 { + as, bs := a[i:i+32:i+32], b[i:i+32:i+32] + d0 := archsimd.LoadFloat64x8Slice(as[0:8]).Sub(archsimd.LoadFloat64x8Slice(bs[0:8])) + d1 := archsimd.LoadFloat64x8Slice(as[8:16]).Sub(archsimd.LoadFloat64x8Slice(bs[8:16])) + d2 := archsimd.LoadFloat64x8Slice(as[16:24]).Sub(archsimd.LoadFloat64x8Slice(bs[16:24])) + d3 := archsimd.LoadFloat64x8Slice(as[24:32]).Sub(archsimd.LoadFloat64x8Slice(bs[24:32])) + acc0 = d0.MulAdd(d0, acc0) + acc1 = d1.MulAdd(d1, acc1) + acc2 = d2.MulAdd(d2, acc2) + acc3 = d3.MulAdd(d3, acc3) + i += 32 + } + sum += sumF64x8(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + d0 := as[0] - bs[0] + d1 := as[1] - bs[1] + d2 := as[2] - bs[2] + d3 := as[3] - bs[3] + d4 := as[4] - bs[4] + d5 := as[5] - bs[5] + d6 := as[6] - bs[6] + d7 := as[7] - bs[7] + sum += (d0*d0 + d1*d1) + (d2*d2 + d3*d3) + (d4*d4 + d5*d5) + (d6*d6 + d7*d7) + i += 8 + } + + for ; i < n; i++ { + diff := a[i] - b[i] + sum += diff * diff + } + return sum, nil +} + +func InnerProductFloat64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var total float64 + i := 0 + if hasAVX512 && n >= 32 { + acc0, acc1, acc2, acc3 := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-32 { + as, bs := a[i:i+32:i+32], b[i:i+32:i+32] + acc0 = archsimd.LoadFloat64x8Slice(as[0:8]).MulAdd(archsimd.LoadFloat64x8Slice(bs[0:8]), acc0) + acc1 = archsimd.LoadFloat64x8Slice(as[8:16]).MulAdd(archsimd.LoadFloat64x8Slice(bs[8:16]), acc1) + acc2 = archsimd.LoadFloat64x8Slice(as[16:24]).MulAdd(archsimd.LoadFloat64x8Slice(bs[16:24]), acc2) + acc3 = archsimd.LoadFloat64x8Slice(as[24:32]).MulAdd(archsimd.LoadFloat64x8Slice(bs[24:32]), acc3) + i += 32 + } + total += sumF64x8(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + total += as[0]*bs[0] + as[1]*bs[1] + as[2]*bs[2] + as[3]*bs[3] + + as[4]*bs[4] + as[5]*bs[5] + as[6]*bs[6] + as[7]*bs[7] + i += 8 + } + + for ; i < n; i++ { + total += a[i] * b[i] + } + return -total, nil +} + +func L2DistanceSq[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := L2DistanceSqFloat32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := L2DistanceSqFloat64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func InnerProduct[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := InnerProductFloat32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := InnerProductFloat64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func L1DistanceFloat32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var sum float32 + i := 0 + if hasAVX512 && n >= 64 { + acc0, acc1, acc2, acc3 := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-64 { + as, bs := a[i:i+64:i+64], b[i:i+64:i+64] + acc0 = acc0.Add(archsimd.LoadFloat32x16Slice(as[0:16]).Sub(archsimd.LoadFloat32x16Slice(bs[0:16])).Max(archsimd.LoadFloat32x16Slice(bs[0:16]).Sub(archsimd.LoadFloat32x16Slice(as[0:16])))) + acc1 = acc1.Add(archsimd.LoadFloat32x16Slice(as[16:32]).Sub(archsimd.LoadFloat32x16Slice(bs[16:32])).Max(archsimd.LoadFloat32x16Slice(bs[16:32]).Sub(archsimd.LoadFloat32x16Slice(as[16:32])))) + acc2 = acc2.Add(archsimd.LoadFloat32x16Slice(as[32:48]).Sub(archsimd.LoadFloat32x16Slice(bs[32:48])).Max(archsimd.LoadFloat32x16Slice(bs[32:48]).Sub(archsimd.LoadFloat32x16Slice(as[32:48])))) + acc3 = acc3.Add(archsimd.LoadFloat32x16Slice(as[48:64]).Sub(archsimd.LoadFloat32x16Slice(bs[48:64])).Max(archsimd.LoadFloat32x16Slice(bs[48:64]).Sub(archsimd.LoadFloat32x16Slice(as[48:64])))) + i += 64 + } + sum += sumF32x16(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + abs := func(x float32) float32 { + return math.Float32frombits(math.Float32bits(x) &^ (1 << 31)) + } + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + sum += abs(as[0]-bs[0]) + abs(as[1]-bs[1]) + abs(as[2]-bs[2]) + abs(as[3]-bs[3]) + + abs(as[4]-bs[4]) + abs(as[5]-bs[5]) + abs(as[6]-bs[6]) + abs(as[7]-bs[7]) + i += 8 + } + + for ; i < n; i++ { + sum += abs(a[i] - b[i]) + } + return sum, nil +} + +func L1DistanceFloat64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var sum float64 + i := 0 + if hasAVX512 && n >= 32 { + acc0, acc1, acc2, acc3 := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-32 { + as, bs := a[i:i+32:i+32], b[i:i+32:i+32] + acc0 = acc0.Add(archsimd.LoadFloat64x8Slice(as[0:8]).Sub(archsimd.LoadFloat64x8Slice(bs[0:8])).Max(archsimd.LoadFloat64x8Slice(bs[0:8]).Sub(archsimd.LoadFloat64x8Slice(as[0:8])))) + acc1 = acc1.Add(archsimd.LoadFloat64x8Slice(as[8:16]).Sub(archsimd.LoadFloat64x8Slice(bs[8:16])).Max(archsimd.LoadFloat64x8Slice(bs[8:16]).Sub(archsimd.LoadFloat64x8Slice(as[8:16])))) + acc2 = acc2.Add(archsimd.LoadFloat64x8Slice(as[16:24]).Sub(archsimd.LoadFloat64x8Slice(bs[16:24])).Max(archsimd.LoadFloat64x8Slice(bs[16:24]).Sub(archsimd.LoadFloat64x8Slice(as[16:24])))) + acc3 = acc3.Add(archsimd.LoadFloat64x8Slice(as[24:32]).Sub(archsimd.LoadFloat64x8Slice(bs[24:32])).Max(archsimd.LoadFloat64x8Slice(bs[24:32]).Sub(archsimd.LoadFloat64x8Slice(as[24:32])))) + i += 32 + } + sum += sumF64x8(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + abs := func(x float64) float64 { + return math.Abs(x) + } + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + sum += abs(as[0]-bs[0]) + abs(as[1]-bs[1]) + abs(as[2]-bs[2]) + abs(as[3]-bs[3]) + + abs(as[4]-bs[4]) + abs(as[5]-bs[5]) + abs(as[6]-bs[6]) + abs(as[7]-bs[7]) + i += 8 + } + + for ; i < n; i++ { + sum += abs(a[i] - b[i]) + } + return sum, nil +} + +func L1Distance[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := L1DistanceFloat32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := L1DistanceFloat64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func CosineDistanceF32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var dot, normA, normB float32 + i := 0 + if n >= 16 && hasAVX512 { + accD, accA, accB := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-16 { + va, vb := archsimd.LoadFloat32x16Slice(a[i:i+16]), archsimd.LoadFloat32x16Slice(b[i:i+16]) + accD = va.MulAdd(vb, accD) + accA = va.MulAdd(va, accA) + accB = vb.MulAdd(vb, accB) + i += 16 + } + dot, normA, normB = sumF32x16(accD), sumF32x16(accA), sumF32x16(accB) + } + + for i <= n-4 { + // BCE Hint + va := a[i : i+4 : i+4] + vb := b[i : i+4 : i+4] + dot += va[0]*vb[0] + va[1]*vb[1] + va[2]*vb[2] + va[3]*vb[3] + normA += va[0]*va[0] + va[1]*va[1] + va[2]*va[2] + va[3]*va[3] + normB += vb[0]*vb[0] + vb[1]*vb[1] + vb[2]*vb[2] + vb[3]*vb[3] + i += 4 + } + + for ; i < n; i++ { + dot, normA, normB = dot+a[i]*b[i], normA+a[i]*a[i], normB+b[i]*b[i] + } + den := math.Sqrt(float64(normA)) * math.Sqrt(float64(normB)) + if den == 0 { + return 1.0, nil + } + return float32(cosineDistClamped(float64(dot), den)), nil +} + +func CosineDistanceF64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var dot, normA, normB float64 + i := 0 + if n >= 8 && hasAVX512 { + accD, accA, accB := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-8 { + va, vb := archsimd.LoadFloat64x8Slice(a[i:i+8]), archsimd.LoadFloat64x8Slice(b[i:i+8]) + accD = va.MulAdd(vb, accD) + accA = va.MulAdd(va, accA) + accB = vb.MulAdd(vb, accB) + i += 8 + } + dot, normA, normB = sumF64x8(accD), sumF64x8(accA), sumF64x8(accB) + } + + for i <= n-4 { + // BCE Hint + va := a[i : i+4 : i+4] + vb := b[i : i+4 : i+4] + dot += va[0]*vb[0] + va[1]*vb[1] + va[2]*vb[2] + va[3]*vb[3] + normA += va[0]*va[0] + va[1]*va[1] + va[2]*va[2] + va[3]*va[3] + normB += vb[0]*vb[0] + vb[1]*vb[1] + vb[2]*vb[2] + vb[3]*vb[3] + i += 4 + } + + for ; i < n; i++ { + dot, normA, normB = dot+a[i]*b[i], normA+a[i]*a[i], normB+b[i]*b[i] + } + den := math.Sqrt(normA) * math.Sqrt(normB) + if den == 0 { + return 1.0, nil + } + return cosineDistClamped(dot, den), nil +} + +func CosineDistance[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := CosineDistanceF32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := CosineDistanceF64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func CosineSimilarityF32(a, b []float32) (float32, error) { + n := len(a) + if n == 0 { + return 0, nil + } + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var dot, normA, normB float32 + i := 0 + if n >= 16 && hasAVX512 { + accD, accA, accB := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-16 { + va, vb := archsimd.LoadFloat32x16Slice(a[i:i+16]), archsimd.LoadFloat32x16Slice(b[i:i+16]) + accD = va.MulAdd(vb, accD) + accA = va.MulAdd(va, accA) + accB = vb.MulAdd(vb, accB) + i += 16 + } + dot, normA, normB = sumF32x16(accD), sumF32x16(accA), sumF32x16(accB) + } + + for i <= n-4 { + // BCE Hint + va := a[i : i+4 : i+4] + vb := b[i : i+4 : i+4] + dot += va[0]*vb[0] + va[1]*vb[1] + va[2]*vb[2] + va[3]*vb[3] + normA += va[0]*va[0] + va[1]*va[1] + va[2]*va[2] + va[3]*va[3] + normB += vb[0]*vb[0] + vb[1]*vb[1] + vb[2]*vb[2] + vb[3]*vb[3] + i += 4 + } + + for ; i < n; i++ { + dot, normA, normB = dot+a[i]*b[i], normA+a[i]*a[i], normB+b[i]*b[i] + } + den := math.Sqrt(float64(normA)) * math.Sqrt(float64(normB)) + if den == 0 { + return 0, moerr.NewInternalErrorNoCtx("cosine similarity zero denominator") + } + return float32(float64(dot) / den), nil +} + +func CosineSimilarityF64(a, b []float64) (float64, error) { + n := len(a) + if n == 0 { + return 0, nil + } + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension mismatch") + } + var dot, normA, normB float64 + i := 0 + if n >= 8 && hasAVX512 { + accD, accA, accB := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-8 { + va, vb := archsimd.LoadFloat64x8Slice(a[i:i+8]), archsimd.LoadFloat64x8Slice(b[i:i+8]) + accD = va.MulAdd(vb, accD) + accA = va.MulAdd(va, accA) + accB = vb.MulAdd(vb, accB) + i += 8 + } + dot, normA, normB = sumF64x8(accD), sumF64x8(accA), sumF64x8(accB) + } + + for i <= n-4 { + // BCE Hint + va := a[i : i+4 : i+4] + vb := b[i : i+4 : i+4] + dot += va[0]*vb[0] + va[1]*vb[1] + va[2]*vb[2] + va[3]*vb[3] + normA += va[0]*va[0] + va[1]*va[1] + va[2]*va[2] + va[3]*va[3] + normB += vb[0]*vb[0] + vb[1]*vb[1] + vb[2]*vb[2] + vb[3]*vb[3] + i += 4 + } + + for ; i < n; i++ { + dot, normA, normB = dot+a[i]*b[i], normA+a[i]*a[i], normB+b[i]*b[i] + } + den := math.Sqrt(normA) * math.Sqrt(normB) + if den == 0 { + return 0, moerr.NewInternalErrorNoCtx("cosine similarity zero denominator") + } + return dot / den, nil +} + +func CosineSimilarity[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := CosineSimilarityF32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := CosineSimilarityF64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func SphericalDistanceFloat32(a, b []float32) (float32, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var total float32 + i := 0 + if hasAVX512 && n >= 64 { + acc0, acc1, acc2, acc3 := archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{}, archsimd.Float32x16{} + for i <= n-64 { + as, bs := a[i:i+64:i+64], b[i:i+64:i+64] + acc0 = archsimd.LoadFloat32x16Slice(as[0:16]).MulAdd(archsimd.LoadFloat32x16Slice(bs[0:16]), acc0) + acc1 = archsimd.LoadFloat32x16Slice(as[16:32]).MulAdd(archsimd.LoadFloat32x16Slice(bs[16:32]), acc1) + acc2 = archsimd.LoadFloat32x16Slice(as[32:48]).MulAdd(archsimd.LoadFloat32x16Slice(bs[32:48]), acc2) + acc3 = archsimd.LoadFloat32x16Slice(as[48:64]).MulAdd(archsimd.LoadFloat32x16Slice(bs[48:64]), acc3) + i += 64 + } + total += sumF32x16(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + total += as[0]*bs[0] + as[1]*bs[1] + as[2]*bs[2] + as[3]*bs[3] + + as[4]*bs[4] + as[5]*bs[5] + as[6]*bs[6] + as[7]*bs[7] + i += 8 + } + + for ; i < n; i++ { + total += a[i] * b[i] + } + if total > 1.0 { + total = 1.0 + } else if total < -1.0 { + total = -1.0 + } + return float32(math.Acos(float64(total)) / math.Pi), nil +} + +func SphericalDistanceFloat64(a, b []float64) (float64, error) { + n := len(a) + if n != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var total float64 + i := 0 + if hasAVX512 && n >= 32 { + acc0, acc1, acc2, acc3 := archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{}, archsimd.Float64x8{} + for i <= n-32 { + as, bs := a[i:i+32:i+32], b[i:i+32:i+32] + acc0 = archsimd.LoadFloat64x8Slice(as[0:8]).MulAdd(archsimd.LoadFloat64x8Slice(bs[0:8]), acc0) + acc1 = archsimd.LoadFloat64x8Slice(as[8:16]).MulAdd(archsimd.LoadFloat64x8Slice(bs[8:16]), acc1) + acc2 = archsimd.LoadFloat64x8Slice(as[16:24]).MulAdd(archsimd.LoadFloat64x8Slice(bs[16:24]), acc2) + acc3 = archsimd.LoadFloat64x8Slice(as[24:32]).MulAdd(archsimd.LoadFloat64x8Slice(bs[24:32]), acc3) + i += 32 + } + total += sumF64x8(acc0.Add(acc1).Add(acc2.Add(acc3))) + } + + for i <= n-8 { + // BCE Hint + as := a[i : i+8 : i+8] + bs := b[i : i+8 : i+8] + total += as[0]*bs[0] + as[1]*bs[1] + as[2]*bs[2] + as[3]*bs[3] + + as[4]*bs[4] + as[5]*bs[5] + as[6]*bs[6] + as[7]*bs[7] + i += 8 + } + + for ; i < n; i++ { + total += a[i] * b[i] + } + if total > 1.0 { + total = 1.0 + } else if total < -1.0 { + total = -1.0 + } + return math.Acos(total) / math.Pi, nil +} + +func SphericalDistance[T types.RealNumbers](p, q []T) (T, error) { + if pf32, ok := any(p).([]float32); ok { + res, err := SphericalDistanceFloat32(pf32, any(q).([]float32)) + return T(res), err + } + if pf64, ok := any(p).([]float64); ok { + res, err := SphericalDistanceFloat64(pf64, any(q).([]float64)) + return T(res), err + } + return 0, moerr.NewInternalErrorNoCtx("vector type not supported") +} + +func NormalizeL2[T types.RealNumbers](v1 []T, normalized []T) error { + if len(v1) == 0 { + return moerr.NewInternalErrorNoCtx("cannot normalize empty vector") + } + var sumSquares float64 + for _, val := range v1 { + sumSquares += float64(val) * float64(val) + } + norm := math.Sqrt(sumSquares) + if norm == 0 { + copy(normalized, v1) + return nil + } + for i, val := range v1 { + normalized[i] = T(float64(val) / norm) + } + return nil +} + +func ScaleInPlace[T types.RealNumbers](v []T, scale T) { + for i := range v { + v[i] *= scale + } +} diff --git a/pkg/vectorindex/metric/distance_func_bench_test.go b/pkg/vectorindex/metric/distance_func_bench_test.go index 506d602d116cd..9a81b4acb6a28 100644 --- a/pkg/vectorindex/metric/distance_func_bench_test.go +++ b/pkg/vectorindex/metric/distance_func_bench_test.go @@ -25,10 +25,10 @@ Benchmark_L2Distance/Normalize_L2-10 1277733 1 Benchmark_L2Distance/L2_Distance(v1,_NormalizeL2)-10 589376 1883 ns/op */ func Benchmark_L2Distance(b *testing.B) { - dim := 128 + dim := 1024 - b.Run("L2 Distance", func(b *testing.B) { - v1, v2 := randomVectors(b.N, dim), randomVectors(b.N, dim) + b.Run("L2 Distance float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -36,34 +36,211 @@ func Benchmark_L2Distance(b *testing.B) { } }) - b.Run("Normalize L2", func(b *testing.B) { - v1 := randomVectors(b.N, dim) + b.Run("L2 Distance float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) b.ResetTimer() for i := 0; i < b.N; i++ { - res := make([]float64, dim) - _ = NormalizeL2[float64](v1[i], res) + _, _ = L2Distance[float32](v1[i], v2[i]) } }) - b.Run("L2 Distance(v1, NormalizeL2)", func(b *testing.B) { - v1, v2 := randomVectors(b.N, dim), randomVectors(b.N, dim) + /* + b.Run("Normalize L2 float64", func(b *testing.B) { + v1 := randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + res := make([]float64, dim) + _ = NormalizeL2[float64](v1[i], res) + } + }) + + b.Run("Normalize L2 float32", func(b *testing.B) { + v1 := randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + res := make([]float32, dim) + _ = NormalizeL2[float32](v1[i], res) + } + }) + + b.Run("L2 Distance(v1, NormalizeL2) float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + res := make([]float64, dim) + _ = NormalizeL2[float64](v2[i], res) + _, _ = L2Distance[float64](v1[i], res) + } + }) + */ +} + +func Benchmark_L2DistanceSq(b *testing.B) { + dim := 1024 + + b.Run("L2 DistanceSq float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = L2DistanceSq[float64](v1[i], v2[i]) + } + }) + + b.Run("L2 DistanceSq float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = L2DistanceSq[float32](v1[i], v2[i]) + } + }) +} + +func Benchmark_L1Distance(b *testing.B) { + dim := 1024 + + b.Run("L1 Distance float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = L1Distance[float64](v1[i], v2[i]) + } + }) + + b.Run("L1 Distance float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) b.ResetTimer() for i := 0; i < b.N; i++ { - res := make([]float64, dim) - _ = NormalizeL2[float64](v2[i], res) - _, _ = L2Distance[float64](v1[i], res) + _, _ = L1Distance[float32](v1[i], v2[i]) } }) +} + +func Benchmark_InnerProduct(b *testing.B) { + dim := 1024 + + b.Run("Inner Product float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = InnerProduct[float64](v1[i], v2[i]) + } + }) + + b.Run("Inner Product float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = InnerProduct[float32](v1[i], v2[i]) + } + }) } -func randomVectors(size, dim int) [][]float64 { - vectors := make([][]float64, size) +func Benchmark_CosineDistance(b *testing.B) { + dim := 1024 + + b.Run("Cosine Distance float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = CosineDistance[float64](v1[i], v2[i]) + } + }) + + b.Run("Cosine Distance float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = CosineDistance[float32](v1[i], v2[i]) + } + }) +} + +func Benchmark_CosineSimilarity(b *testing.B) { + dim := 1024 + + b.Run("Cosine Similarity float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = CosineSimilarity[float64](v1[i], v2[i]) + } + }) + + b.Run("Cosine Similarity float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = CosineSimilarity[float32](v1[i], v2[i]) + } + }) +} + +func Benchmark_SphericalDistance(b *testing.B) { + dim := 1024 + + b.Run("Spherical Distance float64", func(b *testing.B) { + v1, v2 := randomVectors[float64](b.N, dim), randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = SphericalDistance[float64](v1[i], v2[i]) + } + }) + + b.Run("Spherical Distance float32", func(b *testing.B) { + v1, v2 := randomVectors[float32](b.N, dim), randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = SphericalDistance[float32](v1[i], v2[i]) + } + }) +} + +/* +func Benchmark_ScaleInPlace(b *testing.B) { + dim := 1024 + + b.Run("ScaleInPlace float64", func(b *testing.B) { + v1 := randomVectors[float64](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + ScaleInPlace[float64](v1[i], 0.5) + } + }) + + b.Run("ScaleInPlace float32", func(b *testing.B) { + v1 := randomVectors[float32](b.N, dim) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + ScaleInPlace[float32](v1[i], 0.5) + } + }) +} +*/ + +func randomVectors[T float32 | float64](size, dim int) [][]T { + vectors := make([][]T, size) for i := range vectors { + vectors[i] = make([]T, dim) for j := 0; j < dim; j++ { - vectors[i] = append(vectors[i], rand.Float64()) + vectors[i][j] = T(rand.Float64()) } } return vectors diff --git a/pkg/vectorindex/metric/distance_func_f32_test.go b/pkg/vectorindex/metric/distance_func_f32_test.go new file mode 100644 index 0000000000000..098ab6134add8 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_f32_test.go @@ -0,0 +1,583 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/assertx" +) + +func Test_L2Distance_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 1.4142135623730951, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 4.123105625617661, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 3, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 4.242640687119285, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 3, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 3.1622776601683795, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 5.196152422706632, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := L2Distance[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("L2Distance() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_L1Distance_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 2, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 7, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 3, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 6, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 3, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 10, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 27, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := L1Distance[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("L1Distance() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_CosineDistance_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 0.003993481192393733, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 0.0001253573895874105, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 0.1425070742874559, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 0.5294117647058824, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 0.1425070742874559, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 0.0021238962030426523, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0.0025062434610066964, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := CosineDistance[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("CosineDistance() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_CosineSimilarity_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 0.9960065188076063, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 0.9998746426104126, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 0.8574929257125441, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 0.47058823529411764, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 0.8574929257125441, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 0.9978761037969573, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0.9974937565389933, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := CosineSimilarity[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("CosineSimilarity() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_InnerProduct_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: -37, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: -3220, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: -5, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: -8, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: -5, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: -440, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: -1048, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := InnerProduct[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("InnerProduct() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_L2DistanceSq_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 2, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 17, + }, + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 1}, + v2: []float32{4, 1}, + }, + want: 9, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{4, 1}, + v2: []float32{1, 4}, + }, + want: 18, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{1, 4}, + v2: []float32{1, 1}, + }, + want: 9, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 10, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 27, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := L2DistanceSq[float32](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("L2DistanceSq() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_AngularDistance_F32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float32 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 0, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 0, + }, + // Test 3: Triangle Inequality check on **un-normalized** vector + // A(1,0),B(2,2), C(0,1) => AB + AC !>= BC => 0 + 0 !>= 0.5 + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 0}, + v2: []float32{2, 2}, + }, + want: 0, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{2, 2}, + v2: []float32{0, 1}, + }, + want: 0, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{0, 1}, + v2: []float32{1, 0}, + }, + want: 0.5, + }, + { + name: "Test 4", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 0, + }, + { + name: "Test 5", + args: args{ + v1: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float32{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0, + }, + + // Test 4: Triangle Inequality check on **normalized** vector + // A(1,0),B(2,2), C(0,1) => AB + AC >= BC => 0.25 + 0.25 >= 0.5 + //{ + // name: "Test 4.a", + // args: args{ + // v1: moarray.NormalizeMoVecf64([]float32{1, 0}), + // v2: moarray.NormalizeMoVecf64([]float32{2, 2}), + // }, + // want: 0.25000000000000006, + //}, + //{ + // name: "Test 4.b", + // args: args{ + // v1: moarray.NormalizeMoVecf64([]float32{2, 2}), + // v2: moarray.NormalizeMoVecf64([]float32{0, 1}), + // }, + // want: 0.25000000000000006, + //}, + //{ + // name: "Test 4.c", + // args: args{ + // v1: moarray.NormalizeMoVecf64([]float32{0, 1}), + // v2: moarray.NormalizeMoVecf64([]float32{1, 0}), + // }, + // want: 0.5, + //}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + if got, err := SphericalDistance[float32](tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(float64(got), float64(tt.want)) { + t.Errorf("SphericalDistance() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/vectorindex/metric/distance_func_narrow.go b/pkg/vectorindex/metric/distance_func_narrow.go new file mode 100644 index 0000000000000..663c6b6d4d0f4 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow.go @@ -0,0 +1,450 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Distance kernels for the narrow vector element types: vecbf16 (types.BF16), +// vecf16 (types.Float16), vecint8 (int8). Pure Go, loop-unrolled. Untagged so +// it compiles in both the scalar and SIMD builds (a narrow SIMD variant can be +// split out later as distance_func_narrow_amd64.go, with this as the fallback + +// equivalence oracle). +// +// Semantics MATCH ResolveDistanceFn (the float32 path): +// - Metric_L2Distance / Metric_L2sqDistance -> squared L2 (caller sqrts L2) +// - Metric_InnerProduct -> -dot +// - Metric_CosineDistance -> 1 - similarity +// - Metric_L1Distance -> sum|a-b| +// +// bf16/f16 decode to float32 and reuse the float32 kernels (Go has no native +// fp16 arithmetic). int8 uses INTEGER (int64-accumulated) kernels — no float +// upcast in the inner loop — with only the cosine denominator going through +// float for the sqrt/divide. + +package metric + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// The native narrow kernels return float32 — bf16/f16 accumulate in float32, and +// int8/uint8 cast their int64 accumulator down at the end. The merged +// ResolveDistanceFn[T, R] in resolve.go dispatches to resolveBF16Kernel / +// resolveF16Kernel / resolveInt8Kernel / resolveUint8Kernel and casts to R. + +// ---------------------------------------------------------------------------- +// bf16 / f16 CONCRETE fused kernels (unroll-8). NOT generic: a generic +// A generic [T] would share the uint16 gcshape, so .ToFloat32() would become a +// dictionary (virtual) call per element and never inlines. Concrete types let +// .ToFloat32() inline (bf16 = one shift). Decode inline, accumulate in float32, +// no slice materialized -> zero alloc. The multiply/add cannot be done without a +// float (bf16/f16 are floating-point; Go has no 16-bit-float ALU). +// ---------------------------------------------------------------------------- + +// bf16 kernel selection. The SIMD build (distance_func_narrow_amd64.go) swaps +// these to its archsimd implementations in init() when AVX-512 is available; +// otherwise they stay the pure-Go fallbacks defined below (which also remain the +// equivalence oracle the SIMD tests compare against). +var ( + bf16L2sqFn = l2sqBF16 + bf16IPFn = innerProductBF16 + bf16CosineFn = cosineDistanceBF16 + bf16L1Fn = l1DistanceBF16 +) + +func resolveBF16Kernel(metric MetricType) (func(a, b []types.BF16) (float64, error), error) { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + return bf16L2sqFn, nil + case Metric_InnerProduct: + return bf16IPFn, nil + case Metric_CosineDistance: + return bf16CosineFn, nil + case Metric_L1Distance: + return bf16L1Fn, nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +func l2sqBF16(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + d0 := aa[0].ToFloat32() - bb[0].ToFloat32() + d1 := aa[1].ToFloat32() - bb[1].ToFloat32() + d2 := aa[2].ToFloat32() - bb[2].ToFloat32() + d3 := aa[3].ToFloat32() - bb[3].ToFloat32() + d4 := aa[4].ToFloat32() - bb[4].ToFloat32() + d5 := aa[5].ToFloat32() - bb[5].ToFloat32() + d6 := aa[6].ToFloat32() - bb[6].ToFloat32() + d7 := aa[7].ToFloat32() - bb[7].ToFloat32() + sum += (d0*d0 + d1*d1) + (d2*d2 + d3*d3) + (d4*d4 + d5*d5) + (d6*d6 + d7*d7) + } + for ; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + sum += d * d + } + return float64(sum), nil +} + +func innerProductBF16(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += (aa[0].ToFloat32()*bb[0].ToFloat32() + aa[1].ToFloat32()*bb[1].ToFloat32()) + + (aa[2].ToFloat32()*bb[2].ToFloat32() + aa[3].ToFloat32()*bb[3].ToFloat32()) + + (aa[4].ToFloat32()*bb[4].ToFloat32() + aa[5].ToFloat32()*bb[5].ToFloat32()) + + (aa[6].ToFloat32()*bb[6].ToFloat32() + aa[7].ToFloat32()*bb[7].ToFloat32()) + } + for ; i < n; i++ { + sum += a[i].ToFloat32() * b[i].ToFloat32() + } + return float64(-sum), nil +} + +func l1DistanceBF16(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + for i := range a { + d := a[i].ToFloat32() - b[i].ToFloat32() + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +// cosineDistClamped mirrors metric.CosineDistance: clamp the cosine similarity to +// [-1,1] (float32 accumulation / fp16 decode can push it a hair outside) before +// distance = 1 - sim, so a near-parallel pair never yields a tiny negative +// distance that would mis-sort in a top-k scan. +func cosineDistClamped(dot, denom float64) float64 { + sim := dot / denom + if sim > 1 { + sim = 1 + } else if sim < -1 { + sim = -1 + } + return 1.0 - sim +} + +func cosineDistanceBF16(a, b []types.BF16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var dot, na2, nb2 float32 + for i := range a { + ai := a[i].ToFloat32() + bi := b[i].ToFloat32() + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// magic-multiply branchless half->float (Fabian Giesen / rygorous, +// https://gist.github.com/rygorous/2156668). No loop and no fallback call, so it +// inlines into the kernels — unlike types.Float16.ToFloat32, whose subnormal +// normalization loop blocks inlining. The magic multiply rescales the exponent +// and turns half subnormals into the right normal floats in one step; the single +// branch only fixes up Inf/NaN. Verified EXHAUSTIVELY against ToFloat32 over all +// 65536 inputs (TestF16FastExhaustive). +var ( + f16Magic = math.Float32frombits(uint32(254-15) << 23) + f16WasInfNan = math.Float32frombits(uint32(127+16) << 23) +) + +func f16fast(h types.Float16) float32 { + o := uint32(h&0x7fff) << 13 // exponent/mantissa bits, into f32 position + of := math.Float32frombits(o) * f16Magic // rescale exponent; subnormals -> normals + ou := math.Float32bits(of) + if of >= f16WasInfNan { // Inf/NaN -> max exponent + ou |= 255 << 23 + } + ou |= uint32(h&0x8000) << 16 // sign + return math.Float32frombits(ou) +} + +// f16 kernel selection — swapped to archsimd impls by distance_func_narrow_f16_amd64.go. +var ( + f16L2sqFn = l2sqF16 + f16IPFn = innerProductF16 + f16CosineFn = cosineDistanceF16 + f16L1Fn = l1DistanceF16 +) + +func resolveF16Kernel(metric MetricType) (func(a, b []types.Float16) (float64, error), error) { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + return f16L2sqFn, nil + case Metric_InnerProduct: + return f16IPFn, nil + case Metric_CosineDistance: + return f16CosineFn, nil + case Metric_L1Distance: + return f16L1Fn, nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +func l2sqF16(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + d0 := f16fast(aa[0]) - f16fast(bb[0]) + d1 := f16fast(aa[1]) - f16fast(bb[1]) + d2 := f16fast(aa[2]) - f16fast(bb[2]) + d3 := f16fast(aa[3]) - f16fast(bb[3]) + d4 := f16fast(aa[4]) - f16fast(bb[4]) + d5 := f16fast(aa[5]) - f16fast(bb[5]) + d6 := f16fast(aa[6]) - f16fast(bb[6]) + d7 := f16fast(aa[7]) - f16fast(bb[7]) + sum += (d0*d0 + d1*d1) + (d2*d2 + d3*d3) + (d4*d4 + d5*d5) + (d6*d6 + d7*d7) + } + for ; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + sum += d * d + } + return float64(sum), nil +} + +func innerProductF16(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += (f16fast(aa[0])*f16fast(bb[0]) + f16fast(aa[1])*f16fast(bb[1])) + + (f16fast(aa[2])*f16fast(bb[2]) + f16fast(aa[3])*f16fast(bb[3])) + + (f16fast(aa[4])*f16fast(bb[4]) + f16fast(aa[5])*f16fast(bb[5])) + + (f16fast(aa[6])*f16fast(bb[6]) + f16fast(aa[7])*f16fast(bb[7])) + } + for ; i < n; i++ { + sum += f16fast(a[i]) * f16fast(b[i]) + } + return float64(-sum), nil +} + +func l1DistanceF16(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum float32 + for i := range a { + d := f16fast(a[i]) - f16fast(b[i]) + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceF16(a, b []types.Float16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var dot, na2, nb2 float32 + for i := range a { + ai := f16fast(a[i]) + bi := f16fast(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// int8 kernel selection — swapped to archsimd impls by distance_func_narrow_int8_amd64.go. +var ( + int8L2sqFn = l2sqInt8 + int8IPFn = innerProductInt8 + int8CosineFn = cosineDistanceInt8 + int8L1Fn = l1DistanceInt8 +) + +func resolveInt8Kernel(metric MetricType) (func(a, b []int8) (float64, error), error) { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + return int8L2sqFn, nil + case Metric_InnerProduct: + return int8IPFn, nil + case Metric_CosineDistance: + return int8CosineFn, nil + case Metric_L1Distance: + return int8L1Fn, nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +// ---------------------------------------------------------------------------- +// int8 integer kernels (unroll-8). Accumulate in int64: for int8 inputs the +// per-element term is bounded (|d|<=255 -> d*d<=65025, a*b in [-16384,16129]), +// and MaxArrayDimension is 65535, so int32 could overflow — int64 cannot. +// ---------------------------------------------------------------------------- + +func l2sqInt8(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + d0 := int32(aa[0]) - int32(bb[0]) + d1 := int32(aa[1]) - int32(bb[1]) + d2 := int32(aa[2]) - int32(bb[2]) + d3 := int32(aa[3]) - int32(bb[3]) + d4 := int32(aa[4]) - int32(bb[4]) + d5 := int32(aa[5]) - int32(bb[5]) + d6 := int32(aa[6]) - int32(bb[6]) + d7 := int32(aa[7]) - int32(bb[7]) + sum += int64(d0*d0+d1*d1) + int64(d2*d2+d3*d3) + int64(d4*d4+d5*d5) + int64(d6*d6+d7*d7) + } + for ; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductInt8(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += int64(int32(aa[0])*int32(bb[0])+int32(aa[1])*int32(bb[1])) + + int64(int32(aa[2])*int32(bb[2])+int32(aa[3])*int32(bb[3])) + + int64(int32(aa[4])*int32(bb[4])+int32(aa[5])*int32(bb[5])) + + int64(int32(aa[6])*int32(bb[6])+int32(aa[7])*int32(bb[7])) + } + for ; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + // matches metric.InnerProduct: returns -dot + return float64(-sum), nil +} + +func l1DistanceInt8(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + abs := func(x int32) int32 { + if x < 0 { + return -x + } + return x + } + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += int64(abs(int32(aa[0])-int32(bb[0]))+abs(int32(aa[1])-int32(bb[1]))) + + int64(abs(int32(aa[2])-int32(bb[2]))+abs(int32(aa[3])-int32(bb[3]))) + + int64(abs(int32(aa[4])-int32(bb[4]))+abs(int32(aa[5])-int32(bb[5]))) + + int64(abs(int32(aa[6])-int32(bb[6]))+abs(int32(aa[7])-int32(bb[7]))) + } + for ; i < n; i++ { + sum += int64(abs(int32(a[i]) - int32(b[i]))) + } + return float64(sum), nil +} + +func cosineDistanceInt8(a, b []int8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var dot, na2, nb2 int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + for k := 0; k < 8; k++ { + ai := int64(aa[k]) + bi := int64(bb[k]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + } + for ; i < n; i++ { + ai := int64(a[i]) + bi := int64(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + // matches metric.CosineDistance: denominator 0 -> distance 1.0 + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_amd64.go new file mode 100644 index 0000000000000..144f583e57758 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_amd64.go @@ -0,0 +1,187 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX-512 SIMD distance kernels for vecbf16 (types.BF16). +// +// bf16 is the high 16 bits of an IEEE float32, so the decode bf16->f32 is a pure +// bit op: value<<16. Go's archsimd has no bf16 type and no AVX512BF16 detector +// (and the native VDPBF16PS is therefore unreachable from Go), but we don't need +// it: load the raw bf16 bytes as Uint32x16 (32 bf16 per load), split the even and +// odd 16-bit halves into two Float32x16 vectors via one shift + one and-mask + +// AsFloat32x16 bitcast, then reuse the existing AVX-512 float32 reduction +// (sumF32x16 / hasAVX512 from distance_func_amd64.go — same package + build tag). +// +// The pure-Go kernels in distance_func_narrow.go stay the fallback (non-AVX512 +// CPUs) and the equivalence oracle; init() only swaps the selection vars when +// hasAVX512 is true. + +package metric + +import ( + "math" + "unsafe" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +func init() { + switch { + case hasAVX512: + bf16L2sqFn = l2sqBF16SIMD + bf16IPFn = innerProductBF16SIMD + bf16CosineFn = cosineDistanceBF16SIMD + bf16L1Fn = l1DistanceBF16SIMD + case hasAVX2: + bf16L2sqFn = l2sqBF16AVX2 + bf16IPFn = innerProductBF16AVX2 + bf16CosineFn = cosineDistanceBF16AVX2 + bf16L1Fn = l1DistanceBF16AVX2 + } +} + +// bf16AsU32 reinterprets a []types.BF16 (uint16-backed) as []uint32 viewing its +// first len/2 even-aligned pairs. x86 tolerates the unaligned load; the stored +// bf16 bytes originate from an 8-aligned []byte (BytesToArray), so in practice +// the start is 4-aligned. +func bf16AsU32(s []types.BF16) []uint32 { + if len(s) < 2 { + return nil + } + return unsafe.Slice((*uint32)(unsafe.Pointer(unsafe.SliceData(s))), len(s)/2) +} + +func l2sqBF16SIMD(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x16(0xFFFF0000) + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + dE := ua.ShiftAllLeft(16).AsFloat32x16().Sub(ub.ShiftAllLeft(16).AsFloat32x16()) + dO := ua.And(hi).AsFloat32x16().Sub(ub.And(hi).AsFloat32x16()) + acc0 = dE.MulAdd(dE, acc0) + acc1 = dO.MulAdd(dO, acc1) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + sum += d * d + } + return float64(sum), nil +} + +func innerProductBF16SIMD(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x16(0xFFFF0000) + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + acc0 = ua.ShiftAllLeft(16).AsFloat32x16().MulAdd(ub.ShiftAllLeft(16).AsFloat32x16(), acc0) + acc1 = ua.And(hi).AsFloat32x16().MulAdd(ub.And(hi).AsFloat32x16(), acc1) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + sum += a[i].ToFloat32() * b[i].ToFloat32() + } + return float64(-sum), nil +} + +func l1DistanceBF16SIMD(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x16(0xFFFF0000) + absMask := archsimd.BroadcastUint32x16(0x7FFFFFFF) + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + dE := ua.ShiftAllLeft(16).AsFloat32x16().Sub(ub.ShiftAllLeft(16).AsFloat32x16()) + dO := ua.And(hi).AsFloat32x16().Sub(ub.And(hi).AsFloat32x16()) + acc0 = acc0.Add(dE.AsUint32x16().And(absMask).AsFloat32x16()) + acc1 = acc1.Add(dO.AsUint32x16().And(absMask).AsFloat32x16()) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceBF16SIMD(a, b []types.BF16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x16(0xFFFF0000) + dot0, dot1 := archsimd.Float32x16{}, archsimd.Float32x16{} + na0, na1 := archsimd.Float32x16{}, archsimd.Float32x16{} + nb0, nb1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + aE := ua.ShiftAllLeft(16).AsFloat32x16() + aO := ua.And(hi).AsFloat32x16() + bE := ub.ShiftAllLeft(16).AsFloat32x16() + bO := ub.And(hi).AsFloat32x16() + dot0 = aE.MulAdd(bE, dot0) + dot1 = aO.MulAdd(bO, dot1) + na0 = aE.MulAdd(aE, na0) + na1 = aO.MulAdd(aO, na1) + nb0 = bE.MulAdd(bE, nb0) + nb1 = bO.MulAdd(bO, nb1) + } + dot := sumF32x16(dot0.Add(dot1)) + na2 := sumF32x16(na0.Add(na1)) + nb2 := sumF32x16(nb0.Add(nb1)) + for i := j * 2; i < n; i++ { + ai, bi := a[i].ToFloat32(), b[i].ToFloat32() + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_amd64_test.go b/pkg/vectorindex/metric/distance_func_narrow_amd64_test.go new file mode 100644 index 0000000000000..ad5b04461d5c9 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_amd64_test.go @@ -0,0 +1,349 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// SIMD-build-only tests: the narrow archsimd kernels (bf16/f16/int8) coexist with +// their pure-Go twins here, so we can (a) prove they agree and (b) benchmark them +// head to head in one binary. Only built under `GOEXPERIMENT=simd GOAMD64=v3`. + +package metric + +import ( + "math" + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// dims exercise the 16-lane main loop (bf16/f16: 32/iter, int8: 64/iter) plus +// every tail remainder, including odd final elements. +var narrowSIMDDims = []int{1, 2, 3, 4, 7, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 1000, 1024, 1025} + +func randF32(dim int, r *rand.Rand) []float32 { + f := make([]float32, dim) + for i := range f { + f[i] = float32(r.Float64()*16 - 8) // [-8, 8) + } + return f +} +func randBF16(dim int, r *rand.Rand) []types.BF16 { return types.Float32ToBF16Slice(randF32(dim, r)) } +func randF16(dim int, r *rand.Rand) []types.Float16 { + return types.Float32ToFloat16Slice(randF32(dim, r)) +} +func randI8(dim int, r *rand.Rand) []int8 { + v := make([]int8, dim) + for i := range v { + v[i] = int8(r.Intn(255) - 127) + } + return v +} +func randU8(dim int, r *rand.Rand) []uint8 { + v := make([]uint8, dim) + for i := range v { + v[i] = uint8(r.Intn(256)) + } + return v +} + +// checkPair asserts a SIMD kernel matches its scalar oracle. exact=true requires +// bit-equality (integer int8 L2sq/IP/L1); otherwise a magnitude-scaled tolerance +// (float reductions reorder). +func checkPair(t *testing.T, name string, dim int, got, want float64, exact bool) { + t.Helper() + if exact { + require.Equal(t, want, got, "%s dim=%d", name, dim) + return + } + require.InDelta(t, want, got, 1e-4*(1+math.Abs(want)), "%s dim=%d", name, dim) +} + +func TestBF16SIMDMatchesScalar(t *testing.T) { + if !hasAVX512 { + t.Skip("AVX-512 not available") + } + r := rand.New(rand.NewSource(42)) + type k struct { + name string + simd, scalar func(a, b []types.BF16) (float64, error) + } + for _, kn := range []k{ + {"l2sq", l2sqBF16SIMD, l2sqBF16}, + {"innerproduct", innerProductBF16SIMD, innerProductBF16}, + {"l1", l1DistanceBF16SIMD, l1DistanceBF16}, + {"cosine", cosineDistanceBF16SIMD, cosineDistanceBF16}, + } { + for _, dim := range narrowSIMDDims { + a, b := randBF16(dim, r), randBF16(dim, r) + got, err := kn.simd(a, b) + require.NoError(t, err) + want, err := kn.scalar(a, b) + require.NoError(t, err) + checkPair(t, "bf16/"+kn.name, dim, got, want, false) + } + } +} + +func TestF16SIMDMatchesScalar(t *testing.T) { + if !hasAVX512 { + t.Skip("AVX-512 not available") + } + r := rand.New(rand.NewSource(7)) + type k struct { + name string + simd, scalar func(a, b []types.Float16) (float64, error) + } + for _, kn := range []k{ + {"l2sq", l2sqF16SIMD, l2sqF16}, + {"innerproduct", innerProductF16SIMD, innerProductF16}, + {"l1", l1DistanceF16SIMD, l1DistanceF16}, + {"cosine", cosineDistanceF16SIMD, cosineDistanceF16}, + } { + for _, dim := range narrowSIMDDims { + a, b := randF16(dim, r), randF16(dim, r) + got, err := kn.simd(a, b) + require.NoError(t, err) + want, err := kn.scalar(a, b) + require.NoError(t, err) + checkPair(t, "f16/"+kn.name, dim, got, want, false) + } + } +} + +func TestInt8SIMDMatchesScalar(t *testing.T) { + if !hasAVX512 { + t.Skip("AVX-512 not available") + } + r := rand.New(rand.NewSource(9)) + type k struct { + name string + simd, scalar func(a, b []int8) (float64, error) + exact bool // integer kernels are bit-exact; cosine goes through float + } + for _, kn := range []k{ + {"l2sq", l2sqInt8SIMD, l2sqInt8, true}, + {"innerproduct", innerProductInt8SIMD, innerProductInt8, true}, + {"l1", l1DistanceInt8SIMD, l1DistanceInt8, true}, + {"cosine", cosineDistanceInt8SIMD, cosineDistanceInt8, false}, + } { + for _, dim := range narrowSIMDDims { + a, b := randI8(dim, r), randI8(dim, r) + got, err := kn.simd(a, b) + require.NoError(t, err) + want, err := kn.scalar(a, b) + require.NoError(t, err) + checkPair(t, "int8/"+kn.name, dim, got, want, kn.exact) + } + } +} + +func TestUint8SIMDMatchesScalar(t *testing.T) { + if !hasAVX2 { + t.Skip("AVX2 not available") + } + r := rand.New(rand.NewSource(11)) + type k struct { + name string + simd, scalar func(a, b []uint8) (float64, error) + exact bool // integer kernels are bit-exact; cosine goes through float + } + simdSet := func() []k { + if hasAVX512 { + return []k{ + {"l2sq", l2sqUint8SIMD, l2sqUint8, true}, + {"innerproduct", innerProductUint8SIMD, innerProductUint8, true}, + {"l1", l1DistanceUint8SIMD, l1DistanceUint8, true}, + {"cosine", cosineDistanceUint8SIMD, cosineDistanceUint8, false}, + } + } + return []k{ + {"l2sq", l2sqUint8AVX2, l2sqUint8, true}, + {"innerproduct", innerProductUint8AVX2, innerProductUint8, true}, + {"l1", l1DistanceUint8AVX2, l1DistanceUint8, true}, + {"cosine", cosineDistanceUint8AVX2, cosineDistanceUint8, false}, + } + } + for _, kn := range simdSet() { + for _, dim := range narrowSIMDDims { + a, b := randU8(dim, r), randU8(dim, r) + got, err := kn.simd(a, b) + require.NoError(t, err) + want, err := kn.scalar(a, b) + require.NoError(t, err) + checkPair(t, "uint8/"+kn.name, dim, got, want, kn.exact) + } + } +} + +// ---- head-to-head benchmarks (dim=1024, same binary) ---- +// +// GOEXPERIMENT=simd GOAMD64=v3 go test ./pkg/vectorindex/metric/ \ +// -run x -bench Benchmark_Narrow_SIMDvsScalar -benchmem + +func Benchmark_Narrow_SIMDvsScalar(b *testing.B) { + const dim = 1024 + r := rand.New(rand.NewSource(1)) + bf16a, bf16b := randBF16(dim, r), randBF16(dim, r) + f16a, f16b := randF16(dim, r), randF16(dim, r) + i8a, i8b := randI8(dim, r), randI8(dim, r) + u8a, u8b := randU8(dim, r), randU8(dim, r) + f32a, f32b := randF32(dim, r), randF32(dim, r) + f64a := make([]float64, dim) + f64b := make([]float64, dim) + for i := range f64a { + f64a[i] = float64(f32a[i]) + f64b[i] = float64(f32b[i]) + } + + runF32 := func(b *testing.B, fn func(a, c []float32) (float32, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(f32a, f32b) + } + } + runF64 := func(b *testing.B, fn func(a, c []float64) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(f64a, f64b) + } + } + runBF16 := func(b *testing.B, fn func(a, c []types.BF16) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(bf16a, bf16b) + } + } + runF16 := func(b *testing.B, fn func(a, c []types.Float16) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(f16a, f16b) + } + } + runI8 := func(b *testing.B, fn func(a, c []int8) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(i8a, i8b) + } + } + runU8 := func(b *testing.B, fn func(a, c []uint8) (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn(u8a, u8b) + } + } + + // avx512 sub-benchmarks call the x16 kernels directly, so they only run when + // the CPU has AVX-512 (calling them otherwise would fault). avx2 always runs + // (AVX2 is implied by the GOAMD64=v3 build). This bench bypasses the function- + // pointer selection on purpose, so a single run shows all three tiers + // side-by-side regardless of the MO_METRIC_NO_AVX512/AVX2 overrides. + // f32/f64 native baselines (no decode). These auto-dispatch to AVX-512 + // internally via hasAVX512; with MO_METRIC_NO_AVX512=1 they fall to scalar Go. + b.Run("f32", func(b *testing.B) { + b.Run("l2sq", func(b *testing.B) { runF32(b, L2DistanceSq[float32]) }) + b.Run("innerproduct", func(b *testing.B) { runF32(b, InnerProduct[float32]) }) + b.Run("l1", func(b *testing.B) { runF32(b, L1Distance[float32]) }) + b.Run("cosine", func(b *testing.B) { runF32(b, CosineDistance[float32]) }) + }) + b.Run("f64", func(b *testing.B) { + b.Run("l2sq", func(b *testing.B) { runF64(b, L2DistanceSq[float64]) }) + b.Run("innerproduct", func(b *testing.B) { runF64(b, InnerProduct[float64]) }) + b.Run("l1", func(b *testing.B) { runF64(b, L1Distance[float64]) }) + b.Run("cosine", func(b *testing.B) { runF64(b, CosineDistance[float64]) }) + }) + b.Run("bf16", func(b *testing.B) { + b.Run("l2sq/scalar", func(b *testing.B) { runBF16(b, l2sqBF16) }) + b.Run("l2sq/avx2", func(b *testing.B) { runBF16(b, l2sqBF16AVX2) }) + if hasAVX512 { + b.Run("l2sq/avx512", func(b *testing.B) { runBF16(b, l2sqBF16SIMD) }) + } + b.Run("innerproduct/scalar", func(b *testing.B) { runBF16(b, innerProductBF16) }) + b.Run("innerproduct/avx2", func(b *testing.B) { runBF16(b, innerProductBF16AVX2) }) + if hasAVX512 { + b.Run("innerproduct/avx512", func(b *testing.B) { runBF16(b, innerProductBF16SIMD) }) + } + b.Run("l1/scalar", func(b *testing.B) { runBF16(b, l1DistanceBF16) }) + b.Run("l1/avx2", func(b *testing.B) { runBF16(b, l1DistanceBF16AVX2) }) + if hasAVX512 { + b.Run("l1/avx512", func(b *testing.B) { runBF16(b, l1DistanceBF16SIMD) }) + } + b.Run("cosine/scalar", func(b *testing.B) { runBF16(b, cosineDistanceBF16) }) + b.Run("cosine/avx2", func(b *testing.B) { runBF16(b, cosineDistanceBF16AVX2) }) + if hasAVX512 { + b.Run("cosine/avx512", func(b *testing.B) { runBF16(b, cosineDistanceBF16SIMD) }) + } + }) + b.Run("f16", func(b *testing.B) { + b.Run("l2sq/scalar", func(b *testing.B) { runF16(b, l2sqF16) }) + b.Run("l2sq/avx2", func(b *testing.B) { runF16(b, l2sqF16AVX2) }) + if hasAVX512 { + b.Run("l2sq/avx512", func(b *testing.B) { runF16(b, l2sqF16SIMD) }) + } + b.Run("innerproduct/scalar", func(b *testing.B) { runF16(b, innerProductF16) }) + b.Run("innerproduct/avx2", func(b *testing.B) { runF16(b, innerProductF16AVX2) }) + if hasAVX512 { + b.Run("innerproduct/avx512", func(b *testing.B) { runF16(b, innerProductF16SIMD) }) + } + b.Run("l1/scalar", func(b *testing.B) { runF16(b, l1DistanceF16) }) + b.Run("l1/avx2", func(b *testing.B) { runF16(b, l1DistanceF16AVX2) }) + if hasAVX512 { + b.Run("l1/avx512", func(b *testing.B) { runF16(b, l1DistanceF16SIMD) }) + } + b.Run("cosine/scalar", func(b *testing.B) { runF16(b, cosineDistanceF16) }) + b.Run("cosine/avx2", func(b *testing.B) { runF16(b, cosineDistanceF16AVX2) }) + if hasAVX512 { + b.Run("cosine/avx512", func(b *testing.B) { runF16(b, cosineDistanceF16SIMD) }) + } + }) + b.Run("int8", func(b *testing.B) { + b.Run("l2sq/scalar", func(b *testing.B) { runI8(b, l2sqInt8) }) + b.Run("l2sq/avx2", func(b *testing.B) { runI8(b, l2sqInt8AVX2) }) + if hasAVX512 { + b.Run("l2sq/avx512", func(b *testing.B) { runI8(b, l2sqInt8SIMD) }) + } + b.Run("innerproduct/scalar", func(b *testing.B) { runI8(b, innerProductInt8) }) + b.Run("innerproduct/avx2", func(b *testing.B) { runI8(b, innerProductInt8AVX2) }) + if hasAVX512 { + b.Run("innerproduct/avx512", func(b *testing.B) { runI8(b, innerProductInt8SIMD) }) + } + b.Run("l1/scalar", func(b *testing.B) { runI8(b, l1DistanceInt8) }) + b.Run("l1/avx2", func(b *testing.B) { runI8(b, l1DistanceInt8AVX2) }) + if hasAVX512 { + b.Run("l1/avx512", func(b *testing.B) { runI8(b, l1DistanceInt8SIMD) }) + } + b.Run("cosine/scalar", func(b *testing.B) { runI8(b, cosineDistanceInt8) }) + b.Run("cosine/avx2", func(b *testing.B) { runI8(b, cosineDistanceInt8AVX2) }) + if hasAVX512 { + b.Run("cosine/avx512", func(b *testing.B) { runI8(b, cosineDistanceInt8SIMD) }) + } + }) + b.Run("uint8", func(b *testing.B) { + b.Run("l2sq/scalar", func(b *testing.B) { runU8(b, l2sqUint8) }) + b.Run("l2sq/avx2", func(b *testing.B) { runU8(b, l2sqUint8AVX2) }) + if hasAVX512 { + b.Run("l2sq/avx512", func(b *testing.B) { runU8(b, l2sqUint8SIMD) }) + } + b.Run("innerproduct/scalar", func(b *testing.B) { runU8(b, innerProductUint8) }) + b.Run("innerproduct/avx2", func(b *testing.B) { runU8(b, innerProductUint8AVX2) }) + if hasAVX512 { + b.Run("innerproduct/avx512", func(b *testing.B) { runU8(b, innerProductUint8SIMD) }) + } + b.Run("l1/scalar", func(b *testing.B) { runU8(b, l1DistanceUint8) }) + b.Run("l1/avx2", func(b *testing.B) { runU8(b, l1DistanceUint8AVX2) }) + if hasAVX512 { + b.Run("l1/avx512", func(b *testing.B) { runU8(b, l1DistanceUint8SIMD) }) + } + b.Run("cosine/scalar", func(b *testing.B) { runU8(b, cosineDistanceUint8) }) + b.Run("cosine/avx2", func(b *testing.B) { runU8(b, cosineDistanceUint8AVX2) }) + if hasAVX512 { + b.Run("cosine/avx512", func(b *testing.B) { runU8(b, cosineDistanceUint8SIMD) }) + } + }) +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64.go new file mode 100644 index 0000000000000..5583891e18a8c --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64.go @@ -0,0 +1,431 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX2 (256-bit, 8-lane) narrow distance kernels — the middle fallback tier for +// CPUs that have AVX2 but not AVX-512. Mirrors the AVX-512 (x16) kernels exactly +// with Float32x8 / Int32x8 / Uint32x8 ops. Each type's init() (in its x16 file) +// selects AVX-512 -> AVX2 -> scalar. +// +// The narrow types benefit from AVX2 (unlike f32, where AVX2 ~= scalar): their +// decode (bf16 shift / int8 sign-extend / f16 magic-multiply) is pure scalar +// overhead that AVX2 vectorizes. Measured ~7-9x over scalar at dim=1024; AVX-512 +// adds another ~1.3-1.5x for bf16/f16, and ~nothing for int8 (byte-unpack bound). + +package metric + +import ( + "math" + "os" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// hasAVX2 gates the middle tier. AVX512 implies AVX2, so init() checks hasAVX512 +// first; this only decides AVX2-vs-scalar on non-AVX512 CPUs. +// TESTING-ONLY override: set MO_METRIC_NO_AVX2=1 (typically with +// MO_METRIC_NO_AVX512=1) to force the pure-Go scalar fallback for coverage. +var hasAVX2 = archsimd.X86.AVX2() && os.Getenv("MO_METRIC_NO_AVX2") == "" + +func sumF32x8(v archsimd.Float32x8) float32 { + var a [8]float32 + v.Store(&a) + return ((a[0] + a[1]) + (a[2] + a[3])) + ((a[4] + a[5]) + (a[6] + a[7])) +} + +func sumI32x8(v archsimd.Int32x8) int64 { + var a [8]int32 + v.Store(&a) + var s int64 + for _, x := range a { + s += int64(x) + } + return s +} + +// ---- bf16 (AVX2) ---- + +func l2sqBF16AVX2(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x8(0xFFFF0000) + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + dE := ua.ShiftAllLeft(16).AsFloat32x8().Sub(ub.ShiftAllLeft(16).AsFloat32x8()) + dO := ua.And(hi).AsFloat32x8().Sub(ub.And(hi).AsFloat32x8()) + acc0 = dE.MulAdd(dE, acc0) + acc1 = dO.MulAdd(dO, acc1) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + sum += d * d + } + return float64(sum), nil +} + +func innerProductBF16AVX2(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x8(0xFFFF0000) + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + acc0 = ua.ShiftAllLeft(16).AsFloat32x8().MulAdd(ub.ShiftAllLeft(16).AsFloat32x8(), acc0) + acc1 = ua.And(hi).AsFloat32x8().MulAdd(ub.And(hi).AsFloat32x8(), acc1) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + sum += a[i].ToFloat32() * b[i].ToFloat32() + } + return float64(-sum), nil +} + +func l1DistanceBF16AVX2(a, b []types.BF16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x8(0xFFFF0000) + absMask := archsimd.BroadcastUint32x8(0x7FFFFFFF) + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + dE := ua.ShiftAllLeft(16).AsFloat32x8().Sub(ub.ShiftAllLeft(16).AsFloat32x8()) + dO := ua.And(hi).AsFloat32x8().Sub(ub.And(hi).AsFloat32x8()) + acc0 = acc0.Add(dE.AsUint32x8().And(absMask).AsFloat32x8()) + acc1 = acc1.Add(dO.AsUint32x8().And(absMask).AsFloat32x8()) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := a[i].ToFloat32() - b[i].ToFloat32() + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceBF16AVX2(a, b []types.BF16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := bf16AsU32(a), bf16AsU32(b) + hi := archsimd.BroadcastUint32x8(0xFFFF0000) + dot0, dot1 := archsimd.Float32x8{}, archsimd.Float32x8{} + na0, na1 := archsimd.Float32x8{}, archsimd.Float32x8{} + nb0, nb1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + aE := ua.ShiftAllLeft(16).AsFloat32x8() + aO := ua.And(hi).AsFloat32x8() + bE := ub.ShiftAllLeft(16).AsFloat32x8() + bO := ub.And(hi).AsFloat32x8() + dot0 = aE.MulAdd(bE, dot0) + dot1 = aO.MulAdd(bO, dot1) + na0 = aE.MulAdd(aE, na0) + na1 = aO.MulAdd(aO, na1) + nb0 = bE.MulAdd(bE, nb0) + nb1 = bO.MulAdd(bO, nb1) + } + dot := sumF32x8(dot0.Add(dot1)) + na2 := sumF32x8(na0.Add(na1)) + nb2 := sumF32x8(nb0.Add(nb1)) + for i := j * 2; i < n; i++ { + ai, bi := a[i].ToFloat32(), b[i].ToFloat32() + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// ---- int8 (AVX2), integer-exact ---- + +func unpackI8x8(u archsimd.Int32x8) (v0, v1, v2, v3 archsimd.Int32x8) { + v0 = u.ShiftAllLeft(24).ShiftAllRight(24) + v1 = u.ShiftAllLeft(16).ShiftAllRight(24) + v2 = u.ShiftAllLeft(8).ShiftAllRight(24) + v3 = u.ShiftAllRight(24) + return +} + +func l2sqInt8AVX2(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + acc := archsimd.Int32x8{} + nq, j := len(ai), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackI8x8(archsimd.LoadInt32x8Slice(ai[j : j+8])) + b0, b1, b2, b3 := unpackI8x8(archsimd.LoadInt32x8Slice(bi[j : j+8])) + d0, d1, d2, d3 := a0.Sub(b0), a1.Sub(b1), a2.Sub(b2), a3.Sub(b3) + acc = acc.Add(d0.Mul(d0).Add(d1.Mul(d1)).Add(d2.Mul(d2).Add(d3.Mul(d3)))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductInt8AVX2(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + acc := archsimd.Int32x8{} + nq, j := len(ai), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackI8x8(archsimd.LoadInt32x8Slice(ai[j : j+8])) + b0, b1, b2, b3 := unpackI8x8(archsimd.LoadInt32x8Slice(bi[j : j+8])) + acc = acc.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + return float64(-sum), nil +} + +func l1DistanceInt8AVX2(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + zero := archsimd.Int32x8{} + acc := archsimd.Int32x8{} + abs := func(d archsimd.Int32x8) archsimd.Int32x8 { return d.Max(zero.Sub(d)) } + nq, j := len(ai), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackI8x8(archsimd.LoadInt32x8Slice(ai[j : j+8])) + b0, b1, b2, b3 := unpackI8x8(archsimd.LoadInt32x8Slice(bi[j : j+8])) + acc = acc.Add(abs(a0.Sub(b0)).Add(abs(a1.Sub(b1))).Add(abs(a2.Sub(b2)).Add(abs(a3.Sub(b3))))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + if d < 0 { + d = -d + } + sum += int64(d) + } + return float64(sum), nil +} + +func cosineDistanceInt8AVX2(a, b []int8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + dotA, naA, nbA := archsimd.Int32x8{}, archsimd.Int32x8{}, archsimd.Int32x8{} + nq, j := len(ai), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackI8x8(archsimd.LoadInt32x8Slice(ai[j : j+8])) + b0, b1, b2, b3 := unpackI8x8(archsimd.LoadInt32x8Slice(bi[j : j+8])) + dotA = dotA.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + naA = naA.Add(a0.Mul(a0).Add(a1.Mul(a1)).Add(a2.Mul(a2).Add(a3.Mul(a3)))) + nbA = nbA.Add(b0.Mul(b0).Add(b1.Mul(b1)).Add(b2.Mul(b2).Add(b3.Mul(b3)))) + } + dot, na2, nb2 := sumI32x8(dotA), sumI32x8(naA), sumI32x8(nbA) + for i := j * 4; i < n; i++ { + ai8, bi8 := int64(a[i]), int64(b[i]) + dot += ai8 * bi8 + na2 += ai8 * ai8 + nb2 += bi8 * bi8 + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// ---- f16 (AVX2) ---- + +func f16decX8(h, m7fff, m8000, mInf archsimd.Uint32x8, magic, infNan archsimd.Float32x8) archsimd.Float32x8 { + o := h.And(m7fff).ShiftAllLeft(13) + of := o.AsFloat32x8().Mul(magic) + ou := of.AsUint32x8() + ou = ou.Or(mInf).Merge(ou, of.GreaterEqual(infNan)) + return ou.Or(h.And(m8000).ShiftAllLeft(16)).AsFloat32x8() +} + +func f16DecodeConstsX8() (m7fff, m8000, mLo, mInf archsimd.Uint32x8, magic, infNan archsimd.Float32x8) { + m7fff = archsimd.BroadcastUint32x8(0x7fff) + m8000 = archsimd.BroadcastUint32x8(0x8000) + mLo = archsimd.BroadcastUint32x8(0xffff) + mInf = archsimd.BroadcastUint32x8(255 << 23) + magic = archsimd.BroadcastFloat32x8(f16Magic) + infNan = archsimd.BroadcastFloat32x8(f16WasInfNan) + return +} + +func l2sqF16AVX2(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConstsX8() + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + dE := f16decX8(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).Sub(f16decX8(ub.And(mLo), m7fff, m8000, mInf, magic, infNan)) + dO := f16decX8(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).Sub(f16decX8(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan)) + acc0 = dE.MulAdd(dE, acc0) + acc1 = dO.MulAdd(dO, acc1) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + sum += d * d + } + return float64(sum), nil +} + +func innerProductF16AVX2(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConstsX8() + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + acc0 = f16decX8(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).MulAdd(f16decX8(ub.And(mLo), m7fff, m8000, mInf, magic, infNan), acc0) + acc1 = f16decX8(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).MulAdd(f16decX8(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan), acc1) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + sum += f16fast(a[i]) * f16fast(b[i]) + } + return float64(-sum), nil +} + +func l1DistanceF16AVX2(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConstsX8() + absMask := archsimd.BroadcastUint32x8(0x7fffffff) + acc0, acc1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + dE := f16decX8(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).Sub(f16decX8(ub.And(mLo), m7fff, m8000, mInf, magic, infNan)) + dO := f16decX8(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).Sub(f16decX8(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan)) + acc0 = acc0.Add(dE.AsUint32x8().And(absMask).AsFloat32x8()) + acc1 = acc1.Add(dO.AsUint32x8().And(absMask).AsFloat32x8()) + } + sum := sumF32x8(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceF16AVX2(a, b []types.Float16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConstsX8() + dot0, dot1 := archsimd.Float32x8{}, archsimd.Float32x8{} + na0, na1 := archsimd.Float32x8{}, archsimd.Float32x8{} + nb0, nb1 := archsimd.Float32x8{}, archsimd.Float32x8{} + np, j := len(au), 0 + for ; j <= np-8; j += 8 { + ua := archsimd.LoadUint32x8Slice(au[j : j+8]) + ub := archsimd.LoadUint32x8Slice(bu[j : j+8]) + aE := f16decX8(ua.And(mLo), m7fff, m8000, mInf, magic, infNan) + aO := f16decX8(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan) + bE := f16decX8(ub.And(mLo), m7fff, m8000, mInf, magic, infNan) + bO := f16decX8(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan) + dot0 = aE.MulAdd(bE, dot0) + dot1 = aO.MulAdd(bO, dot1) + na0 = aE.MulAdd(aE, na0) + na1 = aO.MulAdd(aO, na1) + nb0 = bE.MulAdd(bE, nb0) + nb1 = bO.MulAdd(bO, nb1) + } + dot := sumF32x8(dot0.Add(dot1)) + na2 := sumF32x8(na0.Add(na1)) + nb2 := sumF32x8(nb0.Add(nb1)) + for i := j * 2; i < n; i++ { + ai, bi := f16fast(a[i]), f16fast(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64_test.go b/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64_test.go new file mode 100644 index 0000000000000..aae466a4b7444 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_avx2_amd64_test.go @@ -0,0 +1,120 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests + benchmark for the AVX2 (256-bit) narrow fallback tier. The AVX2 +// kernels live in distance_func_narrow_avx2_amd64.go (production); here we prove +// they match the scalar oracle and benchmark scalar / AVX2 / AVX-512 side by side +// in one binary. Only built under `GOEXPERIMENT=simd GOAMD64=v3`. + +package metric + +import ( + "math" + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// TestAVX2NarrowMatchesScalar checks all four metrics of each AVX2 narrow kernel +// against the scalar oracle across dims covering the 8-lane loop + every tail. +func TestAVX2NarrowMatchesScalar(t *testing.T) { + r := rand.New(rand.NewSource(11)) + chk := func(name string, dim int, got, want float64, exact bool) { + t.Helper() + if exact { + require.Equal(t, want, got, "%s dim=%d", name, dim) + return + } + require.InDelta(t, want, got, 1e-4*(1+math.Abs(want)), "%s dim=%d", name, dim) + } + for _, dim := range narrowSIMDDims { + bfa, bfb := randBF16(dim, r), randBF16(dim, r) + for _, k := range []struct { + name string + avx2, scalar func(a, b []types.BF16) (float64, error) + }{ + {"bf16/l2sq", l2sqBF16AVX2, l2sqBF16}, + {"bf16/ip", innerProductBF16AVX2, innerProductBF16}, + {"bf16/l1", l1DistanceBF16AVX2, l1DistanceBF16}, + {"bf16/cosine", cosineDistanceBF16AVX2, cosineDistanceBF16}, + } { + g, _ := k.avx2(bfa, bfb) + w, _ := k.scalar(bfa, bfb) + chk(k.name, dim, g, w, false) + } + + fa, fb := randF16(dim, r), randF16(dim, r) + for _, k := range []struct { + name string + avx2, scalar func(a, b []types.Float16) (float64, error) + }{ + {"f16/l2sq", l2sqF16AVX2, l2sqF16}, + {"f16/ip", innerProductF16AVX2, innerProductF16}, + {"f16/l1", l1DistanceF16AVX2, l1DistanceF16}, + {"f16/cosine", cosineDistanceF16AVX2, cosineDistanceF16}, + } { + g, _ := k.avx2(fa, fb) + w, _ := k.scalar(fa, fb) + chk(k.name, dim, g, w, false) + } + + i8a, i8b := randI8(dim, r), randI8(dim, r) + for _, k := range []struct { + name string + avx2, scalar func(a, b []int8) (float64, error) + exact bool + }{ + {"int8/l2sq", l2sqInt8AVX2, l2sqInt8, true}, + {"int8/ip", innerProductInt8AVX2, innerProductInt8, true}, + {"int8/l1", l1DistanceInt8AVX2, l1DistanceInt8, true}, + {"int8/cosine", cosineDistanceInt8AVX2, cosineDistanceInt8, false}, + } { + g, _ := k.avx2(i8a, i8b) + w, _ := k.scalar(i8a, i8b) + chk(k.name, dim, g, w, k.exact) + } + } +} + +// Benchmark_Narrow_AVX2vsAVX512 compares scalar / AVX2 (x8) / AVX-512 (x16) for +// the narrow L2sq kernels in one binary. +// +// GOEXPERIMENT=simd GOAMD64=v3 go test ./pkg/vectorindex/metric/ \ +// -run x -bench Benchmark_Narrow_AVX2vsAVX512 +func Benchmark_Narrow_AVX2vsAVX512(b *testing.B) { + const dim = 1024 + r := rand.New(rand.NewSource(1)) + bfa, bfb := randBF16(dim, r), randBF16(dim, r) + fa, fb := randF16(dim, r), randF16(dim, r) + i8a, i8b := randI8(dim, r), randI8(dim, r) + + run := func(b *testing.B, fn func() (float64, error)) { + for i := 0; i < b.N; i++ { + _, _ = fn() + } + } + b.Run("bf16/scalar", func(b *testing.B) { run(b, func() (float64, error) { return l2sqBF16(bfa, bfb) }) }) + b.Run("bf16/avx2", func(b *testing.B) { run(b, func() (float64, error) { return l2sqBF16AVX2(bfa, bfb) }) }) + b.Run("bf16/avx512", func(b *testing.B) { run(b, func() (float64, error) { return l2sqBF16SIMD(bfa, bfb) }) }) + b.Run("f16/scalar", func(b *testing.B) { run(b, func() (float64, error) { return l2sqF16(fa, fb) }) }) + b.Run("f16/avx2", func(b *testing.B) { run(b, func() (float64, error) { return l2sqF16AVX2(fa, fb) }) }) + b.Run("f16/avx512", func(b *testing.B) { run(b, func() (float64, error) { return l2sqF16SIMD(fa, fb) }) }) + b.Run("int8/scalar", func(b *testing.B) { run(b, func() (float64, error) { return l2sqInt8(i8a, i8b) }) }) + b.Run("int8/avx2", func(b *testing.B) { run(b, func() (float64, error) { return l2sqInt8AVX2(i8a, i8b) }) }) + b.Run("int8/avx512", func(b *testing.B) { run(b, func() (float64, error) { return l2sqInt8SIMD(i8a, i8b) }) }) +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_bench_test.go b/pkg/vectorindex/metric/distance_func_narrow_bench_test.go new file mode 100644 index 0000000000000..de5e312b3803f --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_bench_test.go @@ -0,0 +1,139 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// Compares the narrow element types (bf16/f16/int8) against f32/f64 on the +// realistic workload: distance computed from the raw stored bytes (so the cost +// includes each type's decode — free reinterpret for f32/f64/int8, an +// upcast-to-float32 for bf16/f16). At dim=1024 the per-vector load is: +// f64 8KB · f32 4KB · bf16/f16 2KB · int8 1KB +// so int8 should win on bandwidth + integer ops; bf16/f16 trade half the +// bandwidth for the fp32 upcast. +// +// Run: go test ./pkg/vectorindex/metric/ -run x -bench Benchmark_NarrowVsFloat -benchmem + +const ( + narrowBenchDim = 1024 + narrowBenchPool = 256 +) + +func benchF32Pool(n, dim int) [][]float32 { + out := make([][]float32, n) + for i := range out { + v := make([]float32, dim) + for j := range v { + v[j] = float32(rand.Float64()*16 - 8) // [-8, 8) + } + out[i] = v + } + return out +} + +func benchInt8Pool(n, dim int) [][]int8 { + out := make([][]int8, n) + for i := range out { + v := make([]int8, dim) + for j := range v { + v[j] = int8(rand.Intn(255) - 127) // [-127, 127] + } + out[i] = v + } + return out +} + +func toBytesPool[T types.ArrayElement](vecs [][]T) [][]byte { + out := make([][]byte, len(vecs)) + for i, v := range vecs { + out[i] = append([]byte(nil), types.ArrayToBytes(v)...) + } + return out +} + +func Benchmark_NarrowVsFloat(b *testing.B) { + dim, n := narrowBenchDim, narrowBenchPool + + f32 := benchF32Pool(n, dim) + f64 := make([][]float64, n) + bf16 := make([][]types.BF16, n) + f16 := make([][]types.Float16, n) + for i, v := range f32 { + d := make([]float64, dim) + for j, x := range v { + d[j] = float64(x) + } + f64[i] = d + bf16[i] = types.Float32ToBF16Slice(v) + f16[i] = types.Float32ToFloat16Slice(v) + } + i8 := benchInt8Pool(n, dim) + + f64b := toBytesPool(f64) + f32b := toBytesPool(f32) + bf16b := toBytesPool(bf16) + f16b := toBytesPool(f16) + i8b := toBytesPool(i8) + + metrics := []struct { + name string + mt MetricType + }{ + {"L2sq", Metric_L2sqDistance}, + {"InnerProduct", Metric_InnerProduct}, + {"Cosine", Metric_CosineDistance}, + } + + for _, m := range metrics { + b.Run(m.name, func(b *testing.B) { + b.Run("f64", func(b *testing.B) { benchFloatFromBytes[float64](b, m.mt, f64b) }) + b.Run("f32", func(b *testing.B) { benchFloatFromBytes[float32](b, m.mt, f32b) }) + b.Run("bf16", func(b *testing.B) { benchNarrowFromBytes[types.BF16](b, m.mt, bf16b) }) + b.Run("f16", func(b *testing.B) { benchNarrowFromBytes[types.Float16](b, m.mt, f16b) }) + b.Run("int8", func(b *testing.B) { benchNarrowFromBytes[int8](b, m.mt, i8b) }) + }) + } +} + +func benchFloatFromBytes[T float32 | float64](b *testing.B, m MetricType, pool [][]byte) { + fn, err := ResolveDistanceFn[T, T](m) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + a := types.BytesToArray[T](pool[i%len(pool)]) + c := types.BytesToArray[T](pool[(i+1)%len(pool)]) + _, _ = fn(a, c) + } +} + +func benchNarrowFromBytes[T types.ArrayElement](b *testing.B, m MetricType, pool [][]byte) { + fn, err := ResolveDistanceFn[T, float32](m) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + a := types.BytesToArray[T](pool[i%len(pool)]) + c := types.BytesToArray[T](pool[(i+1)%len(pool)]) + _, _ = fn(a, c) + } +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_f16_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_f16_amd64.go new file mode 100644 index 0000000000000..9b436c40b2745 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_f16_amd64.go @@ -0,0 +1,209 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX-512 SIMD distance kernels for vecf16 (types.Float16). +// +// IEEE half->float32 is NOT a plain shift (exponent rebias + subnormals), and Go +// archsimd exposes no f16 type (this CPU also lacks avx512_fp16, and F16C is not +// surfaced). So we vectorize the same magic-multiply f16fast() the scalar path +// uses (Fabian Giesen / rygorous): rescale the exponent via a float multiply and +// fix up Inf/NaN with a masked Merge. Inputs load as Uint32x16 (32 f16/load), +// even/odd 16-bit halves split out, decoded, then fed to the existing AVX-512 +// float32 reduction (sumF32x16). Matches f16fast bit-for-bit so it agrees with +// the scalar oracle. +// +// PERF: the six decode constants are passed to f16dec as individual vector args, +// NOT bundled in a struct. A by-value struct of vectors gets spilled to the stack +// and reloaded on every field access (77 MOVUPS in the inner loop -> ~7x slower); +// individual args stay in zmm registers. With that, f16 SIMD is ~8x over scalar +// (was ~1.3x with the struct) — the difference between "not worth it" and "worth +// it", so the swap below is ON. + +package metric + +import ( + "math" + "unsafe" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +func init() { + switch { + case hasAVX512: + f16L2sqFn = l2sqF16SIMD + f16IPFn = innerProductF16SIMD + f16CosineFn = cosineDistanceF16SIMD + f16L1Fn = l1DistanceF16SIMD + case hasAVX2: + f16L2sqFn = l2sqF16AVX2 + f16IPFn = innerProductF16AVX2 + f16CosineFn = cosineDistanceF16AVX2 + f16L1Fn = l1DistanceF16AVX2 + } +} + +func f16AsU32(s []types.Float16) []uint32 { + if len(s) < 2 { + return nil + } + return unsafe.Slice((*uint32)(unsafe.Pointer(unsafe.SliceData(s))), len(s)/2) +} + +// f16dec decodes 16 half-floats (each in the low 16 bits of a uint32 lane) to +// float32 — the SIMD form of f16fast(), Inf/NaN fixup included. Constants are +// individual args (see file header: a struct spills; args stay in registers). +func f16dec(h, m7fff, m8000, mInf archsimd.Uint32x16, magic, infNan archsimd.Float32x16) archsimd.Float32x16 { + o := h.And(m7fff).ShiftAllLeft(13) + of := o.AsFloat32x16().Mul(magic) + ou := of.AsUint32x16() + ou = ou.Or(mInf).Merge(ou, of.GreaterEqual(infNan)) // ou|=inf where >=infNan + return ou.Or(h.And(m8000).ShiftAllLeft(16)).AsFloat32x16() +} + +// f16Decode constants, built once per kernel as locals. +func f16DecodeConsts() (m7fff, m8000, mLo, mInf archsimd.Uint32x16, magic, infNan archsimd.Float32x16) { + m7fff = archsimd.BroadcastUint32x16(0x7fff) + m8000 = archsimd.BroadcastUint32x16(0x8000) + mLo = archsimd.BroadcastUint32x16(0xffff) + mInf = archsimd.BroadcastUint32x16(255 << 23) + magic = archsimd.BroadcastFloat32x16(f16Magic) + infNan = archsimd.BroadcastFloat32x16(f16WasInfNan) + return +} + +func l2sqF16SIMD(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConsts() + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + dE := f16dec(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).Sub(f16dec(ub.And(mLo), m7fff, m8000, mInf, magic, infNan)) + dO := f16dec(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).Sub(f16dec(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan)) + acc0 = dE.MulAdd(dE, acc0) + acc1 = dO.MulAdd(dO, acc1) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + sum += d * d + } + return float64(sum), nil +} + +func innerProductF16SIMD(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConsts() + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + acc0 = f16dec(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).MulAdd(f16dec(ub.And(mLo), m7fff, m8000, mInf, magic, infNan), acc0) + acc1 = f16dec(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).MulAdd(f16dec(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan), acc1) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + sum += f16fast(a[i]) * f16fast(b[i]) + } + return float64(-sum), nil +} + +func l1DistanceF16SIMD(a, b []types.Float16) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConsts() + absMask := archsimd.BroadcastUint32x16(0x7fffffff) + acc0, acc1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + dE := f16dec(ua.And(mLo), m7fff, m8000, mInf, magic, infNan).Sub(f16dec(ub.And(mLo), m7fff, m8000, mInf, magic, infNan)) + dO := f16dec(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan).Sub(f16dec(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan)) + acc0 = acc0.Add(dE.AsUint32x16().And(absMask).AsFloat32x16()) + acc1 = acc1.Add(dO.AsUint32x16().And(absMask).AsFloat32x16()) + } + sum := sumF32x16(acc0.Add(acc1)) + for i := j * 2; i < n; i++ { + d := f16fast(a[i]) - f16fast(b[i]) + if d < 0 { + d = -d + } + sum += d + } + return float64(sum), nil +} + +func cosineDistanceF16SIMD(a, b []types.Float16) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := f16AsU32(a), f16AsU32(b) + m7fff, m8000, mLo, mInf, magic, infNan := f16DecodeConsts() + dot0, dot1 := archsimd.Float32x16{}, archsimd.Float32x16{} + na0, na1 := archsimd.Float32x16{}, archsimd.Float32x16{} + nb0, nb1 := archsimd.Float32x16{}, archsimd.Float32x16{} + np, j := len(au), 0 + for ; j <= np-16; j += 16 { + ua := archsimd.LoadUint32x16Slice(au[j : j+16]) + ub := archsimd.LoadUint32x16Slice(bu[j : j+16]) + aE := f16dec(ua.And(mLo), m7fff, m8000, mInf, magic, infNan) + aO := f16dec(ua.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan) + bE := f16dec(ub.And(mLo), m7fff, m8000, mInf, magic, infNan) + bO := f16dec(ub.ShiftAllRight(16), m7fff, m8000, mInf, magic, infNan) + dot0 = aE.MulAdd(bE, dot0) + dot1 = aO.MulAdd(bO, dot1) + na0 = aE.MulAdd(aE, na0) + na1 = aO.MulAdd(aO, na1) + nb0 = bE.MulAdd(bE, nb0) + nb1 = bO.MulAdd(bO, nb1) + } + dot := sumF32x16(dot0.Add(dot1)) + na2 := sumF32x16(na0.Add(na1)) + nb2 := sumF32x16(nb0.Add(nb1)) + for i := j * 2; i < n; i++ { + ai, bi := f16fast(a[i]), f16fast(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_int8_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_int8_amd64.go new file mode 100644 index 0000000000000..1e8dc1d6c9f6f --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_int8_amd64.go @@ -0,0 +1,182 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX-512 SIMD distance kernels for vecint8 ([]int8), INTEGER-EXACT (bit-for-bit +// identical to the int64-accumulating pure-Go oracle). +// +// archsimd has no int8->int32 widening op, so we load the raw bytes as Int32x16 +// (64 int8 per load) and sign-extend the four byte lanes with shifts: +// byteJ = (u << (24-8J)) >>arith 24 (Int32x16.ShiftAllRight is arithmetic). All +// arithmetic then stays in int32 lanes — exact, since for the max dimension +// (65535) a lane accumulates < 1024 terms each <= 255^2, far under 2^31 — and the +// final horizontal reduction is in int64. No float, so results equal the oracle +// exactly (the int8 equivalence test asserts ==, not approx). + +package metric + +import ( + "math" + "unsafe" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +func init() { + switch { + case hasAVX512: + int8L2sqFn = l2sqInt8SIMD + int8IPFn = innerProductInt8SIMD + int8CosineFn = cosineDistanceInt8SIMD + int8L1Fn = l1DistanceInt8SIMD + case hasAVX2: + int8L2sqFn = l2sqInt8AVX2 + int8IPFn = innerProductInt8AVX2 + int8CosineFn = cosineDistanceInt8AVX2 + int8L1Fn = l1DistanceInt8AVX2 + } +} + +// int8AsI32 reinterprets a []int8 as []int32 viewing its first len/4 dwords. +func int8AsI32(s []int8) []int32 { + if len(s) < 4 { + return nil + } + return unsafe.Slice((*int32)(unsafe.Pointer(unsafe.SliceData(s))), len(s)/4) +} + +// sumI32x16 horizontally adds the 16 int32 lanes into an int64 (lane values are +// bounded well under 2^31, but the 16-lane total can exceed it). +func sumI32x16(v archsimd.Int32x16) int64 { + var a [16]int32 + v.Store(&a) + var s int64 + for _, x := range a { + s += int64(x) + } + return s +} + +// unpackI8 sign-extends the four byte lanes of an Int32x16 (64 packed int8) into +// four Int32x16 vectors. Lane k of vJ holds int8[4k+J]. +func unpackI8(u archsimd.Int32x16) (v0, v1, v2, v3 archsimd.Int32x16) { + v0 = u.ShiftAllLeft(24).ShiftAllRight(24) + v1 = u.ShiftAllLeft(16).ShiftAllRight(24) + v2 = u.ShiftAllLeft(8).ShiftAllRight(24) + v3 = u.ShiftAllRight(24) + return +} + +func l2sqInt8SIMD(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + acc := archsimd.Int32x16{} + nq, j := len(ai), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackI8(archsimd.LoadInt32x16Slice(ai[j : j+16])) + b0, b1, b2, b3 := unpackI8(archsimd.LoadInt32x16Slice(bi[j : j+16])) + d0, d1, d2, d3 := a0.Sub(b0), a1.Sub(b1), a2.Sub(b2), a3.Sub(b3) + acc = acc.Add(d0.Mul(d0).Add(d1.Mul(d1)).Add(d2.Mul(d2).Add(d3.Mul(d3)))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductInt8SIMD(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + acc := archsimd.Int32x16{} + nq, j := len(ai), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackI8(archsimd.LoadInt32x16Slice(ai[j : j+16])) + b0, b1, b2, b3 := unpackI8(archsimd.LoadInt32x16Slice(bi[j : j+16])) + acc = acc.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + return float64(-sum), nil +} + +func l1DistanceInt8SIMD(a, b []int8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + zero := archsimd.Int32x16{} + acc := archsimd.Int32x16{} + abs := func(d archsimd.Int32x16) archsimd.Int32x16 { return d.Max(zero.Sub(d)) } + nq, j := len(ai), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackI8(archsimd.LoadInt32x16Slice(ai[j : j+16])) + b0, b1, b2, b3 := unpackI8(archsimd.LoadInt32x16Slice(bi[j : j+16])) + acc = acc.Add(abs(a0.Sub(b0)).Add(abs(a1.Sub(b1))).Add(abs(a2.Sub(b2)).Add(abs(a3.Sub(b3))))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + if d < 0 { + d = -d + } + sum += int64(d) + } + return float64(sum), nil +} + +func cosineDistanceInt8SIMD(a, b []int8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + ai, bi := int8AsI32(a), int8AsI32(b) + dotA, naA, nbA := archsimd.Int32x16{}, archsimd.Int32x16{}, archsimd.Int32x16{} + nq, j := len(ai), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackI8(archsimd.LoadInt32x16Slice(ai[j : j+16])) + b0, b1, b2, b3 := unpackI8(archsimd.LoadInt32x16Slice(bi[j : j+16])) + dotA = dotA.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + naA = naA.Add(a0.Mul(a0).Add(a1.Mul(a1)).Add(a2.Mul(a2).Add(a3.Mul(a3)))) + nbA = nbA.Add(b0.Mul(b0).Add(b1.Mul(b1)).Add(b2.Mul(b2).Add(b3.Mul(b3)))) + } + dot, na2, nb2 := sumI32x16(dotA), sumI32x16(naA), sumI32x16(nbA) + for i := j * 4; i < n; i++ { + ai8, bi8 := int64(a[i]), int64(b[i]) + dot += ai8 * bi8 + na2 += ai8 * ai8 + nb2 += bi8 * bi8 + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_test.go b/pkg/vectorindex/metric/distance_func_narrow_test.go new file mode 100644 index 0000000000000..cf88b39d15089 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_test.go @@ -0,0 +1,403 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "math" + "math/rand" + "testing" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +// resolveNarrowBytes mirrors the (removed) byte-keyed narrow resolver for these +// tests: an oid-keyed, raw-bytes-in, float64-out distance fn built on the merged +// ResolveDistanceFn. R=float64 keeps int8's int64 sum exact (the exact-match +// oracle), and errors for non-narrow oids / invalid metrics exactly as before. +func resolveNarrowBytes(oid types.T, m MetricType) (func(a, b []byte) (float64, error), error) { + switch oid { + case types.T_array_bf16: + fn, err := ResolveDistanceFn[types.BF16, float64](m) + if err != nil { + return nil, err + } + return func(a, b []byte) (float64, error) { + return fn(types.BytesToArray[types.BF16](a), types.BytesToArray[types.BF16](b)) + }, nil + case types.T_array_float16: + fn, err := ResolveDistanceFn[types.Float16, float64](m) + if err != nil { + return nil, err + } + return func(a, b []byte) (float64, error) { + return fn(types.BytesToArray[types.Float16](a), types.BytesToArray[types.Float16](b)) + }, nil + case types.T_array_int8: + fn, err := ResolveDistanceFn[int8, float64](m) + if err != nil { + return nil, err + } + return func(a, b []byte) (float64, error) { + return fn(types.BytesToArray[int8](a), types.BytesToArray[int8](b)) + }, nil + case types.T_array_uint8: + fn, err := ResolveDistanceFn[uint8, float64](m) + if err != nil { + return nil, err + } + return func(a, b []byte) (float64, error) { + return fn(types.BytesToArray[uint8](a), types.BytesToArray[uint8](b)) + }, nil + default: + return nil, moerr.NewInternalErrorNoCtx("resolveNarrowBytes: not a narrow vector type") + } +} + +// reference distance over float64, mirroring ResolveDistanceFn semantics. +func refDist(metric MetricType, a, b []float64) float64 { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + var s float64 + for i := range a { + d := a[i] - b[i] + s += d * d + } + return s + case Metric_InnerProduct: + var s float64 + for i := range a { + s += a[i] * b[i] + } + return -s + case Metric_L1Distance: + var s float64 + for i := range a { + s += math.Abs(a[i] - b[i]) + } + return s + case Metric_CosineDistance: + var dot, na2, nb2 float64 + for i := range a { + dot += a[i] * b[i] + na2 += a[i] * a[i] + nb2 += b[i] * b[i] + } + den := math.Sqrt(na2) * math.Sqrt(nb2) + if den == 0 { + return 1.0 + } + sim := dot / den + if sim > 1 { + sim = 1 + } else if sim < -1 { + sim = -1 + } + return 1.0 - sim + } + return 0 +} + +var narrowMetrics = []MetricType{Metric_L2Distance, Metric_L2sqDistance, Metric_InnerProduct, Metric_CosineDistance, Metric_L1Distance} + +func TestNarrowInt8KernelsExact(t *testing.T) { + // int8 values -> exact integer arithmetic, must match float64 reference exactly. + a := []int8{1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11} + b := []int8{-1, 2, -3, 4, 0, 6, -7, 8, -9, 1, 2} + af := make([]float64, len(a)) + bf := make([]float64, len(b)) + for i := range a { + af[i] = float64(a[i]) + bf[i] = float64(b[i]) + } + ab := types.ArrayToBytes(a) + bb := types.ArrayToBytes(b) + for _, m := range narrowMetrics { + fn, err := resolveNarrowBytes(types.T_array_int8, m) + if err != nil { + t.Fatalf("resolve int8 m=%d: %v", m, err) + } + got, err := fn(ab, bb) + if err != nil { + t.Fatalf("int8 dist m=%d: %v", m, err) + } + want := refDist(m, af, bf) + if math.Abs(got-want) > 1e-9 { + t.Errorf("int8 m=%d: got %v want %v", m, got, want) + } + } +} + +func TestNarrowUint8KernelsExact(t *testing.T) { + // uint8 values (0..255) -> exact integer arithmetic, must match float64 reference. + a := []uint8{1, 2, 3, 4, 5, 250, 7, 8, 9, 10, 255} + b := []uint8{255, 2, 0, 4, 100, 6, 7, 200, 9, 1, 2} + af := make([]float64, len(a)) + bf := make([]float64, len(b)) + for i := range a { + af[i] = float64(a[i]) + bf[i] = float64(b[i]) + } + ab := types.ArrayToBytes(a) + bb := types.ArrayToBytes(b) + for _, m := range narrowMetrics { + fn, err := resolveNarrowBytes(types.T_array_uint8, m) + if err != nil { + t.Fatalf("resolve uint8 m=%d: %v", m, err) + } + got, err := fn(ab, bb) + if err != nil { + t.Fatalf("uint8 dist m=%d: %v", m, err) + } + want := refDist(m, af, bf) + if math.Abs(got-want) > 1e-9 { + t.Errorf("uint8 m=%d: got %v want %v", m, got, want) + } + } +} + +func TestNarrowBF16F16Kernels(t *testing.T) { + src1 := []float32{1, 2, 3, 0.5, -4, 6, 7.5, -8, 9, 10, 11} + src2 := []float32{-1, 2, 0.25, 4, 5, 6, -7, 8, -9, 1, 2} + // bf16 + bf1 := types.Float32ToBF16Slice(src1) + bf2 := types.Float32ToBF16Slice(src2) + af := types.BF16ToFloat32Slice(bf1) + bf := types.BF16ToFloat32Slice(bf2) + af64 := f32to64(af) + bf64 := f32to64(bf) + for _, m := range narrowMetrics { + fn, _ := resolveNarrowBytes(types.T_array_bf16, m) + got, err := fn(types.ArrayToBytes(bf1), types.ArrayToBytes(bf2)) + if err != nil { + t.Fatalf("bf16 m=%d: %v", m, err) + } + want := refDist(m, af64, bf64) + if math.Abs(got-want) > 1e-4 { + t.Errorf("bf16 m=%d: got %v want %v", m, got, want) + } + } + // f16 + h1 := types.Float32ToFloat16Slice(src1) + h2 := types.Float32ToFloat16Slice(src2) + haf := f32to64(types.Float16ToFloat32Slice(h1)) + hbf := f32to64(types.Float16ToFloat32Slice(h2)) + for _, m := range narrowMetrics { + fn, _ := resolveNarrowBytes(types.T_array_float16, m) + got, err := fn(types.ArrayToBytes(h1), types.ArrayToBytes(h2)) + if err != nil { + t.Fatalf("f16 m=%d: %v", m, err) + } + want := refDist(m, haf, hbf) + if math.Abs(got-want) > 1e-4 { + t.Errorf("f16 m=%d: got %v want %v", m, got, want) + } + } +} + +func TestNarrowResolveErrors(t *testing.T) { + if _, err := resolveNarrowBytes(types.T_array_float32, Metric_L2Distance); err == nil { + t.Errorf("expected error for non-narrow oid") + } + if _, err := resolveNarrowBytes(types.T_array_int8, MetricType(999)); err == nil { + t.Errorf("expected error for invalid metric") + } +} + +func f32to64(s []float32) []float64 { + out := make([]float64, len(s)) + for i, v := range s { + out[i] = float64(v) + } + return out +} + +func TestF16FastExhaustive(t *testing.T) { + for u := 0; u < 65536; u++ { + h := types.Float16(uint16(u)) + want := h.ToFloat32() + got := f16fast(h) + if math.IsNaN(float64(want)) { + if !math.IsNaN(float64(got)) { + t.Fatalf("h=0x%04x: want NaN, got %v", u, got) + } + continue + } + if math.Float32bits(got) != math.Float32bits(want) { + t.Fatalf("h=0x%04x: f16fast=%v (0x%08x) ToFloat32=%v (0x%08x)", + u, got, math.Float32bits(got), want, math.Float32bits(want)) + } + } +} + +func TestNarrowKernelEdgeCases(t *testing.T) { + narrowOids := []types.T{types.T_array_bf16, types.T_array_float16, types.T_array_int8} + + // dimension mismatch -> error on every metric/type. + for _, oid := range narrowOids { + for _, m := range narrowMetrics { + fn, err := resolveNarrowBytes(oid, m) + require.NoError(t, err) + var a, b []byte + switch oid { + case types.T_array_int8: + a = types.ArrayToBytes([]int8{1, 2, 3}) + b = types.ArrayToBytes([]int8{1, 2}) + case types.T_array_bf16: + a = types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2, 3})) + b = types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2})) + default: + a = types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2, 3})) + b = types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2})) + } + _, err = fn(a, b) + require.Errorf(t, err, "oid=%d metric=%d dim mismatch", oid, m) + } + } + + // empty vectors: distance 0 for all metrics/types (cosine has an explicit + // empty guard; the rest sum nothing). + for _, oid := range narrowOids { + for _, m := range narrowMetrics { + fn, _ := resolveNarrowBytes(oid, m) + got, err := fn(nil, nil) + require.NoError(t, err) + require.InDeltaf(t, 0.0, got, 1e-9, "oid=%d metric=%d empty", oid, m) + } + } + + // cosine of a zero vector -> 1.0 (denominator 0). + for _, oid := range narrowOids { + fn, _ := resolveNarrowBytes(oid, Metric_CosineDistance) + var z []byte + switch oid { + case types.T_array_int8: + z = types.ArrayToBytes([]int8{0, 0, 0, 0}) + case types.T_array_bf16: + z = types.ArrayToBytes(types.Float32ToBF16Slice([]float32{0, 0, 0, 0})) + default: + z = types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{0, 0, 0, 0})) + } + got, err := fn(z, z) + require.NoError(t, err) + require.InDeltaf(t, 1.0, got, 1e-9, "oid=%d zero cosine", oid) + } + + // int8 extremes over a large dimension: integer accumulation must not overflow. + // L2: dim * (127-(-128))^2 = 1000 * 255^2 = 65025000, exact in int64. + dim := 1000 + amax := make([]int8, dim) + amin := make([]int8, dim) + for i := range amax { + amax[i] = 127 + amin[i] = -128 + } + fn, _ := resolveNarrowBytes(types.T_array_int8, Metric_L2sqDistance) + got, err := fn(types.ArrayToBytes(amax), types.ArrayToBytes(amin)) + require.NoError(t, err) + require.InDelta(t, float64(dim)*255.0*255.0, got, 1e-6) + + // single-element vectors work (loop-remainder path). + for _, oid := range narrowOids { + fn, _ := resolveNarrowBytes(oid, Metric_L2sqDistance) + var a, b []byte + switch oid { + case types.T_array_int8: + a, b = types.ArrayToBytes([]int8{3}), types.ArrayToBytes([]int8{1}) + case types.T_array_bf16: + a, b = types.ArrayToBytes(types.Float32ToBF16Slice([]float32{3})), types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1})) + default: + a, b = types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{3})), types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1})) + } + got, err := fn(a, b) + require.NoError(t, err) + require.InDeltaf(t, 4.0, got, 1e-3, "oid=%d single elem", oid) // (3-1)^2 + } +} + +// TestCosineDistanceClampNonNegative guards the [-1,1] similarity clamp. For two +// identical vectors the true cosine similarity is exactly 1, but sqrt(n)*sqrt(n) +// rounds a hair below n for many inputs, so dot/denom lands slightly above 1 and +// an unclamped kernel returns a tiny NEGATIVE distance (outside cosine's [0,2] +// domain). This exercises whichever kernel is active in the build (scalar in the +// default build, AVX512/AVX2 SIMD under GOEXPERIMENT=simd) — every one must clamp +// to >= 0. Random non-parallel vectors never reach sim>1, so the existing +// equivalence tests miss this; here we sample many identical vectors (fixed seed, +// several dims) so the unclamped path reliably produces negatives and fails. +func TestCosineDistanceClampNonNegative(t *testing.T) { + r := rand.New(rand.NewSource(42)) + const samplesPerDim = 400 + dims := []int{4, 7, 16, 17, 31, 64} // cover SIMD-block (>=16) and tail paths + + genF32 := func(dim int) []float32 { + v := make([]float32, dim) + for i := range v { + v[i] = float32(r.Float64()*16 - 8) + } + return v + } + + bf16, err := ResolveDistanceFn[types.BF16, float32](Metric_CosineDistance) + require.NoError(t, err) + f16, err := ResolveDistanceFn[types.Float16, float32](Metric_CosineDistance) + require.NoError(t, err) + i8, err := ResolveDistanceFn[int8, float32](Metric_CosineDistance) + require.NoError(t, err) + u8, err := ResolveDistanceFn[uint8, float32](Metric_CosineDistance) + require.NoError(t, err) + + for _, dim := range dims { + for s := 0; s < samplesPerDim; s++ { + f := genF32(dim) + + // f32 / f64 native + d32, err := CosineDistance(f, f) + require.NoError(t, err) + require.GreaterOrEqualf(t, d32, float32(0), "f32 cosine of identical vector must be >= 0 (dim=%d)", dim) + + f64v := make([]float64, dim) + i8v := make([]int8, dim) + u8v := make([]uint8, dim) + for i, x := range f { + f64v[i] = float64(x) + i8v[i] = int8(x * 8) // [-64,64) + u8v[i] = uint8(x*8 + 128) // [0,256) + } + d64, err := CosineDistance(f64v, f64v) + require.NoError(t, err) + require.GreaterOrEqualf(t, d64, float64(0), "f64 cosine of identical vector must be >= 0 (dim=%d)", dim) + + // narrow types + bv := types.Float32ToBF16Slice(f) + db, err := bf16(bv, bv) + require.NoError(t, err) + require.GreaterOrEqualf(t, db, float32(0), "bf16 cosine of identical vector must be >= 0 (dim=%d)", dim) + + hv := types.Float32ToFloat16Slice(f) + dh, err := f16(hv, hv) + require.NoError(t, err) + require.GreaterOrEqualf(t, dh, float32(0), "f16 cosine of identical vector must be >= 0 (dim=%d)", dim) + + di, err := i8(i8v, i8v) + require.NoError(t, err) + require.GreaterOrEqualf(t, di, float32(0), "int8 cosine of identical vector must be >= 0 (dim=%d)", dim) + + du, err := u8(u8v, u8v) + require.NoError(t, err) + require.GreaterOrEqualf(t, du, float32(0), "uint8 cosine of identical vector must be >= 0 (dim=%d)", dim) + } + } +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_uint8.go b/pkg/vectorindex/metric/distance_func_narrow_uint8.go new file mode 100644 index 0000000000000..6d9e24864c7f3 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_uint8.go @@ -0,0 +1,163 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +// Pure-Go INTEGER (int64-accumulated) distance kernels for vecuint8 ([]uint8), +// mirroring the vecint8 kernels. uint8 values are promoted to int32/int64 before +// arithmetic so the math is identical to int8 (per-element |d|<=255 -> d*d<=65025, +// a*b in [0,65025]); int64 accumulation never overflows at MaxArrayDimension. +// +// The kernel function pointers are swappable so a future +// distance_func_narrow_uint8_amd64.go can drop in SIMD impls via init(), exactly +// as int8 does — there is no SIMD variant yet, so they point at the Go kernels. +var ( + uint8L2sqFn = l2sqUint8 + uint8IPFn = innerProductUint8 + uint8CosineFn = cosineDistanceUint8 + uint8L1Fn = l1DistanceUint8 +) + +func resolveUint8Kernel(metric MetricType) (func(a, b []uint8) (float64, error), error) { + switch metric { + case Metric_L2Distance, Metric_L2sqDistance: + return uint8L2sqFn, nil + case Metric_InnerProduct: + return uint8IPFn, nil + case Metric_CosineDistance: + return uint8CosineFn, nil + case Metric_L1Distance: + return uint8L1Fn, nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +func l2sqUint8(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + d0 := int32(aa[0]) - int32(bb[0]) + d1 := int32(aa[1]) - int32(bb[1]) + d2 := int32(aa[2]) - int32(bb[2]) + d3 := int32(aa[3]) - int32(bb[3]) + d4 := int32(aa[4]) - int32(bb[4]) + d5 := int32(aa[5]) - int32(bb[5]) + d6 := int32(aa[6]) - int32(bb[6]) + d7 := int32(aa[7]) - int32(bb[7]) + sum += int64(d0*d0+d1*d1) + int64(d2*d2+d3*d3) + int64(d4*d4+d5*d5) + int64(d6*d6+d7*d7) + } + for ; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductUint8(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += int64(int32(aa[0])*int32(bb[0])+int32(aa[1])*int32(bb[1])) + + int64(int32(aa[2])*int32(bb[2])+int32(aa[3])*int32(bb[3])) + + int64(int32(aa[4])*int32(bb[4])+int32(aa[5])*int32(bb[5])) + + int64(int32(aa[6])*int32(bb[6])+int32(aa[7])*int32(bb[7])) + } + for ; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + // matches metric.InnerProduct: returns -dot + return float64(-sum), nil +} + +func l1DistanceUint8(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var sum int64 + n := len(a) + i := 0 + abs := func(x int32) int32 { + if x < 0 { + return -x + } + return x + } + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + sum += int64(abs(int32(aa[0])-int32(bb[0]))+abs(int32(aa[1])-int32(bb[1]))) + + int64(abs(int32(aa[2])-int32(bb[2]))+abs(int32(aa[3])-int32(bb[3]))) + + int64(abs(int32(aa[4])-int32(bb[4]))+abs(int32(aa[5])-int32(bb[5]))) + + int64(abs(int32(aa[6])-int32(bb[6]))+abs(int32(aa[7])-int32(bb[7]))) + } + for ; i < n; i++ { + sum += int64(abs(int32(a[i]) - int32(b[i]))) + } + return float64(sum), nil +} + +func cosineDistanceUint8(a, b []uint8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + var dot, na2, nb2 int64 + n := len(a) + i := 0 + for ; i <= n-8; i += 8 { + aa := a[i : i+8 : i+8] + bb := b[i : i+8 : i+8] + for k := 0; k < 8; k++ { + ai := int64(aa[k]) + bi := int64(bb[k]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + } + for ; i < n; i++ { + ai := int64(a[i]) + bi := int64(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + // matches metric.CosineDistance: denominator 0 -> distance 1.0 + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_narrow_uint8_amd64.go b/pkg/vectorindex/metric/distance_func_narrow_uint8_amd64.go new file mode 100644 index 0000000000000..4f19d44ec00c8 --- /dev/null +++ b/pkg/vectorindex/metric/distance_func_narrow_uint8_amd64.go @@ -0,0 +1,301 @@ +//go:build amd64 && go1.26 && goexperiment.simd + +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// AVX-512 / AVX2 SIMD distance kernels for vecuint8 ([]uint8), INTEGER-EXACT +// (bit-for-bit identical to the int64-accumulating pure-Go oracle). +// +// Like the int8 kernels we load the raw bytes as a 32-bit-lane vector (4 uint8 +// per lane) and split the four byte lanes. The ONLY difference from int8 is the +// unpack: uint8 is ZERO-extended (mask each byte with 0xFF / logical shift), +// whereas int8 sign-extends with arithmetic shifts. The values land in [0,255] +// so reinterpreting the masked Uint32 lanes as Int32 (AsInt32x16) is exact, and +// all subsequent arithmetic (Sub/Mul/Add/Max) stays in signed int32 lanes — +// d=a-b is in [-255,255], d*d <= 65025, a*b in [0,65025]. For the max dimension +// (65535) a lane accumulates < 1100 terms each <= 65025, i.e. < 2^28, far under +// 2^31; the final horizontal reduction is in int64. No float, so results equal +// the oracle exactly (the equivalence test asserts == for L2sq/IP/L1). +// +// The pure-Go kernels in distance_func_narrow_uint8.go stay the fallback +// (non-AVX2 CPUs) and the equivalence oracle; init() only swaps the selection +// vars when AVX-512 / AVX2 is present. hasAVX512 / hasAVX2 / sumI32x16 / +// sumI32x8 are shared with the int8/bf16/f16 kernels (same package + build tag). + +package metric + +import ( + "math" + "unsafe" + + "simd/archsimd" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" +) + +func init() { + switch { + case hasAVX512: + uint8L2sqFn = l2sqUint8SIMD + uint8IPFn = innerProductUint8SIMD + uint8CosineFn = cosineDistanceUint8SIMD + uint8L1Fn = l1DistanceUint8SIMD + case hasAVX2: + uint8L2sqFn = l2sqUint8AVX2 + uint8IPFn = innerProductUint8AVX2 + uint8CosineFn = cosineDistanceUint8AVX2 + uint8L1Fn = l1DistanceUint8AVX2 + } +} + +// uint8AsU32 reinterprets a []uint8 as []uint32 viewing its first len/4 dwords +// (4 packed uint8 per lane). +func uint8AsU32(s []uint8) []uint32 { + if len(s) < 4 { + return nil + } + return unsafe.Slice((*uint32)(unsafe.Pointer(unsafe.SliceData(s))), len(s)/4) +} + +// ---- uint8 (AVX-512), integer-exact ---- + +// unpackU8 zero-extends the four byte lanes of a Uint32x16 (64 packed uint8) +// into four Int32x16 vectors. Lane k of vJ holds uint8[4k+J] as a value in +// [0,255]. mask is BroadcastUint32x16(0xFF); the top byte (>>24) needs no mask. +func unpackU8(u, mask archsimd.Uint32x16) (v0, v1, v2, v3 archsimd.Int32x16) { + v0 = u.And(mask).AsInt32x16() + v1 = u.ShiftAllRight(8).And(mask).AsInt32x16() + v2 = u.ShiftAllRight(16).And(mask).AsInt32x16() + v3 = u.ShiftAllRight(24).AsInt32x16() + return +} + +func l2sqUint8SIMD(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x16(0xFF) + acc := archsimd.Int32x16{} + nq, j := len(au), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackU8(archsimd.LoadUint32x16Slice(au[j:j+16]), mask) + b0, b1, b2, b3 := unpackU8(archsimd.LoadUint32x16Slice(bu[j:j+16]), mask) + d0, d1, d2, d3 := a0.Sub(b0), a1.Sub(b1), a2.Sub(b2), a3.Sub(b3) + acc = acc.Add(d0.Mul(d0).Add(d1.Mul(d1)).Add(d2.Mul(d2).Add(d3.Mul(d3)))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductUint8SIMD(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x16(0xFF) + acc := archsimd.Int32x16{} + nq, j := len(au), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackU8(archsimd.LoadUint32x16Slice(au[j:j+16]), mask) + b0, b1, b2, b3 := unpackU8(archsimd.LoadUint32x16Slice(bu[j:j+16]), mask) + acc = acc.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + return float64(-sum), nil +} + +func l1DistanceUint8SIMD(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x16(0xFF) + zero := archsimd.Int32x16{} + acc := archsimd.Int32x16{} + abs := func(d archsimd.Int32x16) archsimd.Int32x16 { return d.Max(zero.Sub(d)) } + nq, j := len(au), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackU8(archsimd.LoadUint32x16Slice(au[j:j+16]), mask) + b0, b1, b2, b3 := unpackU8(archsimd.LoadUint32x16Slice(bu[j:j+16]), mask) + acc = acc.Add(abs(a0.Sub(b0)).Add(abs(a1.Sub(b1))).Add(abs(a2.Sub(b2)).Add(abs(a3.Sub(b3))))) + } + sum := sumI32x16(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + if d < 0 { + d = -d + } + sum += int64(d) + } + return float64(sum), nil +} + +func cosineDistanceUint8SIMD(a, b []uint8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x16(0xFF) + dotA, naA, nbA := archsimd.Int32x16{}, archsimd.Int32x16{}, archsimd.Int32x16{} + nq, j := len(au), 0 + for ; j <= nq-16; j += 16 { + a0, a1, a2, a3 := unpackU8(archsimd.LoadUint32x16Slice(au[j:j+16]), mask) + b0, b1, b2, b3 := unpackU8(archsimd.LoadUint32x16Slice(bu[j:j+16]), mask) + dotA = dotA.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + naA = naA.Add(a0.Mul(a0).Add(a1.Mul(a1)).Add(a2.Mul(a2).Add(a3.Mul(a3)))) + nbA = nbA.Add(b0.Mul(b0).Add(b1.Mul(b1)).Add(b2.Mul(b2).Add(b3.Mul(b3)))) + } + dot, na2, nb2 := sumI32x16(dotA), sumI32x16(naA), sumI32x16(nbA) + for i := j * 4; i < n; i++ { + ai, bi := int64(a[i]), int64(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} + +// ---- uint8 (AVX2), integer-exact ---- + +// unpackU8x8 is the 256-bit (8-lane) twin of unpackU8. +func unpackU8x8(u, mask archsimd.Uint32x8) (v0, v1, v2, v3 archsimd.Int32x8) { + v0 = u.And(mask).AsInt32x8() + v1 = u.ShiftAllRight(8).And(mask).AsInt32x8() + v2 = u.ShiftAllRight(16).And(mask).AsInt32x8() + v3 = u.ShiftAllRight(24).AsInt32x8() + return +} + +func l2sqUint8AVX2(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x8(0xFF) + acc := archsimd.Int32x8{} + nq, j := len(au), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackU8x8(archsimd.LoadUint32x8Slice(au[j:j+8]), mask) + b0, b1, b2, b3 := unpackU8x8(archsimd.LoadUint32x8Slice(bu[j:j+8]), mask) + d0, d1, d2, d3 := a0.Sub(b0), a1.Sub(b1), a2.Sub(b2), a3.Sub(b3) + acc = acc.Add(d0.Mul(d0).Add(d1.Mul(d1)).Add(d2.Mul(d2).Add(d3.Mul(d3)))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + sum += int64(d * d) + } + return float64(sum), nil +} + +func innerProductUint8AVX2(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x8(0xFF) + acc := archsimd.Int32x8{} + nq, j := len(au), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackU8x8(archsimd.LoadUint32x8Slice(au[j:j+8]), mask) + b0, b1, b2, b3 := unpackU8x8(archsimd.LoadUint32x8Slice(bu[j:j+8]), mask) + acc = acc.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + sum += int64(int32(a[i]) * int32(b[i])) + } + return float64(-sum), nil +} + +func l1DistanceUint8AVX2(a, b []uint8) (float64, error) { + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x8(0xFF) + zero := archsimd.Int32x8{} + acc := archsimd.Int32x8{} + abs := func(d archsimd.Int32x8) archsimd.Int32x8 { return d.Max(zero.Sub(d)) } + nq, j := len(au), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackU8x8(archsimd.LoadUint32x8Slice(au[j:j+8]), mask) + b0, b1, b2, b3 := unpackU8x8(archsimd.LoadUint32x8Slice(bu[j:j+8]), mask) + acc = acc.Add(abs(a0.Sub(b0)).Add(abs(a1.Sub(b1))).Add(abs(a2.Sub(b2)).Add(abs(a3.Sub(b3))))) + } + sum := sumI32x8(acc) + for i := j * 4; i < n; i++ { + d := int32(a[i]) - int32(b[i]) + if d < 0 { + d = -d + } + sum += int64(d) + } + return float64(sum), nil +} + +func cosineDistanceUint8AVX2(a, b []uint8) (float64, error) { + if len(a) == 0 { + return 0, nil + } + if len(a) != len(b) { + return 0, moerr.NewInternalErrorNoCtx("vector dimension not matched") + } + n := len(a) + au, bu := uint8AsU32(a), uint8AsU32(b) + mask := archsimd.BroadcastUint32x8(0xFF) + dotA, naA, nbA := archsimd.Int32x8{}, archsimd.Int32x8{}, archsimd.Int32x8{} + nq, j := len(au), 0 + for ; j <= nq-8; j += 8 { + a0, a1, a2, a3 := unpackU8x8(archsimd.LoadUint32x8Slice(au[j:j+8]), mask) + b0, b1, b2, b3 := unpackU8x8(archsimd.LoadUint32x8Slice(bu[j:j+8]), mask) + dotA = dotA.Add(a0.Mul(b0).Add(a1.Mul(b1)).Add(a2.Mul(b2).Add(a3.Mul(b3)))) + naA = naA.Add(a0.Mul(a0).Add(a1.Mul(a1)).Add(a2.Mul(a2).Add(a3.Mul(a3)))) + nbA = nbA.Add(b0.Mul(b0).Add(b1.Mul(b1)).Add(b2.Mul(b2).Add(b3.Mul(b3)))) + } + dot, na2, nb2 := sumI32x8(dotA), sumI32x8(naA), sumI32x8(nbA) + for i := j * 4; i < n; i++ { + ai, bi := int64(a[i]), int64(b[i]) + dot += ai * bi + na2 += ai * ai + nb2 += bi * bi + } + denom := math.Sqrt(float64(na2)) * math.Sqrt(float64(nb2)) + if denom == 0 { + return 1.0, nil + } + return cosineDistClamped(float64(dot), denom), nil +} diff --git a/pkg/vectorindex/metric/distance_func_test.go b/pkg/vectorindex/metric/distance_func_test.go index 057e47a2c4e30..4dcaa99f100aa 100644 --- a/pkg/vectorindex/metric/distance_func_test.go +++ b/pkg/vectorindex/metric/distance_func_test.go @@ -15,7 +15,6 @@ package metric import ( - "fmt" "math" "testing" @@ -47,10 +46,8 @@ func Test_Blas32(t *testing.T) { distfn, _, err := ResolveKmeansDistanceFn[float32](Metric_L2Distance, false) require.Nil(t, err) - v, err := distfn(v1.Data, v2.Data) + _, err = distfn(v1.Data, v2.Data) require.Nil(t, err) - - fmt.Printf("blas32 v = %v\n", v) } func Test_ResolveFun(t *testing.T) { @@ -213,6 +210,22 @@ func Test_L2Distance(t *testing.T) { }, want: 3.1622776601683795, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 5.196152422706632, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 4.58257569495584, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -281,6 +294,22 @@ func Test_L1Distance(t *testing.T) { }, want: 10, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 27, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 21, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -349,6 +378,22 @@ func Test_CosineDistance(t *testing.T) { }, want: 0.0021238962030426523, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0.0025062434610066964, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 0.002478147161370292, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -359,6 +404,90 @@ func Test_CosineDistance(t *testing.T) { } } +func Test_CosineSimilarity(t *testing.T) { + type args struct { + v1 []float64 + v2 []float64 + } + tests := []struct { + name string + args args + want float64 + }{ + { + name: "Test 1", + args: args{ + v1: []float64{1, 2, 3, 4}, + v2: []float64{1, 2, 4, 5}, + }, + want: 0.9960065188076063, + }, + { + name: "Test 2", + args: args{ + v1: []float64{10, 20, 30, 40}, + v2: []float64{10.5, 21.5, 31.5, 43.5}, + }, + want: 0.9998746426104126, + }, + { + name: "Test 3.a", + args: args{ + v1: []float64{1, 1}, + v2: []float64{4, 1}, + }, + want: 0.8574929257125441, + }, + { + name: "Test 3.b", + args: args{ + v1: []float64{4, 1}, + v2: []float64{1, 4}, + }, + want: 0.47058823529411764, + }, + { + name: "Test 3.c", + args: args{ + v1: []float64{1, 4}, + v2: []float64{1, 1}, + }, + want: 0.8574929257125441, + }, + { + name: "Test 4", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + }, + want: 0.9978761037969573, + }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0.9974937565389933, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 0.9975218528386297, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := CosineSimilarity[float64](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + t.Errorf("CosineSimilarity() = %v, want %v", got, tt.want) + } + }) + } +} + func Test_InnerProduct(t *testing.T) { type args struct { v1 []float64 @@ -417,6 +546,22 @@ func Test_InnerProduct(t *testing.T) { }, want: -440, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: -1048, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: -882, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -485,6 +630,22 @@ func Test_L2DistanceSq(t *testing.T) { }, want: 10, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 27, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 21, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -555,6 +716,22 @@ func Test_AngularDistance(t *testing.T) { }, want: 0, }, + { + name: "Test 5", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8}, + }, + want: 0, + }, + { + name: "Test 6", + args: args{ + v1: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1}, + v2: []float64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 2}, + }, + want: 0, + }, // Test 4: Triangle Inequality check on **normalized** vector // A(1,0),B(2,2), C(0,1) => AB + AC >= BC => 0.25 + 0.25 >= 0.5 diff --git a/pkg/vectorindex/metric/gpu.go b/pkg/vectorindex/metric/gpu.go index 97d15a4babe57..5c474bd0ab76e 100644 --- a/pkg/vectorindex/metric/gpu.go +++ b/pkg/vectorindex/metric/gpu.go @@ -53,7 +53,18 @@ var ( } ) -func PairWiseDistance[T types.RealNumbers]( +// gpuPairwiseSupported reports whether T is a cuVS-supported pairwise element +// type (float32 or types.Float16). bf16/int8/uint8/float64 run on CPU. +func gpuPairwiseSupported[T types.ArrayElement]() bool { + switch any(*new(T)).(type) { + case float32, types.Float16: + return true + default: + return false + } +} + +func PairWiseDistance[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, @@ -78,8 +89,7 @@ func PairWiseDistance[T types.RealNumbers]( return GoPairWiseDistance(x, y, metric) } - var zero T - if _, isF32 := any(zero).(float32); isF32 { + if gpuPairwiseSupported[T]() { res := make([]float32, nX*nY) handle, err := PairwiseDistanceLaunch(x, y, metric, res, GPUThresholdSync, gpuMode) if err != nil { @@ -145,7 +155,7 @@ func (m *gpuJobManager) pop(jobID uint64) *gpuJob { // It flattens the input vectors on the CPU and then launches a CUDA kernel. // This allows for overlapping the CPU-bound flattening work with GPU execution // when pipelined at the reader level. -func PairwiseDistanceLaunch[T types.RealNumbers]( +func PairwiseDistanceLaunch[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, @@ -168,63 +178,87 @@ func PairwiseDistanceLaunch[T types.RealNumbers]( dim := len(x[0]) cuvsMetric, ok := MetricTypeToCuvsMetric[metric] - var zero T - _, isF32 := any(zero).(float32) - - if ok && isF32 && uint64(nX)*uint64(nY)*uint64(dim) >= minWorkSize { - allocator := malloc.NewCAllocator() - - // 1. Flatten Y - yf32Slice, yDeallocator, err := allocator.Allocate(uint64(nY*dim*4), malloc.NoClear) - if err != nil { - return 0, err - } - yf32 := util.UnsafeSliceCast[float32](yf32Slice) - y32 := any(y).([][]float32) - for i, v := range y32 { - copy(yf32[i*dim:(i+1)*dim], v) + if ok && uint64(nX)*uint64(nY)*uint64(dim) >= minWorkSize { + // cuVS pairwise supports float32 and Float16 only. + switch xs := any(x).(type) { + case [][]float32: + return gpuPairwiseLaunch[float32](xs, any(y).([][]float32), dim, cuvsMetric, dist, 4) + case [][]types.Float16: + ys := any(y).([][]types.Float16) + // types.Float16 and cuvs.Float16 are both IEEE binary16 uint16 — a + // per-row reinterpret (no element copy) hands them to the cuVS kernel. + xc := make([][]cuvs.Float16, len(xs)) + for i, v := range xs { + xc[i] = util.UnsafeSliceCast[cuvs.Float16](v) + } + yc := make([][]cuvs.Float16, len(ys)) + for i, v := range ys { + yc[i] = util.UnsafeSliceCast[cuvs.Float16](v) + } + return gpuPairwiseLaunch[cuvs.Float16](xc, yc, dim, cuvsMetric, dist, 2) } + } - // 2. Flatten X - xf32Slice, xDeallocator, err := allocator.Allocate(uint64(nX*dim*4), malloc.NoClear) - if err != nil { - yDeallocator.Deallocate() - return 0, err - } - xf32 := util.UnsafeSliceCast[float32](xf32Slice) - x32 := any(x).([][]float32) - for i, v := range x32 { - copy(xf32[i*dim:(i+1)*dim], v) - } + return PairwiseDistanceLaunchCPU(x, y, metric, dist) +} - // Register job before launch so the slot exists if Wait is called - // concurrently. On launch failure, pop removes it before returning; - // no caller can see the job because cuvsJobID is only set by update() - // below, which is never reached on this error path. - gpuID := globalGpuJobManager.add(dist) - - cuvsID, err := cuvs.PairwiseDistanceLaunch( - xf32, - uint64(nX), - yf32, - uint64(nY), - uint32(dim), - cuvsMetric, - dist, - ) - if err != nil { - xDeallocator.Deallocate() - yDeallocator.Deallocate() - globalGpuJobManager.pop(gpuID) - return 0, err - } +// gpuPairwiseLaunch flattens [][]C into a C-allocator buffer (elemSize bytes per +// element) and launches the async cuVS pairwise distance. C is float32 (4B) or +// cuvs.Float16 (2B). Mirrors the old f32-only path, generalized over the element. +func gpuPairwiseLaunch[C cuvs.VectorType]( + x, y [][]C, + dim int, + cuvsMetric cuvs.DistanceType, + dist []float32, + elemSize int, +) (PairwiseJobHandle, error) { + nX, nY := len(x), len(y) + allocator := malloc.NewCAllocator() + + // 1. Flatten Y + yBuf, yDeallocator, err := allocator.Allocate(uint64(nY*dim*elemSize), malloc.NoClear) + if err != nil { + return 0, err + } + yf := util.UnsafeSliceCast[C](yBuf) + for i, v := range y { + copy(yf[i*dim:(i+1)*dim], v) + } - globalGpuJobManager.update(gpuID, cuvsID, xDeallocator, yDeallocator) + // 2. Flatten X + xBuf, xDeallocator, err := allocator.Allocate(uint64(nX*dim*elemSize), malloc.NoClear) + if err != nil { + yDeallocator.Deallocate() + return 0, err + } + xf := util.UnsafeSliceCast[C](xBuf) + for i, v := range x { + copy(xf[i*dim:(i+1)*dim], v) + } - return PairwiseJobHandle(gpuID), nil + // Register job before launch so the slot exists if Wait is called + // concurrently. On launch failure, pop removes it before returning. + gpuID := globalGpuJobManager.add(dist) + + cuvsID, err := cuvs.PairwiseDistanceLaunch( + xf, + uint64(nX), + yf, + uint64(nY), + uint32(dim), + cuvsMetric, + dist, + ) + if err != nil { + xDeallocator.Deallocate() + yDeallocator.Deallocate() + globalGpuJobManager.pop(gpuID) + return 0, err } - return PairwiseDistanceLaunchCPU(x, y, metric, dist) + globalGpuJobManager.update(gpuID, cuvsID, xDeallocator, yDeallocator) + + return PairwiseJobHandle(gpuID), nil } // PairwiseDistanceWait waits for the completion of the asynchronous GPU distance diff --git a/pkg/vectorindex/metric/pairwise.go b/pkg/vectorindex/metric/pairwise.go index 08ad7135ca600..a012c5f5ade2c 100644 --- a/pkg/vectorindex/metric/pairwise.go +++ b/pkg/vectorindex/metric/pairwise.go @@ -54,13 +54,14 @@ var ( // While this is currently synchronous for CPU (it performs the calculation in Launch), // it follows the asynchronous interface to support the pipelined execution model // used in the block reader. -func PairwiseDistanceLaunchCPU[T types.RealNumbers]( +func PairwiseDistanceLaunchCPU[T types.ArrayElement]( x [][]T, y [][]T, metric MetricType, dist []float32, ) (PairwiseJobHandle, error) { - distFn, err := ResolveDistanceFn[T](metric) + // R=float32: the output is []float32, matching the prior float32(d) truncation. + distFn, err := ResolveDistanceFn[T, float32](metric) if err != nil { return 0, err } @@ -75,38 +76,18 @@ func PairwiseDistanceLaunchCPU[T types.RealNumbers]( dist: dist, } - // Do the calculation in Launch - switch xTyped := any(x).(type) { - case [][]float32: - yTyped := any(y).([][]float32) - dFn := any(distFn).(DistanceFunction[float32]) - for r := 0; r < nX; r++ { - xr := xTyped[r] - for c := 0; c < nY; c++ { - d, err := dFn(xr, yTyped[c]) - if err != nil { - job.err = err - goto DONE - } - dist[r*nY+c] = float32(d) + // One unified loop over any ArrayElement type — the resolver handles f32/f64 + // and the narrow kernels (bf16/f16/int8/uint8) uniformly. + for r := 0; r < nX; r++ { + xr := x[r] + for c := 0; c < nY; c++ { + d, err := distFn(xr, y[c]) + if err != nil { + job.err = err + goto DONE } + dist[r*nY+c] = d } - case [][]float64: - yTyped := any(y).([][]float64) - dFn := any(distFn).(DistanceFunction[float64]) - for r := 0; r < nX; r++ { - xr := xTyped[r] - for c := 0; c < nY; c++ { - d, err := dFn(xr, yTyped[c]) - if err != nil { - job.err = err - goto DONE - } - dist[r*nY+c] = float32(d) - } - } - default: - return 0, moerr.NewInternalErrorNoCtx("unsupported type in PairwiseDistanceLaunchCPU") } if metric == Metric_L2Distance { diff --git a/pkg/vectorindex/metric/pairwise_test.go b/pkg/vectorindex/metric/pairwise_test.go index d12956ee50a06..8aab26132f5b1 100644 --- a/pkg/vectorindex/metric/pairwise_test.go +++ b/pkg/vectorindex/metric/pairwise_test.go @@ -50,7 +50,7 @@ func TestPairWiseDistance(t *testing.T) { require.Equal(t, nX*nY, len(dist)) // Verify against direct calls - distFn, err := ResolveDistanceFn[float32](m) + distFn, err := ResolveDistanceFn[float32, float32](m) require.NoError(t, err) for i := 0; i < nX; i++ { diff --git a/pkg/vectorindex/metric/resolve.go b/pkg/vectorindex/metric/resolve.go new file mode 100644 index 0000000000000..349f80baaf19f --- /dev/null +++ b/pkg/vectorindex/metric/resolve.go @@ -0,0 +1,233 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// NOTE: This file is intentionally UNTAGGED (no //go:build constraint). +// The distance KERNELS live in build-tag alternatives — distance_func.go +// (scalar, !(amd64 && goexperiment.simd)) and distance_func_amd64.go (SIMD, +// amd64 && go1.26 && goexperiment.simd) — so only one compiles per build. The +// resolver / orchestration helpers below are build-tag-independent (they just +// pick and call a kernel), so they must NOT live in a tagged file, or they +// would vanish on the SIMD build and break every caller (kmeans / brute_force / +// ivf). They belong here, where they compile in every build. + +package metric + +import ( + "math" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/types" +) + +// IMPORTANT: Elkans Kmeans always use L2Distance for dense vector or images. After getting the centroids, we can use other distance function +// specified by user to assign vector to corresponding centroids (CENTROIDX JOIN / ProductL2). + +func ResolveKmeansDistanceFn[T types.RealNumbers](metric MetricType, spherical bool) (DistanceFunction[T], bool, error) { + if spherical { + return ResolveKmeansDistanceFnForSparse[T](metric) + } + return ResolveKmeansDistanceFnForDense[T](metric) +} + +func ResolveKmeansDistanceFnForDense[T types.RealNumbers](metric MetricType) (DistanceFunction[T], bool, error) { + var distanceFunction DistanceFunction[T] + normalize := false + switch metric { + case Metric_L2Distance: + distanceFunction = L2Distance[T] + normalize = false + case Metric_L2sqDistance: + // Elkans Kmeans always uses true L2Distance regardless of user metric. + distanceFunction = L2Distance[T] + normalize = false + case Metric_InnerProduct: + distanceFunction = L2Distance[T] + normalize = false + case Metric_CosineDistance: + distanceFunction = L2Distance[T] + normalize = false + case Metric_L1Distance: + distanceFunction = L2Distance[T] + normalize = false + default: + return nil, normalize, moerr.NewInternalErrorNoCtx("invalid distance type") + } + return distanceFunction, normalize, nil +} + +// IMPORTANT: Spherical Kmeans always use Spherical Distance / Cosine Similarity for Sparse vector or text embedding (TD-IDF). +// After getting the centroids, we can use other distance function +// specified by user to assign vector to corresponding centroids (CENTROIDX JOIN / ProductL2). +func ResolveKmeansDistanceFnForSparse[T types.RealNumbers](metric MetricType) (DistanceFunction[T], bool, error) { + var distanceFunction DistanceFunction[T] + normalize := false + switch metric { + case Metric_L2Distance: + distanceFunction = L2Distance[T] + normalize = false + case Metric_L2sqDistance: + distanceFunction = L2Distance[T] + normalize = false + case Metric_InnerProduct: + distanceFunction = SphericalDistance[T] + normalize = true + case Metric_CosineDistance: + distanceFunction = SphericalDistance[T] + normalize = true + case Metric_L1Distance: + distanceFunction = L2Distance[T] + normalize = false + default: + return nil, normalize, moerr.NewInternalErrorNoCtx("invalid distance type") + } + return distanceFunction, normalize, nil +} + +// resolveRealKernel picks the float32/float64 metric kernel (returning the value +// in its own type T). It is the f32/f64 half of ResolveDistanceFn. +func resolveRealKernel[T types.RealNumbers](metric MetricType) (DistanceFunction[T], error) { + switch metric { + case Metric_L2Distance: + return L2DistanceSq[T], nil // caller must sqrt; squared distance + case Metric_L2sqDistance: + return L2DistanceSq[T], nil + case Metric_InnerProduct: + return InnerProduct[T], nil + case Metric_CosineDistance: + return CosineDistance[T], nil + case Metric_L1Distance: + return L1Distance[T], nil + default: + return nil, moerr.NewInternalErrorNoCtx("invalid distance type") + } +} + +// ResolveDistanceFn is the single distance resolver for search / assign-to- +// centroid (CENTROIDX JOIN / ProductL2), brute force, pairwise and topn. It +// works for any storage element type T (types.ArrayElement) and returns the +// distance in a caller-chosen result type R (types.RealNumbers): pass +// R=float32 for the common path and R=float64 only where f64 precision is +// needed (f64 input, topn ordering values). f32/f64 use the metric kernels; +// bf16/f16/int8/uint8 use the native narrow kernels (which compute in +// float32/int64 and are cast to R — casting their float64 down to float32 is +// bit-identical to a native-float32 kernel, since the intermediate is exact). +// +// IMPORTANT: Don't use it for Elkans Kmeans (use ResolveKmeansDistanceFn). +// NOTE: Metric_L2Distance returns squared L2; callers needing true L2 sqrt the +// result (as GoPairWiseDistance does). +func ResolveDistanceFn[T types.ArrayElement, R types.RealNumbers](metric MetricType) (func(a, b []T) (R, error), error) { + // Each case resolves the CONCRETE element kernel, then rebinds it to + // func([]T,...)(R,error) ONCE here (not per call). When R already equals the + // kernel's native result type (e.g. f32 input with R=float32, or a narrow + // kernel's float64 with R=float64), the kernel IS that type — return it + // directly, so the hot loop is a single direct call exactly like before. Only + // when R differs (a cast is genuinely needed, e.g. narrow float64 -> float32) + // do we add a thin casting wrapper. + switch any(*new(T)).(type) { + case float32: + fn, err := resolveRealKernel[float32](metric) + if err != nil { + return nil, err + } + if f, ok := any(fn).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []float32) (R, error) { d, e := fn(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case float64: + fn, err := resolveRealKernel[float64](metric) + if err != nil { + return nil, err + } + if f, ok := any(fn).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []float64) (R, error) { d, e := fn(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case types.BF16: + k, err := resolveBF16Kernel(metric) + if err != nil { + return nil, err + } + if f, ok := any(k).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []types.BF16) (R, error) { d, e := k(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case types.Float16: + k, err := resolveF16Kernel(metric) + if err != nil { + return nil, err + } + if f, ok := any(k).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []types.Float16) (R, error) { d, e := k(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case int8: + k, err := resolveInt8Kernel(metric) + if err != nil { + return nil, err + } + if f, ok := any(k).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []int8) (R, error) { d, e := k(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + case uint8: + k, err := resolveUint8Kernel(metric) + if err != nil { + return nil, err + } + if f, ok := any(k).(func(a, b []T) (R, error)); ok { + return f, nil + } + w := func(a, b []uint8) (R, error) { d, e := k(a, b); return R(d), e } + return any(w).(func(a, b []T) (R, error)), nil + default: + return nil, moerr.NewInternalErrorNoCtx("ResolveDistanceFn: unsupported element type") + } +} + +func GoPairWiseDistance[T types.ArrayElement]( + x [][]T, + y [][]T, + metric MetricType, +) ([]float32, error) { + distFn, err := ResolveDistanceFn[T, float32](metric) + if err != nil { + return nil, err + } + + nX := len(x) + nY := len(y) + res := make([]float32, nX*nY) + for i := 0; i < nX; i++ { + for j := 0; j < nY; j++ { + d, err := distFn(x[i], y[j]) + if err != nil { + return nil, err + } + res[i*nY+j] = d + } + } + + if metric == Metric_L2Distance { + for i := range res { + res[i] = float32(math.Sqrt(float64(res[i]))) + } + } + + return res, nil +} diff --git a/pkg/vectorindex/metric/types.go b/pkg/vectorindex/metric/types.go index 41d4db6715de9..b2dc729fd1b64 100644 --- a/pkg/vectorindex/metric/types.go +++ b/pkg/vectorindex/metric/types.go @@ -65,6 +65,7 @@ const ( const ( Quantization_F32_Str = "float32" Quantization_F16_Str = "float16" + Quantization_BF16_Str = "bf16" Quantization_INT8_Str = "int8" Quantization_UINT8_Str = "uint8" Quantization_F64_Str = "float64" diff --git a/pkg/vectorindex/quantizer/quantizer.go b/pkg/vectorindex/quantizer/quantizer.go new file mode 100644 index 0000000000000..2c5a008df802d --- /dev/null +++ b/pkg/vectorindex/quantizer/quantizer.go @@ -0,0 +1,246 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package quantizer is the single source of truth for ivfflat QUANTIZATION: the +// mapping from a CREATE INDEX QUANTIZATION='...' name to the narrow entry type, +// the cuVS-style asymmetric int8 scalar quantizer (training the [min,max] bounds, +// deriving the q(x)=round(x*mul+add) transform, applying it to a query vector, +// and emitting the equivalent SQL entry projection), and the SQL type names. +// +// The same q(x)=x*mul+add must be applied identically on three sides — the +// synchronous build (compile), the CDC delta writer (iscp), and search — so the +// formula lives here once and each side calls in rather than re-deriving it. +package quantizer + +import ( + "fmt" + "math" + "slices" + "strings" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" +) + +const ( + // Int8Lo/Int8Hi are the signed int8 range the quantizer maps [min,max] onto. + Int8Lo = -128.0 + Int8Hi = 127.0 + // int8Span is the number of quantization steps (Int8Hi-Int8Lo = 255). + int8Span = Int8Hi - Int8Lo + + // Uint8Lo/Uint8Hi are the unsigned uint8 range the quantizer maps [min,max] + // onto (same step count as int8, just shifted to start at 0 — no -128 offset). + Uint8Lo = 0.0 + Uint8Hi = 255.0 + // uint8Span is the number of quantization steps (Uint8Hi-Uint8Lo = 255). + uint8Span = Uint8Hi - Uint8Lo +) + +// ToVectorType maps a CREATE INDEX QUANTIZATION='...' value to the vector element +// type the ivfflat ENTRIES are down-cast to (the base column and centroids are +// unaffected). The accepted names are the canonical metric.Quantization_*_Str +// constants (case-insensitive): float32 -> vecf32, float16 -> vecf16, +// bf16 -> vecbf16, int8 -> vecint8, uint8 -> vecuint8. float32/float16/bf16 are +// float formats (plain cast); int8/uint8 use the trained scalar quantizer. float32 +// is accepted because it is a real down-cast for an f64 base; float64 (an up-cast) +// and "" return ok=false (no quantization; entries keep the base type). +func ToVectorType(q string) (types.T, bool) { + switch strings.ToLower(strings.TrimSpace(q)) { + case metric.Quantization_F32_Str: + return types.T_array_float32, true + case metric.Quantization_F16_Str: + return types.T_array_float16, true + case metric.Quantization_BF16_Str: + return types.T_array_bf16, true + case metric.Quantization_INT8_Str: + return types.T_array_int8, true + case metric.Quantization_UINT8_Str: + return types.T_array_uint8, true + } + return 0, false +} + +// SQLTypeName returns the SQL type name for a vector element type, for use in +// CAST(... AS (dim)). Thin alias over types.T.ArraySQLName (the canonical +// source of the lowercase SQL spellings). +func SQLTypeName(t types.T) string { + return t.ArraySQLName() +} + +// Int8Params returns (mul, add) for the cuVS-style asymmetric int8 scalar +// quantizer that maps [min,max] onto the full int8 range [Int8Lo,Int8Hi]: +// +// q(x) = round(x*mul + add), clamped to [-128,127] +// +// add folds the -min offset and the -128 int8 shift into one constant, so both +// the build (cast(base*mul+add as vecint8)) and search apply it with a single +// multiply-add. A degenerate range (max<=min) falls back to identity. +func Int8Params(min, max float64) (mul, add float64) { + rng := max - min + // !(rng > 0) also rejects NaN (every NaN comparison is false), and the IsInf + // guard rejects a non-finite range — either would otherwise yield NaN/Inf mul + // that poisons the build SQL and the query transform. + if !(rng > 0) || math.IsInf(rng, 0) { + return 1.0, 0.0 + } + mul = int8Span / rng + add = -min*mul + Int8Lo + return mul, add +} + +// TrainInt8 returns (P0.1, P99.9) of the sample values — the bounds for the +// asymmetric int8 scalar quantizer. Percentiles (not raw min/max) clip outliers +// so the quantization grid isn't wasted on a few extreme values. Returns (-1,1) +// for empty data and widens a degenerate range. Subsamples to bound cost. +func TrainInt8[T types.RealNumbers](data [][]T) (min, max float64) { + const maxVals = 2_000_000 + total := 0 + for _, v := range data { + total += len(v) + } + if total == 0 { + return -1, 1 + } + stride := 1 + if total > maxVals { + stride = total/maxVals + 1 + } + vals := make([]float64, 0, total/stride+1) + k := 0 + for _, v := range data { + for _, x := range v { + if k%stride == 0 { + f := float64(x) + // Skip NaN/Inf: they make slices.Sort's order undefined (so a + // percentile pick could land on NaN) and would poison the trained + // bounds and the SQL literal. + if !math.IsNaN(f) && !math.IsInf(f, 0) { + vals = append(vals, f) + } + } + k++ + } + } + if len(vals) == 0 { + return -1, 1 + } + slices.Sort(vals) + lo := vals[int(float64(len(vals)-1)*0.001)] + hi := vals[int(float64(len(vals)-1)*0.999)] + if hi <= lo { + hi = lo + 1 + } + return lo, hi +} + +// ApplyInt8 applies q(x)=x*mul+add to a float32 query vector and narrows to int8 +// (round+clamp), matching the entry build. (mul,add)=(1,0) is identity (no +// quantizer trained), so the raw narrowing cast is used. The multiply-add is done +// in float64 (then narrowed) to match the build side, whose entry SQL +// `cast(base*mul+add as vecint8)` evaluates the f64 literals in the base column's +// arithmetic — doing it in float32 here could bucket a boundary component +// differently. qf32 is never mutated. +func ApplyInt8(qf32 []float32, mul, add float64) []int8 { + if mul == 1.0 && add == 0.0 { + return types.Float32ToInt8Slice(qf32) + } + sq := make([]float32, len(qf32)) + for i, x := range qf32 { + sq[i] = float32(float64(x)*mul + add) + } + return types.Float32ToInt8Slice(sq) +} + +// CastSQL builds `cast( as (dim))` for narrowing an entry to a +// quantization type without scaling — float formats (float16/bf16/float32), or +// int8 when no [min,max] bounds were trained (the implicit cast does identity +// round+clamp). colExpr is the already-quoted column reference or sub-expression. +func CastSQL(colExpr string, t types.T, dim int32) string { + return fmt.Sprintf("cast(%s as %s(%d))", colExpr, SQLTypeName(t), dim) +} + +// Int8EntrySQL builds the int8 entry projection `cast( * mul + add as +// vecint8(dim))` from precomputed literal bounds (the synchronous build path, +// where compile has already read the trained [min,max] from metadata). colExpr is +// the already-quoted column reference. +func Int8EntrySQL(colExpr string, mul, add float64, dim int32) string { + return fmt.Sprintf("cast(%s * %.9g + (%.9g) as vecint8(%d))", colExpr, mul, add, dim) +} + +// Int8EntrySQLFromBounds builds the int8 entry projection where the bounds are SQL +// expressions (e.g. metadata-table subqueries) rather than precomputed literals — +// the CDC delta path, which cannot read metadata into Go before building the +// REPLACE. It inlines q(x)=x*mul+add with mul=int8Span/(max-min) and +// add=-min*mul-128, and wraps mul/add in COALESCE so an absent bound (a pure-async +// index that never trained) falls back to identity (1,0) — matching what search +// does when the bounds are missing. colExpr/minExpr/maxExpr are SQL sub-expressions. +func Int8EntrySQLFromBounds(colExpr, minExpr, maxExpr string, dim int32) string { + // 255.0 == int8Span, 128.0 == -Int8Lo, kept as literals so the division/offset + // evaluate in DOUBLE in SQL (and the generated text is stable). + rng := fmt.Sprintf("(%s - %s)", maxExpr, minExpr) + mul := fmt.Sprintf("COALESCE(255.0 / %s, 1.0)", rng) + add := fmt.Sprintf("COALESCE(0.0 - %s * 255.0 / %s - 128.0, 0.0)", minExpr, rng) + return fmt.Sprintf("cast(%s * %s + %s as vecint8(%d))", colExpr, mul, add, dim) +} + +// Uint8Params returns (mul, add) for the cuVS-style asymmetric uint8 scalar +// quantizer that maps [min,max] onto the full uint8 range [Uint8Lo,Uint8Hi]: +// +// q(x) = round(x*mul + add), clamped to [0,255] +// +// It is the int8 quantizer shifted to start at 0 (add folds only the -min offset; +// there is no -128 shift). A degenerate range (max<=min) falls back to identity. +// The trained bounds come from TrainInt8 (the percentile training is the same). +func Uint8Params(min, max float64) (mul, add float64) { + rng := max - min + if !(rng > 0) || math.IsInf(rng, 0) { + return 1.0, 0.0 + } + mul = uint8Span / rng + add = -min*mul + Uint8Lo + return mul, add +} + +// ApplyUint8 is the uint8 analog of ApplyInt8: applies q(x)=x*mul+add to a float32 +// query vector and narrows to uint8 (round+clamp to [0,255]). (mul,add)=(1,0) is +// identity. The multiply-add is done in float64 to match the build side. qf32 is +// never mutated. +func ApplyUint8(qf32 []float32, mul, add float64) []uint8 { + if mul == 1.0 && add == 0.0 { + return types.Float32ToUint8Slice(qf32) + } + sq := make([]float32, len(qf32)) + for i, x := range qf32 { + sq[i] = float32(float64(x)*mul + add) + } + return types.Float32ToUint8Slice(sq) +} + +// Uint8EntrySQL is the uint8 analog of Int8EntrySQL: the build-side entry +// projection `cast( * mul + add as vecuint8(dim))` from literal bounds. +func Uint8EntrySQL(colExpr string, mul, add float64, dim int32) string { + return fmt.Sprintf("cast(%s * %.9g + (%.9g) as vecuint8(%d))", colExpr, mul, add, dim) +} + +// Uint8EntrySQLFromBounds is the uint8 analog of Int8EntrySQLFromBounds (CDC delta +// path). q(x)=x*mul+add with mul=255/(max-min) and add=-min*255/(max-min) — no -128 +// shift — wrapped in COALESCE for identity fallback when a bound is absent. +func Uint8EntrySQLFromBounds(colExpr, minExpr, maxExpr string, dim int32) string { + // 255.0 == uint8Span; no offset term since Uint8Lo == 0. + rng := fmt.Sprintf("(%s - %s)", maxExpr, minExpr) + mul := fmt.Sprintf("COALESCE(255.0 / %s, 1.0)", rng) + add := fmt.Sprintf("COALESCE(0.0 - %s * 255.0 / %s, 0.0)", minExpr, rng) + return fmt.Sprintf("cast(%s * %s + %s as vecuint8(%d))", colExpr, mul, add, dim) +} diff --git a/pkg/vectorindex/quantizer/quantizer_test.go b/pkg/vectorindex/quantizer/quantizer_test.go new file mode 100644 index 0000000000000..3732863ed9f1f --- /dev/null +++ b/pkg/vectorindex/quantizer/quantizer_test.go @@ -0,0 +1,287 @@ +// Copyright 2026 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quantizer + +import ( + "math" + "testing" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/stretchr/testify/require" +) + +func TestToVectorType(t *testing.T) { + cases := []struct { + in string + want types.T + ok bool + }{ + {"float32", types.T_array_float32, true}, + {"float16", types.T_array_float16, true}, + {"bf16", types.T_array_bf16, true}, + {"int8", types.T_array_int8, true}, + {"uint8", types.T_array_uint8, true}, + // case-insensitive + surrounding space + {"FLOAT16", types.T_array_float16, true}, + {"BF16", types.T_array_bf16, true}, + {" Int8 ", types.T_array_int8, true}, + {"UINT8", types.T_array_uint8, true}, + // not quantization targets + {"float64", 0, false}, + {"f16", 0, false}, // only canonical names + {"bfloat16", 0, false}, + {"", 0, false}, + {"garbage", 0, false}, + } + for _, c := range cases { + got, ok := ToVectorType(c.in) + require.Equalf(t, c.ok, ok, "ok for %q", c.in) + if c.ok { + require.Equalf(t, c.want, got, "type for %q", c.in) + } + } +} + +func TestSQLTypeName(t *testing.T) { + require.Equal(t, "vecf32", SQLTypeName(types.T_array_float32)) + require.Equal(t, "vecf64", SQLTypeName(types.T_array_float64)) + require.Equal(t, "vecbf16", SQLTypeName(types.T_array_bf16)) + require.Equal(t, "vecf16", SQLTypeName(types.T_array_float16)) + require.Equal(t, "vecint8", SQLTypeName(types.T_array_int8)) + require.Equal(t, "", SQLTypeName(types.T_int32)) +} + +func TestInt8Params(t *testing.T) { + // q(x) = round(x*mul + add) must map min -> -128 and max -> +127. + min, max := -2.0, 6.0 + mul, add := Int8Params(min, max) + qmin := min*mul + add + qmax := max*mul + add + require.InDelta(t, -128.0, qmin, 1e-6) + require.InDelta(t, 127.0, qmax, 1e-6) + // midpoint maps near 0 (the [-128,127] center is -0.5) + mid := (min+max)/2*mul + add + require.InDelta(t, -0.5, mid, 1e-6) + + // asymmetric (all-positive) range still spans the full grid. + mul, add = Int8Params(0.07, 0.83) + require.InDelta(t, -128.0, 0.07*mul+add, 1e-6) + require.InDelta(t, 127.0, 0.83*mul+add, 1e-6) + + // degenerate range -> identity (no panic / no inf). + mul, add = Int8Params(1.0, 1.0) + require.Equal(t, 1.0, mul) + require.Equal(t, 0.0, add) + mul, add = Int8Params(5.0, 1.0) + require.Equal(t, 1.0, mul) + require.Equal(t, 0.0, add) + require.False(t, math.IsInf(mul, 0)) +} + +func TestInt8ParamsEdgeCases(t *testing.T) { + // Across a variety of ranges, q(min) must hit -128 and q(max) must hit +127. + ranges := [][2]float64{ + {-10, -2}, // all-negative + {-5, 5}, // symmetric about 0 + {0.999, 1.001}, // tiny range near 1 + {-1e6, 1e6}, // huge range + {0, 255}, // exactly the int8-span width + } + for _, r := range ranges { + mul, add := Int8Params(r[0], r[1]) + require.InDeltaf(t, -128.0, r[0]*mul+add, 1e-6, "min %v", r) + require.InDeltaf(t, 127.0, r[1]*mul+add, 1e-6, "max %v", r) + // a value inside the range stays inside [-128,127]. + mid := (r[0] + r[1]) / 2 + q := mid*mul + add + require.GreaterOrEqualf(t, q, -128.0-1e-6, "mid in range %v", r) + require.LessOrEqualf(t, q, 127.0+1e-6, "mid in range %v", r) + } + + // dequant round-trip: x ~= (q - add) / mul within one quantization step. + min, max := -3.0, 7.0 + mul, add := Int8Params(min, max) + step := (max - min) / 255.0 + for _, x := range []float64{-3, -1.5, 0, 2.2, 6.99} { + q := math.Round(x*mul + add) + deq := (q - add) / mul + require.InDeltaf(t, x, deq, step, "round-trip x=%v", x) + } +} + +func TestTrainInt8(t *testing.T) { + // empty -> (-1, 1) + lo, hi := TrainInt8([][]float32{}) + require.Equal(t, -1.0, lo) + require.Equal(t, 1.0, hi) + + // uniform 0..999: P0.1 near 0, P99.9 near 999 + d := make([][]float32, 1) + d[0] = make([]float32, 1000) + for i := range d[0] { + d[0][i] = float32(i) + } + lo, hi = TrainInt8(d) + require.InDelta(t, 0.0, lo, 2) + require.InDelta(t, 999.0, hi, 2) + + // degenerate (all equal) -> (v, v+1) so the range is never zero. + lo, hi = TrainInt8([][]float32{{5, 5, 5, 5}}) + require.Equal(t, 5.0, lo) + require.Equal(t, 6.0, hi) + + // a single extreme outlier is clipped by the P99.9 percentile. + o := make([][]float32, 1) + o[0] = make([]float32, 1000) + for i := 0; i < 999; i++ { + o[0][i] = 1.0 + } + o[0][999] = 1e6 + _, hi = TrainInt8(o) + require.Less(t, hi, 1e6) + + // works on float64 too (f64-base quantization): bounds are sane and ordered + // (exact percentiles of a 4-element array are not the raw min/max). + lo64, hi64 := TrainInt8([][]float64{{-3, -1, 1, 3}}) + require.GreaterOrEqual(t, lo64, -3.0) + require.LessOrEqual(t, hi64, 3.0) + require.Less(t, lo64, hi64) +} + +func TestTrainInt8Edge(t *testing.T) { + // single value -> degenerate (v, v+1) + lo, hi := TrainInt8([][]float32{{5}}) + require.Equal(t, 5.0, lo) + require.Equal(t, 6.0, hi) + + // all-negative data: bounds stay inside the data range and ordered. + lo, hi = TrainInt8([][]float32{{-10, -8, -5, -3, -2}}) + require.GreaterOrEqual(t, lo, -10.0) + require.LessOrEqual(t, hi, -2.0) + require.Less(t, lo, hi) + + // subsampling path: > 2M values (stride > 1) must not panic and stays in range. + big := make([][]float32, 2500) + for i := range big { + v := make([]float32, 1000) // 2.5M values total + for j := range v { + v[j] = float32((i*1000 + j) % 1000) // cycles 0..999 + } + big[i] = v + } + lo, hi = TrainInt8(big) + require.GreaterOrEqual(t, lo, 0.0) + require.LessOrEqual(t, hi, 999.0) + require.Less(t, lo, hi) + + // NaN/Inf are skipped: a poisoned sample still trains finite, ordered bounds. + lo, hi = TrainInt8([][]float64{{math.NaN(), math.Inf(1), math.Inf(-1), 1, 2, 3, 4}}) + require.False(t, math.IsNaN(lo) || math.IsInf(lo, 0)) + require.False(t, math.IsNaN(hi) || math.IsInf(hi, 0)) + require.Less(t, lo, hi) +} + +func TestApplyInt8(t *testing.T) { + // identity (mul,add)=(1,0): plain round+clamp narrowing, input unchanged. + in := []float32{-130, -1.4, 0.6, 5, 200} + got := ApplyInt8(in, 1.0, 0.0) + require.Equal(t, []int8{-128, -1, 1, 5, 127}, got) + require.Equal(t, []float32{-130, -1.4, 0.6, 5, 200}, in, "input must not be mutated") + + // trained transform matches q(x)=round(x*mul+add): map [0.1,0.99] -> full range. + mul, add := Int8Params(0.10, 0.99) + q := ApplyInt8([]float32{0.10, 0.99, 0.50}, mul, add) + require.Equal(t, int8(-128), q[0]) // min -> -128 + require.Equal(t, int8(127), q[1]) // max -> +127 + // 0.50 matches the float64 multiply-add, rounded. + want := int8(math.Round(0.50*mul + add)) + require.Equal(t, want, q[2]) + + // empty input -> empty output, no panic. + require.Empty(t, ApplyInt8([]float32{}, mul, add)) +} + +func TestEntrySQLBuilders(t *testing.T) { + // literal-bounds (build) projection. + require.Equal(t, + "cast(`v` * 286.516854 + (-156.651685) as vecint8(4))", + Int8EntrySQL("`v`", 286.516854, -156.651685, 4)) + + // metadata-subquery (CDC) projection with COALESCE identity fallback. + min := "(SELECT m FROM meta WHERE k='quantize_min')" + max := "(SELECT m FROM meta WHERE k='quantize_max')" + got := Int8EntrySQLFromBounds("src1", min, max, 4) + require.Equal(t, + "cast(src1 * COALESCE(255.0 / ("+max+" - "+min+"), 1.0) + "+ + "COALESCE(0.0 - "+min+" * 255.0 / ("+max+" - "+min+") - 128.0, 0.0) as vecint8(4))", + got) + + // plain narrowing cast (float formats / untrained int8). + require.Equal(t, "cast(`v` as vecf16(8))", CastSQL("`v`", types.T_array_float16, 8)) + require.Equal(t, "cast(`v` as vecint8(8))", CastSQL("`v`", types.T_array_int8, 8)) + require.Equal(t, "cast(`v` as vecuint8(8))", CastSQL("`v`", types.T_array_uint8, 8)) +} + +func TestUint8Params(t *testing.T) { + // q(x)=round(x*mul+add) must map min -> 0 and max -> 255 (unsigned range). + min, max := -2.0, 6.0 + mul, add := Uint8Params(min, max) + require.InDelta(t, 0.0, min*mul+add, 1e-6) + require.InDelta(t, 255.0, max*mul+add, 1e-6) + // midpoint maps near the center 127.5. + require.InDelta(t, 127.5, (min+max)/2*mul+add, 1e-6) + + // all-positive range still spans the full grid. + mul, add = Uint8Params(0.07, 0.83) + require.InDelta(t, 0.0, 0.07*mul+add, 1e-6) + require.InDelta(t, 255.0, 0.83*mul+add, 1e-6) + + // degenerate range -> identity. + mul, add = Uint8Params(1.0, 1.0) + require.Equal(t, 1.0, mul) + require.Equal(t, 0.0, add) +} + +func TestApplyUint8(t *testing.T) { + // identity: round+clamp to [0,255], input unchanged. + in := []float32{-5, 0.6, 5, 254.5, 300} + got := ApplyUint8(in, 1.0, 0.0) + require.Equal(t, []uint8{0, 1, 5, 255, 255}, got) + require.Equal(t, []float32{-5, 0.6, 5, 254.5, 300}, in, "input must not be mutated") + + // trained transform maps [0.1,0.99] -> [0,255]. + mul, add := Uint8Params(0.10, 0.99) + q := ApplyUint8([]float32{0.10, 0.99, 0.50}, mul, add) + require.Equal(t, uint8(0), q[0]) + require.Equal(t, uint8(255), q[1]) + require.Equal(t, uint8(math.Round(0.50*mul+add)), q[2]) + + require.Empty(t, ApplyUint8([]float32{}, mul, add)) +} + +func TestUint8EntrySQLBuilders(t *testing.T) { + // literal-bounds (build) projection -> vecuint8. + require.Equal(t, + "cast(`v` * 286.516854 + (28.6516854) as vecuint8(4))", + Uint8EntrySQL("`v`", 286.516854, 28.6516854, 4)) + + // metadata-subquery (CDC) projection: no -128 offset, identity COALESCE fallback. + min := "(SELECT m FROM meta WHERE k='quantize_min')" + max := "(SELECT m FROM meta WHERE k='quantize_max')" + require.Equal(t, + "cast(src1 * COALESCE(255.0 / ("+max+" - "+min+"), 1.0) + "+ + "COALESCE(0.0 - "+min+" * 255.0 / ("+max+" - "+min+"), 0.0) as vecuint8(4))", + Uint8EntrySQLFromBounds("src1", min, max, 4)) +} diff --git a/pkg/vectorindex/types.go b/pkg/vectorindex/types.go index 33f875a8463ec..eaab8ed9ca92f 100644 --- a/pkg/vectorindex/types.go +++ b/pkg/vectorindex/types.go @@ -23,6 +23,9 @@ import ( usearch "github.com/unum-cloud/usearch/golang" ) +// QUANTIZATION lives in pkg/vectorindex/quantizer: ToVectorType, Int8Params, +// SQLTypeName, TrainInt8, ApplyInt8, and the SQL entry-projection builders. + /* HNSW vector index using usearch @@ -179,13 +182,19 @@ type CagraParam struct { } type IvfflatIndexConfig struct { - Lists uint - Metric uint16 - InitType uint16 - Dimensions uint - Spherical bool - Version int64 - VectorType int32 + Lists uint + Metric uint16 + InitType uint16 + Dimensions uint + Spherical bool + Version int64 + VectorType int32 + // CentroidType is the element type the centroid hidden table is stored in. + // Entries always keep VectorType (the input/quantization type); centroids may + // be f32 (decoupled — best recall, fast f32 SIMD search, negligible RAM for + // few centroids) or follow VectorType (least RAM, narrow-native search). 0 == + // unset is treated as T_array_float32. (cuVS allows the same choice.) + CentroidType int32 KmeansTrainPercent float64 KmeansMaxIteration int64 } diff --git a/pkg/vm/engine/disttae/logtailreplay/change_handle.go b/pkg/vm/engine/disttae/logtailreplay/change_handle.go index e799c1b0b9800..6d02273f3c4ed 100755 --- a/pkg/vm/engine/disttae/logtailreplay/change_handle.go +++ b/pkg/vm/engine/disttae/logtailreplay/change_handle.go @@ -2512,7 +2512,8 @@ func appendFromEntry(src, vec *vector.Vector, offset int, mp *mpool.MPool) { case types.T_Blockid: val = vector.GetFixedAtNoTypeCheck[types.Blockid](src, offset) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink: val = src.GetBytesAt(offset) default: //return vector.ErrVecTypeNotSupport diff --git a/pkg/vm/engine/disttae/util.go b/pkg/vm/engine/disttae/util.go index fe23578fda4b2..f678d64647da4 100644 --- a/pkg/vm/engine/disttae/util.go +++ b/pkg/vm/engine/disttae/util.go @@ -191,6 +191,26 @@ func LinearSearchOffsetByValFactory(pk *vector.Vector) func(*vector.Vector) []in v := types.ArrayToString[float64](vector.GetArrayAt[float64](pk, i)) mp[v] = true } + case types.T_array_bf16: + for i := 0; i < pk.Length(); i++ { + v := types.ArrayToString[types.BF16](vector.GetArrayAt[types.BF16](pk, i)) + mp[v] = true + } + case types.T_array_float16: + for i := 0; i < pk.Length(); i++ { + v := types.ArrayToString[types.Float16](vector.GetArrayAt[types.Float16](pk, i)) + mp[v] = true + } + case types.T_array_int8: + for i := 0; i < pk.Length(); i++ { + v := types.ArrayToString[int8](vector.GetArrayAt[int8](pk, i)) + mp[v] = true + } + case types.T_array_uint8: + for i := 0; i < pk.Length(); i++ { + v := types.ArrayToString[uint8](vector.GetArrayAt[uint8](pk, i)) + mp[v] = true + } default: panic(moerr.NewInternalErrorNoCtxf("%s not supported", pk.GetType().String())) } @@ -399,6 +419,34 @@ func LinearSearchOffsetByValFactory(pk *vector.Vector) func(*vector.Vector) []in sels = append(sels, int64(i)) } } + case types.T_array_bf16: + for i := 0; i < vec.Length(); i++ { + v := types.ArrayToString[types.BF16](vector.GetArrayAt[types.BF16](vec, i)) + if mp[v] { + sels = append(sels, int64(i)) + } + } + case types.T_array_float16: + for i := 0; i < vec.Length(); i++ { + v := types.ArrayToString[types.Float16](vector.GetArrayAt[types.Float16](vec, i)) + if mp[v] { + sels = append(sels, int64(i)) + } + } + case types.T_array_int8: + for i := 0; i < vec.Length(); i++ { + v := types.ArrayToString[int8](vector.GetArrayAt[int8](vec, i)) + if mp[v] { + sels = append(sels, int64(i)) + } + } + case types.T_array_uint8: + for i := 0; i < vec.Length(); i++ { + v := types.ArrayToString[uint8](vector.GetArrayAt[uint8](vec, i)) + if mp[v] { + sels = append(sels, int64(i)) + } + } default: panic(moerr.NewInternalErrorNoCtxf("%s not supported", vec.GetType().String())) } diff --git a/pkg/vm/engine/disttae/util_test.go b/pkg/vm/engine/disttae/util_test.go index a86bdff26af91..043c6b325379a 100644 --- a/pkg/vm/engine/disttae/util_test.go +++ b/pkg/vm/engine/disttae/util_test.go @@ -73,6 +73,50 @@ func TestLinearSearchOffsetByValFactory_Varchar(t *testing.T) { target2.Free(mp) } +// narrowArrayLinearSearch exercises LinearSearchOffsetByValFactory for a narrow +// vector array key type (vecbf16/vecf16/vecint8/vecuint8). Both the key-side map +// build and the target-side search switch on the element type, so a narrow array +// must be handled in both or this panics "not supported". +func narrowArrayLinearSearch[T types.ArrayElement](t *testing.T, mp *mpool.MPool, oid types.T, a, b, c []T) { + typ := types.New(oid, int32(len(a)), 0) + + keys := vector.NewVec(typ) + require.NoError(t, vector.AppendArray[T](keys, a, false, mp)) + require.NoError(t, vector.AppendArray[T](keys, b, false, mp)) + searchFn := LinearSearchOffsetByValFactory(keys) + + // target with no matching key + target := vector.NewVec(typ) + require.NoError(t, vector.AppendArray[T](target, c, false, mp)) + require.Empty(t, searchFn(target)) + + // target containing key b at index 1 + target2 := vector.NewVec(typ) + require.NoError(t, vector.AppendArray[T](target2, c, false, mp)) + require.NoError(t, vector.AppendArray[T](target2, b, false, mp)) + require.Equal(t, []int64{1}, searchFn(target2)) + + keys.Free(mp) + target.Free(mp) + target2.Free(mp) +} + +func TestLinearSearchOffsetByValFactory_NarrowArray(t *testing.T) { + mp := mpool.MustNewZero() + narrowArrayLinearSearch[types.Float16](t, mp, types.T_array_float16, + types.Float32ToFloat16Slice([]float32{1, 1}), + types.Float32ToFloat16Slice([]float32{2, 2}), + types.Float32ToFloat16Slice([]float32{3, 3})) + narrowArrayLinearSearch[types.BF16](t, mp, types.T_array_bf16, + types.Float32ToBF16Slice([]float32{1, 1}), + types.Float32ToBF16Slice([]float32{2, 2}), + types.Float32ToBF16Slice([]float32{3, 3})) + narrowArrayLinearSearch[int8](t, mp, types.T_array_int8, + []int8{1, 1}, []int8{2, 2}, []int8{3, 3}) + narrowArrayLinearSearch[uint8](t, mp, types.T_array_uint8, + []uint8{1, 1}, []uint8{2, 2}, []uint8{3, 3}) +} + func TestLinearSearchOffsetByValFactory_Int64(t *testing.T) { mp := mpool.MustNewZero() diff --git a/pkg/vm/engine/tae/blockio/read.go b/pkg/vm/engine/tae/blockio/read.go index 34db82a3dfe62..b435c6c5a97cc 100644 --- a/pkg/vm/engine/tae/blockio/read.go +++ b/pkg/vm/engine/tae/blockio/read.go @@ -387,6 +387,20 @@ func BlockDataReadBackup( return } +// topnDistOf builds a per-row distance closure for the topn order-by-limit scan: +// it decodes the query and each row's raw column bytes as element type T and +// returns the float64 distance via the merged ResolveDistanceFn (R=float64). +func topnDistOf[T types.ArrayElement](numVec []byte, m metric.MetricType) (func([]byte) (float64, error), error) { + distFunc, err := metric.ResolveDistanceFn[T, float64](m) + if err != nil { + return nil, err + } + rhs := types.BytesToArray[T](numVec) + return func(b []byte) (float64, error) { + return distFunc(types.BytesToArray[T](b), rhs) + }, nil +} + func HandleOrderByLimitOnIVFFlatIndex( ctx context.Context, selectRows []int64, @@ -411,110 +425,71 @@ func HandleOrderByLimitOnIVFFlatIndex( return nil, nil, err } + // Per-type distance closure: returns the float64 distance between a row's raw + // column bytes and the query vector. The single merged ResolveDistanceFn[T, + // float64] handles f32/f64 and the narrow quantizations uniformly; the bounds + // + top-k heap loop below is shared. + var distOf func(colBytes []byte) (float64, error) switch orderByLimit.Typ { case types.T_array_float32: - distFunc, err := metric.ResolveDistanceFn[float32](orderByLimit.MetricType) + distOf, err = topnDistOf[float32](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_float64: + distOf, err = topnDistOf[float64](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_bf16: + distOf, err = topnDistOf[types.BF16](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_float16: + distOf, err = topnDistOf[types.Float16](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_int8: + distOf, err = topnDistOf[int8](orderByLimit.NumVec, orderByLimit.MetricType) + case types.T_array_uint8: + distOf, err = topnDistOf[uint8](orderByLimit.NumVec, orderByLimit.MetricType) + default: + return nil, nil, moerr.NewInternalError(ctx, fmt.Sprintf("only support float32/float64/bf16/float16/int8/uint8 type for topn: %s", orderByLimit.Typ)) + } + if err != nil { + return nil, nil, err + } + + for _, row := range selectRows { + dist64, err := distOf(vecCol.GetBytesAt(int(row))) if err != nil { return nil, nil, err } - rhs := types.BytesToArray[float32](orderByLimit.NumVec) - - for _, row := range selectRows { - dist, err := distFunc(types.BytesToArray[float32](vecCol.GetBytesAt(int(row))), rhs) - if err != nil { - return nil, nil, err - } - dist64 := float64(dist) - - if orderByLimit.LowerBoundType == plan.BoundType_INCLUSIVE { - if dist64 < orderByLimit.LowerBound { - continue - } - } else if orderByLimit.LowerBoundType == plan.BoundType_EXCLUSIVE { - if dist64 <= orderByLimit.LowerBound { - continue - } + if orderByLimit.LowerBoundType == plan.BoundType_INCLUSIVE { + if dist64 < orderByLimit.LowerBound { + continue } - if orderByLimit.UpperBoundType == plan.BoundType_INCLUSIVE { - if dist64 > orderByLimit.UpperBound { - continue - } - } else if orderByLimit.UpperBoundType == plan.BoundType_EXCLUSIVE { - if dist64 >= orderByLimit.UpperBound { - continue - } - } - - if len(orderByLimit.DistHeap) >= topLimit { - if dist64 < orderByLimit.DistHeap[0] { - orderByLimit.DistHeap[0] = dist64 - heap.Fix(&orderByLimit.DistHeap, 0) - } else { - continue - } - } else { - heap.Push(&orderByLimit.DistHeap, dist64) + } else if orderByLimit.LowerBoundType == plan.BoundType_EXCLUSIVE { + if dist64 <= orderByLimit.LowerBound { + continue } - - searchResults = append(searchResults, vectorindex.SearchResult{ - Id: row, - Distance: dist64, - }) } - - case types.T_array_float64: - distFunc, err := metric.ResolveDistanceFn[float64](orderByLimit.MetricType) - if err != nil { - return nil, nil, err - } - - rhs := types.BytesToArray[float64](orderByLimit.NumVec) - - for _, row := range selectRows { - dist64, err := distFunc(types.BytesToArray[float64](vecCol.GetBytesAt(int(row))), rhs) - if err != nil { - return nil, nil, err - } - - if orderByLimit.LowerBoundType == plan.BoundType_INCLUSIVE { - if dist64 < orderByLimit.LowerBound { - continue - } - } else if orderByLimit.LowerBoundType == plan.BoundType_EXCLUSIVE { - if dist64 <= orderByLimit.LowerBound { - continue - } + if orderByLimit.UpperBoundType == plan.BoundType_INCLUSIVE { + if dist64 > orderByLimit.UpperBound { + continue } - if orderByLimit.UpperBoundType == plan.BoundType_INCLUSIVE { - if dist64 > orderByLimit.UpperBound { - continue - } - } else if orderByLimit.UpperBoundType == plan.BoundType_EXCLUSIVE { - if dist64 >= orderByLimit.UpperBound { - continue - } + } else if orderByLimit.UpperBoundType == plan.BoundType_EXCLUSIVE { + if dist64 >= orderByLimit.UpperBound { + continue } + } - if len(orderByLimit.DistHeap) >= topLimit { - if dist64 < orderByLimit.DistHeap[0] { - orderByLimit.DistHeap[0] = dist64 - heap.Fix(&orderByLimit.DistHeap, 0) - } else { - continue - } + if len(orderByLimit.DistHeap) >= topLimit { + if dist64 < orderByLimit.DistHeap[0] { + orderByLimit.DistHeap[0] = dist64 + heap.Fix(&orderByLimit.DistHeap, 0) } else { - heap.Push(&orderByLimit.DistHeap, dist64) + continue } - - searchResults = append(searchResults, vectorindex.SearchResult{ - Id: row, - Distance: dist64, - }) + } else { + heap.Push(&orderByLimit.DistHeap, dist64) } - default: - return nil, nil, moerr.NewInternalError(ctx, fmt.Sprintf("only support float32/float64 type for topn: %s", orderByLimit.Typ)) + searchResults = append(searchResults, vectorindex.SearchResult{ + Id: row, + Distance: dist64, + }) } searchResults = slices.DeleteFunc(searchResults, func(res vectorindex.SearchResult) bool { diff --git a/pkg/vm/engine/tae/blockio/read_test.go b/pkg/vm/engine/tae/blockio/read_test.go index 26b286c2cc179..60102049d9c39 100644 --- a/pkg/vm/engine/tae/blockio/read_test.go +++ b/pkg/vm/engine/tae/blockio/read_test.go @@ -527,3 +527,65 @@ func TestHandleOrderByLimitOnLiveRowsForOrderedLimit(t *testing.T) { require.Nil(t, dists) require.Equal(t, []int64{1, 2, 3, 5}, rows) } + +// TestHandleOrderByLimitOnSelectRows_Narrow exercises the narrow (bf16/f16/int8) +// branch of the optimized vector top-k scan (the merged distOf path). Same data +// in each type: rows [10,10],[1,2],[5,5] vs query [0,0] -> dists 200,5,50, so the +// top-2 are row 1 then row 2. +func TestHandleOrderByLimitOnSelectRows_Narrow(t *testing.T) { + mp := mpool.MustNewZero() + defer mpool.DeleteMPool(mp) + ctx := context.Background() + + cases := []struct { + name string + oid types.T + rows [][]byte + num []byte + }{ + {"int8", types.T_array_int8, [][]byte{ + types.ArrayToBytes([]int8{10, 10}), + types.ArrayToBytes([]int8{1, 2}), + types.ArrayToBytes([]int8{5, 5}), + }, types.ArrayToBytes([]int8{0, 0})}, + {"bf16", types.T_array_bf16, [][]byte{ + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{10, 10})), + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2})), + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{5, 5})), + }, types.ArrayToBytes(types.Float32ToBF16Slice([]float32{0, 0}))}, + {"f16", types.T_array_float16, [][]byte{ + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{10, 10})), + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2})), + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{5, 5})), + }, types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{0, 0}))}, + } + + for _, c := range cases { + vec0 := vector.NewVec(types.T_int32.ToType()) + vec1 := vector.NewVec(c.oid.ToType()) + for i := 0; i < 3; i++ { + vector.AppendFixed(vec0, int32(i), false, mp) + } + for _, b := range c.rows { + vector.AppendBytes(vec1, b, false, mp) + } + cacheVectors := make(containers.Vectors, 2) + cacheVectors[0] = *vec0 + cacheVectors[1] = *vec1 + + orderByLimit := &objectio.IndexReaderTopOp{ + ColPos: 1, + Limit: 2, + Typ: c.oid, + NumVec: c.num, + MetricType: metric.Metric_L2Distance, + DistHeap: make(objectio.Float64Heap, 0, 2), + } + resSels, resDists, err := handleOrderByLimitOnSelectRows(ctx, []int64{0, 1, 2}, orderByLimit, nil, -1, cacheVectors) + require.NoErrorf(t, err, c.name) + require.Lenf(t, resSels, 2, c.name) + require.Lenf(t, resDists, 2, c.name) + require.Equalf(t, int64(1), resSels[0], "%s closest", c.name) + require.Equalf(t, int64(2), resSels[1], "%s next", c.name) + } +} diff --git a/pkg/vm/engine/tae/compute/compute.go b/pkg/vm/engine/tae/compute/compute.go index b23b3d0ab2bcd..f867281d49769 100644 --- a/pkg/vm/engine/tae/compute/compute.go +++ b/pkg/vm/engine/tae/compute/compute.go @@ -245,7 +245,8 @@ func GetOffsetByVal(data containers.Vector, v any, skipmask *nulls.Bitmap) (offs skipmask) case types.T_char, types.T_varchar, types.T_blob, types.T_binary, types.T_varbinary, types.T_json, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink: // data is retrieved from DN vector, hence T_array can be handled here. val := v.([]byte) start, end := 0, data.Length()-1 diff --git a/pkg/vm/engine/tae/compute/compute_test.go b/pkg/vm/engine/tae/compute/compute_test.go index 1dac3df5f7132..40f1492fe1da1 100644 --- a/pkg/vm/engine/tae/compute/compute_test.go +++ b/pkg/vm/engine/tae/compute/compute_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/RoaringBitmap/roaring/v2" + "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" @@ -26,6 +27,28 @@ import ( "github.com/stretchr/testify/require" ) +// TestGetOffsetByValNarrowArray covers GetOffsetByVal binary search over a +// narrow vector array column (vecint8 here). The array case compares raw bytes; +// it previously listed only vecf32/vecf64, so a narrow array fell through. +func TestGetOffsetByValNarrowArray(t *testing.T) { + defer testutils.AfterTest(t)() + mp := mpool.MustNewZero() + typ := types.New(types.T_array_int8, 2, 0) + vec := containers.MakeVector(typ, mp) + defer vec.Close() + // byte-sorted rows so binary search is well-defined + vec.Append(types.ArrayToBytes([]int8{1, 1}), false) + vec.Append(types.ArrayToBytes([]int8{2, 2}), false) + vec.Append(types.ArrayToBytes([]int8{3, 3}), false) + + off, exist := GetOffsetByVal(vec, types.ArrayToBytes([]int8{2, 2}), nil) + require.True(t, exist) + require.Equal(t, 1, off) + + _, exist = GetOffsetByVal(vec, types.ArrayToBytes([]int8{9, 9}), nil) + require.False(t, exist) +} + func TestSortAndDedup(t *testing.T) { defer testutils.AfterTest(t)() vals := []int{2, 1, 3, 4, 5, 1, 2, 3, 4, 5} diff --git a/pkg/vm/engine/tae/containers/utils.go b/pkg/vm/engine/tae/containers/utils.go index c5f0eee22b7d4..85b7170eef685 100644 --- a/pkg/vm/engine/tae/containers/utils.go +++ b/pkg/vm/engine/tae/containers/utils.go @@ -485,7 +485,8 @@ func getNonNullValue(col *vector.Vector, row uint32) any { case types.T_Blockid: return vector.GetFixedAtNoTypeCheck[types.Blockid](col, int(row)) case types.T_char, types.T_varchar, types.T_binary, types.T_varbinary, types.T_json, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink: return col.GetBytesAt(int(row)) default: //return vector.ErrVecTypeNotSupport @@ -577,7 +578,8 @@ func UpdateValue(col *vector.Vector, row uint32, val any, isNull bool, mp *mpool GenericUpdateFixedValue[types.Blockid](col, row, val, isNull, mp) case types.T_varchar, types.T_char, types.T_json, types.T_binary, types.T_varbinary, types.T_blob, types.T_text, - types.T_array_float32, types.T_array_float64, types.T_datalink: + types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8, types.T_datalink: GenericUpdateBytes(col, row, val, isNull, mp) default: panic(moerr.NewInternalErrorNoCtxf("%v not supported", col.GetType())) diff --git a/pkg/vm/engine/tae/containers/utils_test.go b/pkg/vm/engine/tae/containers/utils_test.go index 85f160e18a837..6df8db50f851b 100644 --- a/pkg/vm/engine/tae/containers/utils_test.go +++ b/pkg/vm/engine/tae/containers/utils_test.go @@ -106,6 +106,46 @@ func TestGeneralBatchBuffer1(t *testing.T) { require.Equal(t, int64(0), mp.CurrNB()) } +// TestNarrowArrayValue covers getNonNullValue + UpdateValue for the narrow +// vector array types (vecbf16/vecf16/vecint8/vecuint8). Both switch on the +// element type and previously listed only vecf32/vecf64, so a narrow array +// panicked ("No Support" / "not supported"). All array types are varlen byte +// storage, so the round-trip is read/update as raw bytes. +func TestNarrowArrayValue(t *testing.T) { + mp := mpool.MustNewZero() + + cases := []struct { + oid types.T + initial []byte + updated []byte + }{ + {types.T_array_float16, + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{1, 2, 3})), + types.ArrayToBytes(types.Float32ToFloat16Slice([]float32{4, 5, 6}))}, + {types.T_array_bf16, + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{1, 2, 3})), + types.ArrayToBytes(types.Float32ToBF16Slice([]float32{4, 5, 6}))}, + {types.T_array_int8, + types.ArrayToBytes([]int8{1, 2, 3}), types.ArrayToBytes([]int8{4, 5, 6})}, + {types.T_array_uint8, + types.ArrayToBytes([]uint8{1, 2, 3}), types.ArrayToBytes([]uint8{4, 5, 6})}, + } + + for _, c := range cases { + vec := vector.NewVec(types.New(c.oid, 3, 0)) + require.NoError(t, vector.AppendBytes(vec, c.initial, false, mp)) + + // read back (was panic "No Support") + require.Equal(t, c.initial, getNonNullValue(vec, 0).([]byte), c.oid.String()) + + // update in place (was panic "not supported"), then read back the new value + UpdateValue(vec, 0, c.updated, false, mp) + require.Equal(t, c.updated, getNonNullValue(vec, 0).([]byte), c.oid.String()) + + vec.Free(mp) + } +} + func TestVectorsCopyToBatch(t *testing.T) { var vecs Vectors require.NoError(t, VectorsCopyToBatch(vecs, nil, nil)) diff --git a/pkg/vm/engine/tae/txn/txnimpl/index.go b/pkg/vm/engine/tae/txn/txnimpl/index.go index 5b7f8e5233ac4..e524975c2e4c0 100644 --- a/pkg/vm/engine/tae/txn/txnimpl/index.go +++ b/pkg/vm/engine/tae/txn/txnimpl/index.go @@ -108,7 +108,8 @@ func (idx *simpleTableIndex) KeyToVector(kType types.Type) containers.Vector { for k := range idx.tree { vec.Append([]byte(k.(string)), false) } - case types.T_array_float32, types.T_array_float64: + case types.T_array_float32, types.T_array_float64, + types.T_array_bf16, types.T_array_float16, types.T_array_int8, types.T_array_uint8: // No usage for this func. for k := range idx.tree { vec.Append(k.([]byte), false) diff --git a/test/distributed/cases/array/array_vecnarrow.result b/test/distributed/cases/array/array_vecnarrow.result new file mode 100644 index 0000000000000..5e94664b8269a --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow.result @@ -0,0 +1,164 @@ +drop database if exists vecnarrowdb; +create database vecnarrowdb; +use vecnarrowdb; +drop table if exists nvec; +create table nvec(a int, bf vecbf16(3), f16 vecf16(3), i8 vecint8(3)); +desc nvec; +Field Type Null Key Default Extra Comment +a INT(32) YES null +bf VECBF16(3) YES null +f16 VECF16(3) YES null +i8 VECINT8(3) YES null +show create table nvec; +Table Create Table +nvec CREATE TABLE `nvec` (\n `a` int DEFAULT NULL,\n `bf` vecbf16(3) DEFAULT NULL,\n `f16` vecf16(3) DEFAULT NULL,\n `i8` vecint8(3) DEFAULT NULL\n) +insert into nvec values(1, "[1,2,3]", "[1,2,3]", "[1,2,3]"); +insert into nvec values(2, "[4,5,6]", "[4,5,6]", "[4,5,6]"); +select * from nvec; +a bf f16 i8 +1 [1, 2, 3] [1, 2, 3] [1, 2, 3] +2 [4, 5, 6] [4, 5, 6] [4, 5, 6] +drop table if exists i8t; +create table i8t(a int, v vecint8(4)); +insert into i8t values(1, "[127,-128,0,5]"); +select * from i8t; +a v +1 [127, -128, 0, 5] +insert into i8t values(2, "[200,-200,0,0]"); +internal error: error while casting 200 to VECINT8 +insert into i8t values(3, "[1.4,2.6,0,0]"); +internal error: error while casting 1.4 to VECINT8 +select cast(cast("[1.6,200,-3.5,-200]" as vecf32(4)) as vecint8(4)); +cast(cast([1.6,200,-3.5,-200] as vecf32(4)) as vecint8(4)) +[2, 127, -4, -128] +select cast("[1,2,3]" as vecbf16(3)); +cast([1,2,3] as vecbf16(3)) +[1, 2, 3] +select cast("[1,2,3]" as vecf16(3)); +cast([1,2,3] as vecf16(3)) +[1, 2, 3] +select cast("[1,2,3]" as vecint8(3)); +cast([1,2,3] as vecint8(3)) +[1, 2, 3] +select cast("[1.4,2.6,-3.5]" as vecint8(3)); +internal error: error while casting 1.4 to VECINT8 +select cast(bf as vecf32(3)), cast(f16 as vecf32(3)), cast(i8 as vecf32(3)) from nvec order by a; +cast(bf as vecf32(3)) cast(f16 as vecf32(3)) cast(i8 as vecf32(3)) +[1, 2, 3] [1, 2, 3] [1, 2, 3] +[4, 5, 6] [4, 5, 6] [4, 5, 6] +select cast(bf as vecf64(3)) from nvec order by a; +cast(bf as vecf64(3)) +[1, 2, 3] +[4, 5, 6] +select cast(cast("[1,2,3]" as vecf32(3)) as vecbf16(3)); +cast(cast([1,2,3] as vecf32(3)) as vecbf16(3)) +[1, 2, 3] +select cast(cast("[1,2,3]" as vecf32(3)) as vecf16(3)); +cast(cast([1,2,3] as vecf32(3)) as vecf16(3)) +[1, 2, 3] +select cast(cast("[1.6,2.4,-3.5]" as vecf32(3)) as vecint8(3)); +cast(cast([1.6,2.4,-3.5] as vecf32(3)) as vecint8(3)) +[2, 2, -4] +select cast(bf as vecf16(3)), cast(f16 as vecint8(3)), cast(i8 as vecbf16(3)) from nvec order by a; +cast(bf as vecf16(3)) cast(f16 as vecint8(3)) cast(i8 as vecbf16(3)) +[1, 2, 3] [1, 2, 3] [1, 2, 3] +[4, 5, 6] [4, 5, 6] [4, 5, 6] +select l2_distance(bf, "[1,2,3]") from nvec order by a; +l2_distance(bf, [1,2,3]) +0.0 +5.196152210235596 +select l2_distance_sq(bf, "[1,2,3]") from nvec order by a; +l2_distance_sq(bf, [1,2,3]) +0.0 +27.0 +select inner_product(bf, "[1,2,3]") from nvec order by a; +inner_product(bf, [1,2,3]) +-14.0 +-32.0 +select cosine_distance(bf, "[1,2,3]") from nvec order by a; +cosine_distance(bf, [1,2,3]) +0.0 +0.025368154048919678 +select cosine_similarity(bf, "[1,2,3]") from nvec order by a; +cosine_similarity(bf, [1,2,3]) +1.0 +0.9746318459510803 +select normalize_l2(bf) from nvec order by a; +normalize_l2(bf) +[0.26757812, 0.53515625, 0.80078125] +[0.45507812, 0.5703125, 0.68359375] +select l2_distance(f16, "[1,2,3]") from nvec order by a; +l2_distance(f16, [1,2,3]) +0.0 +5.196152210235596 +select inner_product(f16, "[1,2,3]") from nvec order by a; +inner_product(f16, [1,2,3]) +-14.0 +-32.0 +select cosine_distance(f16, "[1,2,3]") from nvec order by a; +cosine_distance(f16, [1,2,3]) +0.0 +0.025368154048919678 +select normalize_l2(f16) from nvec order by a; +normalize_l2(f16) +[0.26733398, 0.53466797, 0.8017578] +[0.45581055, 0.5698242, 0.68359375] +select l2_distance(i8, "[1,2,3]") from nvec order by a; +l2_distance(i8, [1,2,3]) +0.0 +5.196152210235596 +select inner_product(i8, "[1,2,3]") from nvec order by a; +inner_product(i8, [1,2,3]) +-14.0 +-32.0 +select cosine_distance(i8, "[1,2,3]") from nvec order by a; +cosine_distance(i8, [1,2,3]) +0.0 +0.025368154048919678 +select l2_distance(bf, cast("[4,5,6]" as vecbf16(3))) from nvec order by a; +l2_distance(bf, cast([4,5,6] as vecbf16(3))) +5.196152210235596 +0.0 +select l2_distance(i8, cast("[4,5,6]" as vecint8(3))) from nvec order by a; +l2_distance(i8, cast([4,5,6] as vecint8(3))) +5.196152210235596 +0.0 +select a FROM nvec ORDER BY l2_distance(bf, '[1,2,3]') LIMIT 5; +a +1 +2 +select a FROM nvec ORDER BY cosine_distance(f16, '[1,2,3]') LIMIT 5; +a +1 +2 +select a FROM nvec ORDER BY inner_product(i8, '[1,2,3]') LIMIT 5; +a +2 +1 +select * from nvec where i8 = "[1,2,3]"; +a bf f16 i8 +1 [1, 2, 3] [1, 2, 3] [1, 2, 3] +select * from nvec order by bf desc; +a bf f16 i8 +2 [4, 5, 6] [4, 5, 6] [4, 5, 6] +1 [1, 2, 3] [1, 2, 3] [1, 2, 3] +select distinct v from i8t order by v; +v +[127, -128, 0, 5] +select bf + bf from nvec; +invalid argument operator +, bad value [VECBF16 VECBF16] +select bf - bf from nvec; +invalid argument operator -, bad value [VECBF16 VECBF16] +select bf * bf from nvec; +invalid argument operator *, bad value [VECBF16 VECBF16] +select sqrt(bf) from nvec; +invalid argument function sqrt, bad value [VECBF16] +select abs(i8) from nvec; +invalid argument function abs, bad value [VECINT8] +select summation(f16) from nvec; +invalid argument function summation, bad value [VECF16] +select cast(bf as vecf32(3)) + cast(bf as vecf32(3)) from nvec order by a; +cast(bf as vecf32(3)) + cast(bf as vecf32(3)) +[2, 4, 6] +[8, 10, 12] +drop database if exists vecnarrowdb; diff --git a/test/distributed/cases/array/array_vecnarrow.sql b/test/distributed/cases/array/array_vecnarrow.sql new file mode 100644 index 0000000000000..e9178b55ad1b2 --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow.sql @@ -0,0 +1,94 @@ +-- vecbf16 / vecf16 / vecint8 narrow vector column types +-- Scope (per design): distance functions + casts + storage only. +-- Elementwise arithmetic (+ - * / sqrt abs summation subvector) is NOT +-- supported on the narrow types and must require an explicit CAST to vecf32. + +-- pre +drop database if exists vecnarrowdb; +create database vecnarrowdb; +use vecnarrowdb; +drop table if exists nvec; + +-- standard: one column of each new type +create table nvec(a int, bf vecbf16(3), f16 vecf16(3), i8 vecint8(3)); +desc nvec; +show create table nvec; +insert into nvec values(1, "[1,2,3]", "[1,2,3]", "[1,2,3]"); +insert into nvec values(2, "[4,5,6]", "[4,5,6]", "[4,5,6]"); +select * from nvec; + +-- int8: a string literal must be an integer in [-128,127]; boundary values OK. +drop table if exists i8t; +create table i8t(a int, v vecint8(4)); +insert into i8t values(1, "[127,-128,0,5]"); +select * from i8t; +-- out-of-range and non-integer string literals error (no silent round/clamp) +insert into i8t values(2, "[200,-200,0,0]"); +insert into i8t values(3, "[1.4,2.6,0,0]"); +-- rounding/clamping IS available, but only via the vecf32 -> vecint8 CAST path +select cast(cast("[1.6,200,-3.5,-200]" as vecf32(4)) as vecint8(4)); + +-- string -> narrow casts +select cast("[1,2,3]" as vecbf16(3)); +select cast("[1,2,3]" as vecf16(3)); +select cast("[1,2,3]" as vecint8(3)); +-- non-integer string literal -> vecint8 errors (strict) +select cast("[1.4,2.6,-3.5]" as vecint8(3)); + +-- narrow -> vecf32 / vecf64 casts (the explicit widening path) +select cast(bf as vecf32(3)), cast(f16 as vecf32(3)), cast(i8 as vecf32(3)) from nvec order by a; +select cast(bf as vecf64(3)) from nvec order by a; + +-- vecf32 -> narrow casts +select cast(cast("[1,2,3]" as vecf32(3)) as vecbf16(3)); +select cast(cast("[1,2,3]" as vecf32(3)) as vecf16(3)); +select cast(cast("[1.6,2.4,-3.5]" as vecf32(3)) as vecint8(3)); + +-- narrow -> narrow casts +select cast(bf as vecf16(3)), cast(f16 as vecint8(3)), cast(i8 as vecbf16(3)) from nvec order by a; + +-- distance functions on bf16 +select l2_distance(bf, "[1,2,3]") from nvec order by a; +select l2_distance_sq(bf, "[1,2,3]") from nvec order by a; +select inner_product(bf, "[1,2,3]") from nvec order by a; +select cosine_distance(bf, "[1,2,3]") from nvec order by a; +select cosine_similarity(bf, "[1,2,3]") from nvec order by a; +select normalize_l2(bf) from nvec order by a; + +-- distance functions on f16 +select l2_distance(f16, "[1,2,3]") from nvec order by a; +select inner_product(f16, "[1,2,3]") from nvec order by a; +select cosine_distance(f16, "[1,2,3]") from nvec order by a; +select normalize_l2(f16) from nvec order by a; + +-- distance functions on int8 +select l2_distance(i8, "[1,2,3]") from nvec order by a; +select inner_product(i8, "[1,2,3]") from nvec order by a; +select cosine_distance(i8, "[1,2,3]") from nvec order by a; + +-- distance between two narrow columns of the same type +select l2_distance(bf, cast("[4,5,6]" as vecbf16(3))) from nvec order by a; +select l2_distance(i8, cast("[4,5,6]" as vecint8(3))) from nvec order by a; + +-- top-K (ORDER BY distance + LIMIT) +select a FROM nvec ORDER BY l2_distance(bf, '[1,2,3]') LIMIT 5; +select a FROM nvec ORDER BY cosine_distance(f16, '[1,2,3]') LIMIT 5; +select a FROM nvec ORDER BY inner_product(i8, '[1,2,3]') LIMIT 5; + +-- filtering / equality / ordering +select * from nvec where i8 = "[1,2,3]"; +select * from nvec order by bf desc; +select distinct v from i8t order by v; + +-- negative: arithmetic is not allowed on narrow types (must CAST to vecf32 first) +select bf + bf from nvec; +select bf - bf from nvec; +select bf * bf from nvec; +select sqrt(bf) from nvec; +select abs(i8) from nvec; +select summation(f16) from nvec; +-- arithmetic IS allowed after an explicit cast to vecf32 +select cast(bf as vecf32(3)) + cast(bf as vecf32(3)) from nvec order by a; + +-- post +drop database if exists vecnarrowdb; diff --git a/test/distributed/cases/array/array_vecnarrow_dims.result b/test/distributed/cases/array/array_vecnarrow_dims.result new file mode 100644 index 0000000000000..2267a5fab3b64 --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow_dims.result @@ -0,0 +1,17 @@ +drop database if exists nvdims; +create database nvdims; +use nvdims; +create table t(a int, bf vecbf16(3), hf vecf16(5), i8 vecint8(4), u8 vecuint8(2)); +insert into t values(1, '[1,2,3]', '[1,2,3,4,5]', '[1,2,3,4]', '[1,2]'); +insert into t values(2, '[4,5,6]', '[6,7,8,9,10]', '[5,6,7,8]', '[9,8]'); +select a, vector_dims(bf) as bf, vector_dims(hf) as hf, vector_dims(i8) as i8, vector_dims(u8) as u8 from t order by a; +a bf hf i8 u8 +1 3 5 4 2 +2 3 5 4 2 +insert into t values(3, null, null, null, null); +select a, vector_dims(bf) as bf, vector_dims(u8) as u8 from t order by a; +a bf u8 +1 3 2 +2 3 2 +3 null null +drop database nvdims; diff --git a/test/distributed/cases/array/array_vecnarrow_dims.sql b/test/distributed/cases/array/array_vecnarrow_dims.sql new file mode 100644 index 0000000000000..e4937f8370a1d --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow_dims.sql @@ -0,0 +1,15 @@ +-- vector_dims on the narrow vector types (vecbf16/vecf16/vecint8/vecuint8). +-- Regression: vector_dims previously only had float32/float64 overloads and +-- errored on narrow types ("invalid argument function vector_dims, bad value +-- [VECINT8]"). It now returns the element count (= content bytes / sizeof(elem)), +-- like vecf32; a NULL vector yields NULL dims, matching vecf32 behavior. +drop database if exists nvdims; +create database nvdims; +use nvdims; +create table t(a int, bf vecbf16(3), hf vecf16(5), i8 vecint8(4), u8 vecuint8(2)); +insert into t values(1, '[1,2,3]', '[1,2,3,4,5]', '[1,2,3,4]', '[1,2]'); +insert into t values(2, '[4,5,6]', '[6,7,8,9,10]', '[5,6,7,8]', '[9,8]'); +select a, vector_dims(bf) as bf, vector_dims(hf) as hf, vector_dims(i8) as i8, vector_dims(u8) as u8 from t order by a; +insert into t values(3, null, null, null, null); +select a, vector_dims(bf) as bf, vector_dims(u8) as u8 from t order by a; +drop database nvdims; diff --git a/test/distributed/cases/array/array_vecnarrow_ops.result b/test/distributed/cases/array/array_vecnarrow_ops.result new file mode 100644 index 0000000000000..395ac6bb9ca27 --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow_ops.result @@ -0,0 +1,98 @@ +drop database if exists nvops; +create database nvops; +use nvops; +create table b(a int, v vecbf16(3)); +insert into b values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'),(4,'[7,8,9]'); +select a, v = cast('[1,2,3]' as vecbf16(3)) as eq, v != cast('[1,2,3]' as vecbf16(3)) as ne, v < cast('[4,5,6]' as vecbf16(3)) as lt, v > cast('[1,2,3]' as vecbf16(3)) as gt, v <= cast('[1,2,3]' as vecbf16(3)) as le, v >= cast('[4,5,6]' as vecbf16(3)) as ge from b order by a; +a eq ne lt gt le ge +1 1 0 1 0 1 0 +2 0 1 0 1 0 1 +3 1 0 1 0 1 0 +4 0 1 0 1 0 1 +select v from b order by v, a; +v +[1, 2, 3] +[1, 2, 3] +[4, 5, 6] +[7, 8, 9] +select distinct v from b order by v; +v +[1, 2, 3] +[4, 5, 6] +[7, 8, 9] +select v, count(*) as c from b group by v order by v; +v c +[1, 2, 3] 2 +[4, 5, 6] 1 +[7, 8, 9] 1 +select a from b where v = cast('[1,2,3]' as vecbf16(3)) order by a; +a +1 +3 +select a from b where v < cast('[4,5,6]' as vecbf16(3)) order by a; +a +1 +3 +select a, round(l2_distance(v, cast('[1,2,3]' as vecbf16(3))),4) as l2, round(l2_distance_sq(v, cast('[1,2,3]' as vecbf16(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecbf16(3))),4) as ip, round(cosine_distance(v, cast('[1,2,3]' as vecbf16(3))),4) as cd, round(cosine_similarity(v, cast('[1,2,3]' as vecbf16(3))),4) as cs from b order by a; +a l2 l2sq ip cd cs +1 0.0 0.0 -14.0 0.0 1.0 +2 5.1962 27.0 -32.0 0.0254 0.9746 +3 0.0 0.0 -14.0 0.0 1.0 +4 10.3923 108.0 -50.0 0.0406 0.9594 +select v + v from b; +invalid argument operator +, bad value [VECBF16 VECBF16] +select v - v from b; +invalid argument operator -, bad value [VECBF16 VECBF16] +select v * 2 from b; +invalid argument operator *, bad value [VECBF16 BIGINT] +select a, cast(v as vecf32(3)) + cast('[1,1,1]' as vecf32(3)) as r from b order by a; +a r +1 [2, 3, 4] +2 [5, 6, 7] +3 [2, 3, 4] +4 [8, 9, 10] +create table f(a int, v vecf16(3)); +insert into f values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'); +select a, v = cast('[1,2,3]' as vecf16(3)) as eq, v < cast('[4,5,6]' as vecf16(3)) as lt from f order by a; +a eq lt +1 1 1 +2 0 0 +3 1 1 +select distinct v from f order by v; +v +[1, 2, 3] +[4, 5, 6] +select v, count(*) as c from f group by v order by v; +v c +[1, 2, 3] 2 +[4, 5, 6] 1 +select a, round(l2_distance_sq(v, cast('[1,2,3]' as vecf16(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecf16(3))),4) as ip from f order by a; +a l2sq ip +1 0.0 -14.0 +2 27.0 -32.0 +3 0.0 -14.0 +select v * 2 from f; +invalid argument operator *, bad value [VECF16 BIGINT] +create table i(a int, v vecint8(3)); +insert into i values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'); +select a, v = cast('[1,2,3]' as vecint8(3)) as eq, v < cast('[4,5,6]' as vecint8(3)) as lt from i order by a; +a eq lt +1 1 1 +2 0 0 +3 1 1 +select distinct v from i order by v; +v +[1, 2, 3] +[4, 5, 6] +select v, count(*) as c from i group by v order by v; +v c +[1, 2, 3] 2 +[4, 5, 6] 1 +select a, round(l2_distance_sq(v, cast('[1,2,3]' as vecint8(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecint8(3))),4) as ip from i order by a; +a l2sq ip +1 0.0 -14.0 +2 27.0 -32.0 +3 0.0 -14.0 +select v + v from i; +invalid argument operator +, bad value [VECINT8 VECINT8] +drop database nvops; diff --git a/test/distributed/cases/array/array_vecnarrow_ops.sql b/test/distributed/cases/array/array_vecnarrow_ops.sql new file mode 100644 index 0000000000000..fb726482e536e --- /dev/null +++ b/test/distributed/cases/array/array_vecnarrow_ops.sql @@ -0,0 +1,48 @@ +-- narrow vector types (vecbf16/vecf16/vecint8): comparison operators, ordering/ +-- grouping/aggregates, distance functions, and the arithmetic rule (elementwise +-- arithmetic errors -- must CAST to vecf32 first). Small integers are exact in all +-- three narrow formats, so results are deterministic. +drop database if exists nvops; +create database nvops; +use nvops; + +-- ===== vecbf16 ===== +create table b(a int, v vecbf16(3)); +insert into b values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'),(4,'[7,8,9]'); +-- comparison operators +select a, v = cast('[1,2,3]' as vecbf16(3)) as eq, v != cast('[1,2,3]' as vecbf16(3)) as ne, v < cast('[4,5,6]' as vecbf16(3)) as lt, v > cast('[1,2,3]' as vecbf16(3)) as gt, v <= cast('[1,2,3]' as vecbf16(3)) as le, v >= cast('[4,5,6]' as vecbf16(3)) as ge from b order by a; +-- ordering / distinct / group by / aggregates (all use comparison) +select v from b order by v, a; +select distinct v from b order by v; +select v, count(*) as c from b group by v order by v; +-- where filter by comparison +select a from b where v = cast('[1,2,3]' as vecbf16(3)) order by a; +select a from b where v < cast('[4,5,6]' as vecbf16(3)) order by a; +-- distance functions +select a, round(l2_distance(v, cast('[1,2,3]' as vecbf16(3))),4) as l2, round(l2_distance_sq(v, cast('[1,2,3]' as vecbf16(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecbf16(3))),4) as ip, round(cosine_distance(v, cast('[1,2,3]' as vecbf16(3))),4) as cd, round(cosine_similarity(v, cast('[1,2,3]' as vecbf16(3))),4) as cs from b order by a; +-- arithmetic is NOT supported on narrow types directly +select v + v from b; +select v - v from b; +select v * 2 from b; +-- ... but works after an explicit CAST to vecf32 +select a, cast(v as vecf32(3)) + cast('[1,1,1]' as vecf32(3)) as r from b order by a; + +-- ===== vecf16 ===== +create table f(a int, v vecf16(3)); +insert into f values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'); +select a, v = cast('[1,2,3]' as vecf16(3)) as eq, v < cast('[4,5,6]' as vecf16(3)) as lt from f order by a; +select distinct v from f order by v; +select v, count(*) as c from f group by v order by v; +select a, round(l2_distance_sq(v, cast('[1,2,3]' as vecf16(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecf16(3))),4) as ip from f order by a; +select v * 2 from f; + +-- ===== vecint8 ===== +create table i(a int, v vecint8(3)); +insert into i values (1,'[1,2,3]'),(2,'[4,5,6]'),(3,'[1,2,3]'); +select a, v = cast('[1,2,3]' as vecint8(3)) as eq, v < cast('[4,5,6]' as vecint8(3)) as lt from i order by a; +select distinct v from i order by v; +select v, count(*) as c from i group by v order by v; +select a, round(l2_distance_sq(v, cast('[1,2,3]' as vecint8(3))),4) as l2sq, round(inner_product(v, cast('[1,2,3]' as vecint8(3))),4) as ip from i order by a; +select v + v from i; + +drop database nvops; diff --git a/test/distributed/cases/array/array_vecuint8.result b/test/distributed/cases/array/array_vecuint8.result new file mode 100644 index 0000000000000..bc207bbbe3aba --- /dev/null +++ b/test/distributed/cases/array/array_vecuint8.result @@ -0,0 +1,125 @@ +drop database if exists vecu8db; +create database vecu8db; +use vecu8db; +create table u8t(a int, v vecuint8(4)); +desc u8t; +Field Type Null Key Default Extra Comment +a INT(32) YES null +v VECUINT8(4) YES null +show create table u8t; +Table Create Table +u8t CREATE TABLE `u8t` (\n `a` int DEFAULT NULL,\n `v` vecuint8(4) DEFAULT NULL\n) +insert into u8t values(1, "[0,1,2,3]"); +insert into u8t values(2, "[255,254,0,128]"); +insert into u8t values(3, "[10,20,30,40]"); +select * from u8t order by a; +a v +1 [0, 1, 2, 3] +2 [255, 254, 0, 128] +3 [10, 20, 30, 40] +insert into u8t values(4, "[300,0,0,0]"); +internal error: error while casting 300 to VECUINT8 +insert into u8t values(5, "[-1,0,0,0]"); +internal error: error while casting -1 to VECUINT8 +insert into u8t values(6, "[1.4,0,0,0]"); +internal error: error while casting 1.4 to VECUINT8 +select cast(cast("[1.6,300,-5,200]" as vecf32(4)) as vecuint8(4)); +cast(cast([1.6,300,-5,200] as vecf32(4)) as vecuint8(4)) +[2, 255, 0, 200] +select cast("[1,2,3]" as vecuint8(3)); +cast([1,2,3] as vecuint8(3)) +[1, 2, 3] +select cast("[1.4,2.6,-3.5]" as vecuint8(3)); +internal error: error while casting 1.4 to VECUINT8 +select cast(v as vecf32(4)), cast(v as vecf64(4)) from u8t order by a; +cast(v as vecf32(4)) cast(v as vecf64(4)) +[0, 1, 2, 3] [0, 1, 2, 3] +[255, 254, 0, 128] [255, 254, 0, 128] +[10, 20, 30, 40] [10, 20, 30, 40] +select cast(cast("[1,2,3]" as vecf32(3)) as vecuint8(3)); +cast(cast([1,2,3] as vecf32(3)) as vecuint8(3)) +[1, 2, 3] +select cast(cast("[1,2,3]" as vecuint8(3)) as vecint8(3)); +cast(cast([1,2,3] as vecuint8(3)) as vecint8(3)) +[1, 2, 3] +select cast(cast("[1,2,3]" as vecuint8(3)) as vecbf16(3)); +cast(cast([1,2,3] as vecuint8(3)) as vecbf16(3)) +[1, 2, 3] +select a, round(l2_distance(v, "[0,1,2,3]"), 4) from u8t order by a; +a round(l2_distance(v, [0,1,2,3]), 4) +1 0.0 +2 380.3459 +3 51.1273 +select a, l2_distance_sq(v, "[0,1,2,3]") from u8t order by a; +a l2_distance_sq(v, [0,1,2,3]) +1 0.0 +2 144663.0 +3 2614.0 +select a, inner_product(v, "[1,1,1,1]") from u8t order by a; +a inner_product(v, [1,1,1,1]) +1 -6.0 +2 -637.0 +3 -100.0 +select a, round(cosine_distance(v, "[0,1,2,3]"), 4) from u8t order by a; +a round(cosine_distance(v, [0,1,2,3]), 4) +1 0.0 +2 0.5536 +3 0.0241 +select a, round(cosine_similarity(v, "[0,1,2,3]"), 4) from u8t order by a; +a round(cosine_similarity(v, [0,1,2,3]), 4) +1 1.0 +2 0.4464 +3 0.9759 +select normalize_l2(v) from u8t order by a; +normalize_l2(v) +[0, 0, 1, 1] +[1, 1, 0, 0] +[0, 0, 1, 1] +select a, round(l2_distance(v, cast("[10,20,30,40]" as vecuint8(4))), 4) from u8t order by a; +a round(l2_distance(v, cast([10,20,30,40] as vecuint8(4))), 4) +1 51.1273 +2 351.3189 +3 0.0 +select a from u8t order by l2_distance(v, '[0,1,2,3]') limit 3; +a +1 +3 +2 +select a from u8t order by inner_product(v, '[1,1,1,1]') limit 3; +a +2 +3 +1 +select a from u8t where v = "[10,20,30,40]"; +a +3 +select * from u8t order by v desc; +a v +2 [255, 254, 0, 128] +3 [10, 20, 30, 40] +1 [0, 1, 2, 3] +select distinct v from u8t order by v; +v +[0, 1, 2, 3] +[10, 20, 30, 40] +[255, 254, 0, 128] +select v + v from u8t; +invalid argument operator +, bad value [VECUINT8 VECUINT8] +select v * v from u8t; +invalid argument operator *, bad value [VECUINT8 VECUINT8] +select abs(v) from u8t; +invalid argument function abs, bad value [VECUINT8] +select summation(v) from u8t; +invalid argument function summation, bad value [VECUINT8] +select cast(v as vecf32(4)) + cast(v as vecf32(4)) from u8t order by a; +cast(v as vecf32(4)) + cast(v as vecf32(4)) +[0, 2, 4, 6] +[510, 508, 0, 256] +[20, 40, 60, 80] +select vecuint8_from_base64('ChQeKA=='); +vecuint8_from_base64(ChQeKA==) +[10, 20, 30, 40] +select vecuint8_from_base64('AP+AAQ=='); +vecuint8_from_base64(AP+AAQ==) +[0, 255, 128, 1] +drop database if exists vecu8db; diff --git a/test/distributed/cases/array/array_vecuint8.sql b/test/distributed/cases/array/array_vecuint8.sql new file mode 100644 index 0000000000000..3caba84a11999 --- /dev/null +++ b/test/distributed/cases/array/array_vecuint8.sql @@ -0,0 +1,78 @@ +-- vecuint8 narrow vector column type (unsigned 8-bit, [0,255]). +-- Mirrors array_vecnarrow for the int8 sibling; scope (per design): distance +-- functions + casts + storage only. Elementwise arithmetic is NOT supported and +-- must go through an explicit CAST to vecf32. + +drop database if exists vecu8db; +create database vecu8db; +use vecu8db; + +-- column type: create / desc / show create / insert / select +create table u8t(a int, v vecuint8(4)); +desc u8t; +show create table u8t; +insert into u8t values(1, "[0,1,2,3]"); +insert into u8t values(2, "[255,254,0,128]"); +insert into u8t values(3, "[10,20,30,40]"); +select * from u8t order by a; + +-- strict string parse: an integer in [0,255]; boundary values OK. +-- out-of-range and non-integer literals error (no silent round/clamp). +insert into u8t values(4, "[300,0,0,0]"); +insert into u8t values(5, "[-1,0,0,0]"); +insert into u8t values(6, "[1.4,0,0,0]"); + +-- rounding/clamping IS available, but only via the vecf32 -> vecuint8 CAST path +select cast(cast("[1.6,300,-5,200]" as vecf32(4)) as vecuint8(4)); + +-- string -> vecuint8 cast (strict) +select cast("[1,2,3]" as vecuint8(3)); +select cast("[1.4,2.6,-3.5]" as vecuint8(3)); + +-- vecuint8 -> vecf32 / vecf64 (explicit widening) +select cast(v as vecf32(4)), cast(v as vecf64(4)) from u8t order by a; + +-- vecuint8 <-> other narrow casts +select cast(cast("[1,2,3]" as vecf32(3)) as vecuint8(3)); +select cast(cast("[1,2,3]" as vecuint8(3)) as vecint8(3)); +select cast(cast("[1,2,3]" as vecuint8(3)) as vecbf16(3)); + +-- distance functions on vecuint8 +-- l2_distance / cosine_* involve sqrt/division and are computed in float64; +-- round to 4 digits so the low-order bits don't diverge across CPU SIMD kernels. +-- l2_distance_sq / inner_product are integer-exact for uint8 (no rounding). +select a, round(l2_distance(v, "[0,1,2,3]"), 4) from u8t order by a; +select a, l2_distance_sq(v, "[0,1,2,3]") from u8t order by a; +select a, inner_product(v, "[1,1,1,1]") from u8t order by a; +select a, round(cosine_distance(v, "[0,1,2,3]"), 4) from u8t order by a; +select a, round(cosine_similarity(v, "[0,1,2,3]"), 4) from u8t order by a; +select normalize_l2(v) from u8t order by a; + +-- distance between two vecuint8 values +select a, round(l2_distance(v, cast("[10,20,30,40]" as vecuint8(4))), 4) from u8t order by a; + +-- top-K (ORDER BY distance + LIMIT) +select a from u8t order by l2_distance(v, '[0,1,2,3]') limit 3; +select a from u8t order by inner_product(v, '[1,1,1,1]') limit 3; + +-- filtering / equality / ordering / distinct +select a from u8t where v = "[10,20,30,40]"; +select * from u8t order by v desc; +select distinct v from u8t order by v; + +-- negative: arithmetic is not allowed on vecuint8 (must CAST to vecf32 first) +select v + v from u8t; +select v * v from u8t; +select abs(v) from u8t; +select summation(v) from u8t; +-- arithmetic IS allowed after an explicit cast to vecf32 +select cast(v as vecf32(4)) + cast(v as vecf32(4)) from u8t order by a; + +-- vecuint8_from_base64: decode raw little-endian uint8 bytes. This is the builtin +-- the ivf search re-rank emits for the (quantized) query vector. A constant +-- argument constant-folds, so this is a direct regression for the elemSize +-- divide-by-zero panic that hid when the uint8 case was missing from the decoder. +select vecuint8_from_base64('ChQeKA=='); +select vecuint8_from_base64('AP+AAQ=='); + +drop database if exists vecu8db; diff --git a/test/distributed/cases/load_data/load_data_narrow_vec.result b/test/distributed/cases/load_data/load_data_narrow_vec.result new file mode 100644 index 0000000000000..8a2c5e550399a --- /dev/null +++ b/test/distributed/cases/load_data/load_data_narrow_vec.result @@ -0,0 +1,32 @@ +drop database if exists load_narrow_vec; +create database load_narrow_vec; +use load_narrow_vec; +create table nvec(id int, a vecbf16(3), b vecf16(3), c vecint8(3), d vecuint8(3)); +load data infile '$resources/load_data/narrow_vec_array.csv' into table nvec fields terminated by ',' ignore 1 lines; +select * from nvec order by id; +id a b c d +1 [1, 2, 3] [0.5, 0.25, -0.5] [-128, 0, 127] [0, 128, 255] +2 [0.5, -0.25, 4] [1, 2, 3] [10, -10, 5] [1, 2, 3] +select id, round(l2_distance(c, '[0,0,0]'), 4) as dist from nvec order by id; +id dist +1 180.3136 +2 15.0 +create table nvec_oor(id int, c vecint8(3)); +load data infile '$resources/load_data/narrow_vec_int8_oor.csv' into table nvec_oor fields terminated by ',' ignore 1 lines; +internal error: error while casting 200 to VECINT8 +select count(*) as cnt from nvec_oor; +cnt +0 +create table nvec_frac(id int, c vecint8(3)); +load data infile '$resources/load_data/narrow_vec_int8_frac.csv' into table nvec_frac fields terminated by ',' ignore 1 lines; +internal error: error while casting 0.5 to VECINT8 +select count(*) as cnt from nvec_frac; +cnt +0 +create table nvec_dim(id int, d vecuint8(3)); +load data infile '$resources/load_data/narrow_vec_dim_bad.csv' into table nvec_dim fields terminated by ',' ignore 1 lines; +invalid input: expected vector dimension 3 != actual dimension 2. +select count(*) as cnt from nvec_dim; +cnt +0 +drop database load_narrow_vec; diff --git a/test/distributed/cases/load_data/load_data_narrow_vec.sql b/test/distributed/cases/load_data/load_data_narrow_vec.sql new file mode 100644 index 0000000000000..ac387d71e1f4c --- /dev/null +++ b/test/distributed/cases/load_data/load_data_narrow_vec.sql @@ -0,0 +1,51 @@ +-- Test: LOAD DATA INFILE into narrow vector base columns +-- (vecbf16 / vecf16 / vecint8 / vecuint8). +-- +-- Before the fix, the external/CSV import switches in external.go only handled +-- T_array_float32/float64, so loading a narrow vector column failed with +-- "the value type N is not support now". INSERT already worked; only the bulk +-- LOAD path was missing the narrow cases. +-- +-- int8/uint8 string parse is strict (integer, in range); fractional or +-- out-of-range values are rejected — mirroring INSERT. + +drop database if exists load_narrow_vec; +create database load_narrow_vec; +use load_narrow_vec; + +-- ============================================================ +-- 1. Happy path: load all four narrow types from one CSV. +-- Values are exactly representable in bf16/f16 (and integer +-- for int8/uint8), so the round-trip is loss-free. +-- ============================================================ +create table nvec(id int, a vecbf16(3), b vecf16(3), c vecint8(3), d vecuint8(3)); +load data infile '$resources/load_data/narrow_vec_array.csv' into table nvec fields terminated by ',' ignore 1 lines; +select * from nvec order by id; + +-- distance functions work on a loaded narrow column +-- round to 4 digits: l2_distance (sqrt, float64) low-order bits vary across SIMD kernels +select id, round(l2_distance(c, '[0,0,0]'), 4) as dist from nvec order by id; + +-- ============================================================ +-- 2. Strict int8 parse: out-of-range value (200) is rejected. +-- ============================================================ +create table nvec_oor(id int, c vecint8(3)); +load data infile '$resources/load_data/narrow_vec_int8_oor.csv' into table nvec_oor fields terminated by ',' ignore 1 lines; +select count(*) as cnt from nvec_oor; + +-- ============================================================ +-- 3. Strict int8 parse: fractional value (0.5) is rejected. +-- ============================================================ +create table nvec_frac(id int, c vecint8(3)); +load data infile '$resources/load_data/narrow_vec_int8_frac.csv' into table nvec_frac fields terminated by ',' ignore 1 lines; +select count(*) as cnt from nvec_frac; + +-- ============================================================ +-- 4. Dimension mismatch is rejected (vecuint8(3) given 2 elems). +-- ============================================================ +create table nvec_dim(id int, d vecuint8(3)); +load data infile '$resources/load_data/narrow_vec_dim_bad.csv' into table nvec_dim fields terminated by ',' ignore 1 lines; +select count(*) as cnt from nvec_dim; + +-- cleanup +drop database load_narrow_vec; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.result b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.result new file mode 100644 index 0000000000000..847dbab3501ff --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.result @@ -0,0 +1,134 @@ +SET probe_limit=10; +create table nbf(a int primary key, v vecbf16(4)); +insert into nbf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xbf using ivfflat on nbf(v) lists=2 op_type 'vector_l2_ops' ASYNC; +create table nhf(a int primary key, v vecf16(4)); +insert into nhf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xhf using ivfflat on nhf(v) lists=2 op_type 'vector_l2_ops' ASYNC; +create table ni8(a int primary key, v vecint8(4)); +insert into ni8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xi8 using ivfflat on ni8(v) lists=2 op_type 'vector_l2_ops' ASYNC; +create table nu8(a int primary key, v vecuint8(4)); +insert into nu8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xu8 using ivfflat on nu8(v) lists=2 op_type 'vector_l2_ops' ASYNC; +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +2 𝄀 +3 +select a from nbf order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +5 𝄀 +4 +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +2 𝄀 +3 +select a from nhf order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +5 𝄀 +4 +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +2 𝄀 +3 +select a from ni8 order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +5 𝄀 +4 +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +2 𝄀 +3 +select a from nu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +5 𝄀 +4 +insert into nbf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into nhf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into ni8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into nu8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +7 𝄀 +2 +select a from nbf order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +8 𝄀 +5 +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +7 𝄀 +2 +select a from nhf order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +8 𝄀 +5 +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +7 𝄀 +2 +select a from ni8 order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +8 𝄀 +5 +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +1 𝄀 +7 𝄀 +2 +select a from nu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +➤ a[4,32,0] 𝄀 +6 𝄀 +8 𝄀 +5 +update nbf set v = '[55,55,55,55]' where a = 1; +update nhf set v = '[55,55,55,55]' where a = 1; +update ni8 set v = '[55,55,55,55]' where a = 1; +update nu8 set v = '[55,55,55,55]' where a = 1; +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +7 𝄀 +2 𝄀 +3 +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +7 𝄀 +2 𝄀 +3 +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +7 𝄀 +2 𝄀 +3 +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +➤ a[4,32,0] 𝄀 +7 𝄀 +2 𝄀 +3 +drop table nbf; +drop table nhf; +drop table ni8; +drop table nu8; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.sql b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.sql new file mode 100644 index 0000000000000..30e53262729e8 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_narrow_base_async.sql @@ -0,0 +1,85 @@ +-- ivfflat NATIVE narrow base columns (vecbf16 / vecf16 / vecint8 / vecuint8) over the +-- ASYNC (ISCP/CDC) maintenance path. The synchronous CREATE INDEX on a narrow base +-- already works (vector_ivf_quantization.sql); this proves the ONGOING CDC delta path +-- carries a narrow base column too. +-- +-- The fix this guards: the ISCP row pipeline (pkg/iscp/util.go) extracts a narrow base +-- column to its native Go slice ([]types.Float16 / []types.BF16 / []int8 / []uint8) and +-- serializes it back into the IvfflatSqlWriter VALUES tuple as CAST('[...]' as vecXXX(n)). +-- Before the fix only vecf32/vecf64 had extract + serialize cases, so any DML on a +-- narrow-base ivfflat index errored ("extractRowFromVector: unsupported type") and the +-- CDC consumer could never apply the delta. +-- +-- Each index is built and maintained entirely by the CDC consumer (first iteration runs +-- the InitSQL build, later inserts/updates ride the delta path). Shared sleep(30) windows +-- let the 10s-tick consumer settle for all four indexes at once. +-- +-- Two well-separated clusters [1..5]/[50..54] and cluster-center query points keep every +-- top-k distance distinct (no ties) so the result is deterministic for every narrow type; +-- integer values 1..55 are exact in bf16/f16 and in int8/uint8 range, so the narrow base +-- stores them without ambiguity. Queries are `ORDER BY l2_distance LIMIT k` with no +-- secondary sort key so the ivfflat index pushdown (ivf_search) actually fires. +SET probe_limit=10; + +create table nbf(a int primary key, v vecbf16(4)); +insert into nbf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xbf using ivfflat on nbf(v) lists=2 op_type 'vector_l2_ops' ASYNC; + +create table nhf(a int primary key, v vecf16(4)); +insert into nhf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xhf using ivfflat on nhf(v) lists=2 op_type 'vector_l2_ops' ASYNC; + +create table ni8(a int primary key, v vecint8(4)); +insert into ni8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xi8 using ivfflat on ni8(v) lists=2 op_type 'vector_l2_ops' ASYNC; + +create table nu8(a int primary key, v vecuint8(4)); +insert into nu8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xu8 using ivfflat on nu8(v) lists=2 op_type 'vector_l2_ops' ASYNC; + +-- 1) initial async build (CDC reindex InitSQL). Low cluster -> 1,2,3 ; high -> 6,5,4. +select sleep(30); +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nbf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nhf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from ni8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nu8 order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 2) incremental rows ride the CDC delta path through the narrow-base extract + +-- serialize. Row 7=[2,2,2,2] joins the low cluster (now 1,7,2) and row 8=[53,53,53,53] +-- the high cluster (now 6,8,5) -- their appearance proves the delta path indexed them. +insert into nbf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into nhf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into ni8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into nu8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +select sleep(30); +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nbf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nhf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from ni8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nu8 order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 3) update row 1 across to the other cluster ([1,1,1,1] -> [55,55,55,55]); the delta path +-- must re-extract + re-bucket the narrow vector. Query the LOW cluster center [1,1,1,1]: +-- row 1 has LEFT it, so the top-3 is now 7,2,3 (row 1 absent) -- proving the delta UPDATE +-- moved it. +update nbf set v = '[55,55,55,55]' where a = 1; +update nhf set v = '[55,55,55,55]' where a = 1; +update ni8 set v = '[55,55,55,55]' where a = 1; +update nu8 set v = '[55,55,55,55]' where a = 1; +select sleep(30); +select a from nbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nhf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from ni8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from nu8 order by l2_distance(v,'[1,1,1,1]') limit 3; + +drop table nbf; +drop table nhf; +drop table ni8; +drop table nu8; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.result b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.result new file mode 100644 index 0000000000000..6484c6f334bae --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.result @@ -0,0 +1,134 @@ +SET probe_limit=10; +create table qi8(a int primary key, v vecf32(4)); +insert into qi8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xi8 using ivfflat on qi8(v) lists=2 op_type 'vector_l2_ops' quantization 'int8' ASYNC; +create table qu8(a int primary key, v vecf32(4)); +insert into qu8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xu8 using ivfflat on qu8(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8' ASYNC; +create table qbf(a int primary key, v vecf32(4)); +insert into qbf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xbf using ivfflat on qbf(v) lists=2 op_type 'vector_l2_ops' quantization 'bf16' ASYNC; +create table qf(a int primary key, v vecf32(4)); +insert into qf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xf using ivfflat on qf(v) lists=2 op_type 'vector_l2_ops' quantization 'float16' ASYNC; +select sleep(30); +sleep(30) +0 +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qi8 order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qbf order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qf order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +insert into qi8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qu8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qbf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +select sleep(30); +sleep(30) +0 +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +7 +2 +select a from qi8 order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +8 +5 +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +7 +2 +select a from qu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +8 +5 +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +7 +2 +select a from qbf order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +8 +5 +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +7 +2 +select a from qf order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +8 +5 +update qi8 set v = '[55,55,55,55]' where a = 1; +update qu8 set v = '[55,55,55,55]' where a = 1; +update qbf set v = '[55,55,55,55]' where a = 1; +update qf set v = '[55,55,55,55]' where a = 1; +select sleep(30); +sleep(30) +0 +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +7 +2 +3 +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +7 +2 +3 +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +7 +2 +3 +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +a +7 +2 +3 +drop table qi8; +drop table qu8; +drop table qbf; +drop table qf; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.sql b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.sql new file mode 100644 index 0000000000000..18ab53bd826f6 --- /dev/null +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_ivf_quant_async.sql @@ -0,0 +1,81 @@ +-- ivfflat QUANTIZATION over the ASYNC (ISCP/CDC) maintenance path, for all four +-- narrow entry types: +-- * int8 / uint8 — the scaled quantizer: the CDC delta path (toIvfflatUpsert) +-- must re-apply the trained [min,max] q(x)=x*mul+add (int8 -> +-- [-128,127], uint8 -> [0,255]), not an identity cast; +-- * bf16 / float16 — lossless narrowing cast on the entry projection. +-- Every async index is built and maintained entirely by the CDC consumer (the +-- first iteration runs ALTER ... REINDEX ... FORCE_SYNC, later inserts/updates +-- ride the delta path). Three shared sleep(30) windows let the 10s-tick consumer +-- settle for all indexes at once. +-- +-- The queries are `ORDER BY l2_distance(v, q) LIMIT k` with NO secondary sort key, +-- so the ivfflat index pushdown fires (the ivf_search table function), actually +-- exercising the quantized re-rank. Two well-separated clusters [1..5]/[50..54] +-- and cluster-center query points keep every top-k distance distinct (no ties), +-- so the result is deterministic without a tiebreaker. +SET probe_limit=10; + +create table qi8(a int primary key, v vecf32(4)); +insert into qi8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xi8 using ivfflat on qi8(v) lists=2 op_type 'vector_l2_ops' quantization 'int8' ASYNC; + +create table qu8(a int primary key, v vecf32(4)); +insert into qu8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xu8 using ivfflat on qu8(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8' ASYNC; + +create table qbf(a int primary key, v vecf32(4)); +insert into qbf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xbf using ivfflat on qbf(v) lists=2 op_type 'vector_l2_ops' quantization 'bf16' ASYNC; + +create table qf(a int primary key, v vecf32(4)); +insert into qf values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index xf using ivfflat on qf(v) lists=2 op_type 'vector_l2_ops' quantization 'float16' ASYNC; + +-- 1) initial async build (CDC reindex InitSQL). Low cluster -> 1,2,3 ; high -> 6,5,4. +select sleep(30); +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qi8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qbf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qf order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 2) incremental rows ride the CDC delta path (toIvfflatUpsert). Row 7=[2,2,2,2] +-- joins the low cluster (now 1,7,2) and row 8=[53,53,53,53] the high cluster +-- (now 6,8,5) -- their appearance proves the delta path indexed them. +insert into qi8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qu8 values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qbf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +insert into qf values (7,'[2,2,2,2]'),(8,'[53,53,53,53]'); +select sleep(30); +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qi8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qu8 order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qbf order by l2_distance(v,'[54,54,54,54]') limit 3; +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qf order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 3) update row 1 across to the other cluster ([1,1,1,1] -> [55,55,55,55]); the +-- delta path must re-quantize + re-bucket it. Query the LOW cluster center +-- [1,1,1,1]: row 1 has LEFT it, so the top-3 is now 7,2,3 (row 1 absent) -- +-- proving the delta UPDATE moved it. (Querying the high side instead would tie +-- under quantization: 55 clamps to the same code as 54.) +update qi8 set v = '[55,55,55,55]' where a = 1; +update qu8 set v = '[55,55,55,55]' where a = 1; +update qbf set v = '[55,55,55,55]' where a = 1; +update qf set v = '[55,55,55,55]' where a = 1; +select sleep(30); +select a from qi8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qu8 order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qbf order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qf order by l2_distance(v,'[1,1,1,1]') limit 3; + +drop table qi8; +drop table qu8; +drop table qbf; +drop table qf; diff --git a/test/distributed/cases/vector/vector_ivf_mode.result b/test/distributed/cases/vector/vector_ivf_mode.result index e8f1072651c77..360806d00d3e0 100644 --- a/test/distributed/cases/vector/vector_ivf_mode.result +++ b/test/distributed/cases/vector/vector_ivf_mode.result @@ -359,22 +359,22 @@ id8 semantic item 0.7760798852372132 id1 hello world 1.0551303640156267 id2 greeting message 1.1550324705439516 id7 random note 1.191049991561001 -(SELECT id, text AS content, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') AS dist +(SELECT id, text AS content, round(l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]'),4) AS dist FROM mini_vector_data ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, content, cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') AS dist +(SELECT id, content, round(cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]'),4) AS dist FROM mini_embed_data ORDER BY cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') LIMIT 2 by rank with option 'mode=pre') -ORDER BY id +ORDER BY dist, id LIMIT 4; id content dist id03 it stores high dimensional vectors 0.0 -id02 sql is structured query language 0.246478870511055 -id10 additional entry 1.4459599256515503 -id1 hello world 1.5163443088531494 +id02 sql is structured query language 0.2465 +id10 additional entry 1.446 +id1 hello world 1.5163 (SELECT id, category, l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') AS dist FROM vec_with_multi_idx WHERE category = 'A' AND status = 1 diff --git a/test/distributed/cases/vector/vector_ivf_mode.sql b/test/distributed/cases/vector/vector_ivf_mode.sql index 6162d9d853d78..a52fb67d34a95 100644 --- a/test/distributed/cases/vector/vector_ivf_mode.sql +++ b/test/distributed/cases/vector/vector_ivf_mode.sql @@ -285,16 +285,21 @@ UNION ORDER BY dist LIMIT 4; -- Test Case: UNION with mode=pre on different tables (mini_vector_data and mini_embed_data) -(SELECT id, text AS content, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') AS dist - FROM mini_vector_data +-- round(dist,4): an exact-match cosine distance is 0 on scalar but ~1.1e-16 with the +-- arch-specific SIMD kernels (FMA); the result comparator treats 0-vs-nonzero as a hard +-- mismatch, so round it. round() wraps only the projection, leaving the ORDER BY on raw +-- distance, so the ivfflat index is still used. ORDER BY dist, id makes the outer row +-- order deterministic (it was under-determined for this UNION shape with ORDER BY id). +(SELECT id, text AS content, round(l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]'),4) AS dist + FROM mini_vector_data ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, content, cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') AS dist - FROM mini_embed_data - ORDER BY cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') +(SELECT id, content, round(cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]'),4) AS dist + FROM mini_embed_data + ORDER BY cosine_distance(embedding, '[0.9,0.8,0.7,0.6,0.5,0.4,0.3,0.2]') LIMIT 2 by rank with option 'mode=pre') -ORDER BY id +ORDER BY dist, id LIMIT 4; -- Test Case: UNION with mode=pre and complex WHERE conditions diff --git a/test/distributed/cases/vector/vector_ivf_quant_ddl.result b/test/distributed/cases/vector/vector_ivf_quant_ddl.result new file mode 100644 index 0000000000000..d5efd3aa7533b --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quant_ddl.result @@ -0,0 +1,75 @@ +drop database if exists ivfqddl; +create database ivfqddl; +use ivfqddl; +create table q(a int primary key, v vecf32(4)); +insert into q values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index qi8 using ivfflat on q(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +alter table q alter reindex qi8 ivfflat lists=2; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +create table qc clone q; +select a from qc order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from qc order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +drop snapshot if exists ivfqsp; +create snapshot ivfqsp for account sys; +drop database if exists ivfqddl2; +create database ivfqddl2 clone ivfqddl {snapshot='ivfqsp'}; +select a from ivfqddl2.q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from ivfqddl2.q order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +drop snapshot ivfqsp; +drop database ivfqddl2; +alter table q add column note varchar(10) default 'x'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q drop index qi8; +create index qi8b using ivfflat on q(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; +a +6 +5 +4 +drop table qc; +drop table q; +drop database ivfqddl; diff --git a/test/distributed/cases/vector/vector_ivf_quant_ddl.sql b/test/distributed/cases/vector/vector_ivf_quant_ddl.sql new file mode 100644 index 0000000000000..d92e270f1695c --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quant_ddl.sql @@ -0,0 +1,54 @@ +-- ivfflat int8 QUANTIZATION across the DDL / maintenance matrix: +-- reindex, table clone, snapshot + db clone, alter table, drop index/table. +-- Two well-separated clusters [1..] and [50..]; queries probe each side and +-- must keep returning the right cluster after every operation (the int8 codes +-- and trained bounds must survive each path, not silently fall back to a raw +-- identity cast). +drop database if exists ivfqddl; +create database ivfqddl; +use ivfqddl; + +create table q(a int primary key, v vecf32(4)); +insert into q values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index qi8 using ivfflat on q(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; + +-- baseline: query near each cluster +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 1) ALTER REINDEX: sync rebuild re-applies the quantizer + re-trains bounds. +alter table q alter reindex qi8 ivfflat lists=2; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 2) CLONE table: block-level physical copy of entries/centroids/metadata. +create table qc clone q; +select a from qc order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from qc order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 3) SNAPSHOT + db clone-from-snapshot: RestoreTable path (empty seed, +-- block clone, FORCE_SYNC reindex InitSQL). +drop snapshot if exists ivfqsp; +create snapshot ivfqsp for account sys; +drop database if exists ivfqddl2; +create database ivfqddl2 clone ivfqddl {snapshot='ivfqsp'}; +select a from ivfqddl2.q order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from ivfqddl2.q order by l2_distance(v,'[54,54,54,54]') limit 3; +drop snapshot ivfqsp; +drop database ivfqddl2; + +-- 4) ALTER TABLE add a non-vector column: index must keep working. +alter table q add column note varchar(10) default 'x'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; + +-- 5) DROP INDEX then recreate int8 on the same column. +alter table q drop index qi8; +create index qi8b using ivfflat on q(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q order by l2_distance(v,'[1,1,1,1]') limit 3; +select a from q order by l2_distance(v,'[54,54,54,54]') limit 3; + +-- 6) DROP TABLE (clone first, then base). +drop table qc; +drop table q; + +drop database ivfqddl; diff --git a/test/distributed/cases/vector/vector_ivf_quant_upcast.result b/test/distributed/cases/vector/vector_ivf_quant_upcast.result new file mode 100644 index 0000000000000..e34aa152c4578 --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quant_upcast.result @@ -0,0 +1,18 @@ +drop database if exists ivf_qup; +create database ivf_qup; +use ivf_qup; +create table i8(a int, v vecint8(4)); +create table bf(a int, v vecbf16(4)); +create table hf(a int, v vecf16(4)); +create table u8(a int, v vecuint8(4)); +create index x using ivfflat on i8(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +not supported: ivfflat QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECINT8 (1 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index x using ivfflat on i8(v) lists=1 op_type 'vector_l2_ops' quantization 'bf16'; +not supported: ivfflat QUANTIZATION 'bf16' (2 bytes/element) cannot upcast base column VECINT8 (1 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index x using ivfflat on bf(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +not supported: ivfflat QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECBF16 (2 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index x using ivfflat on hf(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +not supported: ivfflat QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECF16 (2 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index x using ivfflat on u8(v) lists=1 op_type 'vector_l2_ops' quantization 'float16'; +not supported: ivfflat QUANTIZATION 'float16' (2 bytes/element) cannot upcast base column VECUINT8 (1 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +drop database ivf_qup; diff --git a/test/distributed/cases/vector/vector_ivf_quant_upcast.sql b/test/distributed/cases/vector/vector_ivf_quant_upcast.sql new file mode 100644 index 0000000000000..264877db258ce --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quant_upcast.sql @@ -0,0 +1,23 @@ +-- ivfflat QUANTIZATION is downcast-only. A narrow base column (vecbf16/vecf16/ +-- vecint8/vecuint8) with a QUANTIZATION wider than the base element is rejected at +-- plan time -- it would store upcast entries for no precision gain and force the +-- f32 distance kernel over narrow data. Regression for the schema.go upcast guard. +-- Equal-width / narrower quantization (and omitting it) are allowed. All cases here +-- fail before any index build, so no GPU is required. +drop database if exists ivf_qup; +create database ivf_qup; +use ivf_qup; +create table i8(a int, v vecint8(4)); +create table bf(a int, v vecbf16(4)); +create table hf(a int, v vecf16(4)); +create table u8(a int, v vecuint8(4)); +-- int8 base (1 byte) + wider quantization -> rejected +create index x using ivfflat on i8(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +create index x using ivfflat on i8(v) lists=1 op_type 'vector_l2_ops' quantization 'bf16'; +-- bf16 base (2 bytes) + float32 (4 bytes) -> rejected +create index x using ivfflat on bf(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +-- f16 base (2 bytes) + float32 (4 bytes) -> rejected +create index x using ivfflat on hf(v) lists=1 op_type 'vector_l2_ops' quantization 'float32'; +-- uint8 base (1 byte) + float16 (2 bytes) -> rejected +create index x using ivfflat on u8(v) lists=1 op_type 'vector_l2_ops' quantization 'float16'; +drop database ivf_qup; diff --git a/test/distributed/cases/vector/vector_ivf_quantization.result b/test/distributed/cases/vector/vector_ivf_quantization.result new file mode 100644 index 0000000000000..14cbe5732026f --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quantization.result @@ -0,0 +1,82 @@ +drop database if exists ivfq; +create database ivfq; +use ivfq; +create table tbf16(a int primary key, v vecbf16(4)); +insert into tbf16 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_bf16 using ivfflat on tbf16(v) lists=2 op_type 'vector_l2_ops'; +select a from tbf16 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table tf16(a int primary key, v vecf16(4)); +insert into tf16 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_f16 using ivfflat on tf16(v) lists=2 op_type 'vector_l2_ops'; +select a from tf16 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table ti8(a int primary key, v vecint8(4)); +insert into ti8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_i8 using ivfflat on ti8(v) lists=2 op_type 'vector_l2_ops'; +select a from ti8 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table q32(a int primary key, v vecf32(4)); +insert into q32 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index q32f16 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'float16'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q32 drop index q32f16; +create index q32bf16 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'bf16'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q32 drop index q32bf16; +create index q32i8 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table q64(a int primary key, v vecf64(4)); +insert into q64 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index q64f32 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'float32'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q64 drop index q64f32; +create index q64i8 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q32 drop index q32i8; +create index q32u8 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +alter table q64 drop index q64i8; +create index q64u8 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +a +1 +2 +3 +create table qbad(a int primary key, v vecf32(4)); +create index qb using ivfflat on qbad(v) lists=1 op_type 'vector_l2_ops' quantization 'int16'; +internal error: ivfflat: unsupported quantization 'int16' (supported: 'float32', 'float16', 'bf16', 'int8', 'uint8') +drop database ivfq; diff --git a/test/distributed/cases/vector/vector_ivf_quantization.sql b/test/distributed/cases/vector/vector_ivf_quantization.sql new file mode 100644 index 0000000000000..0709e854e2ada --- /dev/null +++ b/test/distributed/cases/vector/vector_ivf_quantization.sql @@ -0,0 +1,46 @@ +-- ivfflat: narrow base (direct match T->T) and QUANTIZATION down-cast of f32/f64 +-- bases to f32/f16/bf16/int8 entries. Two well-separated clusters (1..5 and 50..54) +-- make the top-3 stable under quantization; query an exact cluster-A point. +drop database if exists ivfq; +create database ivfq; +use ivfq; +create table tbf16(a int primary key, v vecbf16(4)); +insert into tbf16 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_bf16 using ivfflat on tbf16(v) lists=2 op_type 'vector_l2_ops'; +select a from tbf16 order by l2_distance(v,'[1,1,1,1]') limit 3; +create table tf16(a int primary key, v vecf16(4)); +insert into tf16 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_f16 using ivfflat on tf16(v) lists=2 op_type 'vector_l2_ops'; +select a from tf16 order by l2_distance(v,'[1,1,1,1]') limit 3; +create table ti8(a int primary key, v vecint8(4)); +insert into ti8 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index i_i8 using ivfflat on ti8(v) lists=2 op_type 'vector_l2_ops'; +select a from ti8 order by l2_distance(v,'[1,1,1,1]') limit 3; +create table q32(a int primary key, v vecf32(4)); +insert into q32 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index q32f16 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'float16'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +alter table q32 drop index q32f16; +create index q32bf16 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'bf16'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +alter table q32 drop index q32bf16; +create index q32i8 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +create table q64(a int primary key, v vecf64(4)); +insert into q64 values (1,'[1,1,1,1]'),(2,'[3,3,3,3]'),(3,'[5,5,5,5]'),(4,'[50,50,50,50]'),(5,'[52,52,52,52]'),(6,'[54,54,54,54]'); +create index q64f32 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'float32'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +alter table q64 drop index q64f32; +create index q64i8 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'int8'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +-- uint8 QUANTIZATION (unsigned [0,255]) on f32 and f64 bases +alter table q32 drop index q32i8; +create index q32u8 using ivfflat on q32(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8'; +select a from q32 order by l2_distance(v,'[1,1,1,1]') limit 3; +alter table q64 drop index q64i8; +create index q64u8 using ivfflat on q64(v) lists=2 op_type 'vector_l2_ops' quantization 'uint8'; +select a from q64 order by l2_distance(v,'[1,1,1,1]') limit 3; +-- an unsupported quantization name still errors +create table qbad(a int primary key, v vecf32(4)); +create index qb using ivfflat on qbad(v) lists=1 op_type 'vector_l2_ops' quantization 'int16'; +drop database ivfq; diff --git a/test/distributed/cases/vector/vector_reindex_options.result b/test/distributed/cases/vector/vector_reindex_options.result index 180d0f9b7489b..0c1aed165fe9e 100644 --- a/test/distributed/cases/vector/vector_reindex_options.result +++ b/test/distributed/cases/vector/vector_reindex_options.result @@ -19,6 +19,12 @@ alter table ivf_t alter reindex idx ivfflat ef_construction=200; not supported: ALTER REINDEX option "ef_construction" on a ivfflat index alter table ivf_t alter reindex idx ivfflat graph_degree=64; not supported: ALTER REINDEX option "graph_degree" on a ivfflat index +alter table ivf_t alter reindex idx ivfflat quantization 'Float16'; +show create table ivf_t; +Table Create Table +ivf_t CREATE TABLE `ivf_t` (\n `a` int NOT NULL,\n `b` vecf32(4) DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `idx` USING ivfflat (`b`) lists = 4 op_type 'vector_l2_ops' quantization 'float16' kmeans_train_percent = 80 kmeans_max_iteration = 50 \n) +alter table ivf_t alter reindex idx ivfflat quantization 'garbage'; +not supported: ivfflat quantization "garbage" (supported: float32, float16, bf16, int8, uint8) create table hnsw_t(a bigint primary key, b vecf32(4)); insert into hnsw_t values(1,"[1,2,3,4]"),(2,"[5,6,7,8]"),(3,"[9,10,11,12]"),(4,"[2,1,4,3]"); create index hidx using hnsw on hnsw_t(b) op_type "vector_l2_ops" m=48 ef_construction=64 ef_search=64; diff --git a/test/distributed/cases/vector/vector_reindex_options.sql b/test/distributed/cases/vector/vector_reindex_options.sql index dc45c03dd9228..4087f49cfe25b 100644 --- a/test/distributed/cases/vector/vector_reindex_options.sql +++ b/test/distributed/cases/vector/vector_reindex_options.sql @@ -31,6 +31,12 @@ alter table ivf_t alter reindex idx ivfflat m=16; alter table ivf_t alter reindex idx ivfflat ef_construction=200; alter table ivf_t alter reindex idx ivfflat graph_degree=64; +-- IVF-FLAT honors quantization (narrow-type entries); value is normalized to +-- lowercase and an unsupported name is rejected. +alter table ivf_t alter reindex idx ivfflat quantization 'Float16'; +show create table ivf_t; +alter table ivf_t alter reindex idx ivfflat quantization 'garbage'; + -- ---------------------------------------------------------------------------- -- HNSW: honors m + ef_construction + ef_search + max_index_capacity -- ---------------------------------------------------------------------------- diff --git a/test/distributed/gpu_cases/README.md b/test/distributed/gpu_cases/README.md index fe2e7eccffcdd..295ee559f9ac4 100644 --- a/test/distributed/gpu_cases/README.md +++ b/test/distributed/gpu_cases/README.md @@ -11,6 +11,8 @@ CPU-only BVT run is not gated on a GPU. | `vector_ivfpq.sql` | IVF-PQ | `gpu_cases/vector/` | sync CREATE INDEX, DDL surface, exact-match search, drop/recreate lifecycle | | `vector_cagra_quantization.sql` | CAGRA | `gpu_cases/vector/` | `QUANTIZATION 'float16'`, `'int8'` and `'uint8'` — each round-trips through the catalog + exact-match search | | `vector_ivfpq_quantization.sql` | IVF-PQ | `gpu_cases/vector/` | `QUANTIZATION 'float16'`, `'int8'` and `'uint8'` — each round-trips through the catalog + exact-match search | +| `vector_cagra_f16.sql` | CAGRA | `gpu_cases/vector/` | **vecf16 BASE column** (native half end-to-end): direct (half storage), `QUANTIZATION 'int8'`/`'uint8'` (native half→int8/uint8, no f32 detour) — catalog round-trip + exact-match search; query cast to vecf16(8) | +| `vector_ivfpq_f16.sql` | IVF-PQ | `gpu_cases/vector/` | same vecf16 BASE coverage as `vector_cagra_f16.sql` | | `vector_pairwise_scan.sql` | (none) | `gpu_cases/vector/` | GPU **pairwise distance** on a NON-INDEX table scan: `ORDER BY l2_distance/l2_distance_sq/cosine_distance(col, query)` over 10k×128 SIFT rows routes the batch through `metric.PairwiseDistanceLaunch` (exact, deterministic) | | `vector_pairwise_mode.sql` | (none) | `gpu_cases/vector/` | same non-index pairwise scan run under **`gpu_mode=1` (GPU) and `gpu_mode=0` (CPU)** for l2/l2sq/cosine/**inner_product** — results are byte-identical (GPU==CPU), and inner_product shows the negated score | | `vector_ivfflat_mode.sql` | IVF-FLAT | `gpu_cases/vector/` | IVF-FLAT search under **`gpu_mode=1`/`0`** — the productl2 centroid-assignment brute-force (GPU vs CPU) returns identical results | @@ -19,7 +21,9 @@ CPU-only BVT run is not gated on a GPU. | `vector_ivfpq_metric.sql` | IVF-PQ | `gpu_cases/vector/` | same per-metric build/search/score coverage as `vector_cagra_metric.sql` | | `vector_cagra_filter.sql` | CAGRA | `gpu_cases/vector/` | **INCLUDE-column pre-filter** across all 4 supported INCLUDE types — `INCLUDE (c_i32 int, c_i64 bigint, c_f32 float, c_f64 double)`; single- and multi-column `WHERE` predicates are pushed into the GPU search (predsJSON) and restrict the ANN candidate set — verifies both columns round-trip and the filter changes the nearest neighbor | | `vector_ivfpq_filter.sql` | IVF-PQ | `gpu_cases/vector/` | same 4-type INCLUDE pre-filter coverage as `vector_cagra_filter.sql` | -| `vector_gpu_negative.sql` | CAGRA + IVF-PQ | `gpu_cases/vector/` | **validation guard rails** (expected errors): `op_type 'vector_l1_ops'` / unknown op_type rejected, `vecf64` column rejected, `QUANTIZATION 'float64'` rejected, **VARCHAR INCLUDE column** rejected, search dimension-mismatch rejected | +| `vector_cagra_postfilter.sql` | CAGRA | `gpu_cases/vector/` | **post-filter on a NON-INCLUDE column** — a `WHERE` on a column absent from `INCLUDE` cannot be GPU-pushed, so the planner runs the ANN search for a candidate window then JOINs+filters at the DB. Verifies the post-filtered result equals the unfiltered ranked result ∩ predicate (exact when `LIMIT` ≥ rows so the window covers all), plus the mixed pre+post case and the small-`LIMIT` approximate window | +| `vector_ivfpq_postfilter.sql` | IVF-PQ | `gpu_cases/vector/` | same non-INCLUDE post-filter coverage as `vector_cagra_postfilter.sql` | +| `vector_gpu_negative.sql` | CAGRA + IVF-PQ | `gpu_cases/vector/` | **validation guard rails** (expected errors): `op_type 'vector_l1_ops'` / unknown op_type rejected, `vecf64` column rejected, `QUANTIZATION 'float64'` rejected, **VARCHAR INCLUDE column** rejected, search dimension-mismatch rejected, **`vecbf16` base column rejected**, **`vecf16` base + `QUANTIZATION 'float32'` upcast rejected** | | `vector_cagra_delete.sql` | CAGRA | `gpu_cases/pessimistic_transaction/vector/` | **soft-delete**: `DELETE` a row, after CDC catch-up search excludes it and returns the next survivor (per-device deleted bitset) | | `vector_ivfpq_delete.sql` | IVF-PQ | `gpu_cases/pessimistic_transaction/vector/` | same soft-delete coverage as `vector_cagra_delete.sql` | | `vector_cagra_ddl.sql` | CAGRA | `gpu_cases/pessimistic_transaction/vector/` | **DDL/DML lifecycle** on an indexed table: ALTER ADD/DROP COLUMN, TRUNCATE, re-INSERT, reindex — each table-rewrite triggers a CDC rebuild (SLEEP(30)) after which search recovers | diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.result b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.result new file mode 100644 index 0000000000000..9396df8d10a5a --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.result @@ -0,0 +1,63 @@ +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; +drop database if exists cagra_f16_cdc; +create database cagra_f16_cdc; +use cagra_f16_cdc; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +(1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), +(3, '[3,3,3,3,3,3,3,3]'), (4, '[4,4,4,4,4,4,4,4]'), +(5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), +(7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), +(9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'); +create index ix using cagra on t (v) +op_type 'vector_l2_ops' intermediate_graph_degree=8 graph_degree=4 ASYNC; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' async quantization 'float32' distribution_mode 'single' intermediate_graph_degree = 8 graph_degree = 4 +) +insert into t values (100, '[100,100,100,100,100,100,100,100]'); +insert into t values (105, '[105,105,105,105,105,105,105,105]'); +insert into t values (300, '[300,300,300,300,300,300,300,300]'); +insert into t values (700, '[700,700,700,700,700,700,700,700]'); +delete from t where id=105; +delete from t where id=3; +update t set v = '[500,500,500,500,500,500,500,500]' where id=5; +update t set v = '[305,305,305,305,305,305,305,305]' where id=300; +insert into t values (800, '[800,800,800,800,800,800,800,800]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select id from t order by l2_distance(v, cast('[100,100,100,100,100,100,100,100]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +100 +select id from t order by l2_distance(v, cast('[700,700,700,700,700,700,700,700]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +700 +select id from t order by l2_distance(v, cast('[800,800,800,800,800,800,800,800]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +800 +select id from t order by l2_distance(v, cast('[500,500,500,500,500,500,500,500]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t order by l2_distance(v, cast('[305,305,305,305,305,305,305,305]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +300 +select id from t order by l2_distance(v, cast('[105,105,105,105,105,105,105,105]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +100 +set @stbl = (select index_table_name from mo_catalog.mo_indexes +where table_id=(select rel_id from mo_catalog.mo_tables +where relname='t' and reldatabase='cagra_f16_cdc') +and name='ix' and algo_table_type='cagra_index'); +set @q = concat('select distinct tag from `', @stbl, '` order by tag'); +prepare s from @q; execute s; deallocate prepare s; +select count(*) from t; +➤ count(*)[-5,64,0] 𝄀 +13 +drop database cagra_f16_cdc; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.sql b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.sql new file mode 100644 index 0000000000000..a34b88ec51357 --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_cagra_f16_async.sql @@ -0,0 +1,96 @@ +-- ===================================================================== +-- vector_cagra_f16_async.sql — CAGRA vecf16 base + ISCP CDC INSERT/DELETE/UPDATE +-- +-- GPU REQUIRED. The vecf16 twin of vector_cagra_async.sql. It proves the +-- ongoing CDC ingestion path carries a vecf16 base column NATIVELY (2 bytes +-- per element, no f32 widening): the iscp CuvsCdcWriter extracts the source +-- column as []types.Float16, encodes each event record at 2*dim bytes, and +-- CagraSync.AppendRecords steps the stream by that same width; the search-side +-- replayEventChunks[cuvs.Float16] then reinterprets the bytes back to half and +-- feeds the f16 brute-force overflow. +-- +-- ASYNC CREATE INDEX: cagra_create is deferred to the first CDC iteration +-- (stashed as ConsumerInfo.InitSQL). The 10 initial rows build the tag=0 +-- model (native half); every DML below rides the CDC tail into the tag=1 +-- overflow. Like vector_cagra_async / vector_hnsw_async, ALL ops run first, +-- then a single SELECT SLEEP(30) lets the 10s-interval consumer apply the +-- whole batch, and only then do we search. +-- +-- The query literal is cast to vecf16(8) so the native half query path is +-- exercised (CagraSearch over base B == cuvs.Float16). +-- +-- Determinism notes (CAGRA is an approximate index; values are exact in f16): +-- * Every sentinel value (100..800, 305, 500, 105) is an integer < 2048, so +-- it is represented exactly in float16 — no rounding ambiguity. +-- * Exact-match probes always return that row as top-1 — verifies INSERT and +-- UPDATE (new vec replaces old). +-- * The deleted-sentinel probe [105,...] resolves to id=100: once id=105 is +-- gone, id=100 lives in the exact f16 overflow and is the unique nearest +-- neighbor by a wide margin, stable regardless of graph approximation. +-- * id=3 (deleted) is verified via COUNT(*) only — its integer neighbors are +-- L2-equidistant, so a search over them would not be reproducible. +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; + +drop database if exists cagra_f16_cdc; +create database cagra_f16_cdc; +use cagra_f16_cdc; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + (1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), + (3, '[3,3,3,3,3,3,3,3]'), (4, '[4,4,4,4,4,4,4,4]'), + (5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), + (7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), + (9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'); + +-- Async build: cagra_create deferred to first CDC iteration via InitSQL. +create index ix using cagra on t (v) + op_type 'vector_l2_ops' intermediate_graph_degree=8 graph_degree=4 ASYNC; +show create table t; + +-- Batch of DML — all applied before any search. The 10 initial rows go +-- through the InitSQL build (tag=0); everything below rides the f16 CDC tail +-- into the tag=1 overflow. +insert into t values (100, '[100,100,100,100,100,100,100,100]'); +insert into t values (105, '[105,105,105,105,105,105,105,105]'); +insert into t values (300, '[300,300,300,300,300,300,300,300]'); +insert into t values (700, '[700,700,700,700,700,700,700,700]'); +delete from t where id=105; +delete from t where id=3; +update t set v = '[500,500,500,500,500,500,500,500]' where id=5; +update t set v = '[305,305,305,305,305,305,305,305]' where id=300; +insert into t values (800, '[800,800,800,800,800,800,800,800]'); + +-- Single wait for the whole batch to flow through CDC. +select sleep(30); + +-- Surviving inserted sentinels — exact match → that row. +select id from t order by l2_distance(v, cast('[100,100,100,100,100,100,100,100]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[700,700,700,700,700,700,700,700]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[800,800,800,800,800,800,800,800]' as vecf16(8))) limit 1; + +-- Updated rows — exact match on the NEW value returns the moved row. +select id from t order by l2_distance(v, cast('[500,500,500,500,500,500,500,500]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[305,305,305,305,305,305,305,305]' as vecf16(8))) limit 1; + +-- Deleted sentinel — probe its old value; id=105 is gone so the unique +-- nearest survivor id=100 (exact f16 overflow) comes back instead. +select id from t order by l2_distance(v, cast('[105,105,105,105,105,105,105,105]' as vecf16(8))) limit 1; + +-- Storage layout: tag=0 (model from initial build) + tag=1 (CDC overflow). +set @stbl = (select index_table_name from mo_catalog.mo_indexes + where table_id=(select rel_id from mo_catalog.mo_tables + where relname='t' and reldatabase='cagra_f16_cdc') + and name='ix' and algo_table_type='cagra_index'); +set @q = concat('select distinct tag from `', @stbl, '` order by tag'); +prepare s from @q; execute s; deallocate prepare s; + +-- Row count: 10 initial + 5 inserts (100,105,300,700,800) - 2 deletes +-- (105,3) = 13. Confirms id=3 and id=105 are gone. +select count(*) from t; + +drop database cagra_f16_cdc; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.result b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.result new file mode 100644 index 0000000000000..aa8bab83c0d5e --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.result @@ -0,0 +1,60 @@ +SET experimental_cagra_index = 1; +SET experimental_ivfpq_index = 1; +SET cagra_threads_build = 7; +drop database if exists int8scale_cagra; +create database int8scale_cagra; +use int8scale_cagra; +create table t (id bigint primary key, v vecf32(8)); +insert into t values +(1,'[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]'),(2,'[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]'), +(3,'[0.6,0.6,0.6,0.6,0.6,0.6,0.6,0.6]'),(4,'[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8]'), +(5,'[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'),(6,'[1.2,1.2,1.2,1.2,1.2,1.2,1.2,1.2]'), +(7,'[1.4,1.4,1.4,1.4,1.4,1.4,1.4,1.4]'),(8,'[1.6,1.6,1.6,1.6,1.6,1.6,1.6,1.6]'), +(9,'[1.8,1.8,1.8,1.8,1.8,1.8,1.8,1.8]'),(10,'[2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0]'); +create index ix using cagra on t (v) op_type 'vector_l2_ops' +intermediate_graph_degree=8 graph_degree=4 itopk_size=16 QUANTIZATION 'int8'; +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1; +➤ id[-5,64,0] 𝄀 +1 +insert into t values (999, '[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 3; +➤ id[-5,64,0] 𝄀 +1 𝄀 +2 𝄀 +3 +drop database int8scale_cagra; +drop database if exists int8scale_ivfpq; +create database int8scale_ivfpq; +use int8scale_ivfpq; +create table t (id bigint primary key, v vecf32(8)); +insert into t values +(1,'[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]'),(2,'[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]'), +(3,'[0.6,0.6,0.6,0.6,0.6,0.6,0.6,0.6]'),(4,'[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8]'), +(5,'[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'),(6,'[1.2,1.2,1.2,1.2,1.2,1.2,1.2,1.2]'), +(7,'[1.4,1.4,1.4,1.4,1.4,1.4,1.4,1.4]'),(8,'[1.6,1.6,1.6,1.6,1.6,1.6,1.6,1.6]'), +(9,'[1.8,1.8,1.8,1.8,1.8,1.8,1.8,1.8]'),(10,'[2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0]'); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' +lists=2 m=2 bits_per_code=8 QUANTIZATION 'uint8'; +insert into t values (999, '[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select count(*) as top1_is_overflow from +(select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1) x +where x.id = 999; +➤ top1_is_overflow[-5,64,0] 𝄀 +0 +select count(*) as top3_has_overflow from +(select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 3) x +where x.id = 999; +➤ top3_has_overflow[-5,64,0] 𝄀 +0 +drop database int8scale_ivfpq; +SET experimental_cagra_index = 0; +SET experimental_ivfpq_index = 0; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.sql b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.sql new file mode 100644 index 0000000000000..77c442ff47b89 --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_int8_overflow_scale.sql @@ -0,0 +1,79 @@ +-- ===================================================================== +-- vector_int8_overflow_scale.sql — int8/uint8 quantized index + CDC overflow +-- must merge on the SAME distance scale (break-review CRITICAL fix). +-- +-- GPU REQUIRED. A 1-byte (int8/uint8) main index computes L2 over the quantized +-- vectors, i.e. scalar^2 * true_L2 (scalar = 255/(max-min)). The base-typed CDC +-- overflow brute force computes true (base-scale) L2. Before the fix, +-- mergeMultiResults compared the two raw, so a moderately-distant overflow row +-- (small base-scale distance) out-ranked the true-nearest main row (large +-- scalar^2-scaled distance). The fix dequantizes the main distances by +-- 1/scalar^2 inside transform_distance so both tiers share the base scale. +-- +-- Values in [0.2, 2.0] => scalar ~= 255/1.8 ~= 141, scalar^2 ~= 20000. Main rows +-- id 1..10 at [i*0.2]*8. Overflow row id 999 at [1.0]*8 (true dist 4.5 from the +-- query, ~225x farther than id 1). Query [0.25]*8: the TRUE nearest is id 1 +-- (dist ~0.02). Correct top-1 (with the fix) is 1; the bug returned 999. +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET experimental_ivfpq_index = 1; +SET cagra_threads_build = 7; + +-- --------------------------------------------------------------------- +-- CAGRA, QUANTIZATION 'int8' +-- --------------------------------------------------------------------- +drop database if exists int8scale_cagra; +create database int8scale_cagra; +use int8scale_cagra; +create table t (id bigint primary key, v vecf32(8)); +insert into t values + (1,'[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]'),(2,'[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]'), + (3,'[0.6,0.6,0.6,0.6,0.6,0.6,0.6,0.6]'),(4,'[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8]'), + (5,'[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'),(6,'[1.2,1.2,1.2,1.2,1.2,1.2,1.2,1.2]'), + (7,'[1.4,1.4,1.4,1.4,1.4,1.4,1.4,1.4]'),(8,'[1.6,1.6,1.6,1.6,1.6,1.6,1.6,1.6]'), + (9,'[1.8,1.8,1.8,1.8,1.8,1.8,1.8,1.8]'),(10,'[2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0]'); +create index ix using cagra on t (v) op_type 'vector_l2_ops' + intermediate_graph_degree=8 graph_degree=4 itopk_size=16 QUANTIZATION 'int8'; +-- no overflow yet -> id 1 +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1; +-- add the overflow row, then the true nearest is STILL id 1 (not 999) +insert into t values (999, '[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'); +select sleep(30); +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1; +select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 3; +drop database int8scale_cagra; + +-- --------------------------------------------------------------------- +-- IVF-PQ, QUANTIZATION 'uint8' (same scale issue; +128 shift also cancels in L2) +-- --------------------------------------------------------------------- +drop database if exists int8scale_ivfpq; +create database int8scale_ivfpq; +use int8scale_ivfpq; +create table t (id bigint primary key, v vecf32(8)); +insert into t values + (1,'[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]'),(2,'[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]'), + (3,'[0.6,0.6,0.6,0.6,0.6,0.6,0.6,0.6]'),(4,'[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8]'), + (5,'[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'),(6,'[1.2,1.2,1.2,1.2,1.2,1.2,1.2,1.2]'), + (7,'[1.4,1.4,1.4,1.4,1.4,1.4,1.4,1.4]'),(8,'[1.6,1.6,1.6,1.6,1.6,1.6,1.6,1.6]'), + (9,'[1.8,1.8,1.8,1.8,1.8,1.8,1.8,1.8]'),(10,'[2.0,2.0,2.0,2.0,2.0,2.0,2.0,2.0]'); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' + lists=2 m=2 bits_per_code=8 QUANTIZATION 'uint8'; +insert into t values (999, '[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]'); +select sleep(30); +-- IVF-PQ recall on this tiny PQ config picks among the near main ids (1..4) +-- non-deterministically, so the exact top-1 is not stable. The scale bug is +-- about the moderately-distant overflow row 999 (true dist 4.5) out-ranking the +-- near main rows, so assert the stable invariant directly: 999 must NEVER appear +-- in the top results. Pre-fix this returned 1 (999 was top-1); with the +-- dequant fix it is 0. +select count(*) as top1_is_overflow from + (select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 1) x + where x.id = 999; +select count(*) as top3_has_overflow from + (select id from t order by l2_distance(v, '[0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25]') asc limit 3) x + where x.id = 999; +drop database int8scale_ivfpq; + +SET experimental_cagra_index = 0; +SET experimental_ivfpq_index = 0; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_async.result b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_async.result index f14dd68f70aff..9016ce842452c 100644 --- a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_async.result +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_async.result @@ -21,7 +21,7 @@ t ¦ CREATE TABLE `t` ( `id` bigint NOT NULL, `v` vecf32(8) DEFAULT NULL, PRIMARY KEY (`id`), - KEY `ix` USING ivfpq (`v`) lists = 2 m = 2 op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' bits_per_code = 8 + KEY `ix` USING ivfpq (`v`) lists = 2 m = 2 op_type 'vector_l2_ops' async quantization 'float32' distribution_mode 'single' bits_per_code = 8 ) insert into t values (100, '[100,100,100,100,100,100,100,100]'); insert into t values (105, '[105,105,105,105,105,105,105,105]'); diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.result b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.result new file mode 100644 index 0000000000000..ca5395ded29b7 --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.result @@ -0,0 +1,65 @@ +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 37; +SET kmeans_max_iteration = 12; +drop database if exists ivfpq_f16_cdc; +create database ivfpq_f16_cdc; +use ivfpq_f16_cdc; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +(1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), +(3, '[3,3,3,3,3,3,3,3]'), (4, '[4,4,4,4,4,4,4,4]'), +(5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), +(7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), +(9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'); +create index ix using ivfpq on t (v) +op_type 'vector_l2_ops' lists=2 m=2 bits_per_code=8 ASYNC; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 2 op_type 'vector_l2_ops' async quantization 'float32' distribution_mode 'single' bits_per_code = 8 +) +insert into t values (100, '[100,100,100,100,100,100,100,100]'); +insert into t values (105, '[105,105,105,105,105,105,105,105]'); +insert into t values (300, '[300,300,300,300,300,300,300,300]'); +insert into t values (700, '[700,700,700,700,700,700,700,700]'); +delete from t where id=105; +delete from t where id=3; +update t set v = '[500,500,500,500,500,500,500,500]' where id=5; +update t set v = '[305,305,305,305,305,305,305,305]' where id=300; +insert into t values (800, '[800,800,800,800,800,800,800,800]'); +select sleep(30); +➤ sleep(30)[-6,8,0] 𝄀 +0 +select id from t order by l2_distance(v, cast('[100,100,100,100,100,100,100,100]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +100 +select id from t order by l2_distance(v, cast('[700,700,700,700,700,700,700,700]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +700 +select id from t order by l2_distance(v, cast('[800,800,800,800,800,800,800,800]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +800 +select id from t order by l2_distance(v, cast('[500,500,500,500,500,500,500,500]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t order by l2_distance(v, cast('[305,305,305,305,305,305,305,305]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +300 +select id from t order by l2_distance(v, cast('[105,105,105,105,105,105,105,105]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +100 +set @stbl = (select index_table_name from mo_catalog.mo_indexes +where table_id=(select rel_id from mo_catalog.mo_tables +where relname='t' and reldatabase='ivfpq_f16_cdc') +and name='ix' and algo_table_type='ivfpq_index'); +set @q = concat('select distinct tag from `', @stbl, '` order by tag'); +prepare s from @q; execute s; deallocate prepare s; +select count(*) from t; +➤ count(*)[-5,64,0] 𝄀 +13 +drop database ivfpq_f16_cdc; diff --git a/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.sql b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.sql new file mode 100644 index 0000000000000..53d62f760559c --- /dev/null +++ b/test/distributed/gpu_cases/pessimistic_transaction/vector/vector_ivfpq_f16_async.sql @@ -0,0 +1,99 @@ +-- ===================================================================== +-- vector_ivfpq_f16_async.sql — IVF-PQ vecf16 base + ISCP CDC INSERT/DELETE/UPDATE +-- +-- GPU REQUIRED. The vecf16 twin of vector_ivfpq_async.sql. It proves the +-- ongoing CDC ingestion path carries a vecf16 base column NATIVELY (2 bytes +-- per element, no f32 widening): the iscp CuvsCdcWriter extracts the source +-- column as []types.Float16, encodes each event record at 2*dim bytes, and +-- IvfpqSync.AppendRecords steps the stream by that same width; the search-side +-- replayEventChunks[cuvs.Float16] then reinterprets the bytes back to half and +-- feeds the f16 brute-force overflow. +-- +-- ASYNC CREATE INDEX: ivfpq_create is deferred to the first CDC iteration +-- (stashed as ConsumerInfo.InitSQL). The 10 initial rows build the tag=0 +-- model (native half); every DML below rides the CDC tail into the tag=1 +-- overflow. Like vector_ivfpq_async / vector_hnsw_async, ALL ops run first, +-- then a single SELECT SLEEP(30) lets the 10s-interval consumer apply the +-- whole batch, and only then do we search. +-- +-- The query literal is cast to vecf16(8) so the native half query path is +-- exercised (IvfpqSearch over base B == cuvs.Float16). +-- +-- Determinism notes (IVF-PQ is a quantized + clustered approximate index; +-- values are exact in f16): +-- * Every sentinel value (100..800, 305, 500, 105) is an integer < 2048, so +-- it is represented exactly in float16 — no rounding ambiguity. +-- * Exact-match probes always return that row as top-1 — verifies INSERT and +-- UPDATE (new vec replaces old). +-- * The deleted-sentinel probe [105,...] resolves to id=100: once id=105 is +-- gone, id=100 lives in the exact f16 overflow and is the unique nearest +-- neighbor by a wide margin, stable regardless of PQ approximation. +-- * id=3 (deleted) is verified via COUNT(*) only — its integer neighbors are +-- L2-equidistant, so a search over them would not be reproducible. +-- ===================================================================== + +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 37; +SET kmeans_max_iteration = 12; + +drop database if exists ivfpq_f16_cdc; +create database ivfpq_f16_cdc; +use ivfpq_f16_cdc; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + (1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), + (3, '[3,3,3,3,3,3,3,3]'), (4, '[4,4,4,4,4,4,4,4]'), + (5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), + (7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), + (9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'); + +-- Async build: ivfpq_create deferred to first CDC iteration via InitSQL. +create index ix using ivfpq on t (v) + op_type 'vector_l2_ops' lists=2 m=2 bits_per_code=8 ASYNC; +show create table t; + +-- Batch of DML — all applied before any search. The 10 initial rows go +-- through the InitSQL build (tag=0); everything below rides the f16 CDC tail +-- into the tag=1 overflow. +insert into t values (100, '[100,100,100,100,100,100,100,100]'); +insert into t values (105, '[105,105,105,105,105,105,105,105]'); +insert into t values (300, '[300,300,300,300,300,300,300,300]'); +insert into t values (700, '[700,700,700,700,700,700,700,700]'); +delete from t where id=105; +delete from t where id=3; +update t set v = '[500,500,500,500,500,500,500,500]' where id=5; +update t set v = '[305,305,305,305,305,305,305,305]' where id=300; +insert into t values (800, '[800,800,800,800,800,800,800,800]'); + +-- Single wait for the whole batch to flow through CDC. +select sleep(30); + +-- Surviving inserted sentinels — exact match → that row. +select id from t order by l2_distance(v, cast('[100,100,100,100,100,100,100,100]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[700,700,700,700,700,700,700,700]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[800,800,800,800,800,800,800,800]' as vecf16(8))) limit 1; + +-- Updated rows — exact match on the NEW value returns the moved row. +select id from t order by l2_distance(v, cast('[500,500,500,500,500,500,500,500]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[305,305,305,305,305,305,305,305]' as vecf16(8))) limit 1; + +-- Deleted sentinel — probe its old value; id=105 is gone so the unique +-- nearest survivor id=100 (exact f16 overflow) comes back instead. +select id from t order by l2_distance(v, cast('[105,105,105,105,105,105,105,105]' as vecf16(8))) limit 1; + +-- Storage layout: tag=0 (model from initial build) + tag=1 (CDC overflow). +set @stbl = (select index_table_name from mo_catalog.mo_indexes + where table_id=(select rel_id from mo_catalog.mo_tables + where relname='t' and reldatabase='ivfpq_f16_cdc') + and name='ix' and algo_table_type='ivfpq_index'); +set @q = concat('select distinct tag from `', @stbl, '` order by tag'); +prepare s from @q; execute s; deallocate prepare s; + +-- Row count: 10 initial + 5 inserts (100,105,300,700,800) - 2 deletes +-- (105,3) = 13. Confirms id=3 and id=105 are gone. +select count(*) from t; + +drop database ivfpq_f16_cdc; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_f16.result b/test/distributed/gpu_cases/vector/vector_cagra_f16.result new file mode 100644 index 0000000000000..041c967e2d9e9 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_f16.result @@ -0,0 +1,119 @@ +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; +drop database if exists cagra_f16_direct; +create database cagra_f16_direct; +use cagra_f16_direct; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using cagra on t (v) +op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database cagra_f16_direct; +drop database if exists cagra_f16_int8; +create database cagra_f16_int8; +use cagra_f16_int8; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using cagra on t (v) +op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 +QUANTIZATION 'int8'; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database cagra_f16_int8; +drop database if exists cagra_f16_uint8; +create database cagra_f16_uint8; +use cagra_f16_uint8; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using cagra on t (v) +op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 +QUANTIZATION 'uint8'; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database cagra_f16_uint8; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_f16.sql b/test/distributed/gpu_cases/vector/vector_cagra_f16.sql new file mode 100644 index 0000000000000..fe1c0c6ab5568 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_f16.sql @@ -0,0 +1,120 @@ +-- ===================================================================== +-- vector_cagra_f16.sql — CAGRA over a vecf16 (half) BASE column +-- +-- GPU REQUIRED. Unlike vector_cagra_quantization.sql (vecf32 base, the +-- QUANTIZATION clause only changes internal storage), here the COLUMN itself +-- is vecf16 — the native base/query type is half end-to-end: +-- * direct — no QUANTIZATION: the index stores half natively (Q == base). +-- * int8 — vecf16 base quantized half->int8 via the native half-source +-- scalar quantizer (no f32 detour). +-- * uint8 — same, half->uint8. +-- +-- Three databases, one per storage. Each builds a sync CAGRA index and asserts +-- (a) the vecf16 column + index round-trip through SHOW CREATE TABLE and +-- (b) exact-match search returns the right row. The query literal is cast to +-- vecf16(8) so the half query path is exercised. +-- +-- Determinism: integers 1..20 — every value is exact in half, and the int8/ +-- uint8 quantizer trains on [1,20] so each integer maps to a distinct level; +-- the exact-match probe is always the unique top-1. Do not widen the range +-- under int8/uint8 (adjacent levels would collapse). +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; + +-- ===================================================================== +-- vecf16 base, direct (no QUANTIZATION — stored as half) +-- ===================================================================== +drop database if exists cagra_f16_direct; +create database cagra_f16_direct; +use cagra_f16_direct; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using cagra on t (v) + op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database cagra_f16_direct; + +-- ===================================================================== +-- vecf16 base, QUANTIZATION int8 (native half->int8) +-- ===================================================================== +drop database if exists cagra_f16_int8; +create database cagra_f16_int8; +use cagra_f16_int8; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using cagra on t (v) + op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 + QUANTIZATION 'int8'; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database cagra_f16_int8; + +-- ===================================================================== +-- vecf16 base, QUANTIZATION uint8 (native half->uint8) +-- ===================================================================== +drop database if exists cagra_f16_uint8; +create database cagra_f16_uint8; +use cagra_f16_uint8; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using cagra on t (v) + op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 + QUANTIZATION 'uint8'; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database cagra_f16_uint8; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_filter.result b/test/distributed/gpu_cases/vector/vector_cagra_filter.result index 20135d00515c3..4d01e9de8e4f3 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_filter.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_filter.result @@ -44,7 +44,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_filter') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","included_columns":"c_i32,c_i64,c_f32,c_f64","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","included_columns":"c_i32,c_i64,c_f32,c_f64","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; ➤ id[-5,64,0] 𝄀 9 diff --git a/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.result b/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.result new file mode 100644 index 0000000000000..eab105f78ca43 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.result @@ -0,0 +1,345 @@ +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; +drop database if exists cagra_fq_f16; +create database cagra_fq_f16; +use cagra_fq_f16; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'float16' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'float16' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_f16; +drop database if exists cagra_fq_int8; +create database cagra_fq_int8; +use cagra_fq_int8; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_int8; +drop database if exists cagra_fq_uint8; +create database cagra_fq_uint8; +use cagra_fq_uint8; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_uint8; +drop database if exists cagra_fq_f16base; +create database cagra_fq_f16base; +use cagra_fq_f16base; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_f16base; +drop database if exists cagra_fq_f16int8; +create database cagra_fq_f16int8; +use cagra_fq_f16int8; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_f16int8; +drop database if exists cagra_fq_f16uint8; +create database cagra_fq_f16uint8; +use cagra_fq_f16uint8; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 32 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database cagra_fq_f16uint8; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.sql b/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.sql new file mode 100644 index 0000000000000..fad4badfb721e --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_filter_quant.sql @@ -0,0 +1,295 @@ +-- ===================================================================== +-- vector_cagra_filter_quant.sql — CAGRA INCLUDE-column pre-filter combined +-- with quantization and a vecf16 base column. +-- +-- GPU REQUIRED. vector_cagra_filter.sql already covers the INCLUDE pre-filter +-- over a plain vecf32 base (quantization 'float32'). This file proves the SAME +-- predsJSON pre-filter path stays correct when the index storage is compressed +-- or the base column is half: +-- * f32 base + QUANTIZATION 'float16' — supported (query quantized to T) +-- * f32 base + QUANTIZATION 'int8' — supported (learned scalar quantizer) +-- * f32 base + QUANTIZATION 'uint8' — supported +-- * vecf16 base, direct (no QUANTIZATION) — supported (native half query) +-- * vecf16 base + QUANTIZATION 'int8' + filter — supported (the native half +-- query is quantized to int8 inside +-- cuVS via search_quantize_with_filter) +-- * vecf16 base + QUANTIZATION 'uint8' + filter — supported (same path) +-- Every storage routes the SAME predsJSON pre-filter through the const-B* +-- quantize search, so the expected nearest neighbor per predicate is identical. +-- +-- Data/predicates are identical to vector_cagra_filter.sql so the expected +-- nearest neighbor per predicate is unchanged across every storage: +-- id=i -> [i]*8; c_i32=i, c_i64=i*10, c_f32=i.25, c_f64=i.5 (all monotone). +-- Query [12]*8: +-- * c_i32 < 10 -> id 9 +-- * c_i64 >= 100 -> id 12 +-- * c_f32 > 15.25 -> id 16 +-- * c_f64 = 5.5 -> id 5 +-- * c_i32 >= 10 AND c_f64 < 15.5 -> id 12 +-- * c_i64 < 100 AND c_f32 > 5.25 -> id 9 +-- +-- Determinism note: integers 1..20 are exact in float16 and the int8/uint8 +-- quantizer trains on [1,20] so each integer maps to a distinct level; each +-- predicate band keeps a unique nearest. Do NOT widen the range under int8/ +-- uint8 (adjacent levels would collapse and the top-1 would become ambiguous). +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'float16' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists cagra_fq_f16; +create database cagra_fq_f16; +use cagra_fq_f16; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'float16' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database cagra_fq_f16; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'int8' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists cagra_fq_int8; +create database cagra_fq_int8; +use cagra_fq_int8; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database cagra_fq_int8; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'uint8' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists cagra_fq_uint8; +create database cagra_fq_uint8; +use cagra_fq_uint8; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database cagra_fq_uint8; + +-- ===================================================================== +-- vecf16 base, direct (no QUANTIZATION) + INCLUDE pre-filter +-- The query literal is cast to vecf16(8) so the half query path is exercised. +-- ===================================================================== +drop database if exists cagra_fq_f16base; +create database cagra_fq_f16base; +use cagra_fq_f16base; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database cagra_fq_f16base; + +-- ===================================================================== +-- vecf16 base + QUANTIZATION 'int8' + INCLUDE pre-filter +-- The native half query is quantized to int8 inside cuVS (the const-B* +-- search_quantize_with_filter path); same predicates and nearest neighbors +-- as every storage above. +-- ===================================================================== +drop database if exists cagra_fq_f16int8; +create database cagra_fq_f16int8; +use cagra_fq_f16int8; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database cagra_fq_f16int8; + +-- ===================================================================== +-- vecf16 base + QUANTIZATION 'uint8' + INCLUDE pre-filter (same path as int8) +-- ===================================================================== +drop database if exists cagra_fq_f16uint8; +create database cagra_fq_f16uint8; +use cagra_fq_f16uint8; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=32 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database cagra_fq_f16uint8; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_metric.result b/test/distributed/gpu_cases/vector/vector_cagra_metric.result index 4aa3089270840..1f7dec42ec106 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_metric.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_metric.result @@ -23,7 +23,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_metric') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id, l2_distance(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ 0.0 @@ -34,7 +34,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_metric') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2sq_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2sq_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id, l2_distance_sq(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,54,0] 𝄀 1 ¦ 0.0 @@ -45,7 +45,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_metric') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_ip_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_ip_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id, inner_product(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ -1292.0 @@ -56,7 +56,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_metric') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_cosine_ops","quantization":"float32","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_cosine_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id, cosine_distance(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ -1.1920928955078125E-7 diff --git a/test/distributed/gpu_cases/vector/vector_cagra_postfilter.result b/test/distributed/gpu_cases/vector/vector_cagra_postfilter.result new file mode 100644 index 0000000000000..08801f85f71ef --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_postfilter.result @@ -0,0 +1,89 @@ +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; +drop database if exists cagra_postfilter; +create database cagra_postfilter; +use cagra_postfilter; +create table t (id bigint primary key, v vecf32(8), c_inc int, c_noinc int); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 101), +(2, '[2,2,2,2,2,2,2,2]', 2, 102), +(3, '[3,3,3,3,3,3,3,3]', 3, 103), +(4, '[4,4,4,4,4,4,4,4]', 4, 104), +(5, '[5,5,5,5,5,5,5,5]', 5, 105), +(6, '[6,6,6,6,6,6,6,6]', 6, 106), +(7, '[7,7,7,7,7,7,7,7]', 7, 107), +(8, '[8,8,8,8,8,8,8,8]', 8, 108), +(9, '[9,9,9,9,9,9,9,9]', 9, 109), +(10, '[10,10,10,10,10,10,10,10]', 10, 110), +(11, '[11,11,11,11,11,11,11,11]', 11, 111), +(12, '[12,12,12,12,12,12,12,12]', 12, 112), +(13, '[13,13,13,13,13,13,13,13]', 13, 113), +(14, '[14,14,14,14,14,14,14,14]', 14, 114), +(15, '[15,15,15,15,15,15,15,15]', 15, 115), +(16, '[16,16,16,16,16,16,16,16]', 16, 116), +(17, '[17,17,17,17,17,17,17,17]', 17, 117), +(18, '[18,18,18,18,18,18,18,18]', 18, 118), +(19, '[19,19,19,19,19,19,19,19]', 19, 119), +(20, '[20,20,20,20,20,20,20,20]', 20, 120); +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=64 INCLUDE (c_inc); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_inc` int DEFAULT NULL, + `c_noinc` int DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING cagra (`v`) op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' intermediate_graph_degree = 16 graph_degree = 8 itopk_size = 64 INCLUDE (c_inc) +) +select id, c_noinc from t order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] ¦ c_noinc[4,32,0] 𝄀 +12 ¦ 112 𝄀 +13 ¦ 113 𝄀 +11 ¦ 111 𝄀 +10 ¦ 110 𝄀 +14 ¦ 114 𝄀 +9 ¦ 109 𝄀 +15 ¦ 115 𝄀 +16 ¦ 116 𝄀 +8 ¦ 108 𝄀 +7 ¦ 107 𝄀 +17 ¦ 117 𝄀 +6 ¦ 106 𝄀 +18 ¦ 118 𝄀 +5 ¦ 105 𝄀 +19 ¦ 119 𝄀 +20 ¦ 120 𝄀 +4 ¦ 104 𝄀 +3 ¦ 103 𝄀 +2 ¦ 102 𝄀 +1 ¦ 101 +select id from t where c_noinc < 105 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +4 𝄀 +3 𝄀 +2 𝄀 +1 +select id from t where c_noinc >= 116 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +16 𝄀 +17 𝄀 +18 𝄀 +19 𝄀 +20 +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +2 +select id from t where c_inc <= 14 and c_noinc >= 108 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +12 𝄀 +13 𝄀 +11 𝄀 +10 𝄀 +14 𝄀 +9 𝄀 +8 +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] +drop database cagra_postfilter; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_postfilter.sql b/test/distributed/gpu_cases/vector/vector_cagra_postfilter.sql new file mode 100644 index 0000000000000..d894215970443 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_cagra_postfilter.sql @@ -0,0 +1,75 @@ +-- ===================================================================== +-- vector_cagra_postfilter.sql — CAGRA search filtering on a NON-INCLUDE column +-- +-- GPU REQUIRED. A WHERE predicate on a column that is NOT in the index INCLUDE +-- list cannot be pushed into the GPU bitset pre-filter. Instead the planner runs +-- the ANN search to get a candidate window, then JOINs + filters the predicate +-- at the database (post-filter). See the plan: cagra_search (candidate window) +-- INNER JOIN (table scan Filter: ) -> Sort -> Limit. +-- +-- Methodology: first take the UNFILTERED ranked result, then verify the +-- post-filtered result equals exactly the unfiltered rows that satisfy the +-- predicate. The candidate window grows with the query LIMIT, so a LIMIT >= row +-- count makes the window cover every row -> the post-filter is exact. +-- +-- Data: id=i -> [i]*8; c_inc=i (INCLUDE, int), c_noinc=100+i (NOT included). +-- Query [12]*8. With LIMIT 20 the window is all 20 rows, so post-filter is exact: +-- * c_noinc < 105 -> i in 1..4 -> 4,3,2,1 (nearest-first) +-- * c_noinc >= 116 -> i in 16..20 -> 16,17,18,19,20 +-- * c_noinc = 102 -> id 2 (far row still found, full window) +-- * c_inc <= 14 (PRE) AND c_noinc >= 108 (POST) -> i in 8..14 +-- ===================================================================== + +SET experimental_cagra_index = 1; +SET cagra_threads_build = 7; +SET cagra_max_index_capacity = 99999; + +drop database if exists cagra_postfilter; +create database cagra_postfilter; +use cagra_postfilter; + +create table t (id bigint primary key, v vecf32(8), c_inc int, c_noinc int); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 101), + (2, '[2,2,2,2,2,2,2,2]', 2, 102), + (3, '[3,3,3,3,3,3,3,3]', 3, 103), + (4, '[4,4,4,4,4,4,4,4]', 4, 104), + (5, '[5,5,5,5,5,5,5,5]', 5, 105), + (6, '[6,6,6,6,6,6,6,6]', 6, 106), + (7, '[7,7,7,7,7,7,7,7]', 7, 107), + (8, '[8,8,8,8,8,8,8,8]', 8, 108), + (9, '[9,9,9,9,9,9,9,9]', 9, 109), + (10, '[10,10,10,10,10,10,10,10]', 10, 110), + (11, '[11,11,11,11,11,11,11,11]', 11, 111), + (12, '[12,12,12,12,12,12,12,12]', 12, 112), + (13, '[13,13,13,13,13,13,13,13]', 13, 113), + (14, '[14,14,14,14,14,14,14,14]', 14, 114), + (15, '[15,15,15,15,15,15,15,15]', 15, 115), + (16, '[16,16,16,16,16,16,16,16]', 16, 116), + (17, '[17,17,17,17,17,17,17,17]', 17, 117), + (18, '[18,18,18,18,18,18,18,18]', 18, 118), + (19, '[19,19,19,19,19,19,19,19]', 19, 119), + (20, '[20,20,20,20,20,20,20,20]', 20, 120); + +-- Only c_inc is pushed into the GPU pre-filter; c_noinc is post-filtered. +create index ix using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=16 graph_degree=8 itopk_size=64 INCLUDE (c_inc); + +show create table t; + +-- (1) UNFILTERED ranked baseline (window covers all rows at LIMIT 20). +select id, c_noinc from t order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (2) POST-FILTER on the non-INCLUDE column — must equal the baseline rows that +-- satisfy the predicate, in the same distance order. +select id from t where c_noinc < 105 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +select id from t where c_noinc >= 116 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (3) MIXED: c_inc is pushed (pre-filter), c_noinc is post-filtered. Both apply. +select id from t where c_inc <= 14 and c_noinc >= 108 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (4) Small LIMIT shrinks the candidate window: a far post-filter match (id 2, +-- c_noinc=102) falls outside the window and is not returned (approximate). +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database cagra_postfilter; diff --git a/test/distributed/gpu_cases/vector/vector_cagra_quantization.result b/test/distributed/gpu_cases/vector/vector_cagra_quantization.result index 5fb295dc10b53..1b9462fc33ed4 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_quantization.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_quantization.result @@ -32,7 +32,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_q_f16') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float16","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float16","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 @@ -77,7 +77,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_q_int8') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"int8","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"int8","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 @@ -122,7 +122,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_q_uint8') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"uint8","session_vars":{"cfg":{"cagra_max_index_capacity":{"t":"I","v":99999},"cagra_threads_build":{"t":"I","v":7},"experimental_cagra_index":{"t":"I8","v":1},"lower_case_table_names":{"t":"I","v":1}}}} +cagra ¦ cagra_index ¦ {"distribution_mode":"single","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"uint8","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_cagra_replicated.result b/test/distributed/gpu_cases/vector/vector_cagra_replicated.result index 3c812efe292c7..558f4206f5e66 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_replicated.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_replicated.result @@ -33,7 +33,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_replicated') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"replicated","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32"} +cagra ¦ cagra_index ¦ {"distribution_mode":"replicated","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_cagra_sharded.result b/test/distributed/gpu_cases/vector/vector_cagra_sharded.result index 1ff22c7fa8a39..61548c7deea79 100644 --- a/test/distributed/gpu_cases/vector/vector_cagra_sharded.result +++ b/test/distributed/gpu_cases/vector/vector_cagra_sharded.result @@ -87,7 +87,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='cagra_sharded') and name='ix' and algo_table_type='cagra_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -cagra ¦ cagra_index ¦ {"distribution_mode":"sharded","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32"} +cagra ¦ cagra_index ¦ {"distribution_mode":"sharded","graph_degree":"8","intermediate_graph_degree":"16","itopk_size":"32","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"cagra_threads_build":{"t":"I","v":7},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_gpu_negative.result b/test/distributed/gpu_cases/vector/vector_gpu_negative.result index 5214d70421899..03b13ddea0c79 100644 --- a/test/distributed/gpu_cases/vector/vector_gpu_negative.result +++ b/test/distributed/gpu_cases/vector/vector_gpu_negative.result @@ -18,15 +18,49 @@ internal error: invalid op_type. 'vector_l1_ops' create index ix using cagra on t (v) op_type 'vector_bogus_ops'; internal error: invalid op_type. 'vector_bogus_ops' create index ixf using cagra on tf (v) op_type 'vector_l2_ops'; -not supported: Cagra only supports VECF32 column types +not supported: Cagra only supports VECF32 / VECF16 base column types create index ixf using ivfpq on tf (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8; -not supported: IvfPQ only supports VECF32 column types +not supported: IvfPQ only supports VECF32 / VECF16 base column types create index ixq using cagra on t (v) op_type 'vector_l2_ops' QUANTIZATION 'float64'; internal error: invalid quantization. quantization is invalid. f32, f16, int8, uint8 +create index ixbq using cagra on t (v) op_type 'vector_l2_ops' QUANTIZATION 'bf16'; +not supported: Cagra does not support 'bf16' quantization (no GPU bfloat16 storage); use 'float16', 'int8', or 'uint8' +create index ixbq using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'bf16'; +not supported: IvfPQ does not support 'bf16' quantization (no GPU bfloat16 storage); use 'float16', 'int8', or 'uint8' create index ixv using cagra on t (v) op_type 'vector_l2_ops' INCLUDE (lbl); not supported: INCLUDE column 'lbl' has unsupported type VARCHAR (supported: int32, int64, float32, float64) create index ixok using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=8 graph_degree=4 itopk_size=16; select id from t order by l2_distance(v, '[1,2,3]') asc limit 1; invalid input: vector ops between different dimensions (8, 3) is not permitted. +create table tbf (id bigint primary key, v vecbf16(8)); +create index ixbf using cagra on tbf (v) op_type 'vector_l2_ops'; +not supported: Cagra only supports VECF32 / VECF16 base column types +create index ixbf using ivfpq on tbf (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8; +not supported: IvfPQ only supports VECF32 / VECF16 base column types +create table th (id bigint primary key, v vecf16(8)); +create index ixup using cagra on th (v) op_type 'vector_l2_ops' QUANTIZATION 'float32'; +not supported: Cagra QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECF16 (2 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index ixup using ivfpq on th (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'float32'; +not supported: IvfPQ QUANTIZATION 'float32' (4 bytes/element) cannot upcast base column VECF16 (2 bytes/element); use a quantization of equal or smaller width, or omit it to keep the base type +create index ixqi using cagra on t (v) op_type 'vector_ip_ops' QUANTIZATION 'int8'; +not supported: cagra quantization "int8" is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry +create index ixqi using cagra on t (v) op_type 'vector_cosine_ops' QUANTIZATION 'int8'; +not supported: cagra quantization "int8" is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry +create index ixqi using ivfpq on t (v) op_type 'vector_ip_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8'; +not supported: ivfpq quantization "uint8" is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry +create table tre (id bigint primary key, v vecf32(8)); +insert into tre values +(1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), (3, '[3,3,3,3,3,3,3,3]'), +(4, '[4,4,4,4,4,4,4,4]'), (5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), +(7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), (9, '[9,9,9,9,9,9,9,9]'), +(10, '[10,10,10,10,10,10,10,10]'); +create index ixre using cagra on tre (v) op_type 'vector_ip_ops' +intermediate_graph_degree=8 graph_degree=4 itopk_size=16; +alter table tre alter reindex ixre cagra QUANTIZATION 'int8'; +not supported: cagra quantization "int8" is only supported with L2 (op_type 'vector_l2_ops'); the int8/uint8 affine quantizer does not preserve inner-product / cosine geometry +alter table tre alter reindex ixre cagra QUANTIZATION 'bf16'; +not supported: cagra quantization "bf16" (supported: float32, float16, int8, uint8) +alter table tre alter reindex ixre cagra QUANTIZATION 'float64'; +not supported: cagra quantization "float64" (supported: float32, float16, int8, uint8) drop database gpu_negative; diff --git a/test/distributed/gpu_cases/vector/vector_gpu_negative.sql b/test/distributed/gpu_cases/vector/vector_gpu_negative.sql index d6f86f61bd9cf..238a2c7ce9270 100644 --- a/test/distributed/gpu_cases/vector/vector_gpu_negative.sql +++ b/test/distributed/gpu_cases/vector/vector_gpu_negative.sql @@ -9,6 +9,11 @@ -- * op_type 'vector_bogus_ops' — unknown op_type -- * vecf64 column — cuvs has no float64; only VECF32 allowed -- * QUANTIZATION 'float64' — cuvs quantization is f32/f16/int8/uint8 only +-- * QUANTIZATION 'bf16' — no GPU bfloat16 storage; must not silent-fallback to f32 +-- * int8/uint8 + ip/cosine — affine quantizer breaks dot-product/angle (L2-only) at CREATE +-- * REINDEX QUANTIZATION — the (quantization, op_type) pair is gated via the per-algo +-- ValidQuantization hook on the merged config: bad values +-- (bf16/float64) and int8/uint8 on a non-L2 index are rejected -- * dimension mismatch at search — query dim must equal the column dim -- ===================================================================== @@ -42,6 +47,12 @@ create index ixf using ivfpq on tf (v) op_type 'vector_l2_ops' lists=2 m=8 bits_ -- Unsupported QUANTIZATION value. create index ixq using cagra on t (v) op_type 'vector_l2_ops' QUANTIZATION 'float64'; +-- QUANTIZATION 'bf16' has no GPU bfloat16 storage (cuvs has no bfloat16 index or +-- quantizer). It passes the downcast width guard (bf16 is 2 bytes <= f32's 4), +-- so it must be rejected explicitly rather than silently building f32 storage. +create index ixbq using cagra on t (v) op_type 'vector_l2_ops' QUANTIZATION 'bf16'; +create index ixbq using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'bf16'; + -- VARCHAR is not a supported INCLUDE column type (only int32/int64/float32/float64). create index ixv using cagra on t (v) op_type 'vector_l2_ops' INCLUDE (lbl); @@ -50,4 +61,45 @@ create index ixok using cagra on t (v) op_type 'vector_l2_ops' intermediate_graph_degree=8 graph_degree=4 itopk_size=16; select id from t order by l2_distance(v, '[1,2,3]') asc limit 1; +-- Base-column type guard: only vecf32 / vecf16 are valid base columns; +-- vecbf16 (like int8/uint8) is rejected. +create table tbf (id bigint primary key, v vecbf16(8)); +create index ixbf using cagra on tbf (v) op_type 'vector_l2_ops'; +create index ixbf using ivfpq on tbf (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8; + +-- QUANTIZATION is downcast-only: a vecf16 base (2 bytes/element) cannot be +-- upcast to float32 storage (4 bytes/element). +create table th (id bigint primary key, v vecf16(8)); +create index ixup using cagra on th (v) op_type 'vector_l2_ops' QUANTIZATION 'float32'; +create index ixup using ivfpq on th (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'float32'; + +-- int8/uint8 QUANTIZATION is L2-only. The scalar quantizer applies a per-element +-- affine map q(x)=scalar*x+offset; the constant offset is a translation that +-- cancels in an L2 difference but NOT in a dot product (biases IP by component +-- sum) or norm (rotates cosine angles). So int8/uint8 + inner-product / cosine +-- returns wrong rankings and is rejected. (L2 is fine; the scale is corrected +-- on search.) +create index ixqi using cagra on t (v) op_type 'vector_ip_ops' QUANTIZATION 'int8'; +create index ixqi using cagra on t (v) op_type 'vector_cosine_ops' QUANTIZATION 'int8'; +create index ixqi using ivfpq on t (v) op_type 'vector_ip_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8'; + +-- REINDEX gates the (quantization, op_type) pair through the per-algo +-- ValidQuantization hook, evaluated on the MERGED config: the value must be a +-- cuvs storage name (float32/float16/int8/uint8 — bf16/float64 rejected), and +-- int8/uint8 require L2. op_type is immutable across a reindex, so the merged +-- op_type is the index's stored inner-product — hence int8 is rejected here too. +-- Built on its own table, since t already has a CAGRA index on v and two CAGRA +-- indexes may not share a column. +create table tre (id bigint primary key, v vecf32(8)); +insert into tre values + (1, '[1,1,1,1,1,1,1,1]'), (2, '[2,2,2,2,2,2,2,2]'), (3, '[3,3,3,3,3,3,3,3]'), + (4, '[4,4,4,4,4,4,4,4]'), (5, '[5,5,5,5,5,5,5,5]'), (6, '[6,6,6,6,6,6,6,6]'), + (7, '[7,7,7,7,7,7,7,7]'), (8, '[8,8,8,8,8,8,8,8]'), (9, '[9,9,9,9,9,9,9,9]'), + (10, '[10,10,10,10,10,10,10,10]'); +create index ixre using cagra on tre (v) op_type 'vector_ip_ops' + intermediate_graph_degree=8 graph_degree=4 itopk_size=16; +alter table tre alter reindex ixre cagra QUANTIZATION 'int8'; +alter table tre alter reindex ixre cagra QUANTIZATION 'bf16'; +alter table tre alter reindex ixre cagra QUANTIZATION 'float64'; + drop database gpu_negative; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_f16.result b/test/distributed/gpu_cases/vector/vector_ivfpq_f16.result new file mode 100644 index 0000000000000..540d9e48a0097 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_f16.result @@ -0,0 +1,122 @@ +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET kmeans_max_iteration = 12; +SET probe_limit = 16; +drop database if exists ivfpq_f16_direct; +create database ivfpq_f16_direct; +use ivfpq_f16_direct; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using ivfpq on t (v) +op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 10 m = 8 op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' bits_per_code = 8 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database ivfpq_f16_direct; +drop database if exists ivfpq_f16_int8; +create database ivfpq_f16_int8; +use ivfpq_f16_int8; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using ivfpq on t (v) +op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8 +QUANTIZATION 'int8'; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 10 m = 8 op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' bits_per_code = 8 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database ivfpq_f16_int8; +drop database if exists ivfpq_f16_uint8; +create database ivfpq_f16_uint8; +use ivfpq_f16_uint8; +create table t (id bigint primary key, v vecf16(8)); +insert into t values +( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), +( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), +( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), +( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), +( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), +(11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), +(13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), +(15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), +(17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), +(19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); +create index ix using ivfpq on t (v) +op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8 +QUANTIZATION 'uint8'; +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 10 m = 8 op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' bits_per_code = 8 +) +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +1 +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +10 +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +15 +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; +➤ id[-5,64,0] 𝄀 +20 +drop database ivfpq_f16_uint8; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_f16.sql b/test/distributed/gpu_cases/vector/vector_ivfpq_f16.sql new file mode 100644 index 0000000000000..74bf43f9d7b76 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_f16.sql @@ -0,0 +1,123 @@ +-- ===================================================================== +-- vector_ivfpq_f16.sql — IVF-PQ over a vecf16 (half) BASE column +-- +-- GPU REQUIRED. Unlike vector_ivfpq_quantization.sql (vecf32 base, the +-- QUANTIZATION clause only changes internal storage), here the COLUMN itself +-- is vecf16 — the native base/query type is half end-to-end: +-- * direct — no QUANTIZATION: the index stores half natively (Q == base). +-- * int8 — vecf16 base quantized half->int8 via the native half-source +-- scalar quantizer (no f32 detour). +-- * uint8 — same, half->uint8. +-- +-- Three databases, one per storage. Each builds a sync IVF-PQ index and +-- asserts (a) the vecf16 column + index round-trip through SHOW CREATE TABLE / +-- the catalog and (b) exact-match search returns the right row. The query +-- literal is cast to vecf16(8) so the half query path is exercised. +-- +-- Determinism: integers 1..20 — every value is exact in half, and the int8/ +-- uint8 quantizer trains on [1,20] so each integer maps to a distinct level; +-- the exact-match probe is always the unique top-1. Do not widen the range +-- under int8/uint8 (adjacent levels would collapse). +-- ===================================================================== + +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET kmeans_max_iteration = 12; +SET probe_limit = 16; + +-- ===================================================================== +-- vecf16 base, direct (no QUANTIZATION — stored as half) +-- ===================================================================== +drop database if exists ivfpq_f16_direct; +create database ivfpq_f16_direct; +use ivfpq_f16_direct; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using ivfpq on t (v) + op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database ivfpq_f16_direct; + +-- ===================================================================== +-- vecf16 base, QUANTIZATION int8 (native half->int8) +-- ===================================================================== +drop database if exists ivfpq_f16_int8; +create database ivfpq_f16_int8; +use ivfpq_f16_int8; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using ivfpq on t (v) + op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8 + QUANTIZATION 'int8'; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database ivfpq_f16_int8; + +-- ===================================================================== +-- vecf16 base, QUANTIZATION uint8 (native half->uint8) +-- ===================================================================== +drop database if exists ivfpq_f16_uint8; +create database ivfpq_f16_uint8; +use ivfpq_f16_uint8; + +create table t (id bigint primary key, v vecf16(8)); +insert into t values + ( 1, '[1,1,1,1,1,1,1,1]'), ( 2, '[2,2,2,2,2,2,2,2]'), + ( 3, '[3,3,3,3,3,3,3,3]'), ( 4, '[4,4,4,4,4,4,4,4]'), + ( 5, '[5,5,5,5,5,5,5,5]'), ( 6, '[6,6,6,6,6,6,6,6]'), + ( 7, '[7,7,7,7,7,7,7,7]'), ( 8, '[8,8,8,8,8,8,8,8]'), + ( 9, '[9,9,9,9,9,9,9,9]'), (10, '[10,10,10,10,10,10,10,10]'), + (11, '[11,11,11,11,11,11,11,11]'), (12, '[12,12,12,12,12,12,12,12]'), + (13, '[13,13,13,13,13,13,13,13]'), (14, '[14,14,14,14,14,14,14,14]'), + (15, '[15,15,15,15,15,15,15,15]'), (16, '[16,16,16,16,16,16,16,16]'), + (17, '[17,17,17,17,17,17,17,17]'), (18, '[18,18,18,18,18,18,18,18]'), + (19, '[19,19,19,19,19,19,19,19]'), (20, '[20,20,20,20,20,20,20,20]'); + +create index ix using ivfpq on t (v) + op_type 'vector_l2_ops' lists=10 m=8 bits_per_code=8 + QUANTIZATION 'uint8'; + +show create table t; +select id from t order by l2_distance(v, cast('[1,1,1,1,1,1,1,1]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[10,10,10,10,10,10,10,10]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[15,15,15,15,15,15,15,15]' as vecf16(8))) limit 1; +select id from t order by l2_distance(v, cast('[20,20,20,20,20,20,20,20]' as vecf16(8))) limit 1; + +drop database ivfpq_f16_uint8; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_filter.result b/test/distributed/gpu_cases/vector/vector_ivfpq_filter.result index f9ff94aec9cea..2ff55e1f04494 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_filter.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_filter.result @@ -46,7 +46,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_filter') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","included_columns":"c_i32,c_i64,c_f32,c_f64","lists":"2","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","included_columns":"c_i32,c_i64,c_f32,c_f64","lists":"2","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; ➤ id[-5,64,0] 𝄀 9 diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.result b/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.result new file mode 100644 index 0000000000000..e057a9306517d --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.result @@ -0,0 +1,347 @@ +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET probe_limit = 16; +drop database if exists ivfpq_fq_f16; +create database ivfpq_fq_f16; +use ivfpq_fq_f16; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'float16' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'float16' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_f16; +drop database if exists ivfpq_fq_int8; +create database ivfpq_fq_int8; +use ivfpq_fq_int8; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_int8; +drop database if exists ivfpq_fq_uint8; +create database ivfpq_fq_uint8; +use ivfpq_fq_uint8; +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_uint8; +drop database if exists ivfpq_fq_f16base; +create database ivfpq_fq_f16base; +use ivfpq_fq_f16base; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_f16base; +drop database if exists ivfpq_fq_f16int8; +create database ivfpq_fq_f16int8; +use ivfpq_fq_f16int8; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'int8' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_f16int8; +drop database if exists ivfpq_fq_f16uint8; +create database ivfpq_fq_f16uint8; +use ivfpq_fq_f16uint8; +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), +(2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), +(3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), +(4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), +(5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), +(6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), +(7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), +(8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), +(9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), +(10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), +(11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), +(12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), +(13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), +(14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), +(15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), +(16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), +(17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), +(18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), +(19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), +(20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf16(8) DEFAULT NULL, + `c_i32` int DEFAULT NULL, + `c_i64` bigint DEFAULT NULL, + `c_f32` float DEFAULT NULL, + `c_f64` double DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'uint8' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_i32, c_i64, c_f32, c_f64) +) +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +16 +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +5 +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +12 +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +➤ id[-5,64,0] 𝄀 +9 +drop database ivfpq_fq_f16uint8; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.sql b/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.sql new file mode 100644 index 0000000000000..c208b1e2d38e3 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_filter_quant.sql @@ -0,0 +1,297 @@ +-- ===================================================================== +-- vector_ivfpq_filter_quant.sql — IVFPQ INCLUDE-column pre-filter combined +-- with quantization and a vecf16 base column. +-- +-- GPU REQUIRED. vector_ivfpq_filter.sql already covers the INCLUDE pre-filter +-- over a plain vecf32 base (quantization 'float32'). This file proves the SAME +-- predsJSON pre-filter path stays correct when the index storage is compressed +-- or the base column is half: +-- * f32 base + QUANTIZATION 'float16' — supported (query quantized to T) +-- * f32 base + QUANTIZATION 'int8' — supported (learned scalar quantizer) +-- * f32 base + QUANTIZATION 'uint8' — supported +-- * vecf16 base, direct (no QUANTIZATION) — supported (native half query) +-- * vecf16 base + QUANTIZATION 'int8' + filter — supported (the native half +-- query is quantized to int8 inside +-- cuVS via search_quantize_with_filter) +-- * vecf16 base + QUANTIZATION 'uint8' + filter — supported (same path) +-- Every storage routes the SAME predsJSON pre-filter through the const-B* +-- quantize search, so the expected nearest neighbor per predicate is identical. +-- +-- Data/predicates are identical to vector_ivfpq_filter.sql so the expected +-- nearest neighbor per predicate is unchanged across every storage: +-- id=i -> [i]*8; c_i32=i, c_i64=i*10, c_f32=i.25, c_f64=i.5 (all monotone). +-- Query [12]*8: +-- * c_i32 < 10 -> id 9 +-- * c_i64 >= 100 -> id 12 +-- * c_f32 > 15.25 -> id 16 +-- * c_f64 = 5.5 -> id 5 +-- * c_i32 >= 10 AND c_f64 < 15.5 -> id 12 +-- * c_i64 < 100 AND c_f32 > 5.25 -> id 9 +-- +-- Determinism note: integers 1..20 are exact in float16 and the int8/uint8 +-- quantizer trains on [1,20] so each integer maps to a distinct level; each +-- predicate band keeps a unique nearest. Do NOT widen the range under int8/ +-- uint8 (adjacent levels would collapse and the top-1 would become ambiguous). +-- ===================================================================== + +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET probe_limit = 16; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'float16' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists ivfpq_fq_f16; +create database ivfpq_fq_f16; +use ivfpq_fq_f16; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'float16' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database ivfpq_fq_f16; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'int8' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists ivfpq_fq_int8; +create database ivfpq_fq_int8; +use ivfpq_fq_int8; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database ivfpq_fq_int8; + +-- ===================================================================== +-- f32 base + QUANTIZATION 'uint8' + INCLUDE pre-filter +-- ===================================================================== +drop database if exists ivfpq_fq_uint8; +create database ivfpq_fq_uint8; +use ivfpq_fq_uint8; + +create table t (id bigint primary key, v vecf32(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database ivfpq_fq_uint8; + +-- ===================================================================== +-- vecf16 base, direct (no QUANTIZATION) + INCLUDE pre-filter +-- The query literal is cast to vecf16(8) so the half query path is exercised. +-- ===================================================================== +drop database if exists ivfpq_fq_f16base; +create database ivfpq_fq_f16base; +use ivfpq_fq_f16base; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database ivfpq_fq_f16base; + +-- ===================================================================== +-- vecf16 base + QUANTIZATION 'int8' + INCLUDE pre-filter +-- The native half query is quantized to int8 inside cuVS (the const-B* +-- search_quantize_with_filter path); same predicates and nearest neighbors +-- as every storage above. +-- ===================================================================== +drop database if exists ivfpq_fq_f16int8; +create database ivfpq_fq_f16int8; +use ivfpq_fq_f16int8; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'int8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database ivfpq_fq_f16int8; + +-- ===================================================================== +-- vecf16 base + QUANTIZATION 'uint8' + INCLUDE pre-filter (same path as int8) +-- ===================================================================== +drop database if exists ivfpq_fq_f16uint8; +create database ivfpq_fq_f16uint8; +use ivfpq_fq_f16uint8; + +create table t (id bigint primary key, v vecf16(8), c_i32 int, c_i64 bigint, c_f32 float, c_f64 double); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 10, 1.25, 1.5), + (2, '[2,2,2,2,2,2,2,2]', 2, 20, 2.25, 2.5), + (3, '[3,3,3,3,3,3,3,3]', 3, 30, 3.25, 3.5), + (4, '[4,4,4,4,4,4,4,4]', 4, 40, 4.25, 4.5), + (5, '[5,5,5,5,5,5,5,5]', 5, 50, 5.25, 5.5), + (6, '[6,6,6,6,6,6,6,6]', 6, 60, 6.25, 6.5), + (7, '[7,7,7,7,7,7,7,7]', 7, 70, 7.25, 7.5), + (8, '[8,8,8,8,8,8,8,8]', 8, 80, 8.25, 8.5), + (9, '[9,9,9,9,9,9,9,9]', 9, 90, 9.25, 9.5), + (10, '[10,10,10,10,10,10,10,10]', 10, 100, 10.25, 10.5), + (11, '[11,11,11,11,11,11,11,11]', 11, 110, 11.25, 11.5), + (12, '[12,12,12,12,12,12,12,12]', 12, 120, 12.25, 12.5), + (13, '[13,13,13,13,13,13,13,13]', 13, 130, 13.25, 13.5), + (14, '[14,14,14,14,14,14,14,14]', 14, 140, 14.25, 14.5), + (15, '[15,15,15,15,15,15,15,15]', 15, 150, 15.25, 15.5), + (16, '[16,16,16,16,16,16,16,16]', 16, 160, 16.25, 16.5), + (17, '[17,17,17,17,17,17,17,17]', 17, 170, 17.25, 17.5), + (18, '[18,18,18,18,18,18,18,18]', 18, 180, 18.25, 18.5), + (19, '[19,19,19,19,19,19,19,19]', 19, 190, 19.25, 19.5), + (20, '[20,20,20,20,20,20,20,20]', 20, 200, 20.25, 20.5); + +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 QUANTIZATION 'uint8' INCLUDE (c_i32, c_i64, c_f32, c_f64); + +show create table t; +select id from t where c_i32 < 10 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 >= 100 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f32 > 15.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_f64 = 5.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i32 >= 10 and c_f64 < 15.5 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; +select id from t where c_i64 < 100 and c_f32 > 5.25 order by l2_distance(v, cast('[12,12,12,12,12,12,12,12]' as vecf16(8))) asc limit 1; + +drop database ivfpq_fq_f16uint8; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_metric.result b/test/distributed/gpu_cases/vector/vector_ivfpq_metric.result index 2b22fc117e0e7..4323a72d60785 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_metric.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_metric.result @@ -25,7 +25,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_metric') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id, l2_distance(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ 0.0 @@ -36,7 +36,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_metric') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_l2sq_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_l2sq_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id, l2_distance_sq(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,54,0] 𝄀 1 ¦ 0.0 @@ -47,7 +47,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_metric') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_ip_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_ip_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id, inner_product(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ -1292.0 @@ -58,7 +58,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_metric') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_cosine_ops","quantization":"float32","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":20},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"2","m":"8","op_type":"vector_cosine_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id, cosine_distance(v, '[16,15,14,13,12,11,10,9]') as score from t order by score asc limit 1; ➤ id[-5,64,0] ¦ score[8,9,0] 𝄀 1 ¦ 0.0 diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.result b/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.result new file mode 100644 index 0000000000000..c6e69e9be959f --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.result @@ -0,0 +1,91 @@ +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET probe_limit = 16; +drop database if exists ivfpq_postfilter; +create database ivfpq_postfilter; +use ivfpq_postfilter; +create table t (id bigint primary key, v vecf32(8), c_inc int, c_noinc int); +insert into t values +(1, '[1,1,1,1,1,1,1,1]', 1, 101), +(2, '[2,2,2,2,2,2,2,2]', 2, 102), +(3, '[3,3,3,3,3,3,3,3]', 3, 103), +(4, '[4,4,4,4,4,4,4,4]', 4, 104), +(5, '[5,5,5,5,5,5,5,5]', 5, 105), +(6, '[6,6,6,6,6,6,6,6]', 6, 106), +(7, '[7,7,7,7,7,7,7,7]', 7, 107), +(8, '[8,8,8,8,8,8,8,8]', 8, 108), +(9, '[9,9,9,9,9,9,9,9]', 9, 109), +(10, '[10,10,10,10,10,10,10,10]', 10, 110), +(11, '[11,11,11,11,11,11,11,11]', 11, 111), +(12, '[12,12,12,12,12,12,12,12]', 12, 112), +(13, '[13,13,13,13,13,13,13,13]', 13, 113), +(14, '[14,14,14,14,14,14,14,14]', 14, 114), +(15, '[15,15,15,15,15,15,15,15]', 15, 115), +(16, '[16,16,16,16,16,16,16,16]', 16, 116), +(17, '[17,17,17,17,17,17,17,17]', 17, 117), +(18, '[18,18,18,18,18,18,18,18]', 18, 118), +(19, '[19,19,19,19,19,19,19,19]', 19, 119), +(20, '[20,20,20,20,20,20,20,20]', 20, 120); +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 INCLUDE (c_inc); +show create table t; +➤ Table[12,-1,0] ¦ Create Table[12,-1,0] 𝄀 +t ¦ CREATE TABLE `t` ( + `id` bigint NOT NULL, + `v` vecf32(8) DEFAULT NULL, + `c_inc` int DEFAULT NULL, + `c_noinc` int DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `ix` USING ivfpq (`v`) lists = 2 m = 8 op_type 'vector_l2_ops' quantization 'float32' distribution_mode 'single' bits_per_code = 8 INCLUDE (c_inc) +) +select id, c_noinc from t order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] ¦ c_noinc[4,32,0] 𝄀 +12 ¦ 112 𝄀 +13 ¦ 113 𝄀 +11 ¦ 111 𝄀 +10 ¦ 110 𝄀 +14 ¦ 114 𝄀 +9 ¦ 109 𝄀 +15 ¦ 115 𝄀 +16 ¦ 116 𝄀 +8 ¦ 108 𝄀 +7 ¦ 107 𝄀 +17 ¦ 117 𝄀 +6 ¦ 106 𝄀 +18 ¦ 118 𝄀 +5 ¦ 105 𝄀 +19 ¦ 119 𝄀 +20 ¦ 120 𝄀 +4 ¦ 104 𝄀 +3 ¦ 103 𝄀 +2 ¦ 102 𝄀 +1 ¦ 101 +select id from t where c_noinc < 105 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +4 𝄀 +3 𝄀 +2 𝄀 +1 +select id from t where c_noinc >= 116 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +16 𝄀 +17 𝄀 +18 𝄀 +19 𝄀 +20 +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +2 +select id from t where c_inc <= 14 and c_noinc >= 108 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +➤ id[-5,64,0] 𝄀 +12 𝄀 +13 𝄀 +11 𝄀 +10 𝄀 +14 𝄀 +9 𝄀 +8 +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; +➤ id[-5,64,0] +drop database ivfpq_postfilter; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.sql b/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.sql new file mode 100644 index 0000000000000..9b8a648ec70f3 --- /dev/null +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_postfilter.sql @@ -0,0 +1,77 @@ +-- ===================================================================== +-- vector_ivfpq_postfilter.sql — IVF-PQ search filtering on a NON-INCLUDE column +-- +-- GPU REQUIRED. A WHERE predicate on a column that is NOT in the index INCLUDE +-- list cannot be pushed into the GPU bitset pre-filter. Instead the planner runs +-- the ANN search to get a candidate window, then JOINs + filters the predicate +-- at the database (post-filter): ivfpq_search (candidate window) INNER JOIN +-- (table scan Filter: ) -> Sort -> Limit. +-- +-- Methodology: first take the UNFILTERED ranked result, then verify the +-- post-filtered result equals exactly the unfiltered rows that satisfy the +-- predicate. The candidate window grows with the query LIMIT, so a LIMIT >= row +-- count makes the window cover every row -> the post-filter is exact. +-- +-- Data: id=i -> [i]*8; c_inc=i (INCLUDE, int), c_noinc=100+i (NOT included). +-- Query [12]*8. With LIMIT 20 the window is all 20 rows, so post-filter is exact: +-- * c_noinc < 105 -> i in 1..4 -> 4,3,2,1 (nearest-first) +-- * c_noinc >= 116 -> i in 16..20 -> 16,17,18,19,20 +-- * c_noinc = 102 -> id 2 (far row still found, full window) +-- * c_inc <= 14 (PRE) AND c_noinc >= 108 (POST) -> i in 8..14 +-- ===================================================================== + +SET experimental_ivfpq_index = 1; +SET ivfpq_threads_build = 6; +SET ivfpq_max_index_capacity = 99999; +SET kmeans_train_percent = 100; +SET probe_limit = 16; + +drop database if exists ivfpq_postfilter; +create database ivfpq_postfilter; +use ivfpq_postfilter; + +create table t (id bigint primary key, v vecf32(8), c_inc int, c_noinc int); +insert into t values + (1, '[1,1,1,1,1,1,1,1]', 1, 101), + (2, '[2,2,2,2,2,2,2,2]', 2, 102), + (3, '[3,3,3,3,3,3,3,3]', 3, 103), + (4, '[4,4,4,4,4,4,4,4]', 4, 104), + (5, '[5,5,5,5,5,5,5,5]', 5, 105), + (6, '[6,6,6,6,6,6,6,6]', 6, 106), + (7, '[7,7,7,7,7,7,7,7]', 7, 107), + (8, '[8,8,8,8,8,8,8,8]', 8, 108), + (9, '[9,9,9,9,9,9,9,9]', 9, 109), + (10, '[10,10,10,10,10,10,10,10]', 10, 110), + (11, '[11,11,11,11,11,11,11,11]', 11, 111), + (12, '[12,12,12,12,12,12,12,12]', 12, 112), + (13, '[13,13,13,13,13,13,13,13]', 13, 113), + (14, '[14,14,14,14,14,14,14,14]', 14, 114), + (15, '[15,15,15,15,15,15,15,15]', 15, 115), + (16, '[16,16,16,16,16,16,16,16]', 16, 116), + (17, '[17,17,17,17,17,17,17,17]', 17, 117), + (18, '[18,18,18,18,18,18,18,18]', 18, 118), + (19, '[19,19,19,19,19,19,19,19]', 19, 119), + (20, '[20,20,20,20,20,20,20,20]', 20, 120); + +-- Only c_inc is pushed into the GPU pre-filter; c_noinc is post-filtered. +create index ix using ivfpq on t (v) op_type 'vector_l2_ops' lists=2 m=8 bits_per_code=8 INCLUDE (c_inc); + +show create table t; + +-- (1) UNFILTERED ranked baseline (window covers all rows at LIMIT 20). +select id, c_noinc from t order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (2) POST-FILTER on the non-INCLUDE column — must equal the baseline rows that +-- satisfy the predicate, in the same distance order. +select id from t where c_noinc < 105 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +select id from t where c_noinc >= 116 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (3) MIXED: c_inc is pushed (pre-filter), c_noinc is post-filtered. Both apply. +select id from t where c_inc <= 14 and c_noinc >= 108 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 20; + +-- (4) Small LIMIT shrinks the candidate window: a far post-filter match (id 2, +-- c_noinc=102) falls outside the window and is not returned (approximate). +select id from t where c_noinc = 102 order by l2_distance(v, '[12,12,12,12,12,12,12,12]') asc limit 1; + +drop database ivfpq_postfilter; diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_quantization.result b/test/distributed/gpu_cases/vector/vector_ivfpq_quantization.result index 0e3d09db10d83..8bc834c012133 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_quantization.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_quantization.result @@ -35,7 +35,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_q_f16') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float16","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":12},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float16","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 @@ -80,7 +80,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_q_int8') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"int8","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":12},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"int8","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 @@ -125,7 +125,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_q_uint8') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"uint8","session_vars":{"cfg":{"experimental_ivfpq_index":{"t":"I8","v":1},"ivfpq_max_index_capacity":{"t":"I","v":99999},"ivfpq_threads_build":{"t":"I","v":6},"kmeans_max_iteration":{"t":"I","v":12},"kmeans_train_percent":{"t":"F","v":100},"lower_case_table_names":{"t":"I","v":1}}}} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"single","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"uint8","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_replicated.result b/test/distributed/gpu_cases/vector/vector_ivfpq_replicated.result index f3e211c4185b1..c446eed512e7f 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_replicated.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_replicated.result @@ -36,7 +36,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_replicated') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"replicated","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float32"} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"replicated","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/gpu_cases/vector/vector_ivfpq_sharded.result b/test/distributed/gpu_cases/vector/vector_ivfpq_sharded.result index 58f990e2fcda9..8bf09f422700e 100644 --- a/test/distributed/gpu_cases/vector/vector_ivfpq_sharded.result +++ b/test/distributed/gpu_cases/vector/vector_ivfpq_sharded.result @@ -90,7 +90,7 @@ where table_id = (select rel_id from mo_catalog.mo_tables where relname='t' and reldatabase='ivfpq_sharded') and name='ix' and algo_table_type='ivfpq_index'; ➤ algo[12,-1,0] ¦ algo_table_type[12,-1,0] ¦ algo_params[12,-1,0] 𝄀 -ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"sharded","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float32"} +ivfpq ¦ ivfpq_index ¦ {"bits_per_code":"8","distribution_mode":"sharded","lists":"10","m":"8","op_type":"vector_l2_ops","quantization":"float32","session_vars":{"cfg":{"ivfpq_threads_build":{"t":"I","v":6},"lower_case_table_names":{"t":"I","v":1}}}} select id from t order by l2_distance(v, '[1,1,1,1,1,1,1,1]') limit 1; ➤ id[-5,64,0] 𝄀 1 diff --git a/test/distributed/resources/load_data/narrow_vec_array.csv b/test/distributed/resources/load_data/narrow_vec_array.csv new file mode 100644 index 0000000000000..802127603ecab --- /dev/null +++ b/test/distributed/resources/load_data/narrow_vec_array.csv @@ -0,0 +1,3 @@ +id,a,b,c,d +1,"[1, 2, 3]","[0.5, 0.25, -0.5]","[-128, 0, 127]","[0, 128, 255]" +2,"[0.5, -0.25, 4]","[1, 2, 3]","[10, -10, 5]","[1, 2, 3]" diff --git a/test/distributed/resources/load_data/narrow_vec_dim_bad.csv b/test/distributed/resources/load_data/narrow_vec_dim_bad.csv new file mode 100644 index 0000000000000..36c7272447d8b --- /dev/null +++ b/test/distributed/resources/load_data/narrow_vec_dim_bad.csv @@ -0,0 +1,2 @@ +id,d +9,"[1, 2]" diff --git a/test/distributed/resources/load_data/narrow_vec_int8_frac.csv b/test/distributed/resources/load_data/narrow_vec_int8_frac.csv new file mode 100644 index 0000000000000..59001a14f0553 --- /dev/null +++ b/test/distributed/resources/load_data/narrow_vec_int8_frac.csv @@ -0,0 +1,2 @@ +id,c +9,"[0.5, 0, 0]" diff --git a/test/distributed/resources/load_data/narrow_vec_int8_oor.csv b/test/distributed/resources/load_data/narrow_vec_int8_oor.csv new file mode 100644 index 0000000000000..dff5c34d3181f --- /dev/null +++ b/test/distributed/resources/load_data/narrow_vec_int8_oor.csv @@ -0,0 +1,2 @@ +id,c +9,"[200, 0, 0]"