feat(index): support raw-query ivf rq search

docs/src/format/index/vector/index.md

@@ -198,6 +198,7 @@ Compresses vectors using RabitQ with random rotation and binary quantization for
 | `_rabit_codes`       | list<uint8>[dimension / 8]                       | false                    | Binary quantized codes (1 bit per dimension, packed into bytes) |
 | `__add_factors`      | float32                                          | false                    | Additive correction factors for distance computation            |
 | `__scale_factors`    | float32                                          | false                    | Scale correction factors for distance computation               |
+| `__error_factors`    | float32                                          | false for `raw_query`    | Error factors for raw-query lower-bound pruning                 |
 | `__ex_codes`         | list<uint8>[ceil(dimension * (num_bits - 1) / 8)] | false for `num_bits > 1` | Extra RabitQ code bits for multi-bit RQ                         |
 | `__add_factors_ex`   | float32                                          | false for `num_bits > 1` | Additive correction factors for ex-code distance computation    |
 | `__scale_factors_ex` | float32                                          | false for `num_bits > 1` | Scale correction factors for ex-code distance computation       |
@@ -254,6 +255,7 @@ For **RabitQ (RQ)**:
 | `num_bits`            | u8   | Number of bits per dimension, in the range 1..=9     |
 | `code_dim`            | u32  | Rotated vector dimension for the 1-bit binary code   |
 | `packed`              | bool | Whether codes are packed for optimized computation   |
+| `query_estimator`     | string | Distance estimator layout: `residual_query` or `raw_query`. Missing values are read as `residual_query` for compatibility with released 1-bit IVF_RQ indexes. |
 
 #### Lance File Global Buffer
 
@@ -279,8 +281,9 @@ to rotate vectors before binary quantization:
 The rotation matrix has shape `[code_dim, code_dim]` where `code_dim` is the rotated vector
 dimension. IVF_RQ always stores the 1-bit binary sign code in `_rabit_codes`; for `num_bits > 1`,
 the remaining `num_bits - 1` ex-code bits are stored in `__ex_codes` instead of widening the
-binary code path. `num_bits=1` indexes only store the binary-code factor columns; multi-bit indexes
-also store separate ex-code additive and scale factors.
+binary code path. New IVF_RQ indexes store raw-query estimator factors. `num_bits=1` indexes only
+store the binary-code factor columns; multi-bit indexes also store separate ex-code additive and
+scale factors.
 
 ## Appendices
 
@@ -345,7 +348,7 @@ auxiliary schema also includes `__ex_codes`, `__add_factors_ex`, and `__scale_fa
 - Arrow Schema Metadata:
   - `"distance_type"` → `"l2"`
   - `"lance:ivf"` → tracks per-partition `offsets` and `lengths` (no centroids here)
-  - `"lance:rabit"` → `"{"rotate_mat_position":1,"num_bits":1,"packed":true}"`
+  - `"lance:rabit"` → `"{"rotate_mat_position":1,"num_bits":1,"packed":true,"query_estimator":"raw_query"}"`
 - Lance File Global buffer:
   - `Tensor` rotation matrix with shape `[code_dim, code_dim]` = `[128, 128]` (float32)
 - Rows with Arrow schema:
@@ -356,6 +359,7 @@ pa.schema([
     pa.field("_rabit_codes", pa.list(pa.uint8(), list_size=16)), # dimension/8 = 128/8 = 16 bytes
     pa.field("__add_factors", pa.float32()),
     pa.field("__scale_factors", pa.float32()),
+    pa.field("__error_factors", pa.float32()),
 ])
 ```
 

python/python/tests/compat/compat_decorator.py

@@ -153,6 +153,10 @@ def skip_read_after_current_write(self, version: str) -> bool:
         """Return True to skip the old-version read after current-version writes."""
         return False
 
+    def skip_write_after_current_write(self, version: str) -> bool:
+        """Return True to skip the old-version write after current-version writes."""
+        return False
+
     def skip_downgrade(self, version: str) -> bool:
         """Return True to skip the current-write -> old-read downgrade test."""
         return False
@@ -333,8 +337,10 @@ def test_func({sig_params}):
         obj.create()
     # Old version: verify can read
     venv = venv_factory.get_venv(version)
-    venv.execute_method(obj, "check_read", obj.compat_env(version, "check_read"))
-    venv.execute_method(obj, "check_write", obj.compat_env(version, "check_write"))
+    if not obj.skip_read_after_current_write(version):
+        venv.execute_method(obj, "check_read", obj.compat_env(version, "check_read"))
+    if not obj.skip_write_after_current_write(version):
+        venv.execute_method(obj, "check_write", obj.compat_env(version, "check_write"))
 '''
     else:  # upgrade_downgrade
         func_body = f'''

python/python/tests/compat/test_vector_indices.py

@@ -274,6 +274,13 @@
             return {"LANCE_COMPAT_CURRENT_RUNTIME": "1"}
         return {}
 
+    def skip_write_after_current_write(self, version: str) -> bool:
+        # Newly written IVF_RQ indexes carry raw-query estimator metadata and
+        # split-code schema that older runtimes can query but cannot optimize.
+        # The upgrade_downgrade variant still covers old 1-bit residual-query
+        # indexes being read and rewritten by the current runtime.
+        return True
+
     def create(self):
         """Create dataset with IVF_RQ vector index."""
         shutil.rmtree(self.path, ignore_errors=True)

python/python/tests/test_vector_index.py

@@ -1067,32 +1067,46 @@ def test_create_ivf_rq_skip_transpose():
     assert stats["indices"][0]["sub_index"]["packed"] is False
 
 
-@pytest.mark.skip(
-    reason=(
-        "IVF_RQ num_bits>1 creation is gated until split-code search support "
-        "is implemented"
-    )
-)
-def test_create_ivf_rq_multi_bit_gates_search():
-    ds = lance.write_dataset(create_table(), "memory://")
+def _assert_recall_at_least(ds, query, metric=None, k=10, recall_requirement=0.5):
+    nearest = {"column": "vector", "q": query, "k": k}
+    if metric is not None:
+        nearest["metric"] = metric
 
-    ds = ds.create_index(
-        "vector",
-        index_type="IVF_RQ",
-        num_partitions=4,
-        num_bits=9,
+    gt_ids = ds.to_table(nearest=nearest, columns=["id"])["id"].to_numpy()
+    create_index_kwargs = {
+        "index_type": "IVF_RQ",
+        "num_partitions": 4,
+        "num_bits": 9,
+    }
+    if metric is not None:
+        create_index_kwargs["metric"] = metric
+    indexed = ds.create_index("vector", **create_index_kwargs)
+    result_ids = indexed.to_table(nearest=nearest, columns=["id"])["id"].to_numpy()
+
+    assert result_ids.shape[0] == k
+    recall = len(set(gt_ids) & set(result_ids)) / k
+    assert recall >= recall_requirement, (
+        f"recall={recall}, gt={gt_ids}, result={result_ids}"
     )
+    return indexed
+
+
+def test_create_ivf_rq_multi_bit_searches_l2_and_cosine():
+    rng = np.random.default_rng(42)
+    mat = rng.standard_normal((1000, 128)).astype(np.float32)
+    tbl = vec_to_table(data=mat).append_column("id", pa.array(range(len(mat))))
+
+    ds = lance.write_dataset(tbl, "memory://")
+    ds = _assert_recall_at_least(ds, mat[0])
     stats = ds.stats.index_stats("vector_idx")
     assert stats["indices"][0]["sub_index"]["num_bits"] == 9
+    assert stats["indices"][0]["sub_index"]["query_estimator"] == "raw_query"
 
-    with pytest.raises(pa.ArrowInvalid, match="num_bits>1 search is not supported"):
-        ds.to_table(
-            nearest={
-                "column": "vector",
-                "q": np.random.randn(128).astype(np.float32),
-                "k": 10,
-            }
-        )
+    cosine_ds = lance.write_dataset(tbl, "memory://")
+    cosine_ds = _assert_recall_at_least(cosine_ds, mat[1], metric="cosine")
+    cosine_stats = cosine_ds.stats.index_stats("vector_idx")
+    assert cosine_stats["indices"][0]["sub_index"]["num_bits"] == 9
+    assert cosine_stats["indices"][0]["sub_index"]["query_estimator"] == "raw_query"
 
 
 def test_create_ivf_rq_requires_dim_divisible_by_8():

rust/lance-index/src/vector/bq.rs

@@ -128,14 +128,7 @@ pub fn validate_rq_num_bits(num_bits: u8) -> Result<()> {
 }
 
 pub fn validate_supported_rq_num_bits(num_bits: u8) -> Result<()> {
-    validate_rq_num_bits(num_bits)?;
-    if num_bits != RABIT_BINARY_NUM_BITS {
-        return Err(Error::not_supported(format!(
-            "IVF_RQ num_bits={} index creation is not supported until split-code search support is implemented",
-            num_bits
-        )));
-    }
-    Ok(())
+    validate_rq_num_bits(num_bits)
 }
 
 pub fn rabit_ex_bits(num_bits: u8) -> Result<u8> {
@@ -261,13 +254,7 @@ mod tests {
         );
 
         validate_supported_rq_num_bits(1).unwrap();
-        let err = validate_supported_rq_num_bits(9).unwrap_err();
-        assert!(
-            err.to_string()
-                .contains("num_bits=9 index creation is not supported"),
-            "{}",
-            err
-        );
+        validate_supported_rq_num_bits(9).unwrap();
     }
 
     #[test]