feat: support per-base storage options with base_<id>.<key> prefix

docs/src/guide/object_store.md

@@ -41,6 +41,35 @@ These options apply to all object stores.
 | `client_max_retries`         | Number of times for the object store client to retry the request. Default, `3`.                                                                                                                                                                                                                         |
 | `client_retry_timeout`       | Timeout for the object store client to retry the request in seconds. Default, `180`.                                                                                                                                                                                                                    |
 
+## Per-Base Configuration
+
+A dataset can register additional base paths that store part of its data, and each
+base may live in a different bucket, account, or storage provider. A storage option
+key of the form `base_<id>.<key>` applies `<key>` only to the base path with that
+manifest id. Every base inherits the unscoped options; base-scoped entries add to or
+override them for that base only.
+
+```python
+import lance
+ds = lance.dataset(
+    "az://account-a/path",
+    storage_options={
+        # Shared defaults, used by the primary dataset and inherited by bases
+        "account_name": "account-a",
+        "account_key": "key-a",
+        # Overrides for the base path with id 1
+        "base_1.account_name": "account-b",
+        "base_1.account_key": "key-b",
+    },
+)
+```
+
+Base ids are assigned when bases are registered (`initial_bases` ids are assigned
+sequentially starting at 1, in order) and can be inspected through the manifest base
+paths. Keys that do not match `base_<id>.<key>` exactly (e.g. `base_url`) are treated
+as regular storage options. Exact per-base parameter maps (`base_store_params`,
+keyed by base path URI) take precedence over base-scoped keys for that base.
+
 ## S3 Configuration
 
 S3 (and S3-compatible stores) have additional configuration options that configure

java/src/main/java/org/lance/ReadOptions.java

@@ -189,6 +189,13 @@ public Builder setMetadataCacheSize(int metadataCacheSize) {
      * Set storage options. Extra options that make sense for a particular storage connection. This
      * is used to store connection parameters like credentials, endpoint, etc.
      *
+     * <p>For datasets with additional registered base paths, a key of the form {@code
+     * base_<id>.<key>} applies {@code <key>} only to the base path with that manifest id,
+     * overriding the unscoped options that every base inherits. For example {@code
+     * base_1.account_key = abc} makes base 1 use {@code account_key = abc} while all other options
+     * are shared. Exact per-base bindings set via {@link #setBaseStoreParams(Map)} take precedence
+     * over base-scoped keys.
+     *
      * @param storageOptions the storage options
      * @return this builder
      */

java/src/main/java/org/lance/WriteParams.java

@@ -197,6 +197,17 @@ public Builder withDataStorageVersion(String dataStorageVersion) {
       return this;
     }
 
+    /**
+     * Set storage options for the write.
+     *
+     * <p>For writes involving additional registered base paths, a key of the form {@code
+     * base_<id>.<key>} applies {@code <key>} only to the base path with that id, overriding the
+     * unscoped options that every base inherits. Exact per-base bindings set via {@link
+     * #withBaseStoreParams(Map)} take precedence over base-scoped keys.
+     *
+     * @param storageOptions the storage options
+     * @return this builder
+     */
     public Builder withStorageOptions(Map<String, String> storageOptions) {
       this.storageOptions = storageOptions;
       return this;

python/python/lance/__init__.py

@@ -166,6 +166,13 @@ def dataset(
     storage_options : optional, dict
         Extra options that make sense for a particular storage connection. This is
         used to store connection parameters like credentials, endpoint, etc.
+
+        For datasets with additional registered base paths, a key of the form
+        ``base_<id>.<key>`` applies ``<key>`` only to the base path with that
+        manifest id, overriding the unscoped options that every base inherits.
+        For example ``{"account_key": "shared", "base_1.account_key": "abc"}``
+        makes base 1 use ``account_key = abc`` while all other options are
+        shared.
     default_scan_options : optional, dict
         Default scan options that are used when scanning the dataset.  This accepts
         the same arguments described in :py:meth:`lance.LanceDataset.scanner`.  The
@@ -203,9 +210,10 @@ def dataset(
     base_store_params : dict of str to dict, optional
         Runtime-only object store parameters keyed by base path URI. Each key
         is a base path URI (e.g., "s3://bucket/path") and each value is a dict
-        of storage options (credentials, endpoint, etc.) for that base.  When a base
-        has no explicit entry here, the top-level ``storage_options`` is
-        used as a fallback.
+        of storage options (credentials, endpoint, etc.) for that base.  These
+        take precedence over ``base_<id>.<key>`` entries in ``storage_options``.
+        When a base has no explicit entry here, the top-level
+        ``storage_options`` is used as a fallback.
 
     Notes
     -----

python/python/lance/dataset.py

@@ -7236,6 +7236,13 @@ def write_dataset(
     storage_options : optional, dict
         Extra options that make sense for a particular storage connection. This is
         used to store connection parameters like credentials, endpoint, etc.
+
+        For writes involving additional base paths, a key of the form
+        ``base_<id>.<key>`` applies ``<key>`` only to the base path with that
+        id, overriding the unscoped options that every base inherits. For
+        example ``{"account_key": "shared", "base_1.account_key": "abc"}``
+        makes base 1 use ``account_key = abc`` while all other options are
+        shared.
     data_storage_version: optional, str, default None
         The version of the data storage format to use. Newer versions are more
         efficient but require newer versions of lance to read.  The default (None)
@@ -7291,8 +7298,10 @@ def write_dataset(
         Runtime-only object store parameters keyed by base path URI. Each key
         is a base path URI (e.g., "s3://bucket/path") and each value is a dict
         of storage options (credentials, endpoint, etc.) for that base. These
-        are not persisted to the manifest. When a base has no explicit entry
-        here, the top-level ``storage_options`` is used as a fallback.
+        are not persisted to the manifest. These take precedence over
+        ``base_<id>.<key>`` entries in ``storage_options``. When a base has no
+        explicit entry here, the top-level ``storage_options`` is used as a
+        fallback.
     external_blob_mode: {"reference", "ingest"}, default "reference"
         How external blob URIs are handled on write.
 

python/python/tests/test_multi_base.py

@@ -82,6 +82,40 @@ def test_multi_base_create_and_read(self):
             data.sort_values("id").reset_index(drop=True),
         )
 
+    def test_base_scoped_storage_options(self):
+        """base_<id>.<key> storage options flow through write and read."""
+        data = self.create_test_data(200)
+
+        # Local stores ignore these options; this verifies base-scoped entries
+        # are resolved per base without breaking the write or read path.
+        storage_options = {
+            "shared_option": "shared",
+            "base_1.scoped_option": "base1-value",
+        }
+
+        dataset = lance.write_dataset(
+            data,
+            self.primary_uri,
+            mode="create",
+            initial_bases=[DatasetBasePath(self.path1_uri, name="path1")],
+            target_bases=["path1"],
+            max_rows_per_file=100,
+            storage_options=storage_options,
+        )
+        assert dataset.count_rows() == 200
+
+        # Data files land in the scoped base, not the primary path
+        assert list(Path(self.path1_uri).glob("**/*.lance"))
+        assert not list((Path(self.primary_uri) / "data").glob("*.lance"))
+
+        # Reopen with the same flat options and read through the base store
+        dataset = lance.dataset(self.primary_uri, storage_options=storage_options)
+        result = dataset.to_table().to_pandas()
+        pd.testing.assert_frame_equal(
+            result.sort_values("id").reset_index(drop=True),
+            data.sort_values("id").reset_index(drop=True),
+        )
+
     def test_multi_base_append_mode(self):
         """Test appending data to a multi-base dataset."""
         # Create initial dataset

rust/lance-io/src/object_store.rs

@@ -3,6 +3,7 @@
 
 //! Extend [object_store::ObjectStore] functionalities
 
+use std::borrow::Cow;
 use std::collections::HashMap;
 use std::ops::Range;
 use std::pin::Pin;
@@ -83,8 +84,10 @@ pub const DEFAULT_DOWNLOAD_RETRY_COUNT: usize = 3;
 
 pub use providers::{ObjectStoreProvider, ObjectStoreRegistry};
 pub use storage_options::{
-    EXPIRES_AT_MILLIS_KEY, LanceNamespaceStorageOptionsProvider, REFRESH_OFFSET_MILLIS_KEY,
-    StorageOptionsAccessor, StorageOptionsProvider,
+    BASE_SCOPED_OPTION_PREFIX, BaseScopedStorageOptionsProvider, EXPIRES_AT_MILLIS_KEY,
+    LanceNamespaceStorageOptionsProvider, REFRESH_OFFSET_MILLIS_KEY, StorageOptionsAccessor,
+    StorageOptionsProvider, has_base_scoped_options, parse_base_scoped_key,
+    resolve_base_scoped_options,
 };
 
 #[async_trait]
@@ -258,6 +261,27 @@ impl ObjectStoreParams {
             .as_ref()
             .and_then(|a| a.initial_storage_options())
     }
+
+    /// Resolve these params for a single base path scope.
+    ///
+    /// Storage options may carry base-scoped entries (`base_<id>.<key>`) that
+    /// apply only to one registered base path; see
+    /// [`StorageOptionsAccessor::scoped_to_base`]. Returns the params unchanged
+    /// when the storage options contain no base-scoped entries.
+    pub fn scoped_to_base(&self, base_id: Option<u32>) -> Cow<'_, Self> {
+        let Some(accessor) = &self.storage_options_accessor else {
+            return Cow::Borrowed(self);
+        };
+        let scoped = accessor.scoped_to_base(base_id);
+        if Arc::ptr_eq(&scoped, accessor) {
+            Cow::Borrowed(self)
+        } else {
+            Cow::Owned(Self {
+                storage_options_accessor: Some(scoped),
+                ..self.clone()
+            })
+        }
+    }
 }
 
 // We implement hash for caching

rust/lance-io/src/object_store/providers.rs

@@ -215,6 +215,12 @@ impl ObjectStoreRegistry {
         base_path: Url,
         params: &ObjectStoreParams,
     ) -> Result<Arc<ObjectStore>> {
+        // Base-scoped storage options (`base_<id>.<key>`) are directives for
+        // other registered base paths; resolve them away before building or
+        // caching a store for this location. Params already resolved for a
+        // base contain no scoped entries, so this is a no-op for them.
+        let params = params.scoped_to_base(None);
+        let params = params.as_ref();
         let scheme = base_path.scheme();
         let Some(provider) = self.get_provider(scheme) else {
             return Err(self.scheme_not_found_error(scheme));
@@ -394,6 +400,36 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_get_store_resolves_base_scoped_options() {
+        use crate::object_store::StorageOptionsAccessor;
+
+        let registry = ObjectStoreRegistry::default();
+        let url = Url::parse("memory://test").unwrap();
+
+        let with_scoped = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("shared".to_string(), "value".to_string()),
+                    ("base_1.account_key".to_string(), "base1-key".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+        let without_scoped = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([("shared".to_string(), "value".to_string())]),
+            ))),
+            ..Default::default()
+        };
+
+        // Base-scoped entries are resolved away before the store is built and
+        // cached, so params with and without them yield the same cached store.
+        let store_scoped = registry.get_store(url.clone(), &with_scoped).await.unwrap();
+        let store_plain = registry.get_store(url, &without_scoped).await.unwrap();
+        assert!(Arc::ptr_eq(&store_scoped, &store_plain));
+    }
+
     #[test]
     fn test_calculate_object_store_scheme_not_found() {
         let registry = ObjectStoreRegistry::empty();