fix: fix storage options for dataset builder

python/python/tests/test_fragment.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
+import copy
 import json
 import multiprocessing
 import uuid
@@ -23,6 +24,15 @@
 from lance.fragment import write_fragments
 from lance.progress import FileSystemFragmentWriteProgress
 
+CONFIG = {
+    "allow_http": "true",
+    "aws_access_key_id": "ACCESSKEY",
+    "aws_secret_access_key": "SECRETKEY",
+    "aws_endpoint": "http://localhost:9000",
+    "dynamodb_endpoint": "http://localhost:8000",
+    "aws_region": "us-west-2",
+}
+
 
 def test_write_fragment(tmp_path: Path):
     with pytest.raises(OSError):
@@ -354,3 +364,12 @@ def test_create_from_file(tmp_path):
     assert dataset.count_rows() == 1600
     assert len(dataset.get_fragments()) == 1
     assert dataset.get_fragments()[0].fragment_id == 2
+
+
+@pytest.mark.integration
+def test_append(s3_bucket: str):
+    storage_options = copy.deepcopy(CONFIG)
+    table = pa.table({"a": [1, 2], "b": ["a", "b"]})
+    lance.fragment.LanceFragment.create(
+        f"s3://{s3_bucket}/test_append.lance", table, storage_options=storage_options
+    )

rust/lance/src/dataset/fragment/write.rs

@@ -214,7 +214,15 @@ impl<'a> FragmentCreateBuilder<'a> {
     }
 
     async fn existing_dataset_schema(&self) -> Result<Option<Schema>> {
-        match DatasetBuilder::from_uri(self.dataset_uri).load().await {
+        let mut builder = DatasetBuilder::from_uri(self.dataset_uri);
+        let storage_options = self
+            .write_params
+            .and_then(|p| p.store_params.as_ref())
+            .and_then(|p| p.storage_options.clone());
+        if let Some(storage_options) = storage_options {
+            builder = builder.with_storage_options(storage_options);
+        }
+        match builder.load().await {
             Ok(dataset) => {
                 // Use the schema from the dataset, because it has the correct
                 // field ids.