Append to icechunk stores

conftest.py

@@ -35,6 +35,30 @@ def netcdf4_file(tmpdir):
     return filepath
 
 
+@pytest.fixture
+def compressed_netcdf4_files(tmpdir):
+    # without chunks={} we get a compression error: zlib.error: Error -3 while decompressing data: incorrect header check
+    ds = xr.tutorial.open_dataset("air_temperature", chunks={})
+    ds1 = ds.isel(time=slice(None, 1460))
+    ds2 = ds.isel(time=slice(1460, None))
+    # Define compression options for NetCDF
+    encoding = {
+        # without encoding the chunksizes, irregular ones are chosen
+        var: dict(zlib=True, complevel=4, chunksizes=(1460, 25, 53))
+        for var in ds.data_vars
+    }
+
+    # Save it to disk as netCDF (in temporary directory)
+    filepath1 = f"{tmpdir}/air1_compressed.nc"
+    filepath2 = f"{tmpdir}/air2_compressed.nc"
+    ds1.to_netcdf(filepath1, encoding=encoding, engine="h5netcdf")
+    ds2.to_netcdf(filepath2, encoding=encoding, engine="h5netcdf")
+    ds1.close()
+    ds2.close()
+
+    return filepath1, filepath2
+
+
 @pytest.fixture
 def netcdf4_file_with_2d_coords(tmpdir):
     ds = xr.tutorial.open_dataset("ROMS_example")

virtualizarr/accessor.py

@@ -1,10 +1,5 @@
 from pathlib import Path
-from typing import (
-    TYPE_CHECKING,
-    Callable,
-    Literal,
-    overload,
-)
+from typing import TYPE_CHECKING, Callable, Literal, Optional, overload
 
 from xarray import Dataset, register_dataset_accessor
 
@@ -43,7 +38,9 @@ def to_zarr(self, storepath: str) -> None:
         """
         dataset_to_zarr(self.ds, storepath)
 
-    def to_icechunk(self, store: "IcechunkStore") -> None:
+    def to_icechunk(
+        self, store: "IcechunkStore", append_dim: Optional[str] = None
+    ) -> None:
         """
         Write an xarray dataset to an Icechunk store.
 
@@ -55,7 +52,7 @@ def to_icechunk(self, store: "IcechunkStore") -> None:
         """
         from virtualizarr.writers.icechunk import dataset_to_icechunk
 
-        dataset_to_icechunk(self.ds, store)
+        dataset_to_icechunk(self.ds, store, append_dim=append_dim)
 
     @overload
     def to_kerchunk(

virtualizarr/tests/test_writers/test_icechunk.py

@@ -12,7 +12,7 @@
 from zarr import Array, Group, group  # type: ignore[import-untyped]
 
 from virtualizarr.manifests import ChunkManifest, ManifestArray
-from virtualizarr.writers.icechunk import dataset_to_icechunk
+from virtualizarr.writers.icechunk import dataset_to_icechunk, generate_chunk_key
 from virtualizarr.zarr import ZArray
 
 if TYPE_CHECKING:
@@ -127,7 +127,7 @@ def test_set_single_virtual_ref_with_encoding(
     # vds = open_virtual_dataset(netcdf4_file, indexes={})
 
     expected_ds = open_dataset(netcdf4_file).drop_vars(["lon", "lat", "time"])
-    # these atyttirbutes encode floats different and I am not sure why, but its not important enough to block everything
+    # these attributes encode floats different and I am not sure why, but its not important enough to block everything
     expected_ds.air.attrs.pop("actual_range")
 
     # instead for now just write out byte ranges explicitly
@@ -288,3 +288,45 @@ def test_write_loadable_variable(
 # TODO test writing to a group that isn't the root group
 
 # TODO test with S3 / minio
+
+
+def test_generate_chunk_key_no_offset():
+    # Test case without any offset (append_axis and existing_num_chunks are None)
+    index = (1, 2, 3)
+    result = generate_chunk_key(index)
+    assert result == "1/2/3", "The chunk key should match the index without any offset."
+
+
+def test_generate_chunk_key_with_offset():
+    # Test case with offset on append_axis 1
+    index = (1, 2, 3)
+    append_axis = 1
+    existing_num_chunks = 5
+    result = generate_chunk_key(
+        index, append_axis=append_axis, existing_num_chunks=existing_num_chunks
+    )
+    assert result == "1/7/3", "The chunk key should offset the second index by 5."
+
+
+def test_generate_chunk_key_zero_offset():
+    # Test case where existing_num_chunks is 0 (no offset should be applied)
+    index = (4, 5, 6)
+    append_axis = 1
+    existing_num_chunks = 0
+    result = generate_chunk_key(
+        index, append_axis=append_axis, existing_num_chunks=existing_num_chunks
+    )
+    assert (
+        result == "4/5/6"
+    ), "No offset should be applied when existing_num_chunks is 0."
+
+
+def test_generate_chunk_key_append_axis_out_of_bounds():
+    # Edge case where append_axis is out of bounds
+    index = (3, 4)
+    append_axis = 2  # This is out of bounds for a 2D index
+    with pytest.raises(IndexError):
+        generate_chunk_key(index, append_axis=append_axis, existing_num_chunks=1)
+
+
+# Run tests using pytest