Append to icechunk stores

ci/upstream.yml

@@ -28,6 +28,6 @@ dependencies:
   - fsspec
   - pip
   - pip:
-    - icechunk # Installs zarr v3 as dependency
-    # - git+https://github.com/fsspec/kerchunk@main  # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516)
-    - imagecodecs-numcodecs==2024.6.1
+      - icechunk==0.1.0a7 # Installs zarr v3 as dependency
+      # - git+https://github.com/fsspec/kerchunk@main  # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516)
+      - imagecodecs-numcodecs==2024.6.1

conftest.py

@@ -1,3 +1,5 @@
+from typing import Any, Dict, Optional
+
 import h5py
 import numpy as np
 import pytest
@@ -35,6 +37,32 @@ def netcdf4_file(tmpdir):
     return filepath
 
 
+@pytest.fixture
+def netcdf4_files_factory(tmpdir) -> callable:
+    def create_netcdf4_files(
+        encoding: Optional[Dict[str, Dict[str, Any]]] = None,
+    ) -> tuple[str, str]:
+        ds = xr.tutorial.open_dataset("air_temperature")
+
+        # Split dataset into two parts
+        ds1 = ds.isel(time=slice(None, 1460))
+        ds2 = ds.isel(time=slice(1460, None))
+
+        # Save datasets to disk as NetCDF in the temporary directory with the provided encoding
+        filepath1 = f"{tmpdir}/air1.nc"
+        filepath2 = f"{tmpdir}/air2.nc"
+        ds1.to_netcdf(filepath1, encoding=encoding)
+        ds2.to_netcdf(filepath2, encoding=encoding)
+
+        # Close datasets
+        ds1.close()
+        ds2.close()
+
+        return filepath1, filepath2
+
+    return create_netcdf4_files
+
+
 @pytest.fixture
 def netcdf4_file_with_2d_coords(tmpdir):
     ds = xr.tutorial.open_dataset("ROMS_example")
@@ -71,26 +99,6 @@ def hdf5_groups_file(tmpdir):
     return filepath
 
 
-@pytest.fixture
-def netcdf4_files(tmpdir):
-    # Set up example xarray dataset
-    ds = xr.tutorial.open_dataset("air_temperature")
-
-    # split inrto equal chunks so we can concatenate them back together later
-    ds1 = ds.isel(time=slice(None, 1460))
-    ds2 = ds.isel(time=slice(1460, None))
-
-    # Save it to disk as netCDF (in temporary directory)
-    filepath1 = f"{tmpdir}/air1.nc"
-    filepath2 = f"{tmpdir}/air2.nc"
-    ds1.to_netcdf(filepath1)
-    ds2.to_netcdf(filepath2)
-    ds1.close()
-    ds2.close()
-
-    return filepath1, filepath2
-
-
 @pytest.fixture
 def hdf5_empty(tmpdir):
     filepath = f"{tmpdir}/empty.nc"

docs/releases.rst

@@ -11,6 +11,7 @@ New Features
 
 - Add a ``virtual_backend_kwargs`` keyword argument to file readers and to ``open_virtual_dataset``, to allow reader-specific options to be passed down.
   (:pull:`315`) By `Tom Nicholas <https://github.com/TomNicholas>`_.
+- Added append functionality to `to_icechunk` (:pull:`272`) By `Aimee Barciauskas <https://github.com/abarciauskas-bgse>`_.
 
 Breaking changes
 ~~~~~~~~~~~~~~~~