From d3a4048a7439aac598156fbf335a6bdc4fd44b96 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 25 Oct 2024 09:05:08 -0700 Subject: [PATCH 01/90] Initial attempt at appending --- virtualizarr/writers/icechunk.py | 51 +++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 6dadbc08..6ca97da1 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Optional, cast import numpy as np from xarray import Dataset @@ -24,7 +24,9 @@ } -def dataset_to_icechunk(ds: Dataset, store: "IcechunkStore") -> None: +def dataset_to_icechunk( + ds: Dataset, store: "IcechunkStore", append_dim: Optional[str] = None +) -> None: """ Write an xarray dataset whose variables wrap ManifestArrays to an Icechunk store. @@ -51,7 +53,10 @@ def dataset_to_icechunk(ds: Dataset, store: "IcechunkStore") -> None: # TODO only supports writing to the root group currently # TODO pass zarr_format kwarg? - root_group = Group.from_store(store=store) + if store.mode.str == "a": + root_group = Group.open(store=store, zarr_format=3) + else: + root_group = Group.from_store(store=store) # TODO this is Frozen, the API for setting attributes must be something else # root_group.attrs = ds.attrs @@ -63,6 +68,7 @@ def dataset_to_icechunk(ds: Dataset, store: "IcechunkStore") -> None: ds.attrs, store=store, group=root_group, + append_dim=append_dim, ) @@ -71,6 +77,7 @@ def write_variables_to_icechunk_group( attrs, store, group, + append_dim: Optional[str] = None, ): virtual_variables = { name: var @@ -96,6 +103,7 @@ def write_variables_to_icechunk_group( group=group, name=name, var=var, + append_dim=append_dim, ) @@ -104,6 +112,7 @@ def write_variable_to_icechunk( group: "Group", name: str, var: Variable, + append_dim: Optional[str] = None, ) -> None: """Write a single (possibly virtual) variable into an icechunk store""" if isinstance(var.data, ManifestArray): @@ -112,6 +121,7 @@ def write_variable_to_icechunk( group=group, name=name, var=var, + append_dim=append_dim, ) else: raise ValueError( @@ -124,15 +134,37 @@ def write_virtual_variable_to_icechunk( group: "Group", name: str, var: Variable, + append_dim: Optional[str] = None, ) -> None: """Write a single virtual variable into an icechunk store""" ma = cast(ManifestArray, var.data) zarray = ma.zarray + shape = zarray.shape + mode = store.mode.str + + # Aimee: resize the array if it already exists + # TODO: assert chunking and encoding is the same + existing_keys = tuple(group.array_keys()) + append_axis, existing_num_chunks = None, None + if name in existing_keys and mode == "a": + # resize + dims = var.dims + if append_dim in dims: + append_axis = dims.index(append_dim) + existing_array = group[name] + existing_size = existing_array.shape[append_axis] + existing_num_chunks = int( + existing_size / existing_array.chunks[append_axis] + ) + new_shape = list(existing_array.shape) + new_shape[append_axis] += var.shape[append_axis] + shape = tuple(new_shape) + existing_array.resize(new_shape) # creates array if it doesn't already exist arr = group.require_array( name=name, - shape=zarray.shape, + shape=shape, chunk_shape=zarray.chunks, dtype=encode_dtype(zarray.dtype), codecs=zarray._v3_codec_pipeline(), @@ -142,6 +174,7 @@ def write_virtual_variable_to_icechunk( ) # TODO it would be nice if we could assign directly to the .attrs property + # Aimee: assert that new attributes are the same as existing attributes for k, v in var.attrs.items(): arr.attrs[k] = encode_zarr_attr_value(v) arr.attrs["_ARRAY_DIMENSIONS"] = encode_zarr_attr_value(var.dims) @@ -156,6 +189,8 @@ def write_virtual_variable_to_icechunk( group=group, arr_name=name, manifest=ma.manifest, + append_axis=append_axis, + existing_num_chunks=existing_num_chunks, ) @@ -164,6 +199,8 @@ def write_manifest_virtual_refs( group: "Group", arr_name: str, manifest: ChunkManifest, + append_axis: Optional[int] = None, + existing_num_chunks: Optional[int] = None, ) -> None: """Write all the virtual references for one array manifest at once.""" @@ -181,8 +218,14 @@ def write_manifest_virtual_refs( ], op_flags=[["readonly"]] * 3, # type: ignore ) + for path, offset, length in it: index = it.multi_index + if append_axis is not None: + list_index = list(index) + # Offset by the number of existing chunks on the append axis + list_index[append_axis] += existing_num_chunks + index = tuple(list_index) chunk_key = "/".join(str(i) for i in index) # set each reference individually From 5d5f9e2e1e0d9df297b8a8274075301282edb158 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 25 Oct 2024 16:10:11 -0700 Subject: [PATCH 02/90] Working on tests for generate chunk key function --- .../tests/test_writers/test_icechunk.py | 44 ++++++++++++++++++- virtualizarr/writers/icechunk.py | 23 +++++++--- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index f99b2718..44b1fcf6 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -12,7 +12,7 @@ from zarr import Array, Group, group # type: ignore[import-untyped] from virtualizarr.manifests import ChunkManifest, ManifestArray -from virtualizarr.writers.icechunk import dataset_to_icechunk +from virtualizarr.writers.icechunk import dataset_to_icechunk, generate_chunk_key from virtualizarr.zarr import ZArray if TYPE_CHECKING: @@ -288,3 +288,45 @@ def test_write_loadable_variable( # TODO test writing to a group that isn't the root group # TODO test with S3 / minio + + +def test_generate_chunk_key_no_offset(): + # Test case without any offset (append_axis and existing_num_chunks are None) + index = (1, 2, 3) + result = generate_chunk_key(index) + assert result == "1/2/3", "The chunk key should match the index without any offset." + + +def test_generate_chunk_key_with_offset(): + # Test case with offset on append_axis 1 + index = (1, 2, 3) + append_axis = 1 + existing_num_chunks = 5 + result = generate_chunk_key( + index, append_axis=append_axis, existing_num_chunks=existing_num_chunks + ) + assert result == "1/7/3", "The chunk key should offset the second index by 5." + + +def test_generate_chunk_key_zero_offset(): + # Test case where existing_num_chunks is 0 (no offset should be applied) + index = (4, 5, 6) + append_axis = 1 + existing_num_chunks = 0 + result = generate_chunk_key( + index, append_axis=append_axis, existing_num_chunks=existing_num_chunks + ) + assert ( + result == "4/5/6" + ), "No offset should be applied when existing_num_chunks is 0." + + +def test_generate_chunk_key_append_axis_out_of_bounds(): + # Edge case where append_axis is out of bounds + index = (3, 4) + append_axis = 2 # This is out of bounds for a 2D index + with pytest.raises(IndexError): + generate_chunk_key(index, append_axis=append_axis, existing_num_chunks=1) + + +# Run tests using pytest diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 6ca97da1..5bd735c4 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -194,6 +194,19 @@ def write_virtual_variable_to_icechunk( ) +def generate_chunk_key( + index: np.nditer.multi_index, + append_axis: Optional[int] = None, + existing_num_chunks: Optional[int] = None, +) -> str: + if append_axis is not None: + list_index = list(index) + # Offset by the number of existing chunks on the append axis + list_index[append_axis] += existing_num_chunks + index = tuple(list_index) + return "/".join(str(i) for i in index) + + def write_manifest_virtual_refs( store: "IcechunkStore", group: "Group", @@ -209,6 +222,8 @@ def write_manifest_virtual_refs( # loop over every reference in the ChunkManifest for that array # TODO inefficient: this should be replaced with something that sets all (new) references for the array at once # but Icechunk need to expose a suitable API first + # Aimee: the manifest (and it's corresponding paths, offsets and lengths, already has the shape of the datacube's chunks + # so we want to increment the resulting multi index it = np.nditer( [manifest._paths, manifest._offsets, manifest._lengths], # type: ignore[arg-type] flags=[ @@ -220,13 +235,9 @@ def write_manifest_virtual_refs( ) for path, offset, length in it: + # it.multi_index will be an iterator of the chunk shape index = it.multi_index - if append_axis is not None: - list_index = list(index) - # Offset by the number of existing chunks on the append axis - list_index[append_axis] += existing_num_chunks - index = tuple(list_index) - chunk_key = "/".join(str(i) for i in index) + chunk_key = generate_chunk_key(index, append_axis, existing_num_chunks) # set each reference individually store.set_virtual_ref( From 360ea1460ace5e7da12d8bf21843f136c148df4f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sat, 26 Oct 2024 12:24:38 -0700 Subject: [PATCH 03/90] Linting --- virtualizarr/accessor.py | 13 ++- .../test_writers/test_icechunk_append.py | 80 +++++++++++++++++++ 2 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 virtualizarr/tests/test_writers/test_icechunk_append.py diff --git a/virtualizarr/accessor.py b/virtualizarr/accessor.py index 336838f9..ea9e3843 100644 --- a/virtualizarr/accessor.py +++ b/virtualizarr/accessor.py @@ -1,10 +1,5 @@ from pathlib import Path -from typing import ( - TYPE_CHECKING, - Callable, - Literal, - overload, -) +from typing import TYPE_CHECKING, Callable, Literal, Optional, overload from xarray import Dataset, register_dataset_accessor @@ -43,7 +38,9 @@ def to_zarr(self, storepath: str) -> None: """ dataset_to_zarr(self.ds, storepath) - def to_icechunk(self, store: "IcechunkStore") -> None: + def to_icechunk( + self, store: "IcechunkStore", append_dim: Optional[str] = None + ) -> None: """ Write an xarray dataset to an Icechunk store. @@ -55,7 +52,7 @@ def to_icechunk(self, store: "IcechunkStore") -> None: """ from virtualizarr.writers.icechunk import dataset_to_icechunk - dataset_to_icechunk(self.ds, store) + dataset_to_icechunk(self.ds, store, append_dim=append_dim) @overload def to_kerchunk( diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py new file mode 100644 index 00000000..251e9945 --- /dev/null +++ b/virtualizarr/tests/test_writers/test_icechunk_append.py @@ -0,0 +1,80 @@ +from typing import TYPE_CHECKING + +import pytest + +pytest.importorskip("icechunk") + +import numpy as np +import numpy.testing as npt +from xarray import Dataset, open_dataset +from xarray.core.variable import Variable +from zarr import group # type: ignore[import-untyped] + +from virtualizarr.manifests import ChunkManifest, ManifestArray +from virtualizarr.writers.icechunk import dataset_to_icechunk +from virtualizarr.zarr import ZArray + +if TYPE_CHECKING: + from icechunk import StorageConfig # type: ignore[import-not-found] + + +@pytest.fixture(scope="function") +def icechunk_storage(tmpdir) -> "StorageConfig": + from icechunk import StorageConfig + + storage = StorageConfig.filesystem(str(tmpdir)) + + # TODO instead yield store then store.close() ?? + return storage + + +def gen_virtual_dataset(file_uri: str): + manifest = ChunkManifest({"0.0": {"path": file_uri, "offset": 6144, "length": 48}}) + zarray = ZArray( + shape=(3, 4), + chunks=(3, 4), + dtype=np.dtype("int32"), + compressor=None, + filters=None, + fill_value=None, + ) + ma = ManifestArray( + chunkmanifest=manifest, + zarray=zarray, + ) + foo = Variable(data=ma, dims=["x", "y"]) + return Dataset( + {"foo": foo}, + ) + + +def test_set_single_virtual_ref_without_encoding( + icechunk_storage: "StorageConfig", simple_netcdf4: str +): + import xarray as xr + from icechunk import IcechunkStore + + # generate virtual dataset + vds = gen_virtual_dataset(simple_netcdf4) + + # create the icechunk store and commit the first virtual dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds, icechunk_filestore) + icechunk_filestore.commit( + "test commit" + ) # need to commit it in order to append to it in the next lines + + # Append the same dataset to the same store + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") + + root_group = group(store=icechunk_filestore_append) + array = root_group["foo"] + + expected_ds = open_dataset(simple_netcdf4) + expected_array = xr.concat( + [expected_ds["foo"], expected_ds["foo"]], dim="x" + ).to_numpy() + npt.assert_equal(array, expected_array) From d3c2851b22146ce11016ba9f0c1f0d937c1288b9 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sat, 26 Oct 2024 15:02:29 -0700 Subject: [PATCH 04/90] Refactor gen virtual dataset method --- .../test_writers/test_icechunk_append.py | 83 ++++++++++++++++--- virtualizarr/writers/icechunk.py | 29 +++---- 2 files changed, 87 insertions(+), 25 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py index 251e9945..051796ac 100644 --- a/virtualizarr/tests/test_writers/test_icechunk_append.py +++ b/virtualizarr/tests/test_writers/test_icechunk_append.py @@ -28,23 +28,56 @@ def icechunk_storage(tmpdir) -> "StorageConfig": return storage -def gen_virtual_dataset(file_uri: str): - manifest = ChunkManifest({"0.0": {"path": file_uri, "offset": 6144, "length": 48}}) +def generate_chunk_manifest( + netcdf4_file: str, + shape: tuple[int, int] = (3, 4), + chunks: tuple[int, int] = (3, 4), + base_offset=6144, + length=48, +) -> ChunkManifest: + chunk_dict = {} + num_chunks_x = shape[0] // chunks[0] + num_chunks_y = shape[1] // chunks[1] + offset = base_offset + + for i in range(num_chunks_x): + for j in range(num_chunks_y): + chunk_index = f"{i}.{j}" + chunk_dict[chunk_index] = { + "path": netcdf4_file, + "offset": offset, + "length": length, + } + offset += length # Increase offset for each chunk + return ChunkManifest(chunk_dict) + + +def gen_virtual_dataset( + file_uri: str, + shape: tuple[int, int] = (3, 4), + chunks: tuple[int, int] = (3, 4), + dtype: np.dtype = np.dtype("int32"), + compressor: str = None, + filters: str = None, + fill_value: str = None, + variable_name: str = "foo", +): + manifest = generate_chunk_manifest(file_uri, shape, chunks) zarray = ZArray( - shape=(3, 4), - chunks=(3, 4), - dtype=np.dtype("int32"), - compressor=None, - filters=None, - fill_value=None, + shape=shape, + chunks=chunks, + dtype=dtype, + compressor=compressor, + filters=filters, + fill_value=fill_value, ) ma = ManifestArray( chunkmanifest=manifest, zarray=zarray, ) - foo = Variable(data=ma, dims=["x", "y"]) + var = Variable(data=ma, dims=["x", "y"]) return Dataset( - {"foo": foo}, + {variable_name: var}, ) @@ -55,7 +88,7 @@ def test_set_single_virtual_ref_without_encoding( from icechunk import IcechunkStore # generate virtual dataset - vds = gen_virtual_dataset(simple_netcdf4) + vds = gen_virtual_dataset(file_uri=simple_netcdf4) # create the icechunk store and commit the first virtual dataset icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) @@ -78,3 +111,31 @@ def test_set_single_virtual_ref_without_encoding( [expected_ds["foo"], expected_ds["foo"]], dim="x" ).to_numpy() npt.assert_equal(array, expected_array) + + +# def test_append_virtual_ref_with_encoding( +# icechunk_storage: "StorageConfig", netcdf4_files: tuple[str, str] +# ): +# import xarray as xr +# from icechunk import IcechunkStore + +# # generate virtual dataset +# filepath1, filepath2 = netcdf4_files +# vds1, vds2 = open_virtual_dataset(filepath1), open_virtual_dataset(filepath2) + +# # create the icechunk store and commit the first virtual dataset +# icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) +# dataset_to_icechunk(vds1, icechunk_filestore) +# icechunk_filestore.commit( +# "test commit" +# ) # need to commit it in order to append to it in the next lines + +# # Append the same dataset to the same store +# icechunk_filestore_append = IcechunkStore.open_existing( +# storage=icechunk_storage, mode="a" +# ) +# dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") + +# root_group = group(store=icechunk_filestore_append) +# array = root_group["foo"] +# import pdb; pdb.set_trace() diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 5bd735c4..7c5b621c 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -145,7 +145,7 @@ def write_virtual_variable_to_icechunk( # Aimee: resize the array if it already exists # TODO: assert chunking and encoding is the same existing_keys = tuple(group.array_keys()) - append_axis, existing_num_chunks = None, None + append_axis, existing_num_chunks, arr = None, None, None if name in existing_keys and mode == "a": # resize dims = var.dims @@ -159,19 +159,20 @@ def write_virtual_variable_to_icechunk( new_shape = list(existing_array.shape) new_shape[append_axis] += var.shape[append_axis] shape = tuple(new_shape) - existing_array.resize(new_shape) - - # creates array if it doesn't already exist - arr = group.require_array( - name=name, - shape=shape, - chunk_shape=zarray.chunks, - dtype=encode_dtype(zarray.dtype), - codecs=zarray._v3_codec_pipeline(), - dimension_names=var.dims, - fill_value=zarray.fill_value, - # TODO fill_value? - ) + # this doesn't seem to actually resize the array + arr = existing_array.resize(shape) + else: + # create array if it doesn't already exist + arr = group.require_array( + name=name, + shape=shape, + chunk_shape=zarray.chunks, + dtype=encode_dtype(zarray.dtype), + codecs=zarray._v3_codec_pipeline(), + dimension_names=var.dims, + fill_value=zarray.fill_value, + # TODO fill_value? + ) # TODO it would be nice if we could assign directly to the .attrs property # Aimee: assert that new attributes are the same as existing attributes From a7a1e50983c22befc3a19697326eef51cdf7d412 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sun, 27 Oct 2024 16:56:16 -0700 Subject: [PATCH 05/90] Fix spelling --- virtualizarr/tests/test_writers/test_icechunk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 44b1fcf6..878791ad 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -127,7 +127,7 @@ def test_set_single_virtual_ref_with_encoding( # vds = open_virtual_dataset(netcdf4_file, indexes={}) expected_ds = open_dataset(netcdf4_file).drop_vars(["lon", "lat", "time"]) - # these atyttirbutes encode floats different and I am not sure why, but its not important enough to block everything + # these attributes encode floats different and I am not sure why, but its not important enough to block everything expected_ds.air.attrs.pop("actual_range") # instead for now just write out byte ranges explicitly From 0365a4585efc31a7691d24717577930fb0ba1569 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sun, 27 Oct 2024 17:02:17 -0700 Subject: [PATCH 06/90] Linting --- .../test_writers/test_icechunk_append.py | 138 +++++++++++++----- virtualizarr/writers/icechunk.py | 5 +- 2 files changed, 104 insertions(+), 39 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py index 051796ac..e1224d65 100644 --- a/virtualizarr/tests/test_writers/test_icechunk_append.py +++ b/virtualizarr/tests/test_writers/test_icechunk_append.py @@ -38,17 +38,29 @@ def generate_chunk_manifest( chunk_dict = {} num_chunks_x = shape[0] // chunks[0] num_chunks_y = shape[1] // chunks[1] + if len(shape) == 3: + num_chunks_z = shape[2] // chunks[2] offset = base_offset for i in range(num_chunks_x): for j in range(num_chunks_y): - chunk_index = f"{i}.{j}" - chunk_dict[chunk_index] = { - "path": netcdf4_file, - "offset": offset, - "length": length, - } - offset += length # Increase offset for each chunk + if len(shape) == 3: + for k in range(num_chunks_z): + chunk_index = f"{i}.{j}.{k}" + chunk_dict[chunk_index] = { + "path": netcdf4_file, + "offset": offset, + "length": length, + } + offset += length + else: + chunk_index = f"{i}.{j}" + chunk_dict[chunk_index] = { + "path": netcdf4_file, + "offset": offset, + "length": length, + } + offset += length # Increase offset for each chunk return ChunkManifest(chunk_dict) @@ -60,9 +72,19 @@ def gen_virtual_dataset( compressor: str = None, filters: str = None, fill_value: str = None, + encoding: dict = None, variable_name: str = "foo", + base_offset: int = 6144, + length: int = 48, + dims: list[str] = None, ): - manifest = generate_chunk_manifest(file_uri, shape, chunks) + manifest = generate_chunk_manifest( + file_uri, + shape=shape, + chunks=chunks, + base_offset=base_offset, + length=length, + ) zarray = ZArray( shape=shape, chunks=chunks, @@ -71,11 +93,15 @@ def gen_virtual_dataset( filters=filters, fill_value=fill_value, ) - ma = ManifestArray( - chunkmanifest=manifest, - zarray=zarray, + ma = ManifestArray(chunkmanifest=manifest, zarray=zarray) + ds = open_dataset(file_uri) + dims = dims or list(ds.dims.keys()) + var = Variable( + data=ma, + dims=dims, + encoding=encoding, + attrs=ds[variable_name].attrs, ) - var = Variable(data=ma, dims=["x", "y"]) return Dataset( {variable_name: var}, ) @@ -113,29 +139,65 @@ def test_set_single_virtual_ref_without_encoding( npt.assert_equal(array, expected_array) -# def test_append_virtual_ref_with_encoding( -# icechunk_storage: "StorageConfig", netcdf4_files: tuple[str, str] -# ): -# import xarray as xr -# from icechunk import IcechunkStore - -# # generate virtual dataset -# filepath1, filepath2 = netcdf4_files -# vds1, vds2 = open_virtual_dataset(filepath1), open_virtual_dataset(filepath2) - -# # create the icechunk store and commit the first virtual dataset -# icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) -# dataset_to_icechunk(vds1, icechunk_filestore) -# icechunk_filestore.commit( -# "test commit" -# ) # need to commit it in order to append to it in the next lines - -# # Append the same dataset to the same store -# icechunk_filestore_append = IcechunkStore.open_existing( -# storage=icechunk_storage, mode="a" -# ) -# dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") - -# root_group = group(store=icechunk_filestore_append) -# array = root_group["foo"] -# import pdb; pdb.set_trace() +# encoding is applied +def test_append_virtual_ref_with_encoding( + icechunk_storage: "StorageConfig", netcdf4_files: tuple[str, str] +): + import xarray as xr + from icechunk import IcechunkStore + + # generate virtual dataset + filepath1, filepath2 = netcdf4_files + scale_factor = 0.01 + vds1, vds2 = ( + gen_virtual_dataset( + file_uri=filepath1, + shape=(1460, 25, 53), + chunks=(1460, 25, 53), + dims=["time", "lat", "lon"], + dtype=np.dtype("int16"), + variable_name="air", + encoding={"_FillValue": -9999, "scale_factor": scale_factor}, + base_offset=15419, + length=3869000, + ), + gen_virtual_dataset( + file_uri=filepath2, + shape=(1460, 25, 53), + chunks=(1460, 25, 53), + dims=["time", "lat", "lon"], + dtype=np.dtype("int16"), + variable_name="air", + encoding={"_FillValue": -9999, "scale_factor": scale_factor}, + base_offset=15419, + length=3869000, + ), + ) + + # create the icechunk store and commit the first virtual dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds1, icechunk_filestore) + icechunk_filestore.commit( + "test commit" + ) # need to commit it in order to append to it in the next lines + + # Append the same dataset to the same store + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") + + root_group = group(store=icechunk_filestore_append) + array = root_group["air"] + expected_ds1, expected_ds2 = open_dataset(filepath1), open_dataset(filepath2) + expected_array = xr.concat( + [expected_ds1["air"], expected_ds2["air"]], dim="time" + ).to_numpy() + npt.assert_equal(array.get_basic_selection() * scale_factor, expected_array) + + +# when chunking is different it fails +# There's a whole beartrap here around noticing if the last chunk is smaller than the other chunks. We should throw in that case (because zarr can't support it without variable-length chunks). +# when encoding is different it fails +# when using compression it works +# Should also test that it raises a clear error if you try to append with chunks of a different dtype etc. I would hope zarr-python would throw that for us. diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 7c5b621c..64eddc56 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -94,7 +94,9 @@ def write_variables_to_icechunk_group( # will overwrite the root group's attributes with the dataset's attributes. We take advantage # of xarrays zarr integration to ignore having to format the attributes ourselves. ds = Dataset(loadable_variables, attrs=attrs) - ds.to_zarr(store, zarr_format=3, consolidated=False, mode="a") + ds.to_zarr( + store, zarr_format=3, consolidated=False, mode="a", append_dim=append_dim + ) # Then finish by writing the virtual variables to the same group for name, var in virtual_variables.items(): @@ -158,6 +160,7 @@ def write_virtual_variable_to_icechunk( ) new_shape = list(existing_array.shape) new_shape[append_axis] += var.shape[append_axis] + # Tom: I wonder if some axis-handling logic from the concat function I wrote for ManifestArray could be re-used here. shape = tuple(new_shape) # this doesn't seem to actually resize the array arr = existing_array.resize(shape) From 5846d7efc418a3c651e0eb4d874cd42be9225a09 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 28 Oct 2024 16:07:06 -0700 Subject: [PATCH 07/90] Linting --- .../test_writers/test_icechunk_append.py | 123 ++++++++++++++++-- virtualizarr/writers/icechunk.py | 85 +++++++++--- 2 files changed, 177 insertions(+), 31 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py index e1224d65..9ec94d9d 100644 --- a/virtualizarr/tests/test_writers/test_icechunk_append.py +++ b/virtualizarr/tests/test_writers/test_icechunk_append.py @@ -67,7 +67,7 @@ def generate_chunk_manifest( def gen_virtual_dataset( file_uri: str, shape: tuple[int, int] = (3, 4), - chunks: tuple[int, int] = (3, 4), + chunk_shape: tuple[int, int] = (3, 4), dtype: np.dtype = np.dtype("int32"), compressor: str = None, filters: str = None, @@ -81,13 +81,13 @@ def gen_virtual_dataset( manifest = generate_chunk_manifest( file_uri, shape=shape, - chunks=chunks, + chunks=chunk_shape, base_offset=base_offset, length=length, ) zarray = ZArray( shape=shape, - chunks=chunks, + chunks=chunk_shape, dtype=dtype, compressor=compressor, filters=filters, @@ -107,7 +107,11 @@ def gen_virtual_dataset( ) -def test_set_single_virtual_ref_without_encoding( +# Success cases + + +## When appending to a single virtual ref without encoding, it succeeds +def test_append_virtual_ref_without_encoding( icechunk_storage: "StorageConfig", simple_netcdf4: str ): import xarray as xr @@ -139,7 +143,7 @@ def test_set_single_virtual_ref_without_encoding( npt.assert_equal(array, expected_array) -# encoding is applied +## When appending to a virtual ref with encoding, it succeeds def test_append_virtual_ref_with_encoding( icechunk_storage: "StorageConfig", netcdf4_files: tuple[str, str] ): @@ -153,7 +157,7 @@ def test_append_virtual_ref_with_encoding( gen_virtual_dataset( file_uri=filepath1, shape=(1460, 25, 53), - chunks=(1460, 25, 53), + chunk_shape=(1460, 25, 53), dims=["time", "lat", "lon"], dtype=np.dtype("int16"), variable_name="air", @@ -164,7 +168,7 @@ def test_append_virtual_ref_with_encoding( gen_virtual_dataset( file_uri=filepath2, shape=(1460, 25, 53), - chunks=(1460, 25, 53), + chunk_shape=(1460, 25, 53), dims=["time", "lat", "lon"], dtype=np.dtype("int16"), variable_name="air", @@ -196,8 +200,103 @@ def test_append_virtual_ref_with_encoding( npt.assert_equal(array.get_basic_selection() * scale_factor, expected_array) -# when chunking is different it fails -# There's a whole beartrap here around noticing if the last chunk is smaller than the other chunks. We should throw in that case (because zarr can't support it without variable-length chunks). -# when encoding is different it fails -# when using compression it works -# Should also test that it raises a clear error if you try to append with chunks of a different dtype etc. I would hope zarr-python would throw that for us. +## When appending to a virtual ref with compression, it succeeds +@pytest.mark.skip(reason="working on this") +def test_append_with_compression_succeeds( + icechunk_storage: "StorageConfig", simple_netcdf4: str +): + from icechunk import IcechunkStore + + # Generate compressed dataset + vds = gen_virtual_dataset( + file_uri=simple_netcdf4, compressor="zlib", dtype=np.dtype("int16") + ) + + # Create icechunk store and commit the compressed dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds, icechunk_filestore) + icechunk_filestore.commit("test commit") + + # Append another dataset with compatible compression + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") + + +## When chunk shapes are different it fails +@pytest.mark.skip(reason="working on this") +def test_append_with_different_chunking_fails( + icechunk_storage: "StorageConfig", simple_netcdf4: str +): + from icechunk import IcechunkStore + + # Generate a virtual dataset with specific chunking + vds = gen_virtual_dataset(file_uri=simple_netcdf4, chunk_shape=(3, 4)) + + # Create icechunk store and commit the dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds, icechunk_filestore) + icechunk_filestore.commit("test commit") + + # Try to append dataset with different chunking, expect failure + vds_different_chunking = gen_virtual_dataset( + file_uri=simple_netcdf4, chunk_shape=(1, 1) + ) + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + with pytest.raises(ValueError, match="incompatible chunking"): + dataset_to_icechunk( + vds_different_chunking, icechunk_filestore_append, append_dim="x" + ) + + +## When encoding is different it fails +@pytest.mark.skip(reason="working on this") +def test_append_with_different_encoding_fails( + icechunk_storage: "StorageConfig", simple_netcdf4: str +): + from icechunk import IcechunkStore + + # Generate datasets with different encoding + vds1 = gen_virtual_dataset(file_uri=simple_netcdf4, encoding={"scale_factor": 0.1}) + vds2 = gen_virtual_dataset(file_uri=simple_netcdf4, encoding={"scale_factor": 0.01}) + + # Create icechunk store and commit the first dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds1, icechunk_filestore) + icechunk_filestore.commit("test commit") + + # Try to append with different encoding, expect failure + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + with pytest.raises(ValueError, match="incompatible encoding"): + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") + + +# When sizes of other dimensions are different, it fails +@pytest.mark.skip(reason="working on this") +def test_other_dimensions_different_length_fails( + icechunk_storage: "StorageConfig", simple_netcdf4: str +): + from icechunk import IcechunkStore + + # Generate datasets with different lengths in non-append dimensions + vds1 = gen_virtual_dataset(file_uri=simple_netcdf4, shape=(5, 4)) # shape (5, 4) + vds2 = gen_virtual_dataset(file_uri=simple_netcdf4, shape=(6, 4)) # shape (6, 4) + + # Create icechunk store and commit the first dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds1, icechunk_filestore) + icechunk_filestore.commit("test commit") + + # Attempt to append dataset with different length in non-append dimension, expect failure + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + with pytest.raises( + ValueError, match="incompatible lengths in non-append dimensions" + ): + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 64eddc56..dbb2e23b 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -6,6 +6,7 @@ from xarray.core.variable import Variable from virtualizarr.manifests import ChunkManifest, ManifestArray +from virtualizarr.manifests import array_api as manifest_api from virtualizarr.zarr import encode_dtype if TYPE_CHECKING: @@ -131,6 +132,50 @@ def write_variable_to_icechunk( ) +def num_chunks( + array, + axis: int, +): + return array.shape[axis] // array.chunks[axis] + + +def resize_array( + group: "Group", + name: str, + var: Variable, + append_axis: int, +): # -> "Array": + existing_array = group[name] + new_shape = list(existing_array.shape) + new_shape[append_axis] += var.shape[append_axis] + return existing_array.resize(tuple(new_shape)) + + +def get_axis( + dims: list[str], + dim_name: str, +) -> int: + return dims.index(dim_name) + + +import zarr + + +def _check_compatibile_arrays( + ma: ManifestArray, existing_array: zarr.core.array.Array, append_axis: int +): + manifest_api._check_same_dtypes([ma.dtype, existing_array.dtype]) + # this is kind of gross - _v3_codec_pipeline returns a tuple + # Question: Does anything need to be done to apply the codecs to the new manifest array? + manifest_api._check_same_codecs( + [list(ma.zarray._v3_codec_pipeline()), existing_array.metadata.codecs] + ) + manifest_api._check_same_chunk_shapes([ma.chunks, existing_array.chunks]) + manifest_api._check_same_ndims([ma.ndim, existing_array.ndim]) + arr_shapes = [ma.shape, existing_array.shape] + manifest_api._check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) + + def write_virtual_variable_to_icechunk( store: "IcechunkStore", group: "Group", @@ -141,34 +186,36 @@ def write_virtual_variable_to_icechunk( """Write a single virtual variable into an icechunk store""" ma = cast(ManifestArray, var.data) zarray = ma.zarray - shape = zarray.shape mode = store.mode.str # Aimee: resize the array if it already exists # TODO: assert chunking and encoding is the same - existing_keys = tuple(group.array_keys()) + dims = var.dims append_axis, existing_num_chunks, arr = None, None, None - if name in existing_keys and mode == "a": - # resize - dims = var.dims - if append_dim in dims: - append_axis = dims.index(append_dim) - existing_array = group[name] - existing_size = existing_array.shape[append_axis] - existing_num_chunks = int( - existing_size / existing_array.chunks[append_axis] - ) - new_shape = list(existing_array.shape) - new_shape[append_axis] += var.shape[append_axis] - # Tom: I wonder if some axis-handling logic from the concat function I wrote for ManifestArray could be re-used here. - shape = tuple(new_shape) - # this doesn't seem to actually resize the array - arr = existing_array.resize(shape) + if mode == "a" and append_dim in dims: + existing_array = group[name] + append_axis = get_axis(dims, append_dim) + # check if arrays can be concatenated + _check_compatibile_arrays(ma, existing_array, append_axis) + + # determine number of existing chunks along the append axis + existing_num_chunks = num_chunks( + array=group[name], + axis=append_axis, + ) + + # resize the array + arr = resize_array( + group=group, + name=name, + var=var, + append_axis=append_axis, + ) else: # create array if it doesn't already exist arr = group.require_array( name=name, - shape=shape, + shape=zarray.shape, chunk_shape=zarray.chunks, dtype=encode_dtype(zarray.dtype), codecs=zarray._v3_codec_pipeline(), From 66bbd6ee91599b36eed0784e2133b818080e01bc Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 29 Oct 2024 18:54:36 -0700 Subject: [PATCH 08/90] Linting --- conftest.py | 22 +++++++ .../test_writers/test_icechunk_append.py | 57 ++++++++++++++----- virtualizarr/writers/icechunk.py | 18 +++--- virtualizarr/zarr.py | 2 +- 4 files changed, 76 insertions(+), 23 deletions(-) diff --git a/conftest.py b/conftest.py index 810fd833..9f9da924 100644 --- a/conftest.py +++ b/conftest.py @@ -35,6 +35,28 @@ def netcdf4_file(tmpdir): return filepath +@pytest.fixture +def compressed_netcdf4_files(tmpdir): + ds = xr.tutorial.open_dataset("air_temperature") + # Define compression options for NetCDF + encoding = { + var: dict(compression="gzip", compression_opts=4) for var in ds.data_vars + } + + ds1 = ds.isel(time=slice(None, 1460)) + ds2 = ds.isel(time=slice(1460, None)) + + # Save it to disk as netCDF (in temporary directory) + filepath1 = f"{tmpdir}/air1_compressed.nc" + filepath2 = f"{tmpdir}/air2_compressed.nc" + ds1.to_netcdf(filepath1, engine="h5netcdf", encoding=encoding) + ds2.to_netcdf(filepath2, engine="h5netcdf", encoding=encoding) + ds1.close() + ds2.close() + + return filepath1, filepath2 + + @pytest.fixture def netcdf4_virtual_dataset(netcdf4_file): from virtualizarr import open_virtual_dataset diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py index 9ec94d9d..1a9cd659 100644 --- a/virtualizarr/tests/test_writers/test_icechunk_append.py +++ b/virtualizarr/tests/test_writers/test_icechunk_append.py @@ -69,7 +69,7 @@ def gen_virtual_dataset( shape: tuple[int, int] = (3, 4), chunk_shape: tuple[int, int] = (3, 4), dtype: np.dtype = np.dtype("int32"), - compressor: str = None, + compressor: dict = None, filters: str = None, fill_value: str = None, encoding: dict = None, @@ -95,7 +95,7 @@ def gen_virtual_dataset( ) ma = ManifestArray(chunkmanifest=manifest, zarray=zarray) ds = open_dataset(file_uri) - dims = dims or list(ds.dims.keys()) + dims = dims or ds.sizes.keys() var = Variable( data=ma, dims=dims, @@ -161,7 +161,7 @@ def test_append_virtual_ref_with_encoding( dims=["time", "lat", "lon"], dtype=np.dtype("int16"), variable_name="air", - encoding={"_FillValue": -9999, "scale_factor": scale_factor}, + encoding={"scale_factor": scale_factor}, base_offset=15419, length=3869000, ), @@ -172,7 +172,7 @@ def test_append_virtual_ref_with_encoding( dims=["time", "lat", "lon"], dtype=np.dtype("int16"), variable_name="air", - encoding={"_FillValue": -9999, "scale_factor": scale_factor}, + encoding={"scale_factor": scale_factor}, base_offset=15419, length=3869000, ), @@ -201,31 +201,60 @@ def test_append_virtual_ref_with_encoding( ## When appending to a virtual ref with compression, it succeeds -@pytest.mark.skip(reason="working on this") +@pytest.mark.skip(reason="Failing with gzip.BadGzipFile: Not a gzipped file") def test_append_with_compression_succeeds( - icechunk_storage: "StorageConfig", simple_netcdf4: str + icechunk_storage: "StorageConfig", compressed_netcdf4_files: str ): + import xarray as xr from icechunk import IcechunkStore + file1, file2 = compressed_netcdf4_files # Generate compressed dataset - vds = gen_virtual_dataset( - file_uri=simple_netcdf4, compressor="zlib", dtype=np.dtype("int16") + vds1, vds2 = ( + gen_virtual_dataset( + file_uri=file1, + # https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#metadata + compressor={"id": "gzip", "level": 4}, + dtype=np.dtype("int16"), + variable_name="air", + shape=(1460, 25, 53), + chunk_shape=(1460, 25, 53), + base_offset=15419, + length=3869000, + ), + gen_virtual_dataset( + file_uri=file2, + compressor={"id": "gzip", "level": 4}, + dtype=np.dtype("int16"), + variable_name="air", + shape=(1460, 25, 53), + chunk_shape=(1460, 25, 53), + base_offset=15419, + length=3869000, + ), ) # Create icechunk store and commit the compressed dataset icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) - dataset_to_icechunk(vds, icechunk_filestore) + dataset_to_icechunk(vds1, icechunk_filestore) icechunk_filestore.commit("test commit") # Append another dataset with compatible compression icechunk_filestore_append = IcechunkStore.open_existing( storage=icechunk_storage, mode="a" ) - dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") + root_group = group(store=icechunk_filestore_append) + array = root_group["air"] + + expected_ds1, expected_ds2 = open_dataset(file1), open_dataset(file2) + expected_array = xr.concat( + [expected_ds1["air"], expected_ds2["air"]], dim="time" + ).to_numpy() + npt.assert_equal(array, expected_array) ## When chunk shapes are different it fails -@pytest.mark.skip(reason="working on this") def test_append_with_different_chunking_fails( icechunk_storage: "StorageConfig", simple_netcdf4: str ): @@ -246,14 +275,16 @@ def test_append_with_different_chunking_fails( icechunk_filestore_append = IcechunkStore.open_existing( storage=icechunk_storage, mode="a" ) - with pytest.raises(ValueError, match="incompatible chunking"): + with pytest.raises( + ValueError, match="Cannot concatenate arrays with inconsistent chunk shapes" + ): dataset_to_icechunk( vds_different_chunking, icechunk_filestore_append, append_dim="x" ) ## When encoding is different it fails -@pytest.mark.skip(reason="working on this") +# @pytest.mark.skip(reason="working on this") def test_append_with_different_encoding_fails( icechunk_storage: "StorageConfig", simple_netcdf4: str ): diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index dbb2e23b..119f3f37 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -224,16 +224,16 @@ def write_virtual_variable_to_icechunk( # TODO fill_value? ) - # TODO it would be nice if we could assign directly to the .attrs property - # Aimee: assert that new attributes are the same as existing attributes - for k, v in var.attrs.items(): - arr.attrs[k] = encode_zarr_attr_value(v) - arr.attrs["_ARRAY_DIMENSIONS"] = encode_zarr_attr_value(var.dims) - - _encoding_keys = {"_FillValue", "missing_value", "scale_factor", "add_offset"} - for k, v in var.encoding.items(): - if k in _encoding_keys: + # TODO it would be nice if we could assign directly to the .attrs property + # Aimee: Can we assume that the attributes are the same for the new array? + for k, v in var.attrs.items(): arr.attrs[k] = encode_zarr_attr_value(v) + arr.attrs["_ARRAY_DIMENSIONS"] = encode_zarr_attr_value(var.dims) + + _encoding_keys = {"_FillValue", "missing_value", "scale_factor", "add_offset"} + for k, v in var.encoding.items(): + if k in _encoding_keys: + arr.attrs[k] = encode_zarr_attr_value(v) write_manifest_virtual_refs( store=store, diff --git a/virtualizarr/zarr.py b/virtualizarr/zarr.py index e339a3f4..169c03ac 100644 --- a/virtualizarr/zarr.py +++ b/virtualizarr/zarr.py @@ -227,5 +227,5 @@ def _num_codec_config_to_configurable(num_codec: dict) -> dict: return num_codec num_codec_copy = num_codec.copy() - name = "numcodecs." + num_codec_copy.pop("id") + name = num_codec_copy.pop("id") return {"name": name, "configuration": num_codec_copy} From 000c68ffb20f3423ad79184e7ac408c224503fd9 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 1 Nov 2024 15:25:13 -0700 Subject: [PATCH 09/90] Passing compression test --- conftest.py | 16 ++++++----- .../test_writers/test_icechunk_append.py | 28 +++++++++---------- virtualizarr/writers/icechunk.py | 2 -- virtualizarr/zarr.py | 2 +- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/conftest.py b/conftest.py index 9f9da924..7cacb3f0 100644 --- a/conftest.py +++ b/conftest.py @@ -37,20 +37,22 @@ def netcdf4_file(tmpdir): @pytest.fixture def compressed_netcdf4_files(tmpdir): - ds = xr.tutorial.open_dataset("air_temperature") + # without chunks={} we get a compression error: zlib.error: Error -3 while decompressing data: incorrect header check + ds = xr.tutorial.open_dataset("air_temperature", chunks={}) + ds1 = ds.isel(time=slice(None, 1460)) + ds2 = ds.isel(time=slice(1460, None)) # Define compression options for NetCDF encoding = { - var: dict(compression="gzip", compression_opts=4) for var in ds.data_vars + # without encoding the chunksizes, irregular ones are chosen + var: dict(zlib=True, complevel=4, chunksizes=(1460, 25, 53)) + for var in ds.data_vars } - ds1 = ds.isel(time=slice(None, 1460)) - ds2 = ds.isel(time=slice(1460, None)) - # Save it to disk as netCDF (in temporary directory) filepath1 = f"{tmpdir}/air1_compressed.nc" filepath2 = f"{tmpdir}/air2_compressed.nc" - ds1.to_netcdf(filepath1, engine="h5netcdf", encoding=encoding) - ds2.to_netcdf(filepath2, engine="h5netcdf", encoding=encoding) + ds1.to_netcdf(filepath1, encoding=encoding, engine="h5netcdf") + ds2.to_netcdf(filepath2, encoding=encoding, engine="h5netcdf") ds1.close() ds2.close() diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py index 1a9cd659..07938f1a 100644 --- a/virtualizarr/tests/test_writers/test_icechunk_append.py +++ b/virtualizarr/tests/test_writers/test_icechunk_append.py @@ -159,7 +159,7 @@ def test_append_virtual_ref_with_encoding( shape=(1460, 25, 53), chunk_shape=(1460, 25, 53), dims=["time", "lat", "lon"], - dtype=np.dtype("int16"), + dtype=np.dtype("float64"), variable_name="air", encoding={"scale_factor": scale_factor}, base_offset=15419, @@ -170,7 +170,7 @@ def test_append_virtual_ref_with_encoding( shape=(1460, 25, 53), chunk_shape=(1460, 25, 53), dims=["time", "lat", "lon"], - dtype=np.dtype("int16"), + dtype=np.dtype("float64"), variable_name="air", encoding={"scale_factor": scale_factor}, base_offset=15419, @@ -201,7 +201,6 @@ def test_append_virtual_ref_with_encoding( ## When appending to a virtual ref with compression, it succeeds -@pytest.mark.skip(reason="Failing with gzip.BadGzipFile: Not a gzipped file") def test_append_with_compression_succeeds( icechunk_storage: "StorageConfig", compressed_netcdf4_files: str ): @@ -213,24 +212,25 @@ def test_append_with_compression_succeeds( vds1, vds2 = ( gen_virtual_dataset( file_uri=file1, - # https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#metadata - compressor={"id": "gzip", "level": 4}, - dtype=np.dtype("int16"), - variable_name="air", shape=(1460, 25, 53), chunk_shape=(1460, 25, 53), - base_offset=15419, - length=3869000, + compressor={"id": "zlib", "level": 4}, + dims=["time", "lat", "lon"], + dtype=np.dtype("float64"), + variable_name="air", + base_offset=23214, + length=3936114, ), gen_virtual_dataset( file_uri=file2, - compressor={"id": "gzip", "level": 4}, - dtype=np.dtype("int16"), - variable_name="air", shape=(1460, 25, 53), chunk_shape=(1460, 25, 53), - base_offset=15419, - length=3869000, + compressor={"id": "zlib", "level": 4}, + dims=["time", "lat", "lon"], + dtype=np.dtype("float64"), + variable_name="air", + base_offset=23214, + length=3938672, ), ) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 119f3f37..c3ed72b5 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -188,8 +188,6 @@ def write_virtual_variable_to_icechunk( zarray = ma.zarray mode = store.mode.str - # Aimee: resize the array if it already exists - # TODO: assert chunking and encoding is the same dims = var.dims append_axis, existing_num_chunks, arr = None, None, None if mode == "a" and append_dim in dims: diff --git a/virtualizarr/zarr.py b/virtualizarr/zarr.py index 169c03ac..e339a3f4 100644 --- a/virtualizarr/zarr.py +++ b/virtualizarr/zarr.py @@ -227,5 +227,5 @@ def _num_codec_config_to_configurable(num_codec: dict) -> dict: return num_codec num_codec_copy = num_codec.copy() - name = num_codec_copy.pop("id") + name = "numcodecs." + num_codec_copy.pop("id") return {"name": name, "configuration": num_codec_copy} From 5906687728b05510be3279cf2de5f89663f265b4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:39:03 +0000 Subject: [PATCH 10/90] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 75df92d3..f1433b9a 100644 --- a/conftest.py +++ b/conftest.py @@ -58,7 +58,7 @@ def compressed_netcdf4_files(tmpdir): return filepath1, filepath2 - + @pytest.fixture def netcdf4_file_with_2d_coords(tmpdir): ds = xr.tutorial.open_dataset("ROMS_example") From 672e5e1d5bda2e2c5504a883fd0df970edb21daa Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sun, 10 Nov 2024 11:16:14 -0800 Subject: [PATCH 11/90] linting --- .../tests/test_writers/test_icechunk_append.py | 4 +++- virtualizarr/writers/icechunk.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py index 07938f1a..f70c153b 100644 --- a/virtualizarr/tests/test_writers/test_icechunk_append.py +++ b/virtualizarr/tests/test_writers/test_icechunk_append.py @@ -303,7 +303,9 @@ def test_append_with_different_encoding_fails( icechunk_filestore_append = IcechunkStore.open_existing( storage=icechunk_storage, mode="a" ) - with pytest.raises(ValueError, match="incompatible encoding"): + with pytest.raises( + ValueError, match="Cannot concatenate arrays with different values for encoding" + ): dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 8214a0e4..61c2abff 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -176,6 +176,15 @@ def _check_compatibile_arrays( manifest_api._check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) +def check_compatible_encodings(encoding1, encoding2): + for key, value in encoding1.items(): + if key in encoding2: + if encoding2[key] != value: + raise ValueError( + f"Cannot concatenate arrays with different values for encoding key {key}: {encoding2[key]} != {value}" + ) + + def write_virtual_variable_to_icechunk( store: "IcechunkStore", group: "Group", @@ -194,6 +203,7 @@ def write_virtual_variable_to_icechunk( existing_array = group[name] append_axis = get_axis(dims, append_dim) # check if arrays can be concatenated + check_compatible_encodings(var.encoding, existing_array.attrs) _check_compatibile_arrays(ma, existing_array, append_axis) # determine number of existing chunks along the append axis From 0fce71f71d97c5983575baec7c5bdd21499e8834 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sun, 10 Nov 2024 11:23:07 -0800 Subject: [PATCH 12/90] Fix test failing due to incorrect dtype --- virtualizarr/tests/test_writers/test_icechunk_append.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py index f70c153b..58de8dc0 100644 --- a/virtualizarr/tests/test_writers/test_icechunk_append.py +++ b/virtualizarr/tests/test_writers/test_icechunk_append.py @@ -159,7 +159,7 @@ def test_append_virtual_ref_with_encoding( shape=(1460, 25, 53), chunk_shape=(1460, 25, 53), dims=["time", "lat", "lon"], - dtype=np.dtype("float64"), + dtype=np.dtype("int16"), variable_name="air", encoding={"scale_factor": scale_factor}, base_offset=15419, @@ -170,7 +170,7 @@ def test_append_virtual_ref_with_encoding( shape=(1460, 25, 53), chunk_shape=(1460, 25, 53), dims=["time", "lat", "lon"], - dtype=np.dtype("float64"), + dtype=np.dtype("int16"), variable_name="air", encoding={"scale_factor": scale_factor}, base_offset=15419, From e60437f7a2783f9edb9944c401967fb2d3fbb108 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sun, 10 Nov 2024 11:26:05 -0800 Subject: [PATCH 13/90] linting --- virtualizarr/writers/icechunk.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 61c2abff..b0a42aa1 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING, Optional, cast import numpy as np +import zarr from xarray import Dataset from xarray.backends.zarr import encode_zarr_attr_value from xarray.core.variable import Variable @@ -158,10 +159,7 @@ def get_axis( return dims.index(dim_name) -import zarr - - -def _check_compatibile_arrays( +def _check_compatible_arrays( ma: ManifestArray, existing_array: zarr.core.array.Array, append_axis: int ): manifest_api._check_same_dtypes([ma.dtype, existing_array.dtype]) @@ -204,7 +202,7 @@ def write_virtual_variable_to_icechunk( append_axis = get_axis(dims, append_dim) # check if arrays can be concatenated check_compatible_encodings(var.encoding, existing_array.attrs) - _check_compatibile_arrays(ma, existing_array, append_axis) + _check_compatible_arrays(ma, existing_array, append_axis) # determine number of existing chunks along the append axis existing_num_chunks = num_chunks( From de2f135c5475dcb35f8f4ec4d2bd64fd3ead090c Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sun, 10 Nov 2024 14:41:20 -0800 Subject: [PATCH 14/90] Linting --- .../tests/test_writers/test_icechunk.py | 330 +++++++++++++++++- virtualizarr/writers/icechunk.py | 42 +-- 2 files changed, 332 insertions(+), 40 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 2bc21287..6d0c5385 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -1,5 +1,6 @@ +from itertools import product from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Tuple import pytest @@ -16,7 +17,7 @@ from virtualizarr.zarr import ZArray if TYPE_CHECKING: - from icechunk import IcechunkStore # type: ignore[import-not-found] + from icechunk import IcechunkStore, StorageConfig # type: ignore[import-not-found] @pytest.fixture(scope="function") @@ -285,11 +286,6 @@ def test_write_loadable_variable( npt.assert_equal(pres_array, expected_array) -# TODO test writing to a group that isn't the root group - -# TODO test with S3 / minio - - def test_generate_chunk_key_no_offset(): # Test case without any offset (append_axis and existing_num_chunks are None) index = (1, 2, 3) @@ -325,8 +321,324 @@ def test_generate_chunk_key_append_axis_out_of_bounds(): # Edge case where append_axis is out of bounds index = (3, 4) append_axis = 2 # This is out of bounds for a 2D index - with pytest.raises(IndexError): + with pytest.raises(ValueError): generate_chunk_key(index, append_axis=append_axis, existing_num_chunks=1) -# Run tests using pytest +@pytest.fixture(scope="function") +def icechunk_storage(tmpdir) -> "StorageConfig": + from icechunk import StorageConfig + + storage = StorageConfig.filesystem(str(tmpdir)) + + # TODO instead yield store then store.close() ?? + return storage + + +def generate_chunk_manifest( + netcdf4_file: str, + shape: Tuple[int, ...], + chunks: Tuple[int, ...], + base_offset=6144, + length=48, +) -> ChunkManifest: + chunk_dict = {} + num_chunks = [shape[i] // chunks[i] for i in range(len(shape))] + offset = base_offset + + # Generate all possible chunk indices using Cartesian product + for chunk_indices in product(*[range(n) for n in num_chunks]): + chunk_index = ".".join(map(str, chunk_indices)) + chunk_dict[chunk_index] = { + "path": netcdf4_file, + "offset": offset, + "length": length, + } + offset += length # Increase offset for each chunk + + return ChunkManifest(chunk_dict) + + +def gen_virtual_dataset( + file_uri: str, + shape: tuple[int, int] = (3, 4), + chunk_shape: tuple[int, int] = (3, 4), + dtype: np.dtype = np.dtype("int32"), + compressor: dict = None, + filters: str = None, + fill_value: str = None, + encoding: dict = None, + variable_name: str = "foo", + base_offset: int = 6144, + length: int = 48, + dims: list[str] = None, +): + manifest = generate_chunk_manifest( + file_uri, + shape=shape, + chunks=chunk_shape, + base_offset=base_offset, + length=length, + ) + zarray = ZArray( + shape=shape, + chunks=chunk_shape, + dtype=dtype, + compressor=compressor, + filters=filters, + fill_value=fill_value, + ) + ma = ManifestArray(chunkmanifest=manifest, zarray=zarray) + ds = open_dataset(file_uri) + dims = dims or ds.sizes.keys() + var = Variable( + data=ma, + dims=dims, + encoding=encoding, + attrs=ds[variable_name].attrs, + ) + return Dataset( + {variable_name: var}, + ) + + +class TestAppend: + """ + Tests for appending to existing icechunk store. + """ + + # Success cases + ## When appending to a single virtual ref without encoding, it succeeds + def test_append_virtual_ref_without_encoding( + self, icechunk_storage: "StorageConfig", simple_netcdf4: str + ): + import xarray as xr + from icechunk import IcechunkStore + + # generate virtual dataset + vds = gen_virtual_dataset(file_uri=simple_netcdf4) + + # create the icechunk store and commit the first virtual dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds, icechunk_filestore) + icechunk_filestore.commit( + "test commit" + ) # need to commit it in order to append to it in the next lines + + # Append the same dataset to the same store + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") + + root_group = group(store=icechunk_filestore_append) + array = root_group["foo"] + + expected_ds = open_dataset(simple_netcdf4) + expected_array = xr.concat( + [expected_ds["foo"], expected_ds["foo"]], dim="x" + ).to_numpy() + npt.assert_equal(array, expected_array) + + ## When appending to a virtual ref with encoding, it succeeds + def test_append_virtual_ref_with_encoding( + self, icechunk_storage: "StorageConfig", netcdf4_files: tuple[str, str] + ): + import xarray as xr + from icechunk import IcechunkStore + + # generate virtual dataset + filepath1, filepath2 = netcdf4_files + scale_factor = 0.01 + vds1, vds2 = ( + gen_virtual_dataset( + file_uri=filepath1, + shape=(1460, 25, 53), + chunk_shape=(1460, 25, 53), + dims=["time", "lat", "lon"], + dtype=np.dtype("int16"), + variable_name="air", + encoding={"scale_factor": scale_factor}, + base_offset=15419, + length=3869000, + ), + gen_virtual_dataset( + file_uri=filepath2, + shape=(1460, 25, 53), + chunk_shape=(1460, 25, 53), + dims=["time", "lat", "lon"], + dtype=np.dtype("int16"), + variable_name="air", + encoding={"scale_factor": scale_factor}, + base_offset=15419, + length=3869000, + ), + ) + + # create the icechunk store and commit the first virtual dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds1, icechunk_filestore) + icechunk_filestore.commit( + "test commit" + ) # need to commit it in order to append to it in the next lines + + # Append the same dataset to the same store + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") + + root_group = group(store=icechunk_filestore_append) + array = root_group["air"] + expected_ds1, expected_ds2 = open_dataset(filepath1), open_dataset(filepath2) + expected_array = xr.concat( + [expected_ds1["air"], expected_ds2["air"]], dim="time" + ).to_numpy() + npt.assert_equal(array.get_basic_selection() * scale_factor, expected_array) + + ## When appending to a virtual ref with compression, it succeeds + def test_append_with_compression_succeeds( + self, icechunk_storage: "StorageConfig", compressed_netcdf4_files: str + ): + import xarray as xr + from icechunk import IcechunkStore + + file1, file2 = compressed_netcdf4_files + # Generate compressed dataset + vds1, vds2 = ( + gen_virtual_dataset( + file_uri=file1, + shape=(1460, 25, 53), + chunk_shape=(1460, 25, 53), + compressor={"id": "zlib", "level": 4}, + dims=["time", "lat", "lon"], + dtype=np.dtype("float64"), + variable_name="air", + base_offset=23214, + length=3936114, + ), + gen_virtual_dataset( + file_uri=file2, + shape=(1460, 25, 53), + chunk_shape=(1460, 25, 53), + compressor={"id": "zlib", "level": 4}, + dims=["time", "lat", "lon"], + dtype=np.dtype("float64"), + variable_name="air", + base_offset=23214, + length=3938672, + ), + ) + + # Create icechunk store and commit the compressed dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds1, icechunk_filestore) + icechunk_filestore.commit("test commit") + + # Append another dataset with compatible compression + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") + root_group = group(store=icechunk_filestore_append) + array = root_group["air"] + + expected_ds1, expected_ds2 = open_dataset(file1), open_dataset(file2) + expected_array = xr.concat( + [expected_ds1["air"], expected_ds2["air"]], dim="time" + ).to_numpy() + npt.assert_equal(array, expected_array) + + ## When chunk shapes are different it fails + def test_append_with_different_chunking_fails( + self, icechunk_storage: "StorageConfig", simple_netcdf4: str + ): + from icechunk import IcechunkStore + + # Generate a virtual dataset with specific chunking + vds = gen_virtual_dataset(file_uri=simple_netcdf4, chunk_shape=(3, 4)) + + # Create icechunk store and commit the dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds, icechunk_filestore) + icechunk_filestore.commit("test commit") + + # Try to append dataset with different chunking, expect failure + vds_different_chunking = gen_virtual_dataset( + file_uri=simple_netcdf4, chunk_shape=(1, 1) + ) + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + with pytest.raises( + ValueError, match="Cannot concatenate arrays with inconsistent chunk shapes" + ): + dataset_to_icechunk( + vds_different_chunking, icechunk_filestore_append, append_dim="x" + ) + + ## When encoding is different it fails + def test_append_with_different_encoding_fails( + self, icechunk_storage: "StorageConfig", simple_netcdf4: str + ): + from icechunk import IcechunkStore + + # Generate datasets with different encoding + vds1 = gen_virtual_dataset( + file_uri=simple_netcdf4, encoding={"scale_factor": 0.1} + ) + vds2 = gen_virtual_dataset( + file_uri=simple_netcdf4, encoding={"scale_factor": 0.01} + ) + + # Create icechunk store and commit the first dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds1, icechunk_filestore) + icechunk_filestore.commit("test commit") + + # Try to append with different encoding, expect failure + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + with pytest.raises( + ValueError, + match="Cannot concatenate arrays with different values for encoding", + ): + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") + + @pytest.mark.xfail(reason="Working on test") + def test_other_dimensions_different_length_fails( + self, icechunk_storage: "StorageConfig", simple_netcdf4: str + ): + from icechunk import IcechunkStore + + # Generate datasets with different lengths in non-append dimensions + vds1 = gen_virtual_dataset( + file_uri=simple_netcdf4, shape=(5, 4) + ) # shape (5, 4) + vds2 = gen_virtual_dataset( + file_uri=simple_netcdf4, shape=(6, 4) + ) # shape (6, 4) + + # Create icechunk store and commit the first dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds1, icechunk_filestore) + icechunk_filestore.commit("test commit") + + # Attempt to append dataset with different length in non-append dimension, expect failure + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + with pytest.raises( + ValueError, match="incompatible lengths in non-append dimensions" + ): + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") + + @pytest.mark.xfail(reason="Not implemented yet") + def test_no_append_dim_in_append_mode_it_fails(self): + pass + + +# TODO test writing to a group that isn't the root group + +# TODO test with S3 / minio diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index b0a42aa1..5535c872 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -12,7 +12,7 @@ if TYPE_CHECKING: from icechunk import IcechunkStore # type: ignore[import-not-found] - from zarr import Group # type: ignore + from zarr import Array, Group # type: ignore VALID_URI_PREFIXES = { @@ -111,28 +111,6 @@ def write_variables_to_icechunk_group( ) -def write_variable_to_icechunk( - store: "IcechunkStore", - group: "Group", - name: str, - var: Variable, - append_dim: Optional[str] = None, -) -> None: - """Write a single (possibly virtual) variable into an icechunk store""" - if isinstance(var.data, ManifestArray): - write_virtual_variable_to_icechunk( - store=store, - group=group, - name=name, - var=var, - append_dim=append_dim, - ) - else: - raise ValueError( - "Cannot write non-virtual variables as virtual variables to Icechunk stores" - ) - - def num_chunks( array, axis: int, @@ -145,7 +123,7 @@ def resize_array( name: str, var: Variable, append_axis: int, -): # -> "Array": +) -> "Array": existing_array = group[name] new_shape = list(existing_array.shape) new_shape[append_axis] += var.shape[append_axis] @@ -250,16 +228,18 @@ def write_virtual_variable_to_icechunk( def generate_chunk_key( - index: np.nditer.multi_index, + index: tuple[int, ...], append_axis: Optional[int] = None, existing_num_chunks: Optional[int] = None, ) -> str: - if append_axis is not None: - list_index = list(index) - # Offset by the number of existing chunks on the append axis - list_index[append_axis] += existing_num_chunks - index = tuple(list_index) - return "/".join(str(i) for i in index) + if append_axis and append_axis >= len(index): + raise ValueError( + f"append_axis {append_axis} is greater than the number of indices {len(index)}" + ) + return "/".join( + str(ind + existing_num_chunks) if axis is append_axis else str(ind) + for axis, ind in enumerate(index) + ) def write_manifest_virtual_refs( From c8a46a6600fceeb261e7a07724eb35c81c738347 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sun, 10 Nov 2024 14:41:47 -0800 Subject: [PATCH 15/90] Remove obsolete test file for appending --- .../test_writers/test_icechunk_append.py | 335 ------------------ 1 file changed, 335 deletions(-) delete mode 100644 virtualizarr/tests/test_writers/test_icechunk_append.py diff --git a/virtualizarr/tests/test_writers/test_icechunk_append.py b/virtualizarr/tests/test_writers/test_icechunk_append.py deleted file mode 100644 index 58de8dc0..00000000 --- a/virtualizarr/tests/test_writers/test_icechunk_append.py +++ /dev/null @@ -1,335 +0,0 @@ -from typing import TYPE_CHECKING - -import pytest - -pytest.importorskip("icechunk") - -import numpy as np -import numpy.testing as npt -from xarray import Dataset, open_dataset -from xarray.core.variable import Variable -from zarr import group # type: ignore[import-untyped] - -from virtualizarr.manifests import ChunkManifest, ManifestArray -from virtualizarr.writers.icechunk import dataset_to_icechunk -from virtualizarr.zarr import ZArray - -if TYPE_CHECKING: - from icechunk import StorageConfig # type: ignore[import-not-found] - - -@pytest.fixture(scope="function") -def icechunk_storage(tmpdir) -> "StorageConfig": - from icechunk import StorageConfig - - storage = StorageConfig.filesystem(str(tmpdir)) - - # TODO instead yield store then store.close() ?? - return storage - - -def generate_chunk_manifest( - netcdf4_file: str, - shape: tuple[int, int] = (3, 4), - chunks: tuple[int, int] = (3, 4), - base_offset=6144, - length=48, -) -> ChunkManifest: - chunk_dict = {} - num_chunks_x = shape[0] // chunks[0] - num_chunks_y = shape[1] // chunks[1] - if len(shape) == 3: - num_chunks_z = shape[2] // chunks[2] - offset = base_offset - - for i in range(num_chunks_x): - for j in range(num_chunks_y): - if len(shape) == 3: - for k in range(num_chunks_z): - chunk_index = f"{i}.{j}.{k}" - chunk_dict[chunk_index] = { - "path": netcdf4_file, - "offset": offset, - "length": length, - } - offset += length - else: - chunk_index = f"{i}.{j}" - chunk_dict[chunk_index] = { - "path": netcdf4_file, - "offset": offset, - "length": length, - } - offset += length # Increase offset for each chunk - return ChunkManifest(chunk_dict) - - -def gen_virtual_dataset( - file_uri: str, - shape: tuple[int, int] = (3, 4), - chunk_shape: tuple[int, int] = (3, 4), - dtype: np.dtype = np.dtype("int32"), - compressor: dict = None, - filters: str = None, - fill_value: str = None, - encoding: dict = None, - variable_name: str = "foo", - base_offset: int = 6144, - length: int = 48, - dims: list[str] = None, -): - manifest = generate_chunk_manifest( - file_uri, - shape=shape, - chunks=chunk_shape, - base_offset=base_offset, - length=length, - ) - zarray = ZArray( - shape=shape, - chunks=chunk_shape, - dtype=dtype, - compressor=compressor, - filters=filters, - fill_value=fill_value, - ) - ma = ManifestArray(chunkmanifest=manifest, zarray=zarray) - ds = open_dataset(file_uri) - dims = dims or ds.sizes.keys() - var = Variable( - data=ma, - dims=dims, - encoding=encoding, - attrs=ds[variable_name].attrs, - ) - return Dataset( - {variable_name: var}, - ) - - -# Success cases - - -## When appending to a single virtual ref without encoding, it succeeds -def test_append_virtual_ref_without_encoding( - icechunk_storage: "StorageConfig", simple_netcdf4: str -): - import xarray as xr - from icechunk import IcechunkStore - - # generate virtual dataset - vds = gen_virtual_dataset(file_uri=simple_netcdf4) - - # create the icechunk store and commit the first virtual dataset - icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) - dataset_to_icechunk(vds, icechunk_filestore) - icechunk_filestore.commit( - "test commit" - ) # need to commit it in order to append to it in the next lines - - # Append the same dataset to the same store - icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" - ) - dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") - - root_group = group(store=icechunk_filestore_append) - array = root_group["foo"] - - expected_ds = open_dataset(simple_netcdf4) - expected_array = xr.concat( - [expected_ds["foo"], expected_ds["foo"]], dim="x" - ).to_numpy() - npt.assert_equal(array, expected_array) - - -## When appending to a virtual ref with encoding, it succeeds -def test_append_virtual_ref_with_encoding( - icechunk_storage: "StorageConfig", netcdf4_files: tuple[str, str] -): - import xarray as xr - from icechunk import IcechunkStore - - # generate virtual dataset - filepath1, filepath2 = netcdf4_files - scale_factor = 0.01 - vds1, vds2 = ( - gen_virtual_dataset( - file_uri=filepath1, - shape=(1460, 25, 53), - chunk_shape=(1460, 25, 53), - dims=["time", "lat", "lon"], - dtype=np.dtype("int16"), - variable_name="air", - encoding={"scale_factor": scale_factor}, - base_offset=15419, - length=3869000, - ), - gen_virtual_dataset( - file_uri=filepath2, - shape=(1460, 25, 53), - chunk_shape=(1460, 25, 53), - dims=["time", "lat", "lon"], - dtype=np.dtype("int16"), - variable_name="air", - encoding={"scale_factor": scale_factor}, - base_offset=15419, - length=3869000, - ), - ) - - # create the icechunk store and commit the first virtual dataset - icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) - dataset_to_icechunk(vds1, icechunk_filestore) - icechunk_filestore.commit( - "test commit" - ) # need to commit it in order to append to it in the next lines - - # Append the same dataset to the same store - icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" - ) - dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") - - root_group = group(store=icechunk_filestore_append) - array = root_group["air"] - expected_ds1, expected_ds2 = open_dataset(filepath1), open_dataset(filepath2) - expected_array = xr.concat( - [expected_ds1["air"], expected_ds2["air"]], dim="time" - ).to_numpy() - npt.assert_equal(array.get_basic_selection() * scale_factor, expected_array) - - -## When appending to a virtual ref with compression, it succeeds -def test_append_with_compression_succeeds( - icechunk_storage: "StorageConfig", compressed_netcdf4_files: str -): - import xarray as xr - from icechunk import IcechunkStore - - file1, file2 = compressed_netcdf4_files - # Generate compressed dataset - vds1, vds2 = ( - gen_virtual_dataset( - file_uri=file1, - shape=(1460, 25, 53), - chunk_shape=(1460, 25, 53), - compressor={"id": "zlib", "level": 4}, - dims=["time", "lat", "lon"], - dtype=np.dtype("float64"), - variable_name="air", - base_offset=23214, - length=3936114, - ), - gen_virtual_dataset( - file_uri=file2, - shape=(1460, 25, 53), - chunk_shape=(1460, 25, 53), - compressor={"id": "zlib", "level": 4}, - dims=["time", "lat", "lon"], - dtype=np.dtype("float64"), - variable_name="air", - base_offset=23214, - length=3938672, - ), - ) - - # Create icechunk store and commit the compressed dataset - icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) - dataset_to_icechunk(vds1, icechunk_filestore) - icechunk_filestore.commit("test commit") - - # Append another dataset with compatible compression - icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" - ) - dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") - root_group = group(store=icechunk_filestore_append) - array = root_group["air"] - - expected_ds1, expected_ds2 = open_dataset(file1), open_dataset(file2) - expected_array = xr.concat( - [expected_ds1["air"], expected_ds2["air"]], dim="time" - ).to_numpy() - npt.assert_equal(array, expected_array) - - -## When chunk shapes are different it fails -def test_append_with_different_chunking_fails( - icechunk_storage: "StorageConfig", simple_netcdf4: str -): - from icechunk import IcechunkStore - - # Generate a virtual dataset with specific chunking - vds = gen_virtual_dataset(file_uri=simple_netcdf4, chunk_shape=(3, 4)) - - # Create icechunk store and commit the dataset - icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) - dataset_to_icechunk(vds, icechunk_filestore) - icechunk_filestore.commit("test commit") - - # Try to append dataset with different chunking, expect failure - vds_different_chunking = gen_virtual_dataset( - file_uri=simple_netcdf4, chunk_shape=(1, 1) - ) - icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" - ) - with pytest.raises( - ValueError, match="Cannot concatenate arrays with inconsistent chunk shapes" - ): - dataset_to_icechunk( - vds_different_chunking, icechunk_filestore_append, append_dim="x" - ) - - -## When encoding is different it fails -# @pytest.mark.skip(reason="working on this") -def test_append_with_different_encoding_fails( - icechunk_storage: "StorageConfig", simple_netcdf4: str -): - from icechunk import IcechunkStore - - # Generate datasets with different encoding - vds1 = gen_virtual_dataset(file_uri=simple_netcdf4, encoding={"scale_factor": 0.1}) - vds2 = gen_virtual_dataset(file_uri=simple_netcdf4, encoding={"scale_factor": 0.01}) - - # Create icechunk store and commit the first dataset - icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) - dataset_to_icechunk(vds1, icechunk_filestore) - icechunk_filestore.commit("test commit") - - # Try to append with different encoding, expect failure - icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" - ) - with pytest.raises( - ValueError, match="Cannot concatenate arrays with different values for encoding" - ): - dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") - - -# When sizes of other dimensions are different, it fails -@pytest.mark.skip(reason="working on this") -def test_other_dimensions_different_length_fails( - icechunk_storage: "StorageConfig", simple_netcdf4: str -): - from icechunk import IcechunkStore - - # Generate datasets with different lengths in non-append dimensions - vds1 = gen_virtual_dataset(file_uri=simple_netcdf4, shape=(5, 4)) # shape (5, 4) - vds2 = gen_virtual_dataset(file_uri=simple_netcdf4, shape=(6, 4)) # shape (6, 4) - - # Create icechunk store and commit the first dataset - icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) - dataset_to_icechunk(vds1, icechunk_filestore) - icechunk_filestore.commit("test commit") - - # Attempt to append dataset with different length in non-append dimension, expect failure - icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" - ) - with pytest.raises( - ValueError, match="incompatible lengths in non-append dimensions" - ): - dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") From 7663ad7df3f4a606a7af539a0b5ade5a5e82fa82 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 11 Nov 2024 09:45:10 -0800 Subject: [PATCH 16/90] Create netcdf4 files factor in conftest --- conftest.py | 63 +++++++------------ .../tests/test_writers/test_icechunk.py | 54 ++++++++-------- 2 files changed, 52 insertions(+), 65 deletions(-) diff --git a/conftest.py b/conftest.py index f1433b9a..8024214c 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,5 @@ +from typing import Any, Dict, Optional + import h5py import numpy as np import pytest @@ -36,27 +38,30 @@ def netcdf4_file(tmpdir): @pytest.fixture -def compressed_netcdf4_files(tmpdir): - # without chunks={} we get a compression error: zlib.error: Error -3 while decompressing data: incorrect header check - ds = xr.tutorial.open_dataset("air_temperature", chunks={}) - ds1 = ds.isel(time=slice(None, 1460)) - ds2 = ds.isel(time=slice(1460, None)) - # Define compression options for NetCDF - encoding = { - # without encoding the chunksizes, irregular ones are chosen - var: dict(zlib=True, complevel=4, chunksizes=(1460, 25, 53)) - for var in ds.data_vars - } +def netcdf4_files_factory(tmpdir) -> callable: + def create_netcdf4_files( + encoding: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> tuple[str, str]: + # Set up example xarray dataset + ds = xr.tutorial.open_dataset("air_temperature", chunks={}) - # Save it to disk as netCDF (in temporary directory) - filepath1 = f"{tmpdir}/air1_compressed.nc" - filepath2 = f"{tmpdir}/air2_compressed.nc" - ds1.to_netcdf(filepath1, encoding=encoding, engine="h5netcdf") - ds2.to_netcdf(filepath2, encoding=encoding, engine="h5netcdf") - ds1.close() - ds2.close() + # Split dataset into two parts + ds1 = ds.isel(time=slice(None, 1460)) + ds2 = ds.isel(time=slice(1460, None)) + + # Save datasets to disk as NetCDF in the temporary directory with the provided encoding + filepath1 = f"{tmpdir}/air1.nc" + filepath2 = f"{tmpdir}/air2.nc" + ds1.to_netcdf(filepath1, encoding=encoding, engine="h5netcdf") + ds2.to_netcdf(filepath2, encoding=encoding, engine="h5netcdf") + + # Close datasets + ds1.close() + ds2.close() - return filepath1, filepath2 + return filepath1, filepath2 + + return create_netcdf4_files @pytest.fixture @@ -95,26 +100,6 @@ def hdf5_groups_file(tmpdir): return filepath -@pytest.fixture -def netcdf4_files(tmpdir): - # Set up example xarray dataset - ds = xr.tutorial.open_dataset("air_temperature") - - # split inrto equal chunks so we can concatenate them back together later - ds1 = ds.isel(time=slice(None, 1460)) - ds2 = ds.isel(time=slice(1460, None)) - - # Save it to disk as netCDF (in temporary directory) - filepath1 = f"{tmpdir}/air1.nc" - filepath2 = f"{tmpdir}/air2.nc" - ds1.to_netcdf(filepath1) - ds2.to_netcdf(filepath2) - ds1.close() - ds2.close() - - return filepath1, filepath2 - - @pytest.fixture def hdf5_empty(tmpdir): filepath = f"{tmpdir}/empty.nc" diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 6d0c5385..20035314 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -430,26 +430,23 @@ def test_append_virtual_ref_without_encoding( storage=icechunk_storage, mode="a" ) dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") - - root_group = group(store=icechunk_filestore_append) - array = root_group["foo"] + array = open_zarr(icechunk_filestore_append, consolidated=False, zarr_format=3) expected_ds = open_dataset(simple_netcdf4) - expected_array = xr.concat( - [expected_ds["foo"], expected_ds["foo"]], dim="x" - ).to_numpy() - npt.assert_equal(array, expected_array) + expected_array = xr.concat([expected_ds, expected_ds], dim="x") + xr.testing.assert_equal(array.foo, expected_array.foo) ## When appending to a virtual ref with encoding, it succeeds def test_append_virtual_ref_with_encoding( - self, icechunk_storage: "StorageConfig", netcdf4_files: tuple[str, str] + self, icechunk_storage: "StorageConfig", netcdf4_files_factory: callable ): import xarray as xr from icechunk import IcechunkStore # generate virtual dataset - filepath1, filepath2 = netcdf4_files scale_factor = 0.01 + encoding = {"air": {"scale_factor": scale_factor}} + filepath1, filepath2 = netcdf4_files_factory(encoding=encoding) vds1, vds2 = ( gen_virtual_dataset( file_uri=filepath1, @@ -459,7 +456,7 @@ def test_append_virtual_ref_with_encoding( dtype=np.dtype("int16"), variable_name="air", encoding={"scale_factor": scale_factor}, - base_offset=15419, + base_offset=20122, length=3869000, ), gen_virtual_dataset( @@ -470,7 +467,7 @@ def test_append_virtual_ref_with_encoding( dtype=np.dtype("int16"), variable_name="air", encoding={"scale_factor": scale_factor}, - base_offset=15419, + base_offset=20122, length=3869000, ), ) @@ -487,23 +484,25 @@ def test_append_virtual_ref_with_encoding( storage=icechunk_storage, mode="a" ) dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") + new_ds = open_zarr(icechunk_filestore_append, consolidated=False, zarr_format=3) - root_group = group(store=icechunk_filestore_append) - array = root_group["air"] expected_ds1, expected_ds2 = open_dataset(filepath1), open_dataset(filepath2) - expected_array = xr.concat( - [expected_ds1["air"], expected_ds2["air"]], dim="time" - ).to_numpy() - npt.assert_equal(array.get_basic_selection() * scale_factor, expected_array) + expected_ds = xr.concat([expected_ds1, expected_ds2], dim="time").drop_vars( + ["lon", "lat", "time"], errors="ignore" + ) + xr.testing.assert_identical(new_ds.air, expected_ds.air) ## When appending to a virtual ref with compression, it succeeds def test_append_with_compression_succeeds( - self, icechunk_storage: "StorageConfig", compressed_netcdf4_files: str + self, icechunk_storage: "StorageConfig", netcdf4_files_factory: callable ): import xarray as xr from icechunk import IcechunkStore - file1, file2 = compressed_netcdf4_files + encoding = encoding = { + "air": {"zlib": True, "complevel": 4, "chunksizes": (1460, 25, 53)} + } + file1, file2 = netcdf4_files_factory(encoding=encoding) # Generate compressed dataset vds1, vds2 = ( gen_virtual_dataset( @@ -540,14 +539,17 @@ def test_append_with_compression_succeeds( storage=icechunk_storage, mode="a" ) dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") - root_group = group(store=icechunk_filestore_append) - array = root_group["air"] + icechunk_filestore_append.commit("appended data") + updated_ds = xr.open_zarr( + store=icechunk_filestore_append, consolidated=False, zarr_format=3 + ) expected_ds1, expected_ds2 = open_dataset(file1), open_dataset(file2) - expected_array = xr.concat( - [expected_ds1["air"], expected_ds2["air"]], dim="time" - ).to_numpy() - npt.assert_equal(array, expected_array) + expected_ds = xr.concat([expected_ds1, expected_ds2], dim="time") + # Aimee: Double check this is necessary + expected_ds = expected_ds.drop_vars(["lon", "lat", "time"], errors="ignore") + # Aimee: Dataset attributes are not present on the new icechunk store + xr.testing.assert_equal(updated_ds.air, expected_ds.air) ## When chunk shapes are different it fails def test_append_with_different_chunking_fails( @@ -607,7 +609,7 @@ def test_append_with_different_encoding_fails( dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") @pytest.mark.xfail(reason="Working on test") - def test_other_dimensions_different_length_fails( + def test_dimensions_do_not_align( self, icechunk_storage: "StorageConfig", simple_netcdf4: str ): from icechunk import IcechunkStore From 1d704ffd6d32f28c1e507d891c214519225904e5 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 11 Nov 2024 10:01:51 -0800 Subject: [PATCH 17/90] Linting --- .../tests/test_writers/test_icechunk.py | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 20035314..0969cd03 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -399,6 +399,7 @@ def gen_virtual_dataset( ) return Dataset( {variable_name: var}, + attrs=ds.attrs, ) @@ -434,7 +435,8 @@ def test_append_virtual_ref_without_encoding( expected_ds = open_dataset(simple_netcdf4) expected_array = xr.concat([expected_ds, expected_ds], dim="x") - xr.testing.assert_equal(array.foo, expected_array.foo) + # Aimee: attributes for actual_range differ [185.16000366210935, 322.1000061035156] vs [185.16 322.1] + xr.testing.assert_equal(array, expected_array) ## When appending to a virtual ref with encoding, it succeeds def test_append_virtual_ref_with_encoding( @@ -445,30 +447,33 @@ def test_append_virtual_ref_with_encoding( # generate virtual dataset scale_factor = 0.01 - encoding = {"air": {"scale_factor": scale_factor}} + encoding = {"air": {"scale_factor": scale_factor, "dtype": np.dtype("float64")}} filepath1, filepath2 = netcdf4_files_factory(encoding=encoding) + # from virtualizarr import open_virtual_dataset + # vds1, vds2 = open_virtual_dataset(filepath1), open_virtual_dataset(filepath2) + # import pdb; pdb.set_trace() vds1, vds2 = ( gen_virtual_dataset( file_uri=filepath1, shape=(1460, 25, 53), chunk_shape=(1460, 25, 53), dims=["time", "lat", "lon"], - dtype=np.dtype("int16"), + dtype=np.dtype("float64"), variable_name="air", encoding={"scale_factor": scale_factor}, - base_offset=20122, - length=3869000, + base_offset=20146, + length=15476000, ), gen_virtual_dataset( file_uri=filepath2, shape=(1460, 25, 53), chunk_shape=(1460, 25, 53), dims=["time", "lat", "lon"], - dtype=np.dtype("int16"), + dtype=np.dtype("float64"), variable_name="air", encoding={"scale_factor": scale_factor}, - base_offset=20122, - length=3869000, + base_offset=20146, + length=15476000, ), ) @@ -490,7 +495,7 @@ def test_append_virtual_ref_with_encoding( expected_ds = xr.concat([expected_ds1, expected_ds2], dim="time").drop_vars( ["lon", "lat", "time"], errors="ignore" ) - xr.testing.assert_identical(new_ds.air, expected_ds.air) + xr.testing.assert_equal(new_ds, expected_ds) ## When appending to a virtual ref with compression, it succeeds def test_append_with_compression_succeeds( @@ -546,10 +551,8 @@ def test_append_with_compression_succeeds( expected_ds1, expected_ds2 = open_dataset(file1), open_dataset(file2) expected_ds = xr.concat([expected_ds1, expected_ds2], dim="time") - # Aimee: Double check this is necessary expected_ds = expected_ds.drop_vars(["lon", "lat", "time"], errors="ignore") - # Aimee: Dataset attributes are not present on the new icechunk store - xr.testing.assert_equal(updated_ds.air, expected_ds.air) + xr.testing.assert_equal(updated_ds, expected_ds) ## When chunk shapes are different it fails def test_append_with_different_chunking_fails( From af5f57da89e65f397b2353641cbc64b01d40a631 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 11 Nov 2024 16:58:53 -0800 Subject: [PATCH 18/90] Refactor to use combineable zarr arrays --- virtualizarr/manifests/array_api.py | 30 +++++++++++++------ .../tests/test_writers/test_icechunk.py | 1 + virtualizarr/writers/icechunk.py | 12 ++------ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index f5cf220b..015a6d67 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -25,7 +25,7 @@ def decorator(func): return decorator -def _check_combineable_zarr_arrays(arrays: Iterable["ManifestArray"]) -> None: +def check_combineable_zarr_arrays(arrays: Iterable["ManifestArray"]) -> None: """ The downside of the ManifestArray approach compared to the VirtualZarrArray concatenation proposal is that the result must also be a single valid zarr array, implying that the inputs must have the same dtype, codec etc. @@ -34,7 +34,19 @@ def _check_combineable_zarr_arrays(arrays: Iterable["ManifestArray"]) -> None: # Can't combine different codecs in one manifest # see https://github.com/zarr-developers/zarr-specs/issues/288 - _check_same_codecs([arr.zarray.codec for arr in arrays]) + def get_codecs(array): + import zarr + + from .array import ManifestArray + + if isinstance(array, ManifestArray): + if array.zarray.zarr_format == 3: + return list(array.zarray._v3_codec_pipeline()) + return array.zarray.codec + if isinstance(array, zarr.core.array.Array): + return array.metadata.codecs + + _check_same_codecs([get_codecs(arr) for arr in arrays]) # Would require variable-length chunks ZEP _check_same_chunk_shapes([arr.chunks for arr in arrays]) @@ -106,9 +118,9 @@ def concatenate( raise TypeError() # ensure dtypes, shapes, codecs etc. are consistent - _check_combineable_zarr_arrays(arrays) + check_combineable_zarr_arrays(arrays) - _check_same_ndims([arr.ndim for arr in arrays]) + check_same_ndims([arr.ndim for arr in arrays]) # Ensure we handle axis being passed as a negative integer first_arr = arrays[0] @@ -116,7 +128,7 @@ def concatenate( axis = axis % first_arr.ndim arr_shapes = [arr.shape for arr in arrays] - _check_same_shapes_except_on_concat_axis(arr_shapes, axis) + check_same_shapes_except_on_concat_axis(arr_shapes, axis) # find what new array shape must be new_length_along_concat_axis = sum([shape[axis] for shape in arr_shapes]) @@ -151,7 +163,7 @@ def concatenate( return ManifestArray(chunkmanifest=concatenated_manifest, zarray=new_zarray) -def _check_same_ndims(ndims: list[int]) -> None: +def check_same_ndims(ndims: list[int]) -> None: first_ndim, *other_ndims = ndims for other_ndim in other_ndims: if other_ndim != first_ndim: @@ -160,7 +172,7 @@ def _check_same_ndims(ndims: list[int]) -> None: ) -def _check_same_shapes_except_on_concat_axis(shapes: list[tuple[int, ...]], axis: int): +def check_same_shapes_except_on_concat_axis(shapes: list[tuple[int, ...]], axis: int): """Check that shapes are compatible for concatenation""" shapes_without_concat_axis = [ @@ -199,9 +211,9 @@ def stack( raise TypeError() # ensure dtypes, shapes, codecs etc. are consistent - _check_combineable_zarr_arrays(arrays) + check_combineable_zarr_arrays(arrays) - _check_same_ndims([arr.ndim for arr in arrays]) + check_same_ndims([arr.ndim for arr in arrays]) arr_shapes = [arr.shape for arr in arrays] _check_same_shapes(arr_shapes) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 0969cd03..9d4f9a1c 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -387,6 +387,7 @@ def gen_virtual_dataset( compressor=compressor, filters=filters, fill_value=fill_value, + zarr_format=3, ) ma = ManifestArray(chunkmanifest=manifest, zarray=zarray) ds = open_dataset(file_uri) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 5535c872..ca136e13 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -140,16 +140,10 @@ def get_axis( def _check_compatible_arrays( ma: ManifestArray, existing_array: zarr.core.array.Array, append_axis: int ): - manifest_api._check_same_dtypes([ma.dtype, existing_array.dtype]) - # this is kind of gross - _v3_codec_pipeline returns a tuple - # Question: Does anything need to be done to apply the codecs to the new manifest array? - manifest_api._check_same_codecs( - [list(ma.zarray._v3_codec_pipeline()), existing_array.metadata.codecs] - ) - manifest_api._check_same_chunk_shapes([ma.chunks, existing_array.chunks]) - manifest_api._check_same_ndims([ma.ndim, existing_array.ndim]) + manifest_api.check_combineable_zarr_arrays([ma, existing_array]) + manifest_api.check_same_ndims([ma.ndim, existing_array.ndim]) arr_shapes = [ma.shape, existing_array.shape] - manifest_api._check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) + manifest_api.check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) def check_compatible_encodings(encoding1, encoding2): From 6f4cfd92b72bf663a75795e4d4f250f2a93959a0 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 11 Nov 2024 17:08:13 -0800 Subject: [PATCH 19/90] linting --- .../tests/test_writers/test_icechunk.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 9d4f9a1c..da18897a 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -612,19 +612,22 @@ def test_append_with_different_encoding_fails( ): dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") - @pytest.mark.xfail(reason="Working on test") def test_dimensions_do_not_align( self, icechunk_storage: "StorageConfig", simple_netcdf4: str ): from icechunk import IcechunkStore - # Generate datasets with different lengths in non-append dimensions + # Generate datasets with different lengths on the non-append dimension (x) vds1 = gen_virtual_dataset( - file_uri=simple_netcdf4, shape=(5, 4) - ) # shape (5, 4) + # {'x': 5, 'y': 4} + file_uri=simple_netcdf4, + shape=(5, 4), + ) vds2 = gen_virtual_dataset( - file_uri=simple_netcdf4, shape=(6, 4) - ) # shape (6, 4) + # {'x': 6, 'y': 4} + file_uri=simple_netcdf4, + shape=(6, 4), + ) # Create icechunk store and commit the first dataset icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) @@ -635,10 +638,8 @@ def test_dimensions_do_not_align( icechunk_filestore_append = IcechunkStore.open_existing( storage=icechunk_storage, mode="a" ) - with pytest.raises( - ValueError, match="incompatible lengths in non-append dimensions" - ): - dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="x") + with pytest.raises(ValueError, match="Cannot concatenate arrays with shapes"): + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="y") @pytest.mark.xfail(reason="Not implemented yet") def test_no_append_dim_in_append_mode_it_fails(self): From 98c7052f5f4782a9be439f51bb04396ea6b59613 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 11 Nov 2024 17:23:10 -0800 Subject: [PATCH 20/90] Implement no append dim test --- .../tests/test_writers/test_icechunk.py | 63 ++++++++++++------- virtualizarr/writers/icechunk.py | 4 ++ 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index da18897a..160e11a2 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -8,7 +8,7 @@ import numpy as np import numpy.testing as npt -from xarray import Dataset, open_dataset, open_zarr +from xarray import Dataset, concat, open_dataset, open_zarr from xarray.core.variable import Variable from zarr import Array, Group, group # type: ignore[import-untyped] @@ -414,7 +414,7 @@ class TestAppend: def test_append_virtual_ref_without_encoding( self, icechunk_storage: "StorageConfig", simple_netcdf4: str ): - import xarray as xr + import xarray.testing as xrt from icechunk import IcechunkStore # generate virtual dataset @@ -435,24 +435,20 @@ def test_append_virtual_ref_without_encoding( array = open_zarr(icechunk_filestore_append, consolidated=False, zarr_format=3) expected_ds = open_dataset(simple_netcdf4) - expected_array = xr.concat([expected_ds, expected_ds], dim="x") + expected_array = concat([expected_ds, expected_ds], dim="x") # Aimee: attributes for actual_range differ [185.16000366210935, 322.1000061035156] vs [185.16 322.1] - xr.testing.assert_equal(array, expected_array) + xrt.assert_equal(array, expected_array) ## When appending to a virtual ref with encoding, it succeeds def test_append_virtual_ref_with_encoding( self, icechunk_storage: "StorageConfig", netcdf4_files_factory: callable ): - import xarray as xr + import xarray.testing as xrt from icechunk import IcechunkStore - # generate virtual dataset scale_factor = 0.01 encoding = {"air": {"scale_factor": scale_factor, "dtype": np.dtype("float64")}} filepath1, filepath2 = netcdf4_files_factory(encoding=encoding) - # from virtualizarr import open_virtual_dataset - # vds1, vds2 = open_virtual_dataset(filepath1), open_virtual_dataset(filepath2) - # import pdb; pdb.set_trace() vds1, vds2 = ( gen_virtual_dataset( file_uri=filepath1, @@ -493,21 +489,19 @@ def test_append_virtual_ref_with_encoding( new_ds = open_zarr(icechunk_filestore_append, consolidated=False, zarr_format=3) expected_ds1, expected_ds2 = open_dataset(filepath1), open_dataset(filepath2) - expected_ds = xr.concat([expected_ds1, expected_ds2], dim="time").drop_vars( + expected_ds = concat([expected_ds1, expected_ds2], dim="time").drop_vars( ["lon", "lat", "time"], errors="ignore" ) - xr.testing.assert_equal(new_ds, expected_ds) + xrt.assert_equal(new_ds, expected_ds) ## When appending to a virtual ref with compression, it succeeds def test_append_with_compression_succeeds( self, icechunk_storage: "StorageConfig", netcdf4_files_factory: callable ): - import xarray as xr + import xarray.testing as xrt from icechunk import IcechunkStore - encoding = encoding = { - "air": {"zlib": True, "complevel": 4, "chunksizes": (1460, 25, 53)} - } + encoding = {"air": {"zlib": True, "complevel": 4, "chunksizes": (1460, 25, 53)}} file1, file2 = netcdf4_files_factory(encoding=encoding) # Generate compressed dataset vds1, vds2 = ( @@ -546,14 +540,14 @@ def test_append_with_compression_succeeds( ) dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") icechunk_filestore_append.commit("appended data") - updated_ds = xr.open_zarr( + updated_ds = open_zarr( store=icechunk_filestore_append, consolidated=False, zarr_format=3 ) expected_ds1, expected_ds2 = open_dataset(file1), open_dataset(file2) - expected_ds = xr.concat([expected_ds1, expected_ds2], dim="time") + expected_ds = concat([expected_ds1, expected_ds2], dim="time") expected_ds = expected_ds.drop_vars(["lon", "lat", "time"], errors="ignore") - xr.testing.assert_equal(updated_ds, expected_ds) + xrt.assert_equal(updated_ds, expected_ds) ## When chunk shapes are different it fails def test_append_with_different_chunking_fails( @@ -641,9 +635,36 @@ def test_dimensions_do_not_align( with pytest.raises(ValueError, match="Cannot concatenate arrays with shapes"): dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="y") - @pytest.mark.xfail(reason="Not implemented yet") - def test_no_append_dim_in_append_mode_it_fails(self): - pass + def test_no_append_dim_in_append_mode_it_fails( + self, icechunk_storage: "StorageConfig", simple_netcdf4: str + ): + """ + Test case to validate that appending without specifying an append dimension fails. + This test is expected to fail until appropriate handling for missing append dimension is implemented. + """ + from icechunk import IcechunkStore + + # Generate a virtual dataset + vds = gen_virtual_dataset( + file_uri=simple_netcdf4, shape=(5, 4), chunk_shape=(5, 4) + ) + + # Create the icechunk store and commit the initial virtual dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds, icechunk_filestore) + icechunk_filestore.commit("initial commit") + + # Attempt to append the same dataset without specifying an append dimension + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + + with pytest.raises( + ValueError, + match="append_dim must be provided when opening store in append mode", + ): + # This should raise a ValueError because append_dim is not provided + dataset_to_icechunk(vds, icechunk_filestore_append) # TODO test writing to a group that isn't the root group diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index ca136e13..2583f17d 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -56,6 +56,10 @@ def dataset_to_icechunk( # TODO only supports writing to the root group currently # TODO pass zarr_format kwarg? if store.mode.str == "a": + if append_dim is None: + raise ValueError( + "append_dim must be provided when opening store in append mode" + ) root_group = Group.open(store=store, zarr_format=3) else: root_group = Group.from_store(store=store) From f36adf27c012395f947906ae32129dabf83c0de7 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 11 Nov 2024 17:41:22 -0800 Subject: [PATCH 21/90] Add test for when append dim is not in dims --- .../tests/test_writers/test_icechunk.py | 26 +++++++++++++++++++ virtualizarr/writers/icechunk.py | 6 ++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 160e11a2..602a26be 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -666,6 +666,32 @@ def test_no_append_dim_in_append_mode_it_fails( # This should raise a ValueError because append_dim is not provided dataset_to_icechunk(vds, icechunk_filestore_append) + def test_append_dim_not_in_dims_raises_error( + self, icechunk_storage: "StorageConfig", simple_netcdf4: str + ): + """ + Test that attempting to append with an append_dim not present in dims raises a ValueError. + """ + from icechunk import IcechunkStore + + vds = gen_virtual_dataset( + file_uri=simple_netcdf4, shape=(5, 4), chunk_shape=(5, 4), dims=["x", "y"] + ) + + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds, icechunk_filestore) + icechunk_filestore.commit("initial commit") + + # Attempt to append using a non-existent append_dim "z" + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, mode="a" + ) + with pytest.raises( + ValueError, + match="append_dim='z' does not match any existing dataset dimensions", + ): + dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="z") + # TODO test writing to a group that isn't the root group diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 2583f17d..c667ea52 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -173,7 +173,11 @@ def write_virtual_variable_to_icechunk( dims = var.dims append_axis, existing_num_chunks, arr = None, None, None - if mode == "a" and append_dim in dims: + if append_dim and append_dim not in dims: + raise ValueError( + f"append_dim {append_dim} not found in variable dimensions {dims}" + ) + if mode == "a": existing_array = group[name] append_axis = get_axis(dims, append_dim) # check if arrays can be concatenated From e922ccd99b2b74daa7dfd7d0bff320702a4e3287 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 10:52:31 -0800 Subject: [PATCH 22/90] Fix mypy errors --- virtualizarr/manifests/array_api.py | 4 ++- .../tests/test_writers/test_icechunk.py | 27 ++++++++++--------- virtualizarr/writers/icechunk.py | 18 +++++++++---- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index 015a6d67..11796fa5 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -7,6 +7,8 @@ from .manifest import ChunkManifest if TYPE_CHECKING: + from zarr import Array # type: ignore + from .array import ManifestArray @@ -25,7 +27,7 @@ def decorator(func): return decorator -def check_combineable_zarr_arrays(arrays: Iterable["ManifestArray"]) -> None: +def check_combineable_zarr_arrays(arrays: Iterable["ManifestArray" | "Array"]) -> None: """ The downside of the ManifestArray approach compared to the VirtualZarrArray concatenation proposal is that the result must also be a single valid zarr array, implying that the inputs must have the same dtype, codec etc. diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 602a26be..9e3809b2 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -1,6 +1,6 @@ from itertools import product from pathlib import Path -from typing import TYPE_CHECKING, Tuple +from typing import TYPE_CHECKING, Any, Callable, Optional, Tuple, cast import pytest @@ -11,6 +11,7 @@ from xarray import Dataset, concat, open_dataset, open_zarr from xarray.core.variable import Variable from zarr import Array, Group, group # type: ignore[import-untyped] +from zarr.core.metadata import ArrayV3Metadata from virtualizarr.manifests import ChunkManifest, ManifestArray from virtualizarr.writers.icechunk import dataset_to_icechunk, generate_chunk_key @@ -69,7 +70,8 @@ def test_write_new_virtual_variable( # assert dict(arr.attrs) == {"units": "km"} # check dimensions - assert arr.metadata.dimension_names == ("x", "y") + if isinstance(arr.metadata, ArrayV3Metadata): + assert arr.metadata.dimension_names == ("x", "y") def test_set_single_virtual_ref_without_encoding( @@ -361,17 +363,17 @@ def generate_chunk_manifest( def gen_virtual_dataset( file_uri: str, - shape: tuple[int, int] = (3, 4), - chunk_shape: tuple[int, int] = (3, 4), + shape: tuple[int, ...] = (3, 4), + chunk_shape: tuple[int, ...] = (3, 4), dtype: np.dtype = np.dtype("int32"), - compressor: dict = None, - filters: str = None, - fill_value: str = None, - encoding: dict = None, + compressor: Optional[dict] = None, + filters: Optional[list[dict[Any, Any]]] = None, + fill_value: Optional[str] = None, + encoding: Optional[dict] = None, variable_name: str = "foo", base_offset: int = 6144, length: int = 48, - dims: list[str] = None, + dims: Optional[list[str]] = None, ): manifest = generate_chunk_manifest( file_uri, @@ -391,7 +393,8 @@ def gen_virtual_dataset( ) ma = ManifestArray(chunkmanifest=manifest, zarray=zarray) ds = open_dataset(file_uri) - dims = dims or ds.sizes.keys() + ds_dims: list[str] = cast(list[str], list(ds.dims)) + dims = dims or ds_dims var = Variable( data=ma, dims=dims, @@ -441,7 +444,7 @@ def test_append_virtual_ref_without_encoding( ## When appending to a virtual ref with encoding, it succeeds def test_append_virtual_ref_with_encoding( - self, icechunk_storage: "StorageConfig", netcdf4_files_factory: callable + self, icechunk_storage: "StorageConfig", netcdf4_files_factory: Callable ): import xarray.testing as xrt from icechunk import IcechunkStore @@ -496,7 +499,7 @@ def test_append_virtual_ref_with_encoding( ## When appending to a virtual ref with compression, it succeeds def test_append_with_compression_succeeds( - self, icechunk_storage: "StorageConfig", netcdf4_files_factory: callable + self, icechunk_storage: "StorageConfig", netcdf4_files_factory: Callable ): import xarray.testing as xrt from icechunk import IcechunkStore diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index c667ea52..c130d45e 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -129,6 +129,8 @@ def resize_array( append_axis: int, ) -> "Array": existing_array = group[name] + if not isinstance(existing_array, Array): + raise ValueError("Expected existing array to be a zarr.core.Array") new_shape = list(existing_array.shape) new_shape[append_axis] += var.shape[append_axis] return existing_array.resize(tuple(new_shape)) @@ -136,8 +138,10 @@ def resize_array( def get_axis( dims: list[str], - dim_name: str, + dim_name: Optional[str], ) -> int: + if dim_name is None: + raise ValueError("dim_name must be provided") return dims.index(dim_name) @@ -150,7 +154,7 @@ def _check_compatible_arrays( manifest_api.check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) -def check_compatible_encodings(encoding1, encoding2): +def _check_compatible_encodings(encoding1, encoding2): for key, value in encoding1.items(): if key in encoding2: if encoding2[key] != value: @@ -171,7 +175,7 @@ def write_virtual_variable_to_icechunk( zarray = ma.zarray mode = store.mode.str - dims = var.dims + dims: list[str] = cast(list[str], list(var.dims)) append_axis, existing_num_chunks, arr = None, None, None if append_dim and append_dim not in dims: raise ValueError( @@ -179,9 +183,11 @@ def write_virtual_variable_to_icechunk( ) if mode == "a": existing_array = group[name] + if not isinstance(existing_array, Array): + raise ValueError("Expected existing array to be a zarr.core.Array") append_axis = get_axis(dims, append_dim) # check if arrays can be concatenated - check_compatible_encodings(var.encoding, existing_array.attrs) + _check_compatible_encodings(var.encoding, existing_array.attrs) _check_compatible_arrays(ma, existing_array, append_axis) # determine number of existing chunks along the append axis @@ -239,7 +245,9 @@ def generate_chunk_key( f"append_axis {append_axis} is greater than the number of indices {len(index)}" ) return "/".join( - str(ind + existing_num_chunks) if axis is append_axis else str(ind) + str(ind + existing_num_chunks) + if axis is append_axis and existing_num_chunks is not None + else str(ind) for axis, ind in enumerate(index) ) From ca80cb2a8ad7d135a20764b84c531727bb860e79 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 10:56:09 -0800 Subject: [PATCH 23/90] type ignore import untyped zarr --- virtualizarr/tests/test_writers/test_icechunk.py | 2 +- virtualizarr/writers/icechunk.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 9e3809b2..35c3160c 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -11,7 +11,7 @@ from xarray import Dataset, concat, open_dataset, open_zarr from xarray.core.variable import Variable from zarr import Array, Group, group # type: ignore[import-untyped] -from zarr.core.metadata import ArrayV3Metadata +from zarr.core.metadata import ArrayV3Metadata # type: ignore[import-untyped] from virtualizarr.manifests import ChunkManifest, ManifestArray from virtualizarr.writers.icechunk import dataset_to_icechunk, generate_chunk_key diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index c130d45e..72d5bee4 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Optional, cast import numpy as np -import zarr +import zarr # type: ignore[import-untyped] from xarray import Dataset from xarray.backends.zarr import encode_zarr_attr_value from xarray.core.variable import Variable From f186d4f039fa7a07fd9ed9f8c4a515eafa8642f7 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 13:13:53 -0800 Subject: [PATCH 24/90] Use Union type for check_combineable_zarr_arrays arg --- virtualizarr/manifests/array_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index 11796fa5..042b46ed 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Callable, Iterable, cast +from typing import TYPE_CHECKING, Any, Callable, Iterable, Union, cast import numpy as np @@ -27,7 +27,9 @@ def decorator(func): return decorator -def check_combineable_zarr_arrays(arrays: Iterable["ManifestArray" | "Array"]) -> None: +def check_combineable_zarr_arrays( + arrays: Iterable[Union["ManifestArray", "Array"]], +) -> None: """ The downside of the ManifestArray approach compared to the VirtualZarrArray concatenation proposal is that the result must also be a single valid zarr array, implying that the inputs must have the same dtype, codec etc. From 2a90d9cb55664f99c0a30b1486aabe90fce82d40 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 13:17:49 -0800 Subject: [PATCH 25/90] Fix import --- virtualizarr/writers/icechunk.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 72d5bee4..dcfdc24a 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -5,6 +5,7 @@ from xarray import Dataset from xarray.backends.zarr import encode_zarr_attr_value from xarray.core.variable import Variable +from zarr import Array from virtualizarr.manifests import ChunkManifest, ManifestArray from virtualizarr.manifests import array_api as manifest_api @@ -12,7 +13,7 @@ if TYPE_CHECKING: from icechunk import IcechunkStore # type: ignore[import-not-found] - from zarr import Array, Group # type: ignore + from zarr import Group # type: ignore VALID_URI_PREFIXES = { From 0253a8a06cb2a2729936bd4eb5dff80f9c17d34d Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 13:41:18 -0800 Subject: [PATCH 26/90] Fix imports for get_codecs --- virtualizarr/manifests/array_api.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index 042b46ed..c525227c 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -27,6 +27,19 @@ def decorator(func): return decorator +def _get_codecs(array: Union["ManifestArray", "Array"]) -> list[Codec]: + from zarr import Array + + from .array import ManifestArray + + if isinstance(array, ManifestArray): + if array.zarray.zarr_format == 3: + return list(array.zarray._v3_codec_pipeline()) + return [array.zarray.codec] + if isinstance(array, Array): + return array.metadata.codecs + + def check_combineable_zarr_arrays( arrays: Iterable[Union["ManifestArray", "Array"]], ) -> None: @@ -38,19 +51,7 @@ def check_combineable_zarr_arrays( # Can't combine different codecs in one manifest # see https://github.com/zarr-developers/zarr-specs/issues/288 - def get_codecs(array): - import zarr - - from .array import ManifestArray - - if isinstance(array, ManifestArray): - if array.zarray.zarr_format == 3: - return list(array.zarray._v3_codec_pipeline()) - return array.zarray.codec - if isinstance(array, zarr.core.array.Array): - return array.metadata.codecs - - _check_same_codecs([get_codecs(arr) for arr in arrays]) + _check_same_codecs([_get_codecs(arr) for arr in arrays]) # Would require variable-length chunks ZEP _check_same_chunk_shapes([arr.chunks for arr in arrays]) From 7369fcf2aabeec17dacf91941010d4d2baaaa0ec Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 13:57:58 -0800 Subject: [PATCH 27/90] use new factory in test --- virtualizarr/tests/test_xarray.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/virtualizarr/tests/test_xarray.py b/virtualizarr/tests/test_xarray.py index 062eda5f..3b03dc6e 100644 --- a/virtualizarr/tests/test_xarray.py +++ b/virtualizarr/tests/test_xarray.py @@ -1,3 +1,5 @@ +from typing import Callable + import numpy as np import pytest import xarray as xr @@ -225,8 +227,8 @@ def test_concat_dim_coords_along_existing_dim(self): @requires_kerchunk class TestCombineUsingIndexes: - def test_combine_by_coords(self, netcdf4_files): - filepath1, filepath2 = netcdf4_files + def test_combine_by_coords(self, netcdf4_files_factory: Callable): + filepath1, filepath2 = netcdf4_files_factory() with pytest.warns(UserWarning, match="will create in-memory pandas indexes"): vds1 = open_virtual_dataset(filepath1) From 294949334295e040af5138547dd65b651f7b1648 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 14:03:19 -0800 Subject: [PATCH 28/90] Remove need for dask in fixture --- conftest.py | 3 ++- virtualizarr/tests/test_writers/test_icechunk.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index 8024214c..3d18abd7 100644 --- a/conftest.py +++ b/conftest.py @@ -41,9 +41,10 @@ def netcdf4_file(tmpdir): def netcdf4_files_factory(tmpdir) -> callable: def create_netcdf4_files( encoding: Optional[Dict[str, Dict[str, Any]]] = None, + chunks: Optional[Dict[str, int]] = None, ) -> tuple[str, str]: # Set up example xarray dataset - ds = xr.tutorial.open_dataset("air_temperature", chunks={}) + ds = xr.tutorial.open_dataset("air_temperature", chunks=chunks) # Split dataset into two parts ds1 = ds.isel(time=slice(None, 1460)) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 35c3160c..ef99cbc7 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -451,7 +451,7 @@ def test_append_virtual_ref_with_encoding( scale_factor = 0.01 encoding = {"air": {"scale_factor": scale_factor, "dtype": np.dtype("float64")}} - filepath1, filepath2 = netcdf4_files_factory(encoding=encoding) + filepath1, filepath2 = netcdf4_files_factory(encoding=encoding, chunks={}) vds1, vds2 = ( gen_virtual_dataset( file_uri=filepath1, @@ -505,7 +505,7 @@ def test_append_with_compression_succeeds( from icechunk import IcechunkStore encoding = {"air": {"zlib": True, "complevel": 4, "chunksizes": (1460, 25, 53)}} - file1, file2 = netcdf4_files_factory(encoding=encoding) + file1, file2 = netcdf4_files_factory(encoding=encoding, chunks={}) # Generate compressed dataset vds1, vds2 = ( gen_virtual_dataset( From c305dadb6a3344ef431ba83643975a1071853a3d Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 14:07:36 -0800 Subject: [PATCH 29/90] Fix for when zarr is not installed --- virtualizarr/manifests/array_api.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index c525227c..db338e4b 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Callable, Iterable, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional, Union, cast import numpy as np @@ -27,8 +27,11 @@ def decorator(func): return decorator -def _get_codecs(array: Union["ManifestArray", "Array"]) -> list[Codec]: - from zarr import Array +def _get_codecs(array: Union["ManifestArray", "Array"]) -> list[Optional["Codec"]]: + try: + from zarr import Array + except ImportError: + Array = None # Fallback if `zarr` is not installed from .array import ManifestArray @@ -36,9 +39,13 @@ def _get_codecs(array: Union["ManifestArray", "Array"]) -> list[Codec]: if array.zarray.zarr_format == 3: return list(array.zarray._v3_codec_pipeline()) return [array.zarray.codec] - if isinstance(array, Array): + if Array and isinstance(array, Array): return array.metadata.codecs + raise ImportError( + "zarr is not installed, and the provided array is not of type ManifestArray." + ) + def check_combineable_zarr_arrays( arrays: Iterable[Union["ManifestArray", "Array"]], From aaede73c96276d59eec4dd196573a0eb9dff6f60 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 17:17:49 -0800 Subject: [PATCH 30/90] Address test failures --- conftest.py | 2 +- virtualizarr/manifests/array_api.py | 32 +++++-------------- .../tests/test_writers/test_icechunk.py | 5 ++- virtualizarr/writers/icechunk.py | 7 ++-- 4 files changed, 14 insertions(+), 32 deletions(-) diff --git a/conftest.py b/conftest.py index 3d18abd7..430040f1 100644 --- a/conftest.py +++ b/conftest.py @@ -43,7 +43,7 @@ def create_netcdf4_files( encoding: Optional[Dict[str, Dict[str, Any]]] = None, chunks: Optional[Dict[str, int]] = None, ) -> tuple[str, str]: - # Set up example xarray dataset + # Aimee: Figure out why chunks={} is the only way to get the icechunk append tests to pass ds = xr.tutorial.open_dataset("air_temperature", chunks=chunks) # Split dataset into two parts diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index db338e4b..3d4d2b2e 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -1,9 +1,10 @@ -from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Iterable, Union, cast import numpy as np -from virtualizarr.zarr import Codec, determine_chunk_grid_shape +from virtualizarr.zarr import determine_chunk_grid_shape +from .get_codecs import get_codecs from .manifest import ChunkManifest if TYPE_CHECKING: @@ -27,26 +28,6 @@ def decorator(func): return decorator -def _get_codecs(array: Union["ManifestArray", "Array"]) -> list[Optional["Codec"]]: - try: - from zarr import Array - except ImportError: - Array = None # Fallback if `zarr` is not installed - - from .array import ManifestArray - - if isinstance(array, ManifestArray): - if array.zarray.zarr_format == 3: - return list(array.zarray._v3_codec_pipeline()) - return [array.zarray.codec] - if Array and isinstance(array, Array): - return array.metadata.codecs - - raise ImportError( - "zarr is not installed, and the provided array is not of type ManifestArray." - ) - - def check_combineable_zarr_arrays( arrays: Iterable[Union["ManifestArray", "Array"]], ) -> None: @@ -58,7 +39,7 @@ def check_combineable_zarr_arrays( # Can't combine different codecs in one manifest # see https://github.com/zarr-developers/zarr-specs/issues/288 - _check_same_codecs([_get_codecs(arr) for arr in arrays]) + _check_same_codecs([get_codecs(arr) for arr in arrays]) # Would require variable-length chunks ZEP _check_same_chunk_shapes([arr.chunks for arr in arrays]) @@ -75,7 +56,10 @@ def _check_same_dtypes(dtypes: list[np.dtype]) -> None: ) -def _check_same_codecs(codecs: list[Codec]) -> None: +# Aimee: is there a way to type this list? ManifestArrays will return a list of virtualizarr.zarr.Codec objects +# whereas Zarr v3 arrays will return a list of ArrayArrayCodec, etc +# and Zarr v2 arrays will return ?? +def _check_same_codecs(codecs: list[Any]) -> None: first_codec, *other_codecs = codecs for codec in other_codecs: if codec != first_codec: diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index ef99cbc7..4c888bee 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -10,8 +10,8 @@ import numpy.testing as npt from xarray import Dataset, concat, open_dataset, open_zarr from xarray.core.variable import Variable -from zarr import Array, Group, group # type: ignore[import-untyped] -from zarr.core.metadata import ArrayV3Metadata # type: ignore[import-untyped] +from zarr import Array, Group, group # type: ignore +from zarr.core.metadata import ArrayV3Metadata # type: ignore from virtualizarr.manifests import ChunkManifest, ManifestArray from virtualizarr.writers.icechunk import dataset_to_icechunk, generate_chunk_key @@ -422,7 +422,6 @@ def test_append_virtual_ref_without_encoding( # generate virtual dataset vds = gen_virtual_dataset(file_uri=simple_netcdf4) - # create the icechunk store and commit the first virtual dataset icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) dataset_to_icechunk(vds, icechunk_filestore) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index dcfdc24a..315e58ea 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -128,13 +128,14 @@ def resize_array( name: str, var: Variable, append_axis: int, -) -> "Array": +) -> Array: existing_array = group[name] if not isinstance(existing_array, Array): raise ValueError("Expected existing array to be a zarr.core.Array") new_shape = list(existing_array.shape) new_shape[append_axis] += var.shape[append_axis] - return existing_array.resize(tuple(new_shape)) + existing_array.resize(tuple(new_shape)) + return existing_array def get_axis( @@ -268,8 +269,6 @@ def write_manifest_virtual_refs( # loop over every reference in the ChunkManifest for that array # TODO inefficient: this should be replaced with something that sets all (new) references for the array at once # but Icechunk need to expose a suitable API first - # Aimee: the manifest (and it's corresponding paths, offsets and lengths, already has the shape of the datacube's chunks - # so we want to increment the resulting multi index it = np.nditer( [manifest._paths, manifest._offsets, manifest._lengths], # type: ignore[arg-type] flags=[ From 5d685c69558f698db2476ac20bf03d912cb38366 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 17:18:09 -0800 Subject: [PATCH 31/90] Add get_codecs file --- virtualizarr/manifests/get_codecs.py | 79 ++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 virtualizarr/manifests/get_codecs.py diff --git a/virtualizarr/manifests/get_codecs.py b/virtualizarr/manifests/get_codecs.py new file mode 100644 index 00000000..a36515cf --- /dev/null +++ b/virtualizarr/manifests/get_codecs.py @@ -0,0 +1,79 @@ +from typing import TYPE_CHECKING, Any, List, Optional, Union + +from virtualizarr.zarr import Codec + +if TYPE_CHECKING: + from zarr import Array # type: ignore + + from .array import ManifestArray + + +def get_codecs(array: Union["ManifestArray", "Array"]) -> Any: + """ + Get the codecs for either a ManifestArray or a Zarr Array. + + Parameters: + array (Union[ManifestArray, ZarrArray]): The input array, either ManifestArray or Zarr Array. + + Returns: + List[Optional[Codec]]: A list of codecs or an empty list if no codecs are found. + + Raises: + ImportError: If `zarr` is required but not installed. + ValueError: If the array type is unsupported. + """ + if _is_manifest_array(array): + return _get_manifestarray_codecs(array) # type: ignore[arg-type] + + if _is_zarr_array(array): + return _get_zarr_array_codecs(array) # type: ignore[arg-type] + + raise ValueError("Unsupported array type or zarr is not installed.") + + +def _is_manifest_array(array: object) -> bool: + """Check if the array is an instance of ManifestArray.""" + try: + from .array import ManifestArray + + return isinstance(array, ManifestArray) + except ImportError: + return False + + +def _get_manifestarray_codecs(array: "ManifestArray") -> List[Optional["Codec"]]: + """Get codecs for a ManifestArray based on its zarr_format.""" + if array.zarray.zarr_format == 3: + return list(array.zarray._v3_codec_pipeline()) + elif array.zarray.zarr_format == 2: + return [array.zarray.codec] + else: + raise ValueError("Unsupported zarr_format for ManifestArray.") + + +def _is_zarr_array(array: object) -> bool: + """Check if the array is an instance of Zarr Array.""" + try: + from zarr import Array + + return isinstance(array, Array) + except ImportError: + return False + + +def _get_zarr_array_codecs(array: "Array") -> Any: + """Get codecs for a Zarr Array based on its format.""" + try: + from zarr import Array + + if not isinstance(array, Array): + raise ValueError("Provided array is not a Zarr Array.") + + if hasattr(array, "metadata") and hasattr(array.metadata, "codecs"): + return array.metadata.codecs + elif hasattr(array, "compressor") and hasattr(array, "filters"): + return [Codec(compressor=array.compressor, filters=array.filters)] + else: + raise ValueError("Unsupported zarr_format for Zarr Array.") + except ImportError: + raise ImportError("zarr is not installed, but a Zarr Array was provided.") From d704de2851ced9a7b948663056e4f2a1b8d50931 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 15 Nov 2024 17:20:23 -0800 Subject: [PATCH 32/90] Add dask to upstream --- ci/upstream.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/upstream.yml b/ci/upstream.yml index 035d76f8..66c4e7e2 100644 --- a/ci/upstream.yml +++ b/ci/upstream.yml @@ -3,6 +3,7 @@ channels: - conda-forge - nodefaults dependencies: + - dask - xarray>=2024.10.0 - h5netcdf - h5py @@ -25,6 +26,6 @@ dependencies: - fsspec - pip - pip: - - icechunk # Installs zarr v3 as dependency - - git+https://github.com/zarr-developers/numcodecs@zarr3-codecs # zarr-v3 compatibility branch + - icechunk # Installs zarr v3 as dependency + - git+https://github.com/zarr-developers/numcodecs@zarr3-codecs # zarr-v3 compatibility branch # - git+https://github.com/fsspec/kerchunk@main # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516) From d84c58c5af4c343e319cd22cbf35b62aa2242860 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sun, 17 Nov 2024 13:16:53 -0800 Subject: [PATCH 33/90] Remove dependency on dask and h5netcdf engine --- ci/upstream.yml | 1 - conftest.py | 7 +++---- .../tests/test_writers/test_icechunk.py | 21 ++++++++++++------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ci/upstream.yml b/ci/upstream.yml index 66c4e7e2..d1d5e1f7 100644 --- a/ci/upstream.yml +++ b/ci/upstream.yml @@ -3,7 +3,6 @@ channels: - conda-forge - nodefaults dependencies: - - dask - xarray>=2024.10.0 - h5netcdf - h5py diff --git a/conftest.py b/conftest.py index 430040f1..917c0cef 100644 --- a/conftest.py +++ b/conftest.py @@ -41,10 +41,9 @@ def netcdf4_file(tmpdir): def netcdf4_files_factory(tmpdir) -> callable: def create_netcdf4_files( encoding: Optional[Dict[str, Dict[str, Any]]] = None, - chunks: Optional[Dict[str, int]] = None, ) -> tuple[str, str]: # Aimee: Figure out why chunks={} is the only way to get the icechunk append tests to pass - ds = xr.tutorial.open_dataset("air_temperature", chunks=chunks) + ds = xr.tutorial.open_dataset("air_temperature") # Split dataset into two parts ds1 = ds.isel(time=slice(None, 1460)) @@ -53,8 +52,8 @@ def create_netcdf4_files( # Save datasets to disk as NetCDF in the temporary directory with the provided encoding filepath1 = f"{tmpdir}/air1.nc" filepath2 = f"{tmpdir}/air2.nc" - ds1.to_netcdf(filepath1, encoding=encoding, engine="h5netcdf") - ds2.to_netcdf(filepath2, encoding=encoding, engine="h5netcdf") + ds1.to_netcdf(filepath1, encoding=encoding) + ds2.to_netcdf(filepath2, encoding=encoding) # Close datasets ds1.close() diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 4c888bee..9cc38e08 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -450,7 +450,7 @@ def test_append_virtual_ref_with_encoding( scale_factor = 0.01 encoding = {"air": {"scale_factor": scale_factor, "dtype": np.dtype("float64")}} - filepath1, filepath2 = netcdf4_files_factory(encoding=encoding, chunks={}) + filepath1, filepath2 = netcdf4_files_factory(encoding=encoding) vds1, vds2 = ( gen_virtual_dataset( file_uri=filepath1, @@ -460,7 +460,7 @@ def test_append_virtual_ref_with_encoding( dtype=np.dtype("float64"), variable_name="air", encoding={"scale_factor": scale_factor}, - base_offset=20146, + base_offset=15419, length=15476000, ), gen_virtual_dataset( @@ -471,7 +471,7 @@ def test_append_virtual_ref_with_encoding( dtype=np.dtype("float64"), variable_name="air", encoding={"scale_factor": scale_factor}, - base_offset=20146, + base_offset=15419, length=15476000, ), ) @@ -503,8 +503,15 @@ def test_append_with_compression_succeeds( import xarray.testing as xrt from icechunk import IcechunkStore - encoding = {"air": {"zlib": True, "complevel": 4, "chunksizes": (1460, 25, 53)}} - file1, file2 = netcdf4_files_factory(encoding=encoding, chunks={}) + encoding = { + "air": { + "zlib": True, + "complevel": 4, + "chunksizes": (1460, 25, 53), + "shuffle": False, + } + } + file1, file2 = netcdf4_files_factory(encoding=encoding) # Generate compressed dataset vds1, vds2 = ( gen_virtual_dataset( @@ -515,7 +522,7 @@ def test_append_with_compression_succeeds( dims=["time", "lat", "lon"], dtype=np.dtype("float64"), variable_name="air", - base_offset=23214, + base_offset=18043, length=3936114, ), gen_virtual_dataset( @@ -526,7 +533,7 @@ def test_append_with_compression_succeeds( dims=["time", "lat", "lon"], dtype=np.dtype("float64"), variable_name="air", - base_offset=23214, + base_offset=18043, length=3938672, ), ) From 2505d9e96744f1c10f60ece59e8e8dd39a336ff3 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 18 Nov 2024 08:19:32 -0800 Subject: [PATCH 34/90] Remove obsolete comment --- conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conftest.py b/conftest.py index 917c0cef..e86b9244 100644 --- a/conftest.py +++ b/conftest.py @@ -42,7 +42,6 @@ def netcdf4_files_factory(tmpdir) -> callable: def create_netcdf4_files( encoding: Optional[Dict[str, Dict[str, Any]]] = None, ) -> tuple[str, str]: - # Aimee: Figure out why chunks={} is the only way to get the icechunk append tests to pass ds = xr.tutorial.open_dataset("air_temperature") # Split dataset into two parts From aaa7f01d22c56219ea938e87601b209f732b9092 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 18 Nov 2024 08:23:37 -0800 Subject: [PATCH 35/90] Remove duplicate zarr array type check --- virtualizarr/manifests/get_codecs.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/virtualizarr/manifests/get_codecs.py b/virtualizarr/manifests/get_codecs.py index a36515cf..4cd8a254 100644 --- a/virtualizarr/manifests/get_codecs.py +++ b/virtualizarr/manifests/get_codecs.py @@ -64,13 +64,10 @@ def _is_zarr_array(array: object) -> bool: def _get_zarr_array_codecs(array: "Array") -> Any: """Get codecs for a Zarr Array based on its format.""" try: - from zarr import Array - - if not isinstance(array, Array): - raise ValueError("Provided array is not a Zarr Array.") - + # For Zarr v3 if hasattr(array, "metadata") and hasattr(array.metadata, "codecs"): return array.metadata.codecs + # For Zarr v2 elif hasattr(array, "compressor") and hasattr(array, "filters"): return [Codec(compressor=array.compressor, filters=array.filters)] else: From 5071ed76cdae2df17555b03f017d0c451092801e Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 18 Nov 2024 17:13:44 -0800 Subject: [PATCH 36/90] Move codecs module and type output --- virtualizarr/manifests/array_api.py | 4 +- virtualizarr/manifests/get_codecs.py | 76 ---------------------------- 2 files changed, 2 insertions(+), 78 deletions(-) delete mode 100644 virtualizarr/manifests/get_codecs.py diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index 3d4d2b2e..f5435338 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -4,11 +4,11 @@ from virtualizarr.zarr import determine_chunk_grid_shape -from .get_codecs import get_codecs +from ..codecs import get_codecs from .manifest import ChunkManifest if TYPE_CHECKING: - from zarr import Array # type: ignore + from zarr import Array from .array import ManifestArray diff --git a/virtualizarr/manifests/get_codecs.py b/virtualizarr/manifests/get_codecs.py deleted file mode 100644 index 4cd8a254..00000000 --- a/virtualizarr/manifests/get_codecs.py +++ /dev/null @@ -1,76 +0,0 @@ -from typing import TYPE_CHECKING, Any, List, Optional, Union - -from virtualizarr.zarr import Codec - -if TYPE_CHECKING: - from zarr import Array # type: ignore - - from .array import ManifestArray - - -def get_codecs(array: Union["ManifestArray", "Array"]) -> Any: - """ - Get the codecs for either a ManifestArray or a Zarr Array. - - Parameters: - array (Union[ManifestArray, ZarrArray]): The input array, either ManifestArray or Zarr Array. - - Returns: - List[Optional[Codec]]: A list of codecs or an empty list if no codecs are found. - - Raises: - ImportError: If `zarr` is required but not installed. - ValueError: If the array type is unsupported. - """ - if _is_manifest_array(array): - return _get_manifestarray_codecs(array) # type: ignore[arg-type] - - if _is_zarr_array(array): - return _get_zarr_array_codecs(array) # type: ignore[arg-type] - - raise ValueError("Unsupported array type or zarr is not installed.") - - -def _is_manifest_array(array: object) -> bool: - """Check if the array is an instance of ManifestArray.""" - try: - from .array import ManifestArray - - return isinstance(array, ManifestArray) - except ImportError: - return False - - -def _get_manifestarray_codecs(array: "ManifestArray") -> List[Optional["Codec"]]: - """Get codecs for a ManifestArray based on its zarr_format.""" - if array.zarray.zarr_format == 3: - return list(array.zarray._v3_codec_pipeline()) - elif array.zarray.zarr_format == 2: - return [array.zarray.codec] - else: - raise ValueError("Unsupported zarr_format for ManifestArray.") - - -def _is_zarr_array(array: object) -> bool: - """Check if the array is an instance of Zarr Array.""" - try: - from zarr import Array - - return isinstance(array, Array) - except ImportError: - return False - - -def _get_zarr_array_codecs(array: "Array") -> Any: - """Get codecs for a Zarr Array based on its format.""" - try: - # For Zarr v3 - if hasattr(array, "metadata") and hasattr(array.metadata, "codecs"): - return array.metadata.codecs - # For Zarr v2 - elif hasattr(array, "compressor") and hasattr(array, "filters"): - return [Codec(compressor=array.compressor, filters=array.filters)] - else: - raise ValueError("Unsupported zarr_format for Zarr Array.") - except ImportError: - raise ImportError("zarr is not installed, but a Zarr Array was provided.") From 7067b43e69f5a80d9511bc9672b11510a2ddd488 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 18 Nov 2024 17:14:04 -0800 Subject: [PATCH 37/90] Actually add codecs file --- virtualizarr/codecs.py | 90 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 virtualizarr/codecs.py diff --git a/virtualizarr/codecs.py b/virtualizarr/codecs.py new file mode 100644 index 00000000..c365cbc6 --- /dev/null +++ b/virtualizarr/codecs.py @@ -0,0 +1,90 @@ +from typing import TYPE_CHECKING, Union + +from virtualizarr.zarr import Codec + +if TYPE_CHECKING: + from zarr import Array # type: ignore + from zarr.core.abc.codec import ( # type: ignore + ArrayArrayCodec, + ArrayBytesCodec, + BytesBytesCodec, + ) + + from .manifests.array import ManifestArray + + +def get_codecs( + array: Union["ManifestArray", "Array"], +) -> Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]: + """ + Get the codecs for either a ManifestArray or a Zarr Array. + + Parameters: + array (Union[ManifestArray, ZarrArray]): The input array, either ManifestArray or Zarr Array. + + Returns: + List[Optional[Codec]]: A list of codecs or an empty list if no codecs are found. + + Raises: + ImportError: If `zarr` is required but not installed. + ValueError: If the array type is unsupported. + """ + if _is_manifest_array(array): + return _get_manifestarray_codecs(array) # type: ignore[arg-type] + + if _is_zarr_array(array): + return _get_zarr_array_codecs(array) # type: ignore[arg-type] + + raise ValueError("Unsupported array type or zarr is not installed.") + + +def _is_manifest_array(array: object) -> bool: + """Check if the array is an instance of ManifestArray.""" + try: + from .manifests.array import ManifestArray + + return isinstance(array, ManifestArray) + except ImportError: + return False + + +def _get_manifestarray_codecs( + array: "ManifestArray", +) -> Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]: + """Get codecs for a ManifestArray based on its zarr_format.""" + if array.zarray.zarr_format == 3: + # I think this returns a list from a tuple of Zarr Codec objects (zarr.abc.codec) + return array.zarray._v3_codec_pipeline() + elif array.zarray.zarr_format == 2: + # This returns virtualizarr.zarr.ZArray.codec which is + # its own data class with compression as a dict and filters as a list of dicts + return array.zarray.codec + else: + raise ValueError("Unsupported zarr_format for ManifestArray.") + + +def _is_zarr_array(array: object) -> bool: + """Check if the array is an instance of Zarr Array.""" + try: + from zarr import Array + + return isinstance(array, Array) + except ImportError: + return False + + +def _get_zarr_array_codecs( + array: "Array", +) -> Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]: + """Get codecs for a Zarr Array based on its format.""" + try: + # For Zarr v3 + if hasattr(array, "metadata") and array.metadata.zarr_format == 3: + return tuple(array.metadata.codecs) + # For Zarr v2 + elif hasattr(array, "_meta") and array._meta.zarr_format == 2: + return Codec(compressor=array.compressor, filters=array.filters) # type: ignore[attr-defined] + else: + raise ValueError("Unsupported zarr_format for Zarr Array.") + except ImportError: + raise ImportError("zarr is not installed, but a Zarr Array was provided.") From 0ec5084f3a6f8ac2b1dbf560559c49489ac7528f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 18 Nov 2024 17:29:00 -0800 Subject: [PATCH 38/90] Fix merge mistake --- virtualizarr/tests/test_xarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_xarray.py b/virtualizarr/tests/test_xarray.py index 66ec6f60..4d01a9cb 100644 --- a/virtualizarr/tests/test_xarray.py +++ b/virtualizarr/tests/test_xarray.py @@ -229,7 +229,7 @@ def test_concat_dim_coords_along_existing_dim(self): @requires_kerchunk @pytest.mark.parametrize("hdf_backend", [None, HDFVirtualBackend]) class TestCombineUsingIndexes: - def test_combine_by_coords(self, netcdf4_files_factory: Callable): + def test_combine_by_coords(self, netcdf4_files_factory: Callable, hdf_backend): filepath1, filepath2 = netcdf4_files_factory() with pytest.warns(UserWarning, match="will create in-memory pandas indexes"): From 5630b34f3e4b4c0ab48e118a082704aa979855a6 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 18 Nov 2024 17:33:10 -0800 Subject: [PATCH 39/90] Ignore import untyped --- virtualizarr/manifests/array_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index f5435338..18a1bba9 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -8,7 +8,7 @@ from .manifest import ChunkManifest if TYPE_CHECKING: - from zarr import Array + from zarr import Array # type: ignore[import-untyped] from .array import ManifestArray From d93f2ce0317e02c6af25f6ffb843d94243c2d376 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 13:53:55 -0800 Subject: [PATCH 40/90] Add tests for codecs --- virtualizarr/codecs.py | 29 +++--- virtualizarr/tests/test_codecs.py | 154 ++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 11 deletions(-) create mode 100644 virtualizarr/tests/test_codecs.py diff --git a/virtualizarr/codecs.py b/virtualizarr/codecs.py index c365cbc6..9f2492bb 100644 --- a/virtualizarr/codecs.py +++ b/virtualizarr/codecs.py @@ -77,14 +77,21 @@ def _get_zarr_array_codecs( array: "Array", ) -> Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]: """Get codecs for a Zarr Array based on its format.""" - try: - # For Zarr v3 - if hasattr(array, "metadata") and array.metadata.zarr_format == 3: - return tuple(array.metadata.codecs) - # For Zarr v2 - elif hasattr(array, "_meta") and array._meta.zarr_format == 2: - return Codec(compressor=array.compressor, filters=array.filters) # type: ignore[attr-defined] - else: - raise ValueError("Unsupported zarr_format for Zarr Array.") - except ImportError: - raise ImportError("zarr is not installed, but a Zarr Array was provided.") + import zarr + from packaging import version + + # Check that zarr-python v3 is installed + required_version = "3.0.0b" + installed_version = zarr.__version__ + if version.parse(installed_version) < version.parse(required_version): + raise NotImplementedError( + f"zarr-python v3 or higher is required, but version {installed_version} is installed." + ) + # For zarr format v3 + if hasattr(array, "metadata") and array.metadata.zarr_format == 3: + return tuple(array.metadata.codecs) + # For zarr format v2 + elif hasattr(array, "metadata") and array.metadata.zarr_format == 2: + return Codec( + compressor=array.metadata.compressor, filters=array.metadata.filters + ) # type: ignore[attr-defined] diff --git a/virtualizarr/tests/test_codecs.py b/virtualizarr/tests/test_codecs.py new file mode 100644 index 00000000..5580e0c5 --- /dev/null +++ b/virtualizarr/tests/test_codecs.py @@ -0,0 +1,154 @@ +from functools import wraps +from unittest.mock import patch + +import numpy as np +import pytest +import zarr +from numcodecs import Blosc, Delta +from packaging.version import parse + +from virtualizarr import ChunkManifest, ManifestArray +from virtualizarr.codecs import get_codecs +from virtualizarr.zarr import Codec + + +def requires_version(package_name, required_version): + """ + Decorator to skip a test if the specified package is not installed or does not meet the required version. + + Args: + package_name (str): Name of the package to check. + required_version (str): Minimum required version of the package. + """ + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + # Attempt to import the package + package = pytest.importorskip(package_name) + # Check if the version meets the requirement + if parse(package.__version__) < parse(required_version): + pytest.skip( + f"'{package_name}' version >= {required_version} is required." + ) + # Proceed with the test + return func(*args, **kwargs) + + return wrapper + + return decorator + + +class TestCodecs: + def create_manifest_array(self, compressor=None, filters=None, zarr_format=2): + return ManifestArray( + chunkmanifest=ChunkManifest( + entries={"0.0": dict(path="/test.nc", offset=6144, length=48)} + ), + zarray=dict( + shape=(2, 3), + dtype=np.dtype(" Date: Wed, 20 Nov 2024 14:02:54 -0800 Subject: [PATCH 41/90] Resolve mypy errors --- virtualizarr/codecs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/virtualizarr/codecs.py b/virtualizarr/codecs.py index 9f2492bb..8ed87207 100644 --- a/virtualizarr/codecs.py +++ b/virtualizarr/codecs.py @@ -53,11 +53,8 @@ def _get_manifestarray_codecs( ) -> Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]: """Get codecs for a ManifestArray based on its zarr_format.""" if array.zarray.zarr_format == 3: - # I think this returns a list from a tuple of Zarr Codec objects (zarr.abc.codec) return array.zarray._v3_codec_pipeline() elif array.zarray.zarr_format == 2: - # This returns virtualizarr.zarr.ZArray.codec which is - # its own data class with compression as a dict and filters as a list of dicts return array.zarray.codec else: raise ValueError("Unsupported zarr_format for ManifestArray.") @@ -93,5 +90,8 @@ def _get_zarr_array_codecs( # For zarr format v2 elif hasattr(array, "metadata") and array.metadata.zarr_format == 2: return Codec( - compressor=array.metadata.compressor, filters=array.metadata.filters - ) # type: ignore[attr-defined] + compressor=array.metadata.compressor, + filters=list(array.metadata.filters or ()), + ) + else: + raise ValueError("Unsupported zarr_format for Zarr Array.") From 98a676db3a703ba4bbdaaaa5a1071eedb81ddd3e Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 14:05:10 -0800 Subject: [PATCH 42/90] Fix test --- virtualizarr/tests/test_codecs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_codecs.py b/virtualizarr/tests/test_codecs.py index 5580e0c5..8659197a 100644 --- a/virtualizarr/tests/test_codecs.py +++ b/virtualizarr/tests/test_codecs.py @@ -124,7 +124,9 @@ def test_zarr_v2_array_codecs(self): compressor=Blosc( cname="zstd", clevel=5, shuffle=Blosc.SHUFFLE, blocksize=0 ), - filters=(Delta(dtype=" Date: Wed, 20 Nov 2024 14:07:26 -0800 Subject: [PATCH 43/90] Import zarr in function --- virtualizarr/tests/test_codecs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_codecs.py b/virtualizarr/tests/test_codecs.py index 8659197a..9fc4ef66 100644 --- a/virtualizarr/tests/test_codecs.py +++ b/virtualizarr/tests/test_codecs.py @@ -3,7 +3,6 @@ import numpy as np import pytest -import zarr from numcodecs import Blosc, Delta from packaging.version import parse @@ -92,6 +91,8 @@ def create_zarr_array( codecs=None, zarr_format=2, ): + import zarr + shared_kwargs = { "shape": (1000, 1000), "chunks": (100, 100), From 145ed0e37d863f6603b910d06c03bdf761dcf5ce Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 14:16:59 -0800 Subject: [PATCH 44/90] Use existing importorskip function --- virtualizarr/tests/__init__.py | 2 ++ virtualizarr/tests/test_codecs.py | 40 +++++++------------------------ 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/virtualizarr/tests/__init__.py b/virtualizarr/tests/__init__.py index aee82542..db5b980f 100644 --- a/virtualizarr/tests/__init__.py +++ b/virtualizarr/tests/__init__.py @@ -39,6 +39,8 @@ def _importorskip( has_tifffile, requires_tifffile = _importorskip("tifffile") has_imagecodecs, requires_imagecodecs = _importorskip("imagecodecs") has_hdf5plugin, requires_hdf5plugin = _importorskip("hdf5plugin") +has_zarr_python, requires_zarr_python = _importorskip("zarr") +has_zarr_python_v3, requires_zarr_python_v3 = _importorskip("zarr", "3.0.0b") def create_manifestarray( diff --git a/virtualizarr/tests/test_codecs.py b/virtualizarr/tests/test_codecs.py index 9fc4ef66..2625dd12 100644 --- a/virtualizarr/tests/test_codecs.py +++ b/virtualizarr/tests/test_codecs.py @@ -1,43 +1,17 @@ -from functools import wraps from unittest.mock import patch import numpy as np -import pytest from numcodecs import Blosc, Delta -from packaging.version import parse from virtualizarr import ChunkManifest, ManifestArray from virtualizarr.codecs import get_codecs +from virtualizarr.tests import ( + requires_zarr_python, + requires_zarr_python_v3, +) from virtualizarr.zarr import Codec -def requires_version(package_name, required_version): - """ - Decorator to skip a test if the specified package is not installed or does not meet the required version. - - Args: - package_name (str): Name of the package to check. - required_version (str): Minimum required version of the package. - """ - - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - # Attempt to import the package - package = pytest.importorskip(package_name) - # Check if the version meets the requirement - if parse(package.__version__) < parse(required_version): - pytest.skip( - f"'{package_name}' version >= {required_version} is required." - ) - # Proceed with the test - return func(*args, **kwargs) - - return wrapper - - return decorator - - class TestCodecs: def create_manifest_array(self, compressor=None, filters=None, zarr_format=2): return ManifestArray( @@ -72,7 +46,7 @@ def test_manifest_array_codecs_v2(self): ) assert actual_codecs == expected_codecs - @requires_version("zarr", "3.0.0b") + @requires_zarr_python_v3 def test_manifest_array_codecs_v3(self): """Test that get_codecs works for ManifestArray with Zarr v2 metadata.""" from zarr.codecs import BytesCodec @@ -113,6 +87,7 @@ def create_zarr_array( return zarr_array + @requires_zarr_python_v3 def test_zarr_v2_array_codecs(self): # Define your codecs (compressor and filters) compressor = Blosc(cname="zstd", clevel=5, shuffle=Blosc.SHUFFLE) @@ -131,7 +106,7 @@ def test_zarr_v2_array_codecs(self): ) assert actual_codecs == expected_codecs - @requires_version("zarr", "3.0.0b") + @requires_zarr_python_v3 def test_zarr_v3_array_codecs(self): from zarr.codecs import BytesCodec @@ -144,6 +119,7 @@ def test_zarr_v3_array_codecs(self): expected_codecs = tuple([BytesCodec(endian="little")]) assert actual_codecs == expected_codecs + @requires_zarr_python def test_unsupported_zarr_python(self): zarr_array = self.create_zarr_array() unsupported_zarr_version = "2.18.3" From 80cd3580a4832255d8a2b6901ff75f86c905073c Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 14:47:04 -0800 Subject: [PATCH 45/90] Modify comments --- virtualizarr/tests/test_writers/test_icechunk.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 9cc38e08..337d64bd 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -126,8 +126,6 @@ def test_set_single_virtual_ref_with_encoding( icechunk_filestore: "IcechunkStore", netcdf4_file: Path ): import xarray.testing as xrt - # TODO kerchunk doesn't work with zarr-python v3 yet so we can't use open_virtual_dataset and icechunk together! - # vds = open_virtual_dataset(netcdf4_file, indexes={}) expected_ds = open_dataset(netcdf4_file).drop_vars(["lon", "lat", "time"]) # these attributes encode floats different and I am not sure why, but its not important enough to block everything @@ -438,8 +436,7 @@ def test_append_virtual_ref_without_encoding( expected_ds = open_dataset(simple_netcdf4) expected_array = concat([expected_ds, expected_ds], dim="x") - # Aimee: attributes for actual_range differ [185.16000366210935, 322.1000061035156] vs [185.16 322.1] - xrt.assert_equal(array, expected_array) + xrt.assert_identical(array, expected_array) ## When appending to a virtual ref with encoding, it succeeds def test_append_virtual_ref_with_encoding( @@ -449,7 +446,7 @@ def test_append_virtual_ref_with_encoding( from icechunk import IcechunkStore scale_factor = 0.01 - encoding = {"air": {"scale_factor": scale_factor, "dtype": np.dtype("float64")}} + encoding = {"air": {"scale_factor": scale_factor}} filepath1, filepath2 = netcdf4_files_factory(encoding=encoding) vds1, vds2 = ( gen_virtual_dataset( @@ -494,6 +491,9 @@ def test_append_virtual_ref_with_encoding( expected_ds = concat([expected_ds1, expected_ds2], dim="time").drop_vars( ["lon", "lat", "time"], errors="ignore" ) + # Because we encode attributes, attributes may differ, for example + # actual_range for expected_ds.air is array([185.16, 322.1 ], dtype=float32) + # but encoded it is [185.16000366210935, 322.1000061035156] xrt.assert_equal(new_ds, expected_ds) ## When appending to a virtual ref with compression, it succeeds From 3035f0500bcfa30dff514b80d141dd5a0c53ac13 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 15:00:37 -0800 Subject: [PATCH 46/90] Comment updates and spelling of combinable --- virtualizarr/manifests/array_api.py | 6 +++--- virtualizarr/writers/icechunk.py | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index 18a1bba9..d881d659 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -28,7 +28,7 @@ def decorator(func): return decorator -def check_combineable_zarr_arrays( +def check_combinable_zarr_arrays( arrays: Iterable[Union["ManifestArray", "Array"]], ) -> None: """ @@ -114,7 +114,7 @@ def concatenate( raise TypeError() # ensure dtypes, shapes, codecs etc. are consistent - check_combineable_zarr_arrays(arrays) + check_combinable_zarr_arrays(arrays) check_same_ndims([arr.ndim for arr in arrays]) @@ -207,7 +207,7 @@ def stack( raise TypeError() # ensure dtypes, shapes, codecs etc. are consistent - check_combineable_zarr_arrays(arrays) + check_combinable_zarr_arrays(arrays) check_same_ndims([arr.ndim for arr in arrays]) arr_shapes = [arr.shape for arr in arrays] diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 315e58ea..594c7283 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -130,6 +130,8 @@ def resize_array( append_axis: int, ) -> Array: existing_array = group[name] + # This is the second time we check if the array is an instance of zarr.core.Array + # but it's necessary to ensure .shape and .resize are available if not isinstance(existing_array, Array): raise ValueError("Expected existing array to be a zarr.core.Array") new_shape = list(existing_array.shape) @@ -150,10 +152,11 @@ def get_axis( def _check_compatible_arrays( ma: ManifestArray, existing_array: zarr.core.array.Array, append_axis: int ): - manifest_api.check_combineable_zarr_arrays([ma, existing_array]) + manifest_api.check_combinable_zarr_arrays([ma, existing_array]) manifest_api.check_same_ndims([ma.ndim, existing_array.ndim]) arr_shapes = [ma.shape, existing_array.shape] manifest_api.check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) + _check_compatible_encodings(ma.encoding, existing_array.attrs) def _check_compatible_encodings(encoding1, encoding2): @@ -188,8 +191,8 @@ def write_virtual_variable_to_icechunk( if not isinstance(existing_array, Array): raise ValueError("Expected existing array to be a zarr.core.Array") append_axis = get_axis(dims, append_dim) + # check if arrays can be concatenated - _check_compatible_encodings(var.encoding, existing_array.attrs) _check_compatible_arrays(ma, existing_array, append_axis) # determine number of existing chunks along the append axis @@ -215,7 +218,6 @@ def write_virtual_variable_to_icechunk( codecs=zarray._v3_codec_pipeline(), dimension_names=var.dims, fill_value=zarray.fill_value, - # TODO fill_value? ) # TODO it would be nice if we could assign directly to the .attrs property From 28e05db063642ebbf85cd5dfb523c6b18a30e729 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 15:07:17 -0800 Subject: [PATCH 47/90] Revert change to check compatible encoding --- virtualizarr/writers/icechunk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 594c7283..ed71fee5 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -156,7 +156,6 @@ def _check_compatible_arrays( manifest_api.check_same_ndims([ma.ndim, existing_array.ndim]) arr_shapes = [ma.shape, existing_array.shape] manifest_api.check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) - _check_compatible_encodings(ma.encoding, existing_array.attrs) def _check_compatible_encodings(encoding1, encoding2): @@ -194,6 +193,7 @@ def write_virtual_variable_to_icechunk( # check if arrays can be concatenated _check_compatible_arrays(ma, existing_array, append_axis) + _check_compatible_encodings(var.encoding, existing_array.attrs) # determine number of existing chunks along the append axis existing_num_chunks = num_chunks( From 2c07cdfc4f86c1d648957a5e024e2a80979218ae Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 15:13:36 -0800 Subject: [PATCH 48/90] Ignore zarr untyped import errors --- virtualizarr/tests/test_codecs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/virtualizarr/tests/test_codecs.py b/virtualizarr/tests/test_codecs.py index 2625dd12..cb874618 100644 --- a/virtualizarr/tests/test_codecs.py +++ b/virtualizarr/tests/test_codecs.py @@ -49,7 +49,7 @@ def test_manifest_array_codecs_v2(self): @requires_zarr_python_v3 def test_manifest_array_codecs_v3(self): """Test that get_codecs works for ManifestArray with Zarr v2 metadata.""" - from zarr.codecs import BytesCodec + from zarr.codecs import BytesCodec # type: ignore[import-untyped] manifest_array = self.create_manifest_array(zarr_format=3) @@ -65,7 +65,7 @@ def create_zarr_array( codecs=None, zarr_format=2, ): - import zarr + import zarr # type: ignore[import-untyped] shared_kwargs = { "shape": (1000, 1000), @@ -108,7 +108,7 @@ def test_zarr_v2_array_codecs(self): @requires_zarr_python_v3 def test_zarr_v3_array_codecs(self): - from zarr.codecs import BytesCodec + from zarr.codecs import BytesCodec # type: ignore[import-untyped] zarr_array_v3 = self.create_zarr_array( codecs=[BytesCodec(endian="little")], zarr_format=3 From 19837b2c5d348e22e3b45725c4ab67ac86e1aee3 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 18:11:37 -0800 Subject: [PATCH 49/90] Implement a manifest.utils module --- virtualizarr/manifests/array_api.py | 106 ++----------------------- virtualizarr/manifests/utils.py | 118 ++++++++++++++++++++++++++++ virtualizarr/writers/icechunk.py | 28 +++---- 3 files changed, 138 insertions(+), 114 deletions(-) create mode 100644 virtualizarr/manifests/utils.py diff --git a/virtualizarr/manifests/array_api.py b/virtualizarr/manifests/array_api.py index d881d659..4950b48c 100644 --- a/virtualizarr/manifests/array_api.py +++ b/virtualizarr/manifests/array_api.py @@ -1,15 +1,18 @@ -from typing import TYPE_CHECKING, Any, Callable, Iterable, Union, cast +from typing import TYPE_CHECKING, Any, Callable, cast import numpy as np from virtualizarr.zarr import determine_chunk_grid_shape -from ..codecs import get_codecs from .manifest import ChunkManifest +from .utils import ( + check_combinable_zarr_arrays, + check_same_ndims, + check_same_shapes, + check_same_shapes_except_on_concat_axis, +) if TYPE_CHECKING: - from zarr import Array # type: ignore[import-untyped] - from .array import ManifestArray @@ -28,60 +31,6 @@ def decorator(func): return decorator -def check_combinable_zarr_arrays( - arrays: Iterable[Union["ManifestArray", "Array"]], -) -> None: - """ - The downside of the ManifestArray approach compared to the VirtualZarrArray concatenation proposal is that - the result must also be a single valid zarr array, implying that the inputs must have the same dtype, codec etc. - """ - _check_same_dtypes([arr.dtype for arr in arrays]) - - # Can't combine different codecs in one manifest - # see https://github.com/zarr-developers/zarr-specs/issues/288 - _check_same_codecs([get_codecs(arr) for arr in arrays]) - - # Would require variable-length chunks ZEP - _check_same_chunk_shapes([arr.chunks for arr in arrays]) - - -def _check_same_dtypes(dtypes: list[np.dtype]) -> None: - """Check all the dtypes are the same""" - - first_dtype, *other_dtypes = dtypes - for other_dtype in other_dtypes: - if other_dtype != first_dtype: - raise ValueError( - f"Cannot concatenate arrays with inconsistent dtypes: {other_dtype} vs {first_dtype}" - ) - - -# Aimee: is there a way to type this list? ManifestArrays will return a list of virtualizarr.zarr.Codec objects -# whereas Zarr v3 arrays will return a list of ArrayArrayCodec, etc -# and Zarr v2 arrays will return ?? -def _check_same_codecs(codecs: list[Any]) -> None: - first_codec, *other_codecs = codecs - for codec in other_codecs: - if codec != first_codec: - raise NotImplementedError( - "The ManifestArray class cannot concatenate arrays which were stored using different codecs, " - f"But found codecs {first_codec} vs {codec} ." - "See https://github.com/zarr-developers/zarr-specs/issues/288" - ) - - -def _check_same_chunk_shapes(chunks_list: list[tuple[int, ...]]) -> None: - """Check all the chunk shapes are the same""" - - first_chunks, *other_chunks_list = chunks_list - for other_chunks in other_chunks_list: - if other_chunks != first_chunks: - raise ValueError( - f"Cannot concatenate arrays with inconsistent chunk shapes: {other_chunks} vs {first_chunks} ." - "Requires ZEP003 (Variable-length Chunks)." - ) - - @implements(np.result_type) def result_type(*arrays_and_dtypes) -> np.dtype: """Called by xarray to ensure all arguments to concat have the same dtype.""" @@ -159,36 +108,6 @@ def concatenate( return ManifestArray(chunkmanifest=concatenated_manifest, zarray=new_zarray) -def check_same_ndims(ndims: list[int]) -> None: - first_ndim, *other_ndims = ndims - for other_ndim in other_ndims: - if other_ndim != first_ndim: - raise ValueError( - f"Cannot concatenate arrays with differing number of dimensions: {first_ndim} vs {other_ndim}" - ) - - -def check_same_shapes_except_on_concat_axis(shapes: list[tuple[int, ...]], axis: int): - """Check that shapes are compatible for concatenation""" - - shapes_without_concat_axis = [ - _remove_element_at_position(shape, axis) for shape in shapes - ] - - first_shape, *other_shapes = shapes_without_concat_axis - for other_shape in other_shapes: - if other_shape != first_shape: - raise ValueError( - f"Cannot concatenate arrays with shapes {[shape for shape in shapes]}" - ) - - -def _remove_element_at_position(t: tuple[int, ...], pos: int) -> tuple[int, ...]: - new_l = list(t) - new_l.pop(pos) - return tuple(new_l) - - @implements(np.stack) def stack( arrays: tuple["ManifestArray", ...] | list["ManifestArray"], @@ -211,7 +130,7 @@ def stack( check_same_ndims([arr.ndim for arr in arrays]) arr_shapes = [arr.shape for arr in arrays] - _check_same_shapes(arr_shapes) + check_same_shapes(arr_shapes) # Ensure we handle axis being passed as a negative integer first_arr = arrays[0] @@ -259,15 +178,6 @@ def stack( return ManifestArray(chunkmanifest=stacked_manifest, zarray=new_zarray) -def _check_same_shapes(shapes: list[tuple[int, ...]]) -> None: - first_shape, *other_shapes = shapes - for other_shape in other_shapes: - if other_shape != first_shape: - raise ValueError( - f"Cannot concatenate arrays with differing shapes: {first_shape} vs {other_shape}" - ) - - @implements(np.expand_dims) def expand_dims(x: "ManifestArray", /, *, axis: int = 0) -> "ManifestArray": """Expands the shape of an array by inserting a new axis (dimension) of size one at the position specified by axis.""" diff --git a/virtualizarr/manifests/utils.py b/virtualizarr/manifests/utils.py new file mode 100644 index 00000000..03604204 --- /dev/null +++ b/virtualizarr/manifests/utils.py @@ -0,0 +1,118 @@ +from typing import TYPE_CHECKING, Any, Iterable, Union + +import numpy as np + +from virtualizarr.codecs import get_codecs + +if TYPE_CHECKING: + from zarr import Array # type: ignore + + from .array import ManifestArray + + +def _check_same_dtypes(dtypes: list[np.dtype]) -> None: + """Check all the dtypes are the same""" + + first_dtype, *other_dtypes = dtypes + for other_dtype in other_dtypes: + if other_dtype != first_dtype: + raise ValueError( + f"Cannot concatenate arrays with inconsistent dtypes: {other_dtype} vs {first_dtype}" + ) + + +def check_compatible_encodings(encoding1, encoding2): + for key, value in encoding1.items(): + if key in encoding2: + if encoding2[key] != value: + raise ValueError( + f"Cannot concatenate arrays with different values for encoding key {key}: {encoding2[key]} != {value}" + ) + + +def _check_same_codecs(codecs: list[Any]) -> None: + first_codec, *other_codecs = codecs + for codec in other_codecs: + if codec != first_codec: + raise NotImplementedError( + "The ManifestArray class cannot concatenate arrays which were stored using different codecs, " + f"But found codecs {first_codec} vs {codec} ." + "See https://github.com/zarr-developers/zarr-specs/issues/288" + ) + + +def _check_same_chunk_shapes(chunks_list: list[tuple[int, ...]]) -> None: + """Check all the chunk shapes are the same""" + + first_chunks, *other_chunks_list = chunks_list + for other_chunks in other_chunks_list: + if other_chunks != first_chunks: + raise ValueError( + f"Cannot concatenate arrays with inconsistent chunk shapes: {other_chunks} vs {first_chunks} ." + "Requires ZEP003 (Variable-length Chunks)." + ) + + +def check_same_ndims(ndims: list[int]) -> None: + first_ndim, *other_ndims = ndims + for other_ndim in other_ndims: + if other_ndim != first_ndim: + raise ValueError( + f"Cannot concatenate arrays with differing number of dimensions: {first_ndim} vs {other_ndim}" + ) + + +def check_same_shapes(shapes: list[tuple[int, ...]]) -> None: + first_shape, *other_shapes = shapes + for other_shape in other_shapes: + if other_shape != first_shape: + raise ValueError( + f"Cannot concatenate arrays with differing shapes: {first_shape} vs {other_shape}" + ) + + +def _remove_element_at_position(t: tuple[int, ...], pos: int) -> tuple[int, ...]: + new_l = list(t) + new_l.pop(pos) + return tuple(new_l) + + +def check_same_shapes_except_on_concat_axis(shapes: list[tuple[int, ...]], axis: int): + """Check that shapes are compatible for concatenation""" + + shapes_without_concat_axis = [ + _remove_element_at_position(shape, axis) for shape in shapes + ] + + first_shape, *other_shapes = shapes_without_concat_axis + for other_shape in other_shapes: + if other_shape != first_shape: + raise ValueError( + f"Cannot concatenate arrays with shapes {[shape for shape in shapes]}" + ) + + +def check_combinable_zarr_arrays( + arrays: Iterable[Union["ManifestArray", "Array"]], +) -> None: + """ + The downside of the ManifestArray approach compared to the VirtualZarrArray concatenation proposal is that + the result must also be a single valid zarr array, implying that the inputs must have the same dtype, codec etc. + """ + _check_same_dtypes([arr.dtype for arr in arrays]) + + # Can't combine different codecs in one manifest + # see https://github.com/zarr-developers/zarr-specs/issues/288 + _check_same_codecs([get_codecs(arr) for arr in arrays]) + + # Would require variable-length chunks ZEP + _check_same_chunk_shapes([arr.chunks for arr in arrays]) + + +def check_compatible_arrays( + ma: "ManifestArray", existing_array: "Array", append_axis: int +): + check_combinable_zarr_arrays([ma, existing_array]) + check_same_ndims([ma.ndim, existing_array.ndim]) + arr_shapes = [ma.shape, existing_array.shape] + check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index ed71fee5..31a6063b 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -8,7 +8,12 @@ from zarr import Array from virtualizarr.manifests import ChunkManifest, ManifestArray -from virtualizarr.manifests import array_api as manifest_api +from virtualizarr.manifests.utils import ( + check_combinable_zarr_arrays, + check_compatible_encodings, + check_same_ndims, + check_same_shapes_except_on_concat_axis, +) from virtualizarr.zarr import encode_dtype if TYPE_CHECKING: @@ -149,22 +154,13 @@ def get_axis( return dims.index(dim_name) -def _check_compatible_arrays( +def check_compatible_arrays( ma: ManifestArray, existing_array: zarr.core.array.Array, append_axis: int ): - manifest_api.check_combinable_zarr_arrays([ma, existing_array]) - manifest_api.check_same_ndims([ma.ndim, existing_array.ndim]) + check_combinable_zarr_arrays([ma, existing_array]) + check_same_ndims([ma.ndim, existing_array.ndim]) arr_shapes = [ma.shape, existing_array.shape] - manifest_api.check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) - - -def _check_compatible_encodings(encoding1, encoding2): - for key, value in encoding1.items(): - if key in encoding2: - if encoding2[key] != value: - raise ValueError( - f"Cannot concatenate arrays with different values for encoding key {key}: {encoding2[key]} != {value}" - ) + check_same_shapes_except_on_concat_axis(arr_shapes, append_axis) def write_virtual_variable_to_icechunk( @@ -192,8 +188,8 @@ def write_virtual_variable_to_icechunk( append_axis = get_axis(dims, append_dim) # check if arrays can be concatenated - _check_compatible_arrays(ma, existing_array, append_axis) - _check_compatible_encodings(var.encoding, existing_array.attrs) + check_compatible_arrays(ma, existing_array, append_axis) + check_compatible_encodings(var.encoding, existing_array.attrs) # determine number of existing chunks along the append axis existing_num_chunks = num_chunks( From 532ff38c33f4fa217f840d55620210b81f881837 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 18:13:35 -0800 Subject: [PATCH 50/90] pass the array into resize_array Co-authored-by: Tom Nicholas --- virtualizarr/writers/icechunk.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 31a6063b..a7aac0f5 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -197,11 +197,12 @@ def write_virtual_variable_to_icechunk( axis=append_axis, ) + arr = group[name] + # resize the array - arr = resize_array( - group=group, - name=name, - var=var, + resize_array( + arr, + shape_to_append=var.shape, append_axis=append_axis, ) else: From 24f727401da12559db1c1e3f062bd984d38f75b8 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 18:21:59 -0800 Subject: [PATCH 51/90] Refactor resize_array --- virtualizarr/writers/icechunk.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index a7aac0f5..73ae51dc 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -129,20 +129,12 @@ def num_chunks( def resize_array( - group: "Group", - name: str, - var: Variable, + arr: "Array", append_axis: int, -) -> Array: - existing_array = group[name] - # This is the second time we check if the array is an instance of zarr.core.Array - # but it's necessary to ensure .shape and .resize are available - if not isinstance(existing_array, Array): - raise ValueError("Expected existing array to be a zarr.core.Array") - new_shape = list(existing_array.shape) - new_shape[append_axis] += var.shape[append_axis] - existing_array.resize(tuple(new_shape)) - return existing_array +) -> None: + new_shape = list(arr.shape) + new_shape[append_axis] += arr.shape[append_axis] + arr.resize(tuple(new_shape)) def get_axis( @@ -176,7 +168,7 @@ def write_virtual_variable_to_icechunk( mode = store.mode.str dims: list[str] = cast(list[str], list(var.dims)) - append_axis, existing_num_chunks, arr = None, None, None + existing_num_chunks = 0 if append_dim and append_dim not in dims: raise ValueError( f"append_dim {append_dim} not found in variable dimensions {dims}" @@ -197,15 +189,13 @@ def write_virtual_variable_to_icechunk( axis=append_axis, ) - arr = group[name] - # resize the array resize_array( - arr, - shape_to_append=var.shape, + existing_array, append_axis=append_axis, ) else: + append_axis = None # create array if it doesn't already exist arr = group.require_array( name=name, From 113cd2cd251ea2ca4cebfdbbbf7dd4324a4ae334 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 20 Nov 2024 18:27:14 -0800 Subject: [PATCH 52/90] Remove unnecessary zarr imports --- virtualizarr/writers/icechunk.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 73ae51dc..e0240ac1 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -1,11 +1,9 @@ from typing import TYPE_CHECKING, Optional, cast import numpy as np -import zarr # type: ignore[import-untyped] from xarray import Dataset from xarray.backends.zarr import encode_zarr_attr_value from xarray.core.variable import Variable -from zarr import Array from virtualizarr.manifests import ChunkManifest, ManifestArray from virtualizarr.manifests.utils import ( @@ -18,7 +16,7 @@ if TYPE_CHECKING: from icechunk import IcechunkStore # type: ignore[import-not-found] - from zarr import Group # type: ignore + from zarr import Array, Group # type: ignore VALID_URI_PREFIXES = { @@ -147,7 +145,7 @@ def get_axis( def check_compatible_arrays( - ma: ManifestArray, existing_array: zarr.core.array.Array, append_axis: int + ma: ManifestArray, existing_array: "Array", append_axis: int ): check_combinable_zarr_arrays([ma, existing_array]) check_same_ndims([ma.ndim, existing_array.ndim]) @@ -163,6 +161,8 @@ def write_virtual_variable_to_icechunk( append_dim: Optional[str] = None, ) -> None: """Write a single virtual variable into an icechunk store""" + from zarr import Array + ma = cast(ManifestArray, var.data) zarray = ma.zarray mode = store.mode.str From 61ce01a60debbaae284ba96ff21a7003434499ce Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 21 Nov 2024 09:23:25 -0800 Subject: [PATCH 53/90] Add pinned version of icechunk as an optional dependency --- ci/upstream.yml | 6 +++--- pyproject.toml | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/upstream.yml b/ci/upstream.yml index 91272fab..46af1474 100644 --- a/ci/upstream.yml +++ b/ci/upstream.yml @@ -28,6 +28,6 @@ dependencies: - fsspec - pip - pip: - - icechunk # Installs zarr v3 as dependency - # - git+https://github.com/fsspec/kerchunk@main # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516) - - imagecodecs-numcodecs==2024.6.1 + - icechunk==0.1.0a4 # Installs zarr v3 as dependency + # - git+https://github.com/fsspec/kerchunk@main # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516) + - imagecodecs-numcodecs==2024.6.1 diff --git a/pyproject.toml b/pyproject.toml index 77998076..649d0b4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ hdf_reader = [ "imagecodecs-numcodecs==2024.6.1", "numcodecs" ] +icechunk = [ + "icechunk==0.1.0a4", +] test = [ "codecov", "fastparquet", From defe7d95cad332233d9d196149d0fd9194a1a5cc Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 22 Nov 2024 09:37:09 -0800 Subject: [PATCH 54/90] Add append_dim in docstring --- virtualizarr/accessor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/virtualizarr/accessor.py b/virtualizarr/accessor.py index ea9e3843..594c2c0a 100644 --- a/virtualizarr/accessor.py +++ b/virtualizarr/accessor.py @@ -49,6 +49,7 @@ def to_icechunk( Parameters ---------- store: IcechunkStore + append_dim: str, optional """ from virtualizarr.writers.icechunk import dataset_to_icechunk From cb82d4002bf07e63ae6ae90d2dec128cb10cf630 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 22 Nov 2024 10:08:55 -0800 Subject: [PATCH 55/90] Kludgy solution to v2 v3 codecs difference --- virtualizarr/tests/test_writers/test_icechunk.py | 11 +++++++++-- virtualizarr/writers/icechunk.py | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 337d64bd..633499f1 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -372,6 +372,7 @@ def gen_virtual_dataset( base_offset: int = 6144, length: int = 48, dims: Optional[list[str]] = None, + zarr_format: int = 2, ): manifest = generate_chunk_manifest( file_uri, @@ -387,7 +388,7 @@ def gen_virtual_dataset( compressor=compressor, filters=filters, fill_value=fill_value, - zarr_format=3, + zarr_format=zarr_format, ) ma = ManifestArray(chunkmanifest=manifest, zarray=zarray) ds = open_dataset(file_uri) @@ -497,8 +498,12 @@ def test_append_virtual_ref_with_encoding( xrt.assert_equal(new_ds, expected_ds) ## When appending to a virtual ref with compression, it succeeds + @pytest.mark.parametrize("zarr_format", [2, 3]) def test_append_with_compression_succeeds( - self, icechunk_storage: "StorageConfig", netcdf4_files_factory: Callable + self, + icechunk_storage: "StorageConfig", + netcdf4_files_factory: Callable, + zarr_format: int, ): import xarray.testing as xrt from icechunk import IcechunkStore @@ -524,6 +529,7 @@ def test_append_with_compression_succeeds( variable_name="air", base_offset=18043, length=3936114, + zarr_format=zarr_format, ), gen_virtual_dataset( file_uri=file2, @@ -535,6 +541,7 @@ def test_append_with_compression_succeeds( variable_name="air", base_offset=18043, length=3938672, + zarr_format=zarr_format, ), ) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index e0240ac1..cadc8b30 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -180,6 +180,8 @@ def write_virtual_variable_to_icechunk( append_axis = get_axis(dims, append_dim) # check if arrays can be concatenated + # if the manifest array has zarr format v2 compression and filter, we need to convert it to v3 + ma.zarray.zarr_format = 3 check_compatible_arrays(ma, existing_array, append_axis) check_compatible_encodings(var.encoding, existing_array.attrs) From 2f6cbc2b71e7d0c84901ab201676c3e27775d678 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 22 Nov 2024 10:51:17 -0800 Subject: [PATCH 56/90] Add normalize to v3 parameter --- virtualizarr/codecs.py | 29 +++++++++++++------- virtualizarr/manifests/utils.py | 12 ++++----- virtualizarr/tests/test_codecs.py | 45 ++++++++++++++++++++++++------- virtualizarr/writers/icechunk.py | 12 ++++++--- 4 files changed, 69 insertions(+), 29 deletions(-) diff --git a/virtualizarr/codecs.py b/virtualizarr/codecs.py index 8ed87207..4d29bd29 100644 --- a/virtualizarr/codecs.py +++ b/virtualizarr/codecs.py @@ -15,6 +15,7 @@ def get_codecs( array: Union["ManifestArray", "Array"], + normalize_to_zarr_v3: bool = False, ) -> Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]: """ Get the codecs for either a ManifestArray or a Zarr Array. @@ -30,10 +31,10 @@ def get_codecs( ValueError: If the array type is unsupported. """ if _is_manifest_array(array): - return _get_manifestarray_codecs(array) # type: ignore[arg-type] + return _get_manifestarray_codecs(array, normalize_to_zarr_v3) # type: ignore[arg-type] if _is_zarr_array(array): - return _get_zarr_array_codecs(array) # type: ignore[arg-type] + return _get_zarr_array_codecs(array, normalize_to_zarr_v3) # type: ignore[arg-type] raise ValueError("Unsupported array type or zarr is not installed.") @@ -50,9 +51,10 @@ def _is_manifest_array(array: object) -> bool: def _get_manifestarray_codecs( array: "ManifestArray", + normalize_to_zarr_v3: bool = False, ) -> Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]: """Get codecs for a ManifestArray based on its zarr_format.""" - if array.zarray.zarr_format == 3: + if normalize_to_zarr_v3 or array.zarray.zarr_format == 3: return array.zarray._v3_codec_pipeline() elif array.zarray.zarr_format == 2: return array.zarray.codec @@ -72,6 +74,7 @@ def _is_zarr_array(array: object) -> bool: def _get_zarr_array_codecs( array: "Array", + normalize_to_zarr_v3: bool = False, ) -> Union[Codec, tuple["ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec", ...]]: """Get codecs for a Zarr Array based on its format.""" import zarr @@ -84,14 +87,22 @@ def _get_zarr_array_codecs( raise NotImplementedError( f"zarr-python v3 or higher is required, but version {installed_version} is installed." ) + from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata + # For zarr format v3 - if hasattr(array, "metadata") and array.metadata.zarr_format == 3: + if isinstance(array.metadata, ArrayV3Metadata): return tuple(array.metadata.codecs) # For zarr format v2 - elif hasattr(array, "metadata") and array.metadata.zarr_format == 2: - return Codec( - compressor=array.metadata.compressor, - filters=list(array.metadata.filters or ()), - ) + elif isinstance(array.metadata, ArrayV2Metadata): + if normalize_to_zarr_v3: + # we could potentially normalize to v3 using ZArray._v3_codec_pipeline, but we don't have a use case for that. + raise NotImplementedError( + "Normalization to zarr v3 is not supported for zarr v2 array." + ) + else: + return Codec( + compressor=array.metadata.compressor, + filters=list(array.metadata.filters or ()), + ) else: raise ValueError("Unsupported zarr_format for Zarr Array.") diff --git a/virtualizarr/manifests/utils.py b/virtualizarr/manifests/utils.py index 03604204..07cf2baf 100644 --- a/virtualizarr/manifests/utils.py +++ b/virtualizarr/manifests/utils.py @@ -10,7 +10,7 @@ from .array import ManifestArray -def _check_same_dtypes(dtypes: list[np.dtype]) -> None: +def check_same_dtypes(dtypes: list[np.dtype]) -> None: """Check all the dtypes are the same""" first_dtype, *other_dtypes = dtypes @@ -30,7 +30,7 @@ def check_compatible_encodings(encoding1, encoding2): ) -def _check_same_codecs(codecs: list[Any]) -> None: +def check_same_codecs(codecs: list[Any]) -> None: first_codec, *other_codecs = codecs for codec in other_codecs: if codec != first_codec: @@ -41,7 +41,7 @@ def _check_same_codecs(codecs: list[Any]) -> None: ) -def _check_same_chunk_shapes(chunks_list: list[tuple[int, ...]]) -> None: +def check_same_chunk_shapes(chunks_list: list[tuple[int, ...]]) -> None: """Check all the chunk shapes are the same""" first_chunks, *other_chunks_list = chunks_list @@ -99,14 +99,14 @@ def check_combinable_zarr_arrays( The downside of the ManifestArray approach compared to the VirtualZarrArray concatenation proposal is that the result must also be a single valid zarr array, implying that the inputs must have the same dtype, codec etc. """ - _check_same_dtypes([arr.dtype for arr in arrays]) + check_same_dtypes([arr.dtype for arr in arrays]) # Can't combine different codecs in one manifest # see https://github.com/zarr-developers/zarr-specs/issues/288 - _check_same_codecs([get_codecs(arr) for arr in arrays]) + check_same_codecs([get_codecs(arr) for arr in arrays]) # Would require variable-length chunks ZEP - _check_same_chunk_shapes([arr.chunks for arr in arrays]) + check_same_chunk_shapes([arr.chunks for arr in arrays]) def check_compatible_arrays( diff --git a/virtualizarr/tests/test_codecs.py b/virtualizarr/tests/test_codecs.py index cb874618..3fae3c00 100644 --- a/virtualizarr/tests/test_codecs.py +++ b/virtualizarr/tests/test_codecs.py @@ -1,6 +1,7 @@ from unittest.mock import patch import numpy as np +import pytest from numcodecs import Blosc, Delta from virtualizarr import ChunkManifest, ManifestArray @@ -30,7 +31,7 @@ def create_manifest_array(self, compressor=None, filters=None, zarr_format=2): ), ) - def test_manifest_array_codecs_v2(self): + def test_manifest_array_zarr_v2(self): """Test that get_codecs works for ManifestArray with Zarr v2 metadata.""" compressor = {"id": "blosc", "cname": "zstd", "clevel": 5, "shuffle": 1} filters = [{"id": "delta", "dtype": " Date: Fri, 22 Nov 2024 10:53:16 -0800 Subject: [PATCH 57/90] Add more info to docstring --- virtualizarr/accessor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/virtualizarr/accessor.py b/virtualizarr/accessor.py index 594c2c0a..d9b9d6a2 100644 --- a/virtualizarr/accessor.py +++ b/virtualizarr/accessor.py @@ -46,6 +46,8 @@ def to_icechunk( Any variables backed by ManifestArray objects will be be written as virtual references, any other variables will be loaded into memory before their binary chunk data is written into the store. + If `append_dim` is provided, the virtual dataset will be appended on the append_dim axis to the existing IcechunkStore. + Parameters ---------- store: IcechunkStore From 7d1bb361c291f42f2738b0b1dc681a032ec75b5f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 22 Nov 2024 11:02:00 -0800 Subject: [PATCH 58/90] Fix typing issues --- virtualizarr/tests/test_writers/test_icechunk.py | 6 +++--- virtualizarr/writers/icechunk.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 633499f1..1af74fe9 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -1,6 +1,6 @@ from itertools import product from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Tuple, cast +from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Tuple, cast import pytest @@ -372,7 +372,7 @@ def gen_virtual_dataset( base_offset: int = 6144, length: int = 48, dims: Optional[list[str]] = None, - zarr_format: int = 2, + zarr_format: Literal[2, 3] = 2, ): manifest = generate_chunk_manifest( file_uri, @@ -503,7 +503,7 @@ def test_append_with_compression_succeeds( self, icechunk_storage: "StorageConfig", netcdf4_files_factory: Callable, - zarr_format: int, + zarr_format: Literal[2, 3], ): import xarray.testing as xrt from icechunk import IcechunkStore diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index dcb283e1..7443bd64 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -148,9 +148,9 @@ def get_axis( def check_compatible_arrays( - ma: ManifestArray, existing_array: "Array", append_axis: int + ma: "ManifestArray", existing_array: "Array", append_axis: int ): - arrays = [ma, existing_array] + arrays = [ma, existing_array] # type: List[Union[ManifestArray, Array]] check_same_dtypes([arr.dtype for arr in arrays]) check_same_codecs([get_codecs(arr, normalize_to_zarr_v3=True) for arr in arrays]) check_same_chunk_shapes([arr.chunks for arr in arrays]) From 39677e877ebcab6c75a087920bf9923eaedeff03 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 22 Nov 2024 11:04:53 -0800 Subject: [PATCH 59/90] Add decorator for zarr python v3 test --- virtualizarr/tests/test_codecs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/virtualizarr/tests/test_codecs.py b/virtualizarr/tests/test_codecs.py index 3fae3c00..41d16b5b 100644 --- a/virtualizarr/tests/test_codecs.py +++ b/virtualizarr/tests/test_codecs.py @@ -47,6 +47,7 @@ def test_manifest_array_zarr_v2(self): ) assert actual_codecs == expected_codecs + @requires_zarr_python_v3 def test_manifest_array_zarr_v2_normalized(self): """Test that get_codecs works for ManifestArray with Zarr v2 metadata.""" compressor = {"id": "blosc", "cname": "zstd", "clevel": 5, "shuffle": 1} From 5fa717760baf2af8656a0268537c4aa54687edb2 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Fri, 22 Nov 2024 11:08:11 -0800 Subject: [PATCH 60/90] Fix mypy and ruff errors --- virtualizarr/codecs.py | 5 ++++- virtualizarr/writers/icechunk.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/virtualizarr/codecs.py b/virtualizarr/codecs.py index 4d29bd29..ad2a3d9b 100644 --- a/virtualizarr/codecs.py +++ b/virtualizarr/codecs.py @@ -87,7 +87,10 @@ def _get_zarr_array_codecs( raise NotImplementedError( f"zarr-python v3 or higher is required, but version {installed_version} is installed." ) - from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata + from zarr.core.metadata import ( # type: ignore[import-untyped] + ArrayV2Metadata, + ArrayV3Metadata, + ) # For zarr format v3 if isinstance(array.metadata, ArrayV3Metadata): diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 7443bd64..c1796cce 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, cast +from typing import TYPE_CHECKING, List, Optional, Union, cast import numpy as np from xarray import Dataset @@ -150,7 +150,7 @@ def get_axis( def check_compatible_arrays( ma: "ManifestArray", existing_array: "Array", append_axis: int ): - arrays = [ma, existing_array] # type: List[Union[ManifestArray, Array]] + arrays: List[Union[ManifestArray, Array]] = [ma, existing_array] check_same_dtypes([arr.dtype for arr in arrays]) check_same_codecs([get_codecs(arr, normalize_to_zarr_v3=True) for arr in arrays]) check_same_chunk_shapes([arr.chunks for arr in arrays]) From e109c0d43184cdcd561f1e85d1f6f21c8e8b1c18 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Sat, 23 Nov 2024 12:39:06 -0800 Subject: [PATCH 61/90] Only append if append_dim in dims --- virtualizarr/writers/icechunk.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index c1796cce..39c9f1b4 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -67,6 +67,10 @@ def dataset_to_icechunk( raise ValueError( "append_dim must be provided when opening store in append mode" ) + if append_dim not in ds.dims: + raise ValueError( + f"append_dim {append_dim} not found in dataset dimensions {ds.dims}" + ) root_group = Group.open(store=store, zarr_format=3) else: root_group = Group.from_store(store=store) @@ -175,19 +179,16 @@ def write_virtual_variable_to_icechunk( dims: list[str] = cast(list[str], list(var.dims)) existing_num_chunks = 0 - if append_dim and append_dim not in dims: - raise ValueError( - f"append_dim {append_dim} not found in variable dimensions {dims}" - ) - if mode == "a": - existing_array = group[name] - if not isinstance(existing_array, Array): + if mode == "a" and append_dim in dims: + # TODO: MRP - zarr, or icechunk zarr, array assignment to a variable doesn't work to point to the same object + # for example, if you resize an array, it resizes the array but not the bound variable. + if not isinstance(group[name], Array): raise ValueError("Expected existing array to be a zarr.core.Array") append_axis = get_axis(dims, append_dim) # check if arrays can be concatenated - check_compatible_arrays(ma, existing_array, append_axis) - check_compatible_encodings(var.encoding, existing_array.attrs) + check_compatible_arrays(ma, group[name], append_axis) + check_compatible_encodings(var.encoding, group[name].attrs) # determine number of existing chunks along the append axis existing_num_chunks = num_chunks( @@ -197,12 +198,13 @@ def write_virtual_variable_to_icechunk( # resize the array resize_array( - existing_array, + group[name], append_axis=append_axis, ) else: append_axis = None # create array if it doesn't already exist + arr = group.require_array( name=name, shape=zarray.shape, From eb0e8f253b5e241176a5df37b005cdb5f7f3567b Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 25 Nov 2024 10:35:48 -0800 Subject: [PATCH 62/90] Add example notebook --- noaa-cdr-sst.ipynb | 1242 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1242 insertions(+) create mode 100644 noaa-cdr-sst.ipynb diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb new file mode 100644 index 00000000..0f89b899 --- /dev/null +++ b/noaa-cdr-sst.ipynb @@ -0,0 +1,1242 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "id": "d09bbff3-4e96-4490-b837-14b78b64df35", + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install -e \".[icechunk]\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "import fsspec\n", + "import xarray as xr\n", + "from icechunk import IcechunkStore, StorageConfig, StoreConfig, VirtualRefConfig\n", + "\n", + "from virtualizarr import open_virtual_dataset\n", + "\n", + "warnings.filterwarnings(\"ignore\", category=UserWarning)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", + "metadata": {}, + "outputs": [], + "source": [ + "fs = fsspec.filesystem(\"s3\", anon=True)\n", + "\n", + "oisst_files = fs.glob(\n", + " \"s3://noaa-cdr-sea-surface-temp-optimum-interpolation-pds/data/v2.1/avhrr/202408/oisst-avhrr-v02r01.*.nc\"\n", + ")\n", + "\n", + "oisst_files = sorted([\"s3://\" + f for f in oisst_files])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "06bbec92-3974-4859-8bda-353afc7800b9", + "metadata": {}, + "outputs": [], + "source": [ + "so = dict(anon=True, default_fill_cache=False, default_cache_type=\"none\")\n", + "\n", + "virtual_datasets = [\n", + " open_virtual_dataset(url, indexes={}, reader_options={\"storage_options\": so})\n", + " for url in oisst_files[0:2]\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_ds = xr.concat(\n", + " virtual_datasets,\n", + " dim=\"time\",\n", + " coords=\"minimal\",\n", + " compat=\"override\",\n", + " combine_attrs=\"override\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "abefd6fa-386a-4e07-a7c8-219d3730eeeb", + "metadata": {}, + "outputs": [], + "source": [ + "#!rm -rf ./noaa-cdr-icechunk/" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "79a4228a-0e17-4b07-9144-f24fe06db832", + "metadata": {}, + "outputs": [], + "source": [ + "storage_config = StorageConfig.filesystem(\"./noaa-cdr-icechunk\")\n", + "virtual_ref_store_config = StoreConfig(\n", + " virtual_ref_config=VirtualRefConfig.s3_anonymous(region=\"us-east-1\"),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "5fd0c082-8d5e-46a8-a994-fee80baa4ecc", + "metadata": {}, + "outputs": [], + "source": [ + "store = IcechunkStore.create(storage=storage_config, config=virtual_ref_store_config)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "55ebbc5f-add2-4de8-81f6-5aaf64d9e2b6", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_ds.virtualize.to_icechunk(store)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cae7e34f-d6dd-42aa-9e7a-f0d5420ba0b9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'B89RQMAK4SPQR2S6G2DG'" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.commit(\"first 2 days of 202408 data\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b6271bd1-bc0b-4901-9901-91aabe508cf7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 66MB\n",
+       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
+       "Coordinates:\n",
+       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
+       "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
+       "  * zlev     (zlev) float32 4B 0.0\n",
+       "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
+       "Data variables:\n",
+       "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
+       "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
+       "    anom     (time, zlev, lat, lon) float64 17MB ...\n",
+       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
+       "Attributes: (12/37)\n",
+       "    Conventions:                CF-1.6, ACDD-1.3\n",
+       "    cdm_data_type:              Grid\n",
+       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
+       "    creator_email:              oisst-help@noaa.gov\n",
+       "    creator_url:                https://www.ncei.noaa.gov/\n",
+       "    date_created:               2024-08-16T09:12:00Z\n",
+       "    ...                         ...\n",
+       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
+       "    standard_name_vocabulary:   CF Standard Name Table (v40, 25 January 2017)\n",
+       "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
+       "    time_coverage_end:          2024-08-01T23:59:59Z\n",
+       "    time_coverage_start:        2024-08-01T00:00:00Z\n",
+       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
" + ], + "text/plain": [ + " Size: 66MB\n", + "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", + "Coordinates:\n", + " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", + " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", + " * zlev (zlev) float32 4B 0.0\n", + " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", + "Data variables:\n", + " ice (time, zlev, lat, lon) float64 17MB ...\n", + " sst (time, zlev, lat, lon) float64 17MB ...\n", + " anom (time, zlev, lat, lon) float64 17MB ...\n", + " err (time, zlev, lat, lon) float64 17MB ...\n", + "Attributes: (12/37)\n", + " Conventions: CF-1.6, ACDD-1.3\n", + " cdm_data_type: Grid\n", + " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", + " creator_email: oisst-help@noaa.gov\n", + " creator_url: https://www.ncei.noaa.gov/\n", + " date_created: 2024-08-16T09:12:00Z\n", + " ... ...\n", + " source: ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n", + " standard_name_vocabulary: CF Standard Name Table (v40, 25 January 2017)\n", + " summary: NOAAs 1/4-degree Daily Optimum Interpolation ...\n", + " time_coverage_end: 2024-08-01T23:59:59Z\n", + " time_coverage_start: 2024-08-01T00:00:00Z\n", + " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds = xr.open_zarr(store, consolidated=False, zarr_format=3)\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "id": "23dd5a13-9c0e-4132-9073-474c0af65920", + "metadata": {}, + "source": [ + "## Append" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "190c25f9-e000-4b17-83eb-cf551141dfea", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_datasets_a = [\n", + " open_virtual_dataset(\n", + " url, indexes={}, reader_options={\"storage_options\": {\"anon\": True}}\n", + " )\n", + " for url in oisst_files[2:4]\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "af330082-207a-4f08-aefe-fc15aa8b2eb3", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_ds_a = xr.concat(\n", + " virtual_datasets_a,\n", + " dim=\"time\",\n", + " coords=\"minimal\",\n", + " compat=\"override\",\n", + " combine_attrs=\"override\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "103b44d2-124a-4de5-8074-e997fd5a1698", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 17MB\n",
+       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
+       "Coordinates:\n",
+       "    time     (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n",
+       "    lon      (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n",
+       "    zlev     (zlev) float32 4B ManifestArray<shape=(1,), dtype=float32, chunk...\n",
+       "    lat      (lat) float32 3kB ManifestArray<shape=(720,), dtype=float32, chu...\n",
+       "Data variables:\n",
+       "    anom     (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
+       "    ice      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
+       "    err      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
+       "    sst      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
+       "Attributes: (12/37)\n",
+       "    Conventions:                CF-1.6, ACDD-1.3\n",
+       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...\n",
+       "    references:                 Reynolds, et al.(2007) Daily High-Resolution-...\n",
+       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
+       "    id:                         oisst-avhrr-v02r01.20240803.nc\n",
+       "    naming_authority:           gov.noaa.ncei\n",
+       "    ...                         ...\n",
+       "    time_coverage_start:        2024-08-03T00:00:00Z\n",
+       "    time_coverage_end:          2024-08-03T23:59:59Z\n",
+       "    metadata_link:              https://doi.org/10.25921/RE9P-PT57\n",
+       "    ncei_template_version:      NCEI_NetCDF_Grid_Template_v2.0\n",
+       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
+       "    sensor:                     Thermometer, AVHRR
" + ], + "text/plain": [ + " Size: 17MB\n", + "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", + "Coordinates:\n", + " time (time) float32 8B ManifestArray 451\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m \u001b[43mdecode_cf_variable\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 452\u001b[0m \u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 453\u001b[0m \u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 454\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 456\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mstack_char_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstack_char_dim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 459\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 460\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:292\u001b[0m, in \u001b[0;36mdecode_cf_variable\u001b[0;34m(name, var, concat_characters, mask_and_scale, decode_times, decode_endianness, stack_char_dim, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_times:\n\u001b[0;32m--> 292\u001b[0m var \u001b[38;5;241m=\u001b[39m \u001b[43mtimes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCFDatetimeCoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 294\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_endianness \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39misnative:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/coding/times.py:1001\u001b[0m, in \u001b[0;36mCFDatetimeCoder.decode\u001b[0;34m(self, variable, name)\u001b[0m\n\u001b[1;32m 1000\u001b[0m calendar \u001b[38;5;241m=\u001b[39m pop_to(attrs, encoding, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalendar\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1001\u001b[0m dtype \u001b[38;5;241m=\u001b[39m \u001b[43m_decode_cf_datetime_dtype\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munits\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcalendar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1002\u001b[0m transform \u001b[38;5;241m=\u001b[39m partial(\n\u001b[1;32m 1003\u001b[0m decode_cf_datetime,\n\u001b[1;32m 1004\u001b[0m units\u001b[38;5;241m=\u001b[39munits,\n\u001b[1;32m 1005\u001b[0m calendar\u001b[38;5;241m=\u001b[39mcalendar,\n\u001b[1;32m 1006\u001b[0m use_cftime\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_cftime,\n\u001b[1;32m 1007\u001b[0m )\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/coding/times.py:214\u001b[0m, in \u001b[0;36m_decode_cf_datetime_dtype\u001b[0;34m(data, units, calendar, use_cftime)\u001b[0m\n\u001b[1;32m 212\u001b[0m values \u001b[38;5;241m=\u001b[39m indexing\u001b[38;5;241m.\u001b[39mImplicitToExplicitIndexingAdapter(indexing\u001b[38;5;241m.\u001b[39mas_indexable(data))\n\u001b[1;32m 213\u001b[0m example_value \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate(\n\u001b[0;32m--> 214\u001b[0m [\u001b[43mfirst_n_items\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m], last_item(values) \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m 215\u001b[0m )\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/formatting.py:97\u001b[0m, in \u001b[0;36mfirst_n_items\u001b[0;34m(array, n_desired)\u001b[0m\n\u001b[1;32m 96\u001b[0m array \u001b[38;5;241m=\u001b[39m array\u001b[38;5;241m.\u001b[39m_data\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mravel(\u001b[43mto_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43marray\u001b[49m\u001b[43m)\u001b[49m)[:n_desired]\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/namedarray/pycompat.py:138\u001b[0m, in \u001b[0;36mto_duck_array\u001b[0;34m(data, **kwargs)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 138\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43masarray\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:578\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.__array__\u001b[0;34m(self, dtype, copy)\u001b[0m\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m Version(np\u001b[38;5;241m.\u001b[39m__version__) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m Version(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.0.0\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 578\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, dtype\u001b[38;5;241m=\u001b[39mdtype, copy\u001b[38;5;241m=\u001b[39mcopy)\n\u001b[1;32m 579\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:583\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 583\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:657\u001b[0m, in \u001b[0;36mLazilyIndexedArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 655\u001b[0m \u001b[38;5;66;03m# If the array is not an ExplicitlyIndexedNDArrayMixin,\u001b[39;00m\n\u001b[1;32m 656\u001b[0m \u001b[38;5;66;03m# it may wrap a BackendArray so use its __getitem__\u001b[39;00m\n\u001b[0;32m--> 657\u001b[0m array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 659\u001b[0m \u001b[38;5;66;03m# self.array[self.key] is now a numpy array when\u001b[39;00m\n\u001b[1;32m 660\u001b[0m \u001b[38;5;66;03m# self.array is a BackendArray subclass\u001b[39;00m\n\u001b[1;32m 661\u001b[0m \u001b[38;5;66;03m# and self.key is BasicIndexer((slice(None, None, None),))\u001b[39;00m\n\u001b[1;32m 662\u001b[0m \u001b[38;5;66;03m# so we need the explicit check for ExplicitlyIndexed\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:175\u001b[0m, in \u001b[0;36mZarrArrayWrapper.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 174\u001b[0m method \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oindex\n\u001b[0;32m--> 175\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexplicit_indexing_adapter\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 176\u001b[0m \u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mIndexingSupport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mVECTORIZED\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\n\u001b[1;32m 177\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:1018\u001b[0m, in \u001b[0;36mexplicit_indexing_adapter\u001b[0;34m(key, shape, indexing_support, raw_indexing_method)\u001b[0m\n\u001b[1;32m 1017\u001b[0m raw_key, numpy_indices \u001b[38;5;241m=\u001b[39m decompose_indexer(key, shape, indexing_support)\n\u001b[0;32m-> 1018\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mraw_indexing_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_key\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtuple\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1019\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m numpy_indices\u001b[38;5;241m.\u001b[39mtuple:\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;66;03m# index the loaded np.ndarray\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:158\u001b[0m, in \u001b[0;36mZarrArrayWrapper._getitem\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_getitem\u001b[39m(\u001b[38;5;28mself\u001b[39m, key):\n\u001b[1;32m 156\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 157\u001b[0m \u001b[38;5;66;03m# try:\u001b[39;00m\n\u001b[0;32m--> 158\u001b[0m item \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_array\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;66;03m# except Exception as e:\u001b[39;00m\n\u001b[1;32m 160\u001b[0m \u001b[38;5;66;03m# import traceback\u001b[39;00m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;66;03m# print(f\"An error occurred: {e}\")\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[38;5;66;03m# print(\"Stack trace:\")\u001b[39;00m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;66;03m# traceback.print_exc() # Prints the full traceback \u001b[39;00m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:1688\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 1687\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_pure_orthogonal_indexing(pure_selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim):\n\u001b[0;32m-> 1688\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_orthogonal_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1689\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/_compat.py:43\u001b[0m, in \u001b[0;36m_deprecate_positional_args.._inner_deprecate_positional_args..inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m extra_args \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# extra_args > 0\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:2130\u001b[0m, in \u001b[0;36mArray.get_orthogonal_selection\u001b[0;34m(self, selection, out, fields, prototype)\u001b[0m\n\u001b[1;32m 2129\u001b[0m indexer \u001b[38;5;241m=\u001b[39m OrthogonalIndexer(selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mchunk_grid)\n\u001b[0;32m-> 2130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2131\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_async_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2132\u001b[0m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprototype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprototype\u001b[49m\n\u001b[1;32m 2133\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2134\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:141\u001b[0m, in \u001b[0;36msync\u001b[0;34m(coro, loop, timeout)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 141\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:100\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 100\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:961\u001b[0m, in \u001b[0;36mAsyncArray._get_selection\u001b[0;34m(self, indexer, prototype, out, fields)\u001b[0m\n\u001b[1;32m 959\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m product(indexer\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 960\u001b[0m \u001b[38;5;66;03m# reading chunks and decoding them\u001b[39;00m\n\u001b[0;32m--> 961\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcodec_pipeline\u001b[38;5;241m.\u001b[39mread(\n\u001b[1;32m 962\u001b[0m [\n\u001b[1;32m 963\u001b[0m (\n\u001b[1;32m 964\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_path \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mencode_chunk_key(chunk_coords),\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget_chunk_spec(chunk_coords, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morder, prototype\u001b[38;5;241m=\u001b[39mprototype),\n\u001b[1;32m 966\u001b[0m chunk_selection,\n\u001b[1;32m 967\u001b[0m out_selection,\n\u001b[1;32m 968\u001b[0m )\n\u001b[1;32m 969\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_coords, chunk_selection, out_selection \u001b[38;5;129;01min\u001b[39;00m indexer\n\u001b[1;32m 970\u001b[0m ],\n\u001b[1;32m 971\u001b[0m out_buffer,\n\u001b[1;32m 972\u001b[0m drop_axes\u001b[38;5;241m=\u001b[39mindexer\u001b[38;5;241m.\u001b[39mdrop_axes,\n\u001b[1;32m 973\u001b[0m )\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_buffer\u001b[38;5;241m.\u001b[39mas_ndarray_like()\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:440\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread\u001b[39m(\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 436\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]],\n\u001b[1;32m 437\u001b[0m out: NDBuffer,\n\u001b[1;32m 438\u001b[0m drop_axes: \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (),\n\u001b[1;32m 439\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 440\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 441\u001b[0m [\n\u001b[1;32m 442\u001b[0m (single_batch_info, out, drop_axes)\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m single_batch_info \u001b[38;5;129;01min\u001b[39;00m batched(batch_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m 444\u001b[0m ],\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_batch,\n\u001b[1;32m 446\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 447\u001b[0m )\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:270\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read_batch\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 262\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 263\u001b[0m [\n\u001b[1;32m 264\u001b[0m (byte_getter, array_spec\u001b[38;5;241m.\u001b[39mprototype)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 268\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 269\u001b[0m )\n\u001b[0;32m--> 270\u001b[0m chunk_array_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode_batch(\n\u001b[1;32m 271\u001b[0m [\n\u001b[1;32m 272\u001b[0m (chunk_bytes, chunk_spec)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_bytes, (_, chunk_spec, _, _) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 274\u001b[0m chunk_bytes_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 275\u001b[0m )\n\u001b[1;32m 276\u001b[0m ],\n\u001b[1;32m 277\u001b[0m )\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_array, (_, chunk_spec, chunk_selection, out_selection) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 279\u001b[0m chunk_array_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 280\u001b[0m ):\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:172\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.decode_batch\u001b[0;34m(self, chunk_bytes_and_specs)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m bb_codec, chunk_spec_batch \u001b[38;5;129;01min\u001b[39;00m bb_codecs_with_spec[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 172\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m bb_codec\u001b[38;5;241m.\u001b[39mdecode(\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mzip\u001b[39m(chunk_bytes_batch, chunk_spec_batch, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 174\u001b[0m )\n\u001b[1;32m 176\u001b[0m ab_codec, chunk_spec_batch \u001b[38;5;241m=\u001b[39m ab_codec_with_spec\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:130\u001b[0m, in \u001b[0;36mBaseCodec.decode\u001b[0;34m(self, chunks_and_specs)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decodes a batch of chunks.\u001b[39;00m\n\u001b[1;32m 119\u001b[0m \u001b[38;5;124;03mChunks can be None in which case they are ignored by the codec.\u001b[39;00m\n\u001b[1;32m 120\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124;03mIterable[CodecInput | None]\u001b[39;00m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m _batching_helper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decode_single, chunks_and_specs)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:408\u001b[0m, in \u001b[0;36m_batching_helper\u001b[0;34m(func, batch_info)\u001b[0m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_batching_helper\u001b[39m(\n\u001b[1;32m 405\u001b[0m func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]],\n\u001b[1;32m 406\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[CodecInput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, ArraySpec]],\n\u001b[1;32m 407\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]:\n\u001b[0;32m--> 408\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 409\u001b[0m \u001b[38;5;28mlist\u001b[39m(batch_info),\n\u001b[1;32m 410\u001b[0m _noop_for_none(func),\n\u001b[1;32m 411\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 412\u001b[0m )\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:421\u001b[0m, in \u001b[0;36m_noop_for_none..wrap\u001b[0;34m(chunk, chunk_spec)\u001b[0m\n\u001b[1;32m 420\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 421\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(chunk, chunk_spec)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zarr3.py:125\u001b[0m, in \u001b[0;36m_NumcodecsBytesBytesCodec._decode_single\u001b[0;34m(self, chunk_bytes, chunk_spec)\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_decode_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, chunk_bytes: Buffer, chunk_spec: ArraySpec) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Buffer:\n\u001b[0;32m--> 125\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mto_thread(\n\u001b[1;32m 126\u001b[0m as_numpy_array_wrapper,\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_codec\u001b[38;5;241m.\u001b[39mdecode,\n\u001b[1;32m 128\u001b[0m chunk_bytes,\n\u001b[1;32m 129\u001b[0m chunk_spec\u001b[38;5;241m.\u001b[39mprototype,\n\u001b[1;32m 130\u001b[0m )\n", + "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/asyncio/threads.py:25\u001b[0m, in \u001b[0;36mto_thread\u001b[0;34m(func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m func_call \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mpartial(ctx\u001b[38;5;241m.\u001b[39mrun, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m loop\u001b[38;5;241m.\u001b[39mrun_in_executor(\u001b[38;5;28;01mNone\u001b[39;00m, func_call)\n", + "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/buffer/cpu.py:213\u001b[0m, in \u001b[0;36mas_numpy_array_wrapper\u001b[0;34m(func, buf, prototype)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Converts the input of `func` to a numpy array and the output back to `Buffer`.\u001b[39;00m\n\u001b[1;32m 192\u001b[0m \n\u001b[1;32m 193\u001b[0m \u001b[38;5;124;03mThis function is useful when calling a `func` that only support host memory such\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[38;5;124;03m The result of `func` converted to a `Buffer`\u001b[39;00m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m prototype\u001b[38;5;241m.\u001b[39mbuffer\u001b[38;5;241m.\u001b[39mfrom_bytes(\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_numpy_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zlib.py:37\u001b[0m, in \u001b[0;36mZlib.decode\u001b[0;34m(self, buf, out)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# do decompression\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m dec \u001b[38;5;241m=\u001b[39m \u001b[43m_zlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecompress\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;66;03m# handle destination - Python standard library zlib module does not\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;66;03m# support direct decompression into buffer, so we have to copy into\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# out if given\u001b[39;00m\n", + "\u001b[0;31merror\u001b[0m: Error -3 while decompressing data: incorrect header check", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31merror\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[26], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_zarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mread_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconsolidated\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mzarr_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1346\u001b[0m, in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, zarr_version, zarr_format, use_zarr_fill_value_as_mask, chunked_array_type, from_array_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1335\u001b[0m backend_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 1336\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msynchronizer\u001b[39m\u001b[38;5;124m\"\u001b[39m: synchronizer,\n\u001b[1;32m 1337\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconsolidated\u001b[39m\u001b[38;5;124m\"\u001b[39m: consolidated,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1343\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_format\u001b[39m\u001b[38;5;124m\"\u001b[39m: zarr_format,\n\u001b[1;32m 1344\u001b[0m }\n\u001b[1;32m 1345\u001b[0m \u001b[38;5;66;03m#import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[0;32m-> 1346\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1347\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1348\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1349\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_cf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_cf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1350\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1351\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1352\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1353\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1354\u001b[0m \u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mzarr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1355\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1356\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1357\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked_array_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked_array_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1358\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1359\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1360\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1361\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1362\u001b[0m \u001b[43m \u001b[49m\u001b[43mzarr_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzarr_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1363\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1364\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1365\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/api.py:671\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 659\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 660\u001b[0m decode_cf,\n\u001b[1;32m 661\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 667\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 668\u001b[0m )\n\u001b[1;32m 670\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 671\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 677\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 678\u001b[0m backend_ds,\n\u001b[1;32m 679\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 689\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 690\u001b[0m )\n\u001b[1;32m 691\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1436\u001b[0m, in \u001b[0;36mZarrBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask)\u001b[0m\n\u001b[1;32m 1434\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 1435\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n\u001b[0;32m-> 1436\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mstore_entrypoint\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1439\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1440\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1441\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1442\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1443\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1444\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1445\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/store.py:46\u001b[0m, in \u001b[0;36mStoreBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mload()\n\u001b[1;32m 44\u001b[0m encoding \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mget_encoding()\n\u001b[0;32m---> 46\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs, coord_names \u001b[38;5;241m=\u001b[39m \u001b[43mconventions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_cf_variables\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 47\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mvars\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 56\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 58\u001b[0m ds \u001b[38;5;241m=\u001b[39m Dataset(\u001b[38;5;28mvars\u001b[39m, attrs\u001b[38;5;241m=\u001b[39mattrs)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:462\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 451\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m decode_cf_variable(\n\u001b[1;32m 452\u001b[0m k,\n\u001b[1;32m 453\u001b[0m v,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 459\u001b[0m decode_timedelta\u001b[38;5;241m=\u001b[39m_item_or_default(decode_timedelta, k, \u001b[38;5;28;01mNone\u001b[39;00m),\n\u001b[1;32m 460\u001b[0m )\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 462\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(e)(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to decode variable \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_coords \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcoordinates\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mall\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 464\u001b[0m var_attrs \u001b[38;5;241m=\u001b[39m new_vars[k]\u001b[38;5;241m.\u001b[39mattrs\n", + "\u001b[0;31merror\u001b[0m: Failed to decode variable 'time': Error -3 while decompressing data: incorrect header check" + ] + } + ], + "source": [ + "ds = xr.open_zarr(read_store, consolidated=False, zarr_format=3)\n", + "ds" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "virtualizarr", + "language": "python", + "name": "venv" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From fd2df4ef088879aa265d4ad06f35da1c52b4fa7f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 25 Nov 2024 10:40:32 -0800 Subject: [PATCH 63/90] Add a runtime --- runtime.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 runtime.txt diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 00000000..67ebc4e9 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.11 From 1659d21374ae6afe1e56ba4105841d911e507d5c Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 25 Nov 2024 11:05:01 -0800 Subject: [PATCH 64/90] Add failing test --- noaa-cdr-sst.ipynb | 268 ++++++++++-------- .../tests/test_writers/test_icechunk.py | 5 +- 2 files changed, 151 insertions(+), 122 deletions(-) diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb index 0f89b899..a01bff91 100644 --- a/noaa-cdr-sst.ipynb +++ b/noaa-cdr-sst.ipynb @@ -2,17 +2,19 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "d09bbff3-4e96-4490-b837-14b78b64df35", "metadata": {}, "outputs": [], "source": [ - "#!pip install -e \".[icechunk]\"" + "# !pip install -e \".[icechunk]\"\n", + "# !pip install git+https://github.com/mpiannucci/kerchunk@v3\n", + "# !pip install fsspec s3fs" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", "metadata": {}, "outputs": [], @@ -30,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", "metadata": {}, "outputs": [], @@ -46,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "06bbec92-3974-4859-8bda-353afc7800b9", "metadata": {}, "outputs": [], @@ -61,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", "metadata": {}, "outputs": [], @@ -87,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "id": "79a4228a-0e17-4b07-9144-f24fe06db832", "metadata": {}, "outputs": [], @@ -100,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "id": "5fd0c082-8d5e-46a8-a994-fee80baa4ecc", "metadata": {}, "outputs": [], @@ -110,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 9, "id": "55ebbc5f-add2-4de8-81f6-5aaf64d9e2b6", "metadata": {}, "outputs": [], @@ -120,17 +122,17 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 10, "id": "cae7e34f-d6dd-42aa-9e7a-f0d5420ba0b9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'B89RQMAK4SPQR2S6G2DG'" + "'MZWAPPV1HD75JFQFCZ2G'" ] }, - "execution_count": 12, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -141,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, "id": "b6271bd1-bc0b-4901-9901-91aabe508cf7", "metadata": {}, "outputs": [ @@ -178,14 +180,14 @@ " --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n", "}\n", "\n", - "html[theme=dark],\n", - "html[data-theme=dark],\n", - "body[data-theme=dark],\n", + "html[theme=\"dark\"],\n", + "html[data-theme=\"dark\"],\n", + "body[data-theme=\"dark\"],\n", "body.vscode-dark {\n", " --xr-font-color0: rgba(255, 255, 255, 1);\n", " --xr-font-color2: rgba(255, 255, 255, 0.54);\n", " --xr-font-color3: rgba(255, 255, 255, 0.38);\n", - " --xr-border-color: #1F1F1F;\n", + " --xr-border-color: #1f1f1f;\n", " --xr-disabled-color: #515151;\n", " --xr-background-color: #111111;\n", " --xr-background-color-row-even: #111111;\n", @@ -240,6 +242,7 @@ ".xr-section-item input {\n", " display: inline-block;\n", " opacity: 0;\n", + " height: 0;\n", "}\n", "\n", ".xr-section-item input + label {\n", @@ -276,7 +279,7 @@ "\n", ".xr-section-summary-in + label:before {\n", " display: inline-block;\n", - " content: '►';\n", + " content: \"►\";\n", " font-size: 11px;\n", " width: 15px;\n", " text-align: center;\n", @@ -287,7 +290,7 @@ "}\n", "\n", ".xr-section-summary-in:checked + label:before {\n", - " content: '▼';\n", + " content: \"▼\";\n", "}\n", "\n", ".xr-section-summary-in:checked + label > span {\n", @@ -359,15 +362,15 @@ "}\n", "\n", ".xr-dim-list:before {\n", - " content: '(';\n", + " content: \"(\";\n", "}\n", "\n", ".xr-dim-list:after {\n", - " content: ')';\n", + " content: \")\";\n", "}\n", "\n", ".xr-dim-list li:not(:last-child):after {\n", - " content: ',';\n", + " content: \",\";\n", " padding-right: 5px;\n", "}\n", "\n", @@ -518,17 +521,17 @@ " fill: currentColor;\n", "}\n", "
<xarray.Dataset> Size: 66MB\n",
-       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
+       "Dimensions:  (lon: 1440, time: 2, zlev: 1, lat: 720)\n",
        "Coordinates:\n",
-       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
        "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
-       "  * zlev     (zlev) float32 4B 0.0\n",
        "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
+       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
+       "  * zlev     (zlev) float32 4B 0.0\n",
        "Data variables:\n",
-       "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
+       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
        "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
        "    anom     (time, zlev, lat, lon) float64 17MB ...\n",
-       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
+       "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
        "Attributes: (12/37)\n",
        "    Conventions:                CF-1.6, ACDD-1.3\n",
        "    cdm_data_type:              Grid\n",
@@ -542,34 +545,34 @@
        "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
        "    time_coverage_end:          2024-08-01T23:59:59Z\n",
        "    time_coverage_start:        2024-08-01T00:00:00Z\n",
-       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
  • Conventions :
    CF-1.6, ACDD-1.3
    cdm_data_type :
    Grid
    comment :
    Data was converted from NetCDF-3 to NetCDF-4 format with metadata updates in November 2017.
    creator_email :
    oisst-help@noaa.gov
    creator_url :
    https://www.ncei.noaa.gov/
    date_created :
    2024-08-16T09:12:00Z
    date_modified :
    2024-08-16T09:12:00Z
    geospatial_lat_max :
    90.0
    geospatial_lat_min :
    -90.0
    geospatial_lat_resolution :
    0.25
    geospatial_lat_units :
    degrees_north
    geospatial_lon_max :
    360.0
    geospatial_lon_min :
    0.0
    geospatial_lon_resolution :
    0.25
    geospatial_lon_units :
    degrees_east
    history :
    Final file created using preliminary as first guess, and 3 days of AVHRR data. Preliminary uses only 1 day of AVHRR data.
    id :
    oisst-avhrr-v02r01.20240801.nc
    institution :
    NOAA/National Centers for Environmental Information
    instrument :
    Earth Remote Sensing Instruments > Passive Remote Sensing > Spectrometers/Radiometers > Imaging Spectrometers/Radiometers > AVHRR > Advanced Very High Resolution Radiometer
    instrument_vocabulary :
    Global Change Master Directory (GCMD) Instrument Keywords
    keywords :
    Earth Science > Oceans > Ocean Temperature > Sea Surface Temperature
    keywords_vocabulary :
    Global Change Master Directory (GCMD) Earth Science Keywords
    metadata_link :
    https://doi.org/10.25921/RE9P-PT57
    naming_authority :
    gov.noaa.ncei
    ncei_template_version :
    NCEI_NetCDF_Grid_Template_v2.0
    platform :
    Ships, buoys, Argo floats, MetOp-A, MetOp-B
    platform_vocabulary :
    Global Change Master Directory (GCMD) Platform Keywords
    processing_level :
    NOAA Level 4
    product_version :
    Version v02r01
    references :
    Reynolds, et al.(2007) Daily High-Resolution-Blended Analyses for Sea Surface Temperature (available at https://doi.org/10.1175/2007JCLI1824.1). Banzon, et al.(2016) A long-term record of blended satellite and in situ sea-surface temperature for climate monitoring, modeling and environmental studies (available at https://doi.org/10.5194/essd-8-165-2016). Huang et al. (2020) Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version v02r01, submitted.Climatology is based on 1971-2000 OI.v2 SST. Satellite data: Pathfinder AVHRR SST, Navy AVHRR SST, and NOAA ACSPO SST. Ice data: NCEP Ice and GSFC Ice.
    sensor :
    Thermometer, AVHRR
    source :
    ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfinder_AVHRR, Navy_AVHRR, NOAA_ACSP
    standard_name_vocabulary :
    CF Standard Name Table (v40, 25 January 2017)
    summary :
    NOAAs 1/4-degree Daily Optimum Interpolation Sea Surface Temperature (OISST) (sometimes referred to as Reynolds SST, which however also refers to earlier products at different resolution), currently available as version v02r01, is created by interpolating and extrapolating SST observations from different sources, resulting in a smoothed complete field. The sources of data are satellite (AVHRR) and in situ platforms (i.e., ships and buoys), and the specific datasets employed may change over time. At the marginal ice zone, sea ice concentrations are used to generate proxy SSTs. A preliminary version of this file is produced in near-real time (1-day latency), and then replaced with a final version after 2 weeks. Note that this is the AVHRR-ONLY DOISST, available from Oct 1981, but there is a companion DOISST product that includes microwave satellite data, available from June 2002
    time_coverage_end :
    2024-08-01T23:59:59Z
    time_coverage_start :
    2024-08-01T00:00:00Z
    title :
    NOAA/NCEI 1/4 Degree Daily Optimum Interpolation Sea Surface Temperature (OISST) Analysis, Version 2.1 - Final
  • " ], "text/plain": [ " Size: 66MB\n", - "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", + "Dimensions: (lon: 1440, time: 2, zlev: 1, lat: 720)\n", "Coordinates:\n", - " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", - " * zlev (zlev) float32 4B 0.0\n", " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", + " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", + " * zlev (zlev) float32 4B 0.0\n", "Data variables:\n", - " ice (time, zlev, lat, lon) float64 17MB ...\n", + " err (time, zlev, lat, lon) float64 17MB ...\n", " sst (time, zlev, lat, lon) float64 17MB ...\n", " anom (time, zlev, lat, lon) float64 17MB ...\n", - " err (time, zlev, lat, lon) float64 17MB ...\n", + " ice (time, zlev, lat, lon) float64 17MB ...\n", "Attributes: (12/37)\n", " Conventions: CF-1.6, ACDD-1.3\n", " cdm_data_type: Grid\n", @@ -586,7 +589,7 @@ " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." ] }, - "execution_count": 13, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -606,7 +609,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 12, "id": "190c25f9-e000-4b17-83eb-cf551141dfea", "metadata": {}, "outputs": [], @@ -621,7 +624,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 13, "id": "af330082-207a-4f08-aefe-fc15aa8b2eb3", "metadata": {}, "outputs": [], @@ -637,7 +640,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 14, "id": "103b44d2-124a-4de5-8074-e997fd5a1698", "metadata": {}, "outputs": [ @@ -674,14 +677,14 @@ " --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n", "}\n", "\n", - "html[theme=dark],\n", - "html[data-theme=dark],\n", - "body[data-theme=dark],\n", + "html[theme=\"dark\"],\n", + "html[data-theme=\"dark\"],\n", + "body[data-theme=\"dark\"],\n", "body.vscode-dark {\n", " --xr-font-color0: rgba(255, 255, 255, 1);\n", " --xr-font-color2: rgba(255, 255, 255, 0.54);\n", " --xr-font-color3: rgba(255, 255, 255, 0.38);\n", - " --xr-border-color: #1F1F1F;\n", + " --xr-border-color: #1f1f1f;\n", " --xr-disabled-color: #515151;\n", " --xr-background-color: #111111;\n", " --xr-background-color-row-even: #111111;\n", @@ -736,6 +739,7 @@ ".xr-section-item input {\n", " display: inline-block;\n", " opacity: 0;\n", + " height: 0;\n", "}\n", "\n", ".xr-section-item input + label {\n", @@ -772,7 +776,7 @@ "\n", ".xr-section-summary-in + label:before {\n", " display: inline-block;\n", - " content: '►';\n", + " content: \"►\";\n", " font-size: 11px;\n", " width: 15px;\n", " text-align: center;\n", @@ -783,7 +787,7 @@ "}\n", "\n", ".xr-section-summary-in:checked + label:before {\n", - " content: '▼';\n", + " content: \"▼\";\n", "}\n", "\n", ".xr-section-summary-in:checked + label > span {\n", @@ -855,15 +859,15 @@ "}\n", "\n", ".xr-dim-list:before {\n", - " content: '(';\n", + " content: \"(\";\n", "}\n", "\n", ".xr-dim-list:after {\n", - " content: ')';\n", + " content: \")\";\n", "}\n", "\n", ".xr-dim-list li:not(:last-child):after {\n", - " content: ',';\n", + " content: \",\";\n", " padding-right: 5px;\n", "}\n", "\n", @@ -1016,15 +1020,15 @@ "
    <xarray.Dataset> Size: 17MB\n",
            "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
            "Coordinates:\n",
    -       "    time     (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n",
    -       "    lon      (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n",
            "    zlev     (zlev) float32 4B ManifestArray<shape=(1,), dtype=float32, chunk...\n",
            "    lat      (lat) float32 3kB ManifestArray<shape=(720,), dtype=float32, chu...\n",
    +       "    time     (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n",
    +       "    lon      (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n",
            "Data variables:\n",
    -       "    anom     (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "    ice      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
            "    err      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    +       "    ice      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
            "    sst      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    +       "    anom     (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
            "Attributes: (12/37)\n",
            "    Conventions:                CF-1.6, ACDD-1.3\n",
            "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...\n",
    @@ -1038,21 +1042,21 @@
            "    metadata_link:              https://doi.org/10.25921/RE9P-PT57\n",
            "    ncei_template_version:      NCEI_NetCDF_Grid_Template_v2.0\n",
            "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    -       "    sensor:                     Thermometer, AVHRR
    " + " sensor: Thermometer, AVHRR" ], "text/plain": [ " Size: 17MB\n", "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", "Coordinates:\n", - " time (time) float32 8B ManifestArray\\x00\\x00\\xc0>\\x00\\x00 ?\\x00\\x00`?\\x00\\x00\\x90?\\x00\\x00\\xb0?\\x00\\x00\\xd0?\\x00\\x00\\xf0?\\x00\\x00\\x08@\\x00\\x00\\x18@\\x00\\x00(@\\x00\\x008@\\x00\\x00H@\\x00\\x00X@\\x00\\x00h@\\x00\\x00x@\\x00\\x00\\x84@\\x00\\x00\\x8c@\\x00\\x00\\x94@\\x00\\x00\\x9c@\\x00\\x00\\xa4@\\x00\\x00\\xac@\\x00\\x00\\xb4@\\x00\\x00\\xbc@\\x00\\x00\\xc4@\\x00\\x00\\xcc@\\x00\\x00\\xd4@\\x00\\x00\\xdc@\\x00\\x00\\xe4@\\x00\\x00\\xec@\\x00\\x00\\xf4@\\x00\\x00\\xfc@\\x00\\x00\\x02A\\x00\\x00\\x06A\\x00\\x00\\nA\\x00\\x00\\x0eA\\x00\\x00\\x12A\\x00\\x00\\x16A\\x00\\x00\\x1aA\\x00\\x00\\x1eA\\x00\\x00\"A\\x00\\x00&A\\x00\\x00*A\\x00\\x00.A\\x00\\x002A\\x00\\x006A\\x00\\x00:A\\x00\\x00>A\\x00\\x00BA\\x00\\x00FA\\x00\\x00JA\\x00\\x00NA\\x00\\x00RA\\x00\\x00VA\\x00\\x00ZA\\x00\\x00^A\\x00\\x00bA\\x00\\x00fA\\x00\\x00jA\\x00\\x00nA\\x00\\x00rA\\x00\\x00vA\\x00\\x00zA\\x00\\x00~A\\x00\\x00\\x81A\\x00\\x00\\x83A\\x00\\x00\\x85A\\x00\\x00\\x87A\\x00\\x00\\x89A\\x00\\x00\\x8bA\\x00\\x00\\x8dA\\x00\\x00\\x8fA\\x00\\x00\\x91A\\x00\\x00\\x93A\\x00\\x00\\x95A\\x00\\x00\\x97A\\x00\\x00\\x99A\\x00\\x00\\x9bA\\x00\\x00\\x9dA\\x00\\x00\\x9fA\\x00\\x00\\xa1A\\x00\\x00\\xa3A\\x00\\x00\\xa5A\\x00\\x00\\xa7A\\x00\\x00\\xa9A\\x00\\x00\\xabA\\x00\\x00\\xadA\\x00\\x00\\xafA\\x00\\x00\\xb1A\\x00\\x00\\xb3A\\x00\\x00\\xb5A\\x00\\x00\\xb7A\\x00\\x00\\xb9A\\x00\\x00\\xbbA\\x00\\x00\\xbdA\\x00\\x00\\xbfA\\x00\\x00\\xc1A\\x00\\x00\\xc3A\\x00\\x00\\xc5A\\x00\\x00\\xc7A\\x00\\x00\\xc9A\\x00\\x00\\xcbA\\x00\\x00\\xcdA\\x00\\x00\\xcfA\\x00\\x00\\xd1A\\x00\\x00\\xd3A\\x00\\x00\\xd5A\\x00\\x00\\xd7A\\x00\\x00\\xd9A\\x00\\x00\\xdbA\\x00\\x00\\xddA\\x00\\x00\\xdfA\\x00\\x00\\xe1A\\x00\\x00\\xe3A\\x00\\x00\\xe5A\\x00\\x00\\xe7A\\x00\\x00\\xe9A\\x00\\x00\\xebA\\x00\\x00\\xedA\\x00\\x00\\xefA\\x00\\x00\\xf1A\\x00\\x00\\xf3A\\x00\\x00\\xf5A\\x00\\x00\\xf7A\\x00\\x00\\xf9A\\x00\\x00\\xfbA\\x00\\x00\\xfdA\\x00\\x00\\xffA\\x00\\x80\\x00B\\x00\\x80\\x01B\\x00\\x80\\x02B\\x00\\x80\\x03B\\x00\\x80\\x04B\\x00\\x80\\x05B\\x00\\x80\\x06B\\x00\\x80\\x07B\\x00\\x80\\x08B\\x00\\x80\\tB\\x00\\x80\\nB\\x00\\x80\\x0bB\\x00\\x80\\x0cB\\x00\\x80\\rB\\x00\\x80\\x0eB\\x00\\x80\\x0fB\\x00\\x80\\x10B\\x00\\x80\\x11B\\x00\\x80\\x12B\\x00\\x80\\x13B\\x00\\x80\\x14B\\x00\\x80\\x15B\\x00\\x80\\x16B\\x00\\x80\\x17B\\x00\\x80\\x18B\\x00\\x80\\x19B\\x00\\x80\\x1aB\\x00\\x80\\x1bB\\x00\\x80\\x1cB\\x00\\x80\\x1dB\\x00\\x80\\x1eB\\x00\\x80\\x1fB\\x00\\x80 B\\x00\\x80!B\\x00\\x80\"B\\x00\\x80#B\\x00\\x80$B\\x00\\x80%B\\x00\\x80&B\\x00\\x80\\'B\\x00\\x80(B\\x00\\x80)B\\x00\\x80*B\\x00\\x80+B\\x00\\x80,B\\x00\\x80-B\\x00\\x80.B\\x00\\x80/B\\x00\\x800B\\x00\\x801B\\x00\\x802B\\x00\\x803B\\x00\\x804B\\x00\\x805B\\x00\\x806B\\x00\\x807B\\x00\\x808B\\x00\\x809B\\x00\\x80:B\\x00\\x80;B\\x00\\x80B\\x00\\x80?B\\x00\\x80@B\\x00\\x80AB\\x00\\x80BB\\x00\\x80CB\\x00\\x80DB\\x00\\x80EB\\x00\\x80FB\\x00\\x80GB\\x00\\x80HB\\x00\\x80IB\\x00\\x80JB\\x00\\x80KB\\x00\\x80LB\\x00\\x80MB\\x00\\x80NB\\x00\\x80OB\\x00\\x80PB\\x00\\x80QB\\x00\\x80RB\\x00\\x80SB\\x00\\x80TB\\x00\\x80UB\\x00\\x80VB\\x00\\x80WB\\x00\\x80XB\\x00\\x80YB\\x00\\x80ZB\\x00\\x80[B\\x00\\x80\\\\B\\x00\\x80]B\\x00\\x80^B\\x00\\x80_B\\x00\\x80`B\\x00\\x80aB\\x00\\x80bB\\x00\\x80cB\\x00\\x80dB\\x00\\x80eB\\x00\\x80fB\\x00\\x80gB\\x00\\x80hB\\x00\\x80iB\\x00\\x80jB\\x00\\x80kB\\x00\\x80lB\\x00\\x80mB\\x00\\x80nB\\x00\\x80oB\\x00\\x80pB\\x00\\x80qB\\x00\\x80rB\\x00\\x80sB\\x00\\x80tB\\x00\\x80uB\\x00\\x80vB\\x00\\x80wB\\x00\\x80xB\\x00\\x80yB\\x00\\x80zB\\x00\\x80{B\\x00\\x80|B\\x00\\x80}B\\x00\\x80~B\\x00\\x80\\x7fB\\x00@\\x80B\\x00\\xc0\\x80B\\x00@\\x81B\\x00\\xc0\\x81B\\x00@\\x82B\\x00\\xc0\\x82B\\x00@\\x83B\\x00\\xc0\\x83B\\x00@\\x84B\\x00\\xc0\\x84B\\x00@\\x85B\\x00\\xc0\\x85B\\x00@\\x86B\\x00\\xc0\\x86B\\x00@\\x87B\\x00\\xc0\\x87B\\x00@\\x88B\\x00\\xc0\\x88B\\x00@\\x89B\\x00\\xc0\\x89B\\x00@\\x8aB\\x00\\xc0\\x8aB\\x00@\\x8bB\\x00\\xc0\\x8bB\\x00@\\x8cB\\x00\\xc0\\x8cB\\x00@\\x8dB\\x00\\xc0\\x8dB\\x00@\\x8eB\\x00\\xc0\\x8eB\\x00@\\x8fB\\x00\\xc0\\x8fB\\x00@\\x90B\\x00\\xc0\\x90B\\x00@\\x91B\\x00\\xc0\\x91B\\x00@\\x92B\\x00\\xc0\\x92B\\x00@\\x93B\\x00\\xc0\\x93B\\x00@\\x94B\\x00\\xc0\\x94B\\x00@\\x95B\\x00\\xc0\\x95B\\x00@\\x96B\\x00\\xc0\\x96B\\x00@\\x97B\\x00\\xc0\\x97B\\x00@\\x98B\\x00\\xc0\\x98B\\x00@\\x99B\\x00\\xc0\\x99B\\x00@\\x9aB\\x00\\xc0\\x9aB\\x00@\\x9bB\\x00\\xc0\\x9bB\\x00@\\x9cB\\x00\\xc0\\x9cB\\x00@\\x9dB\\x00\\xc0\\x9dB\\x00@\\x9eB\\x00\\xc0\\x9eB\\x00@\\x9fB\\x00\\xc0\\x9fB\\x00@\\xa0B\\x00\\xc0\\xa0B\\x00@\\xa1B\\x00\\xc0\\xa1B\\x00@\\xa2B\\x00\\xc0\\xa2B\\x00@\\xa3B\\x00\\xc0\\xa3B\\x00@\\xa4B\\x00\\xc0\\xa4B\\x00@\\xa5B\\x00\\xc0\\xa5B\\x00@\\xa6B\\x00\\xc0\\xa6B\\x00@\\xa7B\\x00\\xc0\\xa7B\\x00@\\xa8B\\x00\\xc0\\xa8B\\x00@\\xa9B\\x00\\xc0\\xa9B\\x00@\\xaaB\\x00\\xc0\\xaaB\\x00@\\xabB\\x00\\xc0\\xabB\\x00@\\xacB\\x00\\xc0\\xacB\\x00@\\xadB\\x00\\xc0\\xadB\\x00@\\xaeB\\x00\\xc0\\xaeB\\x00@\\xafB\\x00\\xc0\\xafB\\x00@\\xb0B\\x00\\xc0\\xb0B\\x00@\\xb1B\\x00\\xc0\\xb1B\\x00@\\xb2B\\x00\\xc0\\xb2B\\x00@\\xb3B\\x00\\xc0\\xb3B\\x00@\\xb4B\\x00\\xc0\\xb4B\\x00@\\xb5B\\x00\\xc0\\xb5B\\x00@\\xb6B\\x00\\xc0\\xb6B\\x00@\\xb7B\\x00\\xc0\\xb7B\\x00@\\xb8B\\x00\\xc0\\xb8B\\x00@\\xb9B\\x00\\xc0\\xb9B\\x00@\\xbaB\\x00\\xc0\\xbaB\\x00@\\xbbB\\x00\\xc0\\xbbB\\x00@\\xbcB\\x00\\xc0\\xbcB\\x00@\\xbdB\\x00\\xc0\\xbdB\\x00@\\xbeB\\x00\\xc0\\xbeB\\x00@\\xbfB\\x00\\xc0\\xbfB\\x00@\\xc0B\\x00\\xc0\\xc0B\\x00@\\xc1B\\x00\\xc0\\xc1B\\x00@\\xc2B\\x00\\xc0\\xc2B\\x00@\\xc3B\\x00\\xc0\\xc3B\\x00@\\xc4B\\x00\\xc0\\xc4B\\x00@\\xc5B\\x00\\xc0\\xc5B\\x00@\\xc6B\\x00\\xc0\\xc6B\\x00@\\xc7B\\x00\\xc0\\xc7B\\x00@\\xc8B\\x00\\xc0\\xc8B\\x00@\\xc9B\\x00\\xc0\\xc9B\\x00@\\xcaB\\x00\\xc0\\xcaB\\x00@\\xcbB\\x00\\xc0\\xcbB\\x00@\\xccB\\x00\\xc0\\xccB\\x00@\\xcdB\\x00\\xc0\\xcdB\\x00@\\xceB\\x00\\xc0\\xceB\\x00@\\xcfB\\x00\\xc0\\xcfB\\x00@\\xd0B\\x00\\xc0\\xd0B\\x00@\\xd1B\\x00\\xc0\\xd1B\\x00@\\xd2B\\x00\\xc0\\xd2B\\x00@\\xd3B\\x00\\xc0\\xd3B\\x00@\\xd4B\\x00\\xc0\\xd4B\\x00@\\xd5B\\x00\\xc0\\xd5B\\x00@\\xd6B\\x00\\xc0\\xd6B\\x00@\\xd7B\\x00\\xc0\\xd7B\\x00@\\xd8B\\x00\\xc0\\xd8B\\x00@\\xd9B\\x00\\xc0\\xd9B\\x00@\\xdaB\\x00\\xc0\\xdaB\\x00@\\xdbB\\x00\\xc0\\xdbB\\x00@\\xdcB\\x00\\xc0\\xdcB\\x00@\\xddB\\x00\\xc0\\xddB\\x00@\\xdeB\\x00\\xc0\\xdeB\\x00@\\xdfB\\x00\\xc0\\xdfB\\x00@\\xe0B\\x00\\xc0\\xe0B\\x00@\\xe1B\\x00\\xc0\\xe1B\\x00@\\xe2B\\x00\\xc0\\xe2B\\x00@\\xe3B\\x00\\xc0\\xe3B\\x00@\\xe4B\\x00\\xc0\\xe4B\\x00@\\xe5B\\x00\\xc0\\xe5B\\x00@\\xe6B\\x00\\xc0\\xe6B\\x00@\\xe7B\\x00\\xc0\\xe7B\\x00@\\xe8B\\x00\\xc0\\xe8B\\x00@\\xe9B\\x00\\xc0\\xe9B\\x00@\\xeaB\\x00\\xc0\\xeaB\\x00@\\xebB\\x00\\xc0\\xebB\\x00@\\xecB\\x00\\xc0\\xecB\\x00@\\xedB\\x00\\xc0\\xedB\\x00@\\xeeB\\x00\\xc0\\xeeB\\x00@\\xefB\\x00\\xc0\\xefB\\x00@\\xf0B\\x00\\xc0\\xf0B\\x00@\\xf1B\\x00\\xc0\\xf1B\\x00@\\xf2B\\x00\\xc0\\xf2B\\x00@\\xf3B\\x00\\xc0\\xf3B\\x00@\\xf4B\\x00\\xc0\\xf4B\\x00@\\xf5B\\x00\\xc0\\xf5B\\x00@\\xf6B\\x00\\xc0\\xf6B\\x00@\\xf7B\\x00\\xc0\\xf7B\\x00@\\xf8B\\x00\\xc0\\xf8B\\x00@\\xf9B\\x00\\xc0\\xf9B\\x00@\\xfaB\\x00\\xc0\\xfaB\\x00@\\xfbB\\x00\\xc0\\xfbB\\x00@\\xfcB\\x00\\xc0\\xfcB\\x00@\\xfdB\\x00\\xc0\\xfdB\\x00@\\xfeB\\x00\\xc0\\xfeB\\x00@\\xffB\\x00\\xc0\\xffB\\x00 \\x00C\\x00`\\x00C\\x00\\xa0\\x00C\\x00\\xe0\\x00C\\x00 \\x01C\\x00`\\x01C\\x00\\xa0\\x01C\\x00\\xe0\\x01C\\x00 \\x02C\\x00`\\x02C\\x00\\xa0\\x02C\\x00\\xe0\\x02C\\x00 \\x03C\\x00`\\x03C\\x00\\xa0\\x03C\\x00\\xe0\\x03C\\x00 \\x04C\\x00`\\x04C\\x00\\xa0\\x04C\\x00\\xe0\\x04C\\x00 \\x05C\\x00`\\x05C\\x00\\xa0\\x05C\\x00\\xe0\\x05C\\x00 \\x06C\\x00`\\x06C\\x00\\xa0\\x06C\\x00\\xe0\\x06C\\x00 \\x07C\\x00`\\x07C\\x00\\xa0\\x07C\\x00\\xe0\\x07C\\x00 \\x08C\\x00`\\x08C\\x00\\xa0\\x08C\\x00\\xe0\\x08C\\x00 \\tC\\x00`\\tC\\x00\\xa0\\tC\\x00\\xe0\\tC\\x00 \\nC\\x00`\\nC\\x00\\xa0\\nC\\x00\\xe0\\nC\\x00 \\x0bC\\x00`\\x0bC\\x00\\xa0\\x0bC\\x00\\xe0\\x0bC\\x00 \\x0cC\\x00`\\x0cC\\x00\\xa0\\x0cC\\x00\\xe0\\x0cC\\x00 \\rC\\x00`\\rC\\x00\\xa0\\rC\\x00\\xe0\\rC\\x00 \\x0eC\\x00`\\x0eC\\x00\\xa0\\x0eC\\x00\\xe0\\x0eC\\x00 \\x0fC\\x00`\\x0fC\\x00\\xa0\\x0fC\\x00\\xe0\\x0fC\\x00 \\x10C\\x00`\\x10C\\x00\\xa0\\x10C\\x00\\xe0\\x10C\\x00 \\x11C\\x00`\\x11C\\x00\\xa0\\x11C\\x00\\xe0\\x11C\\x00 \\x12C\\x00`\\x12C\\x00\\xa0\\x12C\\x00\\xe0\\x12C\\x00 \\x13C\\x00`\\x13C\\x00\\xa0\\x13C\\x00\\xe0\\x13C\\x00 \\x14C\\x00`\\x14C\\x00\\xa0\\x14C\\x00\\xe0\\x14C\\x00 \\x15C\\x00`\\x15C\\x00\\xa0\\x15C\\x00\\xe0\\x15C\\x00 \\x16C\\x00`\\x16C\\x00\\xa0\\x16C\\x00\\xe0\\x16C\\x00 \\x17C\\x00`\\x17C\\x00\\xa0\\x17C\\x00\\xe0\\x17C\\x00 \\x18C\\x00`\\x18C\\x00\\xa0\\x18C\\x00\\xe0\\x18C\\x00 \\x19C\\x00`\\x19C\\x00\\xa0\\x19C\\x00\\xe0\\x19C\\x00 \\x1aC\\x00`\\x1aC\\x00\\xa0\\x1aC\\x00\\xe0\\x1aC\\x00 \\x1bC\\x00`\\x1bC\\x00\\xa0\\x1bC\\x00\\xe0\\x1bC\\x00 \\x1cC\\x00`\\x1cC\\x00\\xa0\\x1cC\\x00\\xe0\\x1cC\\x00 \\x1dC\\x00`\\x1dC\\x00\\xa0\\x1dC\\x00\\xe0\\x1dC\\x00 \\x1eC\\x00`\\x1eC\\x00\\xa0\\x1eC\\x00\\xe0\\x1eC\\x00 \\x1fC\\x00`\\x1fC\\x00\\xa0\\x1fC\\x00\\xe0\\x1fC\\x00 C\\x00` C\\x00\\xa0 C\\x00\\xe0 C\\x00 !C\\x00`!C\\x00\\xa0!C\\x00\\xe0!C\\x00 \"C\\x00`\"C\\x00\\xa0\"C\\x00\\xe0\"C\\x00 #C\\x00`#C\\x00\\xa0#C\\x00\\xe0#C\\x00 $C\\x00`$C\\x00\\xa0$C\\x00\\xe0$C\\x00 %C\\x00`%C\\x00\\xa0%C\\x00\\xe0%C\\x00 &C\\x00`&C\\x00\\xa0&C\\x00\\xe0&C\\x00 \\'C\\x00`\\'C\\x00\\xa0\\'C\\x00\\xe0\\'C\\x00 (C\\x00`(C\\x00\\xa0(C\\x00\\xe0(C\\x00 )C\\x00`)C\\x00\\xa0)C\\x00\\xe0)C\\x00 *C\\x00`*C\\x00\\xa0*C\\x00\\xe0*C\\x00 +C\\x00`+C\\x00\\xa0+C\\x00\\xe0+C\\x00 ,C\\x00`,C\\x00\\xa0,C\\x00\\xe0,C\\x00 -C\\x00`-C\\x00\\xa0-C\\x00\\xe0-C\\x00 .C\\x00`.C\\x00\\xa0.C\\x00\\xe0.C\\x00 /C\\x00`/C\\x00\\xa0/C\\x00\\xe0/C\\x00 0C\\x00`0C\\x00\\xa00C\\x00\\xe00C\\x00 1C\\x00`1C\\x00\\xa01C\\x00\\xe01C\\x00 2C\\x00`2C\\x00\\xa02C\\x00\\xe02C\\x00 3C\\x00`3C\\x00\\xa03C\\x00\\xe03C\\x00 4C\\x00`4C\\x00\\xa04C\\x00\\xe04C\\x00 5C\\x00`5C\\x00\\xa05C\\x00\\xe05C\\x00 6C\\x00`6C\\x00\\xa06C\\x00\\xe06C\\x00 7C\\x00`7C\\x00\\xa07C\\x00\\xe07C\\x00 8C\\x00`8C\\x00\\xa08C\\x00\\xe08C\\x00 9C\\x00`9C\\x00\\xa09C\\x00\\xe09C\\x00 :C\\x00`:C\\x00\\xa0:C\\x00\\xe0:C\\x00 ;C\\x00`;C\\x00\\xa0;C\\x00\\xe0;C\\x00 C\\x00`>C\\x00\\xa0>C\\x00\\xe0>C\\x00 ?C\\x00`?C\\x00\\xa0?C\\x00\\xe0?C\\x00 @C\\x00`@C\\x00\\xa0@C\\x00\\xe0@C\\x00 AC\\x00`AC\\x00\\xa0AC\\x00\\xe0AC\\x00 BC\\x00`BC\\x00\\xa0BC\\x00\\xe0BC\\x00 CC\\x00`CC\\x00\\xa0CC\\x00\\xe0CC\\x00 DC\\x00`DC\\x00\\xa0DC\\x00\\xe0DC\\x00 EC\\x00`EC\\x00\\xa0EC\\x00\\xe0EC\\x00 FC\\x00`FC\\x00\\xa0FC\\x00\\xe0FC\\x00 GC\\x00`GC\\x00\\xa0GC\\x00\\xe0GC\\x00 HC\\x00`HC\\x00\\xa0HC\\x00\\xe0HC\\x00 IC\\x00`IC\\x00\\xa0IC\\x00\\xe0IC\\x00 JC\\x00`JC\\x00\\xa0JC\\x00\\xe0JC\\x00 KC\\x00`KC\\x00\\xa0KC\\x00\\xe0KC\\x00 LC\\x00`LC\\x00\\xa0LC\\x00\\xe0LC\\x00 MC\\x00`MC\\x00\\xa0MC\\x00\\xe0MC\\x00 NC\\x00`NC\\x00\\xa0NC\\x00\\xe0NC\\x00 OC\\x00`OC\\x00\\xa0OC\\x00\\xe0OC\\x00 PC\\x00`PC\\x00\\xa0PC\\x00\\xe0PC\\x00 QC\\x00`QC\\x00\\xa0QC\\x00\\xe0QC\\x00 RC\\x00`RC\\x00\\xa0RC\\x00\\xe0RC\\x00 SC\\x00`SC\\x00\\xa0SC\\x00\\xe0SC\\x00 TC\\x00`TC\\x00\\xa0TC\\x00\\xe0TC\\x00 UC\\x00`UC\\x00\\xa0UC\\x00\\xe0UC\\x00 VC\\x00`VC\\x00\\xa0VC\\x00\\xe0VC\\x00 WC\\x00`WC\\x00\\xa0WC\\x00\\xe0WC\\x00 XC\\x00`XC\\x00\\xa0XC\\x00\\xe0XC\\x00 YC\\x00`YC\\x00\\xa0YC\\x00\\xe0YC\\x00 ZC\\x00`ZC\\x00\\xa0ZC\\x00\\xe0ZC\\x00 [C\\x00`[C\\x00\\xa0[C\\x00\\xe0[C\\x00 \\\\C\\x00`\\\\C\\x00\\xa0\\\\C\\x00\\xe0\\\\C\\x00 ]C\\x00`]C\\x00\\xa0]C\\x00\\xe0]C\\x00 ^C\\x00`^C\\x00\\xa0^C\\x00\\xe0^C\\x00 _C\\x00`_C\\x00\\xa0_C\\x00\\xe0_C\\x00 `C\\x00``C\\x00\\xa0`C\\x00\\xe0`C\\x00 aC\\x00`aC\\x00\\xa0aC\\x00\\xe0aC\\x00 bC\\x00`bC\\x00\\xa0bC\\x00\\xe0bC\\x00 cC\\x00`cC\\x00\\xa0cC\\x00\\xe0cC\\x00 dC\\x00`dC\\x00\\xa0dC\\x00\\xe0dC\\x00 eC\\x00`eC\\x00\\xa0eC\\x00\\xe0eC\\x00 fC\\x00`fC\\x00\\xa0fC\\x00\\xe0fC\\x00 gC\\x00`gC\\x00\\xa0gC\\x00\\xe0gC\\x00 hC\\x00`hC\\x00\\xa0hC\\x00\\xe0hC\\x00 iC\\x00`iC\\x00\\xa0iC\\x00\\xe0iC\\x00 jC\\x00`jC\\x00\\xa0jC\\x00\\xe0jC\\x00 kC\\x00`kC\\x00\\xa0kC\\x00\\xe0kC\\x00 lC\\x00`lC\\x00\\xa0lC\\x00\\xe0lC\\x00 mC\\x00`mC\\x00\\xa0mC\\x00\\xe0mC\\x00 nC\\x00`nC\\x00\\xa0nC\\x00\\xe0nC\\x00 oC\\x00`oC\\x00\\xa0oC\\x00\\xe0oC\\x00 pC\\x00`pC\\x00\\xa0pC\\x00\\xe0pC\\x00 qC\\x00`qC\\x00\\xa0qC\\x00\\xe0qC\\x00 rC\\x00`rC\\x00\\xa0rC\\x00\\xe0rC\\x00 sC\\x00`sC\\x00\\xa0sC\\x00\\xe0sC\\x00 tC\\x00`tC\\x00\\xa0tC\\x00\\xe0tC\\x00 uC\\x00`uC\\x00\\xa0uC\\x00\\xe0uC\\x00 vC\\x00`vC\\x00\\xa0vC\\x00\\xe0vC\\x00 wC\\x00`wC\\x00\\xa0wC\\x00\\xe0wC\\x00 xC\\x00`xC\\x00\\xa0xC\\x00\\xe0xC\\x00 yC\\x00`yC\\x00\\xa0yC\\x00\\xe0yC\\x00 zC\\x00`zC\\x00\\xa0zC\\x00\\xe0zC\\x00 {C\\x00`{C\\x00\\xa0{C\\x00\\xe0{C\\x00 |C\\x00`|C\\x00\\xa0|C\\x00\\xe0|C\\x00 }C\\x00`}C\\x00\\xa0}C\\x00\\xe0}C\\x00 ~C\\x00`~C\\x00\\xa0~C\\x00\\xe0~C\\x00 \\x7fC\\x00`\\x7fC\\x00\\xa0\\x7fC\\x00\\xe0\\x7fC\\x00\\x10\\x80C\\x000\\x80C\\x00P\\x80C\\x00p\\x80C\\x00\\x90\\x80C\\x00\\xb0\\x80C\\x00\\xd0\\x80C\\x00\\xf0\\x80C\\x00\\x10\\x81C\\x000\\x81C\\x00P\\x81C\\x00p\\x81C\\x00\\x90\\x81C\\x00\\xb0\\x81C\\x00\\xd0\\x81C\\x00\\xf0\\x81C\\x00\\x10\\x82C\\x000\\x82C\\x00P\\x82C\\x00p\\x82C\\x00\\x90\\x82C\\x00\\xb0\\x82C\\x00\\xd0\\x82C\\x00\\xf0\\x82C\\x00\\x10\\x83C\\x000\\x83C\\x00P\\x83C\\x00p\\x83C\\x00\\x90\\x83C\\x00\\xb0\\x83C\\x00\\xd0\\x83C\\x00\\xf0\\x83C\\x00\\x10\\x84C\\x000\\x84C\\x00P\\x84C\\x00p\\x84C\\x00\\x90\\x84C\\x00\\xb0\\x84C\\x00\\xd0\\x84C\\x00\\xf0\\x84C\\x00\\x10\\x85C\\x000\\x85C\\x00P\\x85C\\x00p\\x85C\\x00\\x90\\x85C\\x00\\xb0\\x85C\\x00\\xd0\\x85C\\x00\\xf0\\x85C\\x00\\x10\\x86C\\x000\\x86C\\x00P\\x86C\\x00p\\x86C\\x00\\x90\\x86C\\x00\\xb0\\x86C\\x00\\xd0\\x86C\\x00\\xf0\\x86C\\x00\\x10\\x87C\\x000\\x87C\\x00P\\x87C\\x00p\\x87C\\x00\\x90\\x87C\\x00\\xb0\\x87C\\x00\\xd0\\x87C\\x00\\xf0\\x87C\\x00\\x10\\x88C\\x000\\x88C\\x00P\\x88C\\x00p\\x88C\\x00\\x90\\x88C\\x00\\xb0\\x88C\\x00\\xd0\\x88C\\x00\\xf0\\x88C\\x00\\x10\\x89C\\x000\\x89C\\x00P\\x89C\\x00p\\x89C\\x00\\x90\\x89C\\x00\\xb0\\x89C\\x00\\xd0\\x89C\\x00\\xf0\\x89C\\x00\\x10\\x8aC\\x000\\x8aC\\x00P\\x8aC\\x00p\\x8aC\\x00\\x90\\x8aC\\x00\\xb0\\x8aC\\x00\\xd0\\x8aC\\x00\\xf0\\x8aC\\x00\\x10\\x8bC\\x000\\x8bC\\x00P\\x8bC\\x00p\\x8bC\\x00\\x90\\x8bC\\x00\\xb0\\x8bC\\x00\\xd0\\x8bC\\x00\\xf0\\x8bC\\x00\\x10\\x8cC\\x000\\x8cC\\x00P\\x8cC\\x00p\\x8cC\\x00\\x90\\x8cC\\x00\\xb0\\x8cC\\x00\\xd0\\x8cC\\x00\\xf0\\x8cC\\x00\\x10\\x8dC\\x000\\x8dC\\x00P\\x8dC\\x00p\\x8dC\\x00\\x90\\x8dC\\x00\\xb0\\x8dC\\x00\\xd0\\x8dC\\x00\\xf0\\x8dC\\x00\\x10\\x8eC\\x000\\x8eC\\x00P\\x8eC\\x00p\\x8eC\\x00\\x90\\x8eC\\x00\\xb0\\x8eC\\x00\\xd0\\x8eC\\x00\\xf0\\x8eC\\x00\\x10\\x8fC\\x000\\x8fC\\x00P\\x8fC\\x00p\\x8fC\\x00\\x90\\x8fC\\x00\\xb0\\x8fC\\x00\\xd0\\x8fC\\x00\\xf0\\x8fC\\x00\\x10\\x90C\\x000\\x90C\\x00P\\x90C\\x00p\\x90C\\x00\\x90\\x90C\\x00\\xb0\\x90C\\x00\\xd0\\x90C\\x00\\xf0\\x90C\\x00\\x10\\x91C\\x000\\x91C\\x00P\\x91C\\x00p\\x91C\\x00\\x90\\x91C\\x00\\xb0\\x91C\\x00\\xd0\\x91C\\x00\\xf0\\x91C\\x00\\x10\\x92C\\x000\\x92C\\x00P\\x92C\\x00p\\x92C\\x00\\x90\\x92C\\x00\\xb0\\x92C\\x00\\xd0\\x92C\\x00\\xf0\\x92C\\x00\\x10\\x93C\\x000\\x93C\\x00P\\x93C\\x00p\\x93C\\x00\\x90\\x93C\\x00\\xb0\\x93C\\x00\\xd0\\x93C\\x00\\xf0\\x93C\\x00\\x10\\x94C\\x000\\x94C\\x00P\\x94C\\x00p\\x94C\\x00\\x90\\x94C\\x00\\xb0\\x94C\\x00\\xd0\\x94C\\x00\\xf0\\x94C\\x00\\x10\\x95C\\x000\\x95C\\x00P\\x95C\\x00p\\x95C\\x00\\x90\\x95C\\x00\\xb0\\x95C\\x00\\xd0\\x95C\\x00\\xf0\\x95C\\x00\\x10\\x96C\\x000\\x96C\\x00P\\x96C\\x00p\\x96C\\x00\\x90\\x96C\\x00\\xb0\\x96C\\x00\\xd0\\x96C\\x00\\xf0\\x96C\\x00\\x10\\x97C\\x000\\x97C\\x00P\\x97C\\x00p\\x97C\\x00\\x90\\x97C\\x00\\xb0\\x97C\\x00\\xd0\\x97C\\x00\\xf0\\x97C\\x00\\x10\\x98C\\x000\\x98C\\x00P\\x98C\\x00p\\x98C\\x00\\x90\\x98C\\x00\\xb0\\x98C\\x00\\xd0\\x98C\\x00\\xf0\\x98C\\x00\\x10\\x99C\\x000\\x99C\\x00P\\x99C\\x00p\\x99C\\x00\\x90\\x99C\\x00\\xb0\\x99C\\x00\\xd0\\x99C\\x00\\xf0\\x99C\\x00\\x10\\x9aC\\x000\\x9aC\\x00P\\x9aC\\x00p\\x9aC\\x00\\x90\\x9aC\\x00\\xb0\\x9aC\\x00\\xd0\\x9aC\\x00\\xf0\\x9aC\\x00\\x10\\x9bC\\x000\\x9bC\\x00P\\x9bC\\x00p\\x9bC\\x00\\x90\\x9bC\\x00\\xb0\\x9bC\\x00\\xd0\\x9bC\\x00\\xf0\\x9bC\\x00\\x10\\x9cC\\x000\\x9cC\\x00P\\x9cC\\x00p\\x9cC\\x00\\x90\\x9cC\\x00\\xb0\\x9cC\\x00\\xd0\\x9cC\\x00\\xf0\\x9cC\\x00\\x10\\x9dC\\x000\\x9dC\\x00P\\x9dC\\x00p\\x9dC\\x00\\x90\\x9dC\\x00\\xb0\\x9dC\\x00\\xd0\\x9dC\\x00\\xf0\\x9dC\\x00\\x10\\x9eC\\x000\\x9eC\\x00P\\x9eC\\x00p\\x9eC\\x00\\x90\\x9eC\\x00\\xb0\\x9eC\\x00\\xd0\\x9eC\\x00\\xf0\\x9eC\\x00\\x10\\x9fC\\x000\\x9fC\\x00P\\x9fC\\x00p\\x9fC\\x00\\x90\\x9fC\\x00\\xb0\\x9fC\\x00\\xd0\\x9fC\\x00\\xf0\\x9fC\\x00\\x10\\xa0C\\x000\\xa0C\\x00P\\xa0C\\x00p\\xa0C\\x00\\x90\\xa0C\\x00\\xb0\\xa0C\\x00\\xd0\\xa0C\\x00\\xf0\\xa0C\\x00\\x10\\xa1C\\x000\\xa1C\\x00P\\xa1C\\x00p\\xa1C\\x00\\x90\\xa1C\\x00\\xb0\\xa1C\\x00\\xd0\\xa1C\\x00\\xf0\\xa1C\\x00\\x10\\xa2C\\x000\\xa2C\\x00P\\xa2C\\x00p\\xa2C\\x00\\x90\\xa2C\\x00\\xb0\\xa2C\\x00\\xd0\\xa2C\\x00\\xf0\\xa2C\\x00\\x10\\xa3C\\x000\\xa3C\\x00P\\xa3C\\x00p\\xa3C\\x00\\x90\\xa3C\\x00\\xb0\\xa3C\\x00\\xd0\\xa3C\\x00\\xf0\\xa3C\\x00\\x10\\xa4C\\x000\\xa4C\\x00P\\xa4C\\x00p\\xa4C\\x00\\x90\\xa4C\\x00\\xb0\\xa4C\\x00\\xd0\\xa4C\\x00\\xf0\\xa4C\\x00\\x10\\xa5C\\x000\\xa5C\\x00P\\xa5C\\x00p\\xa5C\\x00\\x90\\xa5C\\x00\\xb0\\xa5C\\x00\\xd0\\xa5C\\x00\\xf0\\xa5C\\x00\\x10\\xa6C\\x000\\xa6C\\x00P\\xa6C\\x00p\\xa6C\\x00\\x90\\xa6C\\x00\\xb0\\xa6C\\x00\\xd0\\xa6C\\x00\\xf0\\xa6C\\x00\\x10\\xa7C\\x000\\xa7C\\x00P\\xa7C\\x00p\\xa7C\\x00\\x90\\xa7C\\x00\\xb0\\xa7C\\x00\\xd0\\xa7C\\x00\\xf0\\xa7C\\x00\\x10\\xa8C\\x000\\xa8C\\x00P\\xa8C\\x00p\\xa8C\\x00\\x90\\xa8C\\x00\\xb0\\xa8C\\x00\\xd0\\xa8C\\x00\\xf0\\xa8C\\x00\\x10\\xa9C\\x000\\xa9C\\x00P\\xa9C\\x00p\\xa9C\\x00\\x90\\xa9C\\x00\\xb0\\xa9C\\x00\\xd0\\xa9C\\x00\\xf0\\xa9C\\x00\\x10\\xaaC\\x000\\xaaC\\x00P\\xaaC\\x00p\\xaaC\\x00\\x90\\xaaC\\x00\\xb0\\xaaC\\x00\\xd0\\xaaC\\x00\\xf0\\xaaC\\x00\\x10\\xabC\\x000\\xabC\\x00P\\xabC\\x00p\\xabC\\x00\\x90\\xabC\\x00\\xb0\\xabC\\x00\\xd0\\xabC\\x00\\xf0\\xabC\\x00\\x10\\xacC\\x000\\xacC\\x00P\\xacC\\x00p\\xacC\\x00\\x90\\xacC\\x00\\xb0\\xacC\\x00\\xd0\\xacC\\x00\\xf0\\xacC\\x00\\x10\\xadC\\x000\\xadC\\x00P\\xadC\\x00p\\xadC\\x00\\x90\\xadC\\x00\\xb0\\xadC\\x00\\xd0\\xadC\\x00\\xf0\\xadC\\x00\\x10\\xaeC\\x000\\xaeC\\x00P\\xaeC\\x00p\\xaeC\\x00\\x90\\xaeC\\x00\\xb0\\xaeC\\x00\\xd0\\xaeC\\x00\\xf0\\xaeC\\x00\\x10\\xafC\\x000\\xafC\\x00P\\xafC\\x00p\\xafC\\x00\\x90\\xafC\\x00\\xb0\\xafC\\x00\\xd0\\xafC\\x00\\xf0\\xafC\\x00\\x10\\xb0C\\x000\\xb0C\\x00P\\xb0C\\x00p\\xb0C\\x00\\x90\\xb0C\\x00\\xb0\\xb0C\\x00\\xd0\\xb0C\\x00\\xf0\\xb0C\\x00\\x10\\xb1C\\x000\\xb1C\\x00P\\xb1C\\x00p\\xb1C\\x00\\x90\\xb1C\\x00\\xb0\\xb1C\\x00\\xd0\\xb1C\\x00\\xf0\\xb1C\\x00\\x10\\xb2C\\x000\\xb2C\\x00P\\xb2C\\x00p\\xb2C\\x00\\x90\\xb2C\\x00\\xb0\\xb2C\\x00\\xd0\\xb2C\\x00\\xf0\\xb2C\\x00\\x10\\xb3C\\x000\\xb3C\\x00P\\xb3C\\x00p\\xb3C\\x00\\x90\\xb3C\\x00\\xb0\\xb3C\\x00\\xd0\\xb3C\\x00\\xf0\\xb3C'" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "await read_store._store.get(\"time/c/0\")" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 21, "id": "47a53027-dbae-48aa-85d2-dcbc04e01e61", "metadata": {}, "outputs": [ { "ename": "error", - "evalue": "Failed to decode variable 'time': Error -3 while decompressing data: incorrect header check", + "evalue": "Failed to decode variable 'time': Error -3 while decompressing data: unknown compression method", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31merror\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:451\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 450\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 451\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m \u001b[43mdecode_cf_variable\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 452\u001b[0m \u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 453\u001b[0m \u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 454\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 456\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mstack_char_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstack_char_dim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 459\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 460\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:292\u001b[0m, in \u001b[0;36mdecode_cf_variable\u001b[0;34m(name, var, concat_characters, mask_and_scale, decode_times, decode_endianness, stack_char_dim, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_times:\n\u001b[0;32m--> 292\u001b[0m var \u001b[38;5;241m=\u001b[39m \u001b[43mtimes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCFDatetimeCoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 294\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_endianness \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39misnative:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/coding/times.py:1001\u001b[0m, in \u001b[0;36mCFDatetimeCoder.decode\u001b[0;34m(self, variable, name)\u001b[0m\n\u001b[1;32m 1000\u001b[0m calendar \u001b[38;5;241m=\u001b[39m pop_to(attrs, encoding, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalendar\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1001\u001b[0m dtype \u001b[38;5;241m=\u001b[39m \u001b[43m_decode_cf_datetime_dtype\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munits\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcalendar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1002\u001b[0m transform \u001b[38;5;241m=\u001b[39m partial(\n\u001b[1;32m 1003\u001b[0m decode_cf_datetime,\n\u001b[1;32m 1004\u001b[0m units\u001b[38;5;241m=\u001b[39munits,\n\u001b[1;32m 1005\u001b[0m calendar\u001b[38;5;241m=\u001b[39mcalendar,\n\u001b[1;32m 1006\u001b[0m use_cftime\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_cftime,\n\u001b[1;32m 1007\u001b[0m )\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/coding/times.py:214\u001b[0m, in \u001b[0;36m_decode_cf_datetime_dtype\u001b[0;34m(data, units, calendar, use_cftime)\u001b[0m\n\u001b[1;32m 212\u001b[0m values \u001b[38;5;241m=\u001b[39m indexing\u001b[38;5;241m.\u001b[39mImplicitToExplicitIndexingAdapter(indexing\u001b[38;5;241m.\u001b[39mas_indexable(data))\n\u001b[1;32m 213\u001b[0m example_value \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate(\n\u001b[0;32m--> 214\u001b[0m [\u001b[43mfirst_n_items\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m], last_item(values) \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m 215\u001b[0m )\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/formatting.py:97\u001b[0m, in \u001b[0;36mfirst_n_items\u001b[0;34m(array, n_desired)\u001b[0m\n\u001b[1;32m 96\u001b[0m array \u001b[38;5;241m=\u001b[39m array\u001b[38;5;241m.\u001b[39m_data\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mravel(\u001b[43mto_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43marray\u001b[49m\u001b[43m)\u001b[49m)[:n_desired]\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/namedarray/pycompat.py:138\u001b[0m, in \u001b[0;36mto_duck_array\u001b[0;34m(data, **kwargs)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 138\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43masarray\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:578\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.__array__\u001b[0;34m(self, dtype, copy)\u001b[0m\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m Version(np\u001b[38;5;241m.\u001b[39m__version__) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m Version(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.0.0\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 578\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, dtype\u001b[38;5;241m=\u001b[39mdtype, copy\u001b[38;5;241m=\u001b[39mcopy)\n\u001b[1;32m 579\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:583\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 583\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:657\u001b[0m, in \u001b[0;36mLazilyIndexedArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 655\u001b[0m \u001b[38;5;66;03m# If the array is not an ExplicitlyIndexedNDArrayMixin,\u001b[39;00m\n\u001b[1;32m 656\u001b[0m \u001b[38;5;66;03m# it may wrap a BackendArray so use its __getitem__\u001b[39;00m\n\u001b[0;32m--> 657\u001b[0m array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 659\u001b[0m \u001b[38;5;66;03m# self.array[self.key] is now a numpy array when\u001b[39;00m\n\u001b[1;32m 660\u001b[0m \u001b[38;5;66;03m# self.array is a BackendArray subclass\u001b[39;00m\n\u001b[1;32m 661\u001b[0m \u001b[38;5;66;03m# and self.key is BasicIndexer((slice(None, None, None),))\u001b[39;00m\n\u001b[1;32m 662\u001b[0m \u001b[38;5;66;03m# so we need the explicit check for ExplicitlyIndexed\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:175\u001b[0m, in \u001b[0;36mZarrArrayWrapper.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 174\u001b[0m method \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oindex\n\u001b[0;32m--> 175\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexplicit_indexing_adapter\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 176\u001b[0m \u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mIndexingSupport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mVECTORIZED\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\n\u001b[1;32m 177\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:1018\u001b[0m, in \u001b[0;36mexplicit_indexing_adapter\u001b[0;34m(key, shape, indexing_support, raw_indexing_method)\u001b[0m\n\u001b[1;32m 1017\u001b[0m raw_key, numpy_indices \u001b[38;5;241m=\u001b[39m decompose_indexer(key, shape, indexing_support)\n\u001b[0;32m-> 1018\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mraw_indexing_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_key\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtuple\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1019\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m numpy_indices\u001b[38;5;241m.\u001b[39mtuple:\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;66;03m# index the loaded np.ndarray\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:158\u001b[0m, in \u001b[0;36mZarrArrayWrapper._getitem\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_getitem\u001b[39m(\u001b[38;5;28mself\u001b[39m, key):\n\u001b[1;32m 156\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 157\u001b[0m \u001b[38;5;66;03m# try:\u001b[39;00m\n\u001b[0;32m--> 158\u001b[0m item \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_array\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;66;03m# except Exception as e:\u001b[39;00m\n\u001b[1;32m 160\u001b[0m \u001b[38;5;66;03m# import traceback\u001b[39;00m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;66;03m# print(f\"An error occurred: {e}\")\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[38;5;66;03m# print(\"Stack trace:\")\u001b[39;00m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;66;03m# traceback.print_exc() # Prints the full traceback \u001b[39;00m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:1688\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 1687\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_pure_orthogonal_indexing(pure_selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim):\n\u001b[0;32m-> 1688\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_orthogonal_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1689\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/_compat.py:43\u001b[0m, in \u001b[0;36m_deprecate_positional_args.._inner_deprecate_positional_args..inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m extra_args \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# extra_args > 0\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:2130\u001b[0m, in \u001b[0;36mArray.get_orthogonal_selection\u001b[0;34m(self, selection, out, fields, prototype)\u001b[0m\n\u001b[1;32m 2129\u001b[0m indexer \u001b[38;5;241m=\u001b[39m OrthogonalIndexer(selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mchunk_grid)\n\u001b[0;32m-> 2130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2131\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_async_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2132\u001b[0m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprototype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprototype\u001b[49m\n\u001b[1;32m 2133\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2134\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:141\u001b[0m, in \u001b[0;36msync\u001b[0;34m(coro, loop, timeout)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 141\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:100\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 100\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:961\u001b[0m, in \u001b[0;36mAsyncArray._get_selection\u001b[0;34m(self, indexer, prototype, out, fields)\u001b[0m\n\u001b[1;32m 959\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m product(indexer\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 960\u001b[0m \u001b[38;5;66;03m# reading chunks and decoding them\u001b[39;00m\n\u001b[0;32m--> 961\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcodec_pipeline\u001b[38;5;241m.\u001b[39mread(\n\u001b[1;32m 962\u001b[0m [\n\u001b[1;32m 963\u001b[0m (\n\u001b[1;32m 964\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_path \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mencode_chunk_key(chunk_coords),\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget_chunk_spec(chunk_coords, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morder, prototype\u001b[38;5;241m=\u001b[39mprototype),\n\u001b[1;32m 966\u001b[0m chunk_selection,\n\u001b[1;32m 967\u001b[0m out_selection,\n\u001b[1;32m 968\u001b[0m )\n\u001b[1;32m 969\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_coords, chunk_selection, out_selection \u001b[38;5;129;01min\u001b[39;00m indexer\n\u001b[1;32m 970\u001b[0m ],\n\u001b[1;32m 971\u001b[0m out_buffer,\n\u001b[1;32m 972\u001b[0m drop_axes\u001b[38;5;241m=\u001b[39mindexer\u001b[38;5;241m.\u001b[39mdrop_axes,\n\u001b[1;32m 973\u001b[0m )\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_buffer\u001b[38;5;241m.\u001b[39mas_ndarray_like()\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:440\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread\u001b[39m(\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 436\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]],\n\u001b[1;32m 437\u001b[0m out: NDBuffer,\n\u001b[1;32m 438\u001b[0m drop_axes: \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (),\n\u001b[1;32m 439\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 440\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 441\u001b[0m [\n\u001b[1;32m 442\u001b[0m (single_batch_info, out, drop_axes)\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m single_batch_info \u001b[38;5;129;01min\u001b[39;00m batched(batch_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m 444\u001b[0m ],\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_batch,\n\u001b[1;32m 446\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 447\u001b[0m )\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:270\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read_batch\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 262\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 263\u001b[0m [\n\u001b[1;32m 264\u001b[0m (byte_getter, array_spec\u001b[38;5;241m.\u001b[39mprototype)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 268\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 269\u001b[0m )\n\u001b[0;32m--> 270\u001b[0m chunk_array_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode_batch(\n\u001b[1;32m 271\u001b[0m [\n\u001b[1;32m 272\u001b[0m (chunk_bytes, chunk_spec)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_bytes, (_, chunk_spec, _, _) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 274\u001b[0m chunk_bytes_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 275\u001b[0m )\n\u001b[1;32m 276\u001b[0m ],\n\u001b[1;32m 277\u001b[0m )\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_array, (_, chunk_spec, chunk_selection, out_selection) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 279\u001b[0m chunk_array_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 280\u001b[0m ):\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:172\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.decode_batch\u001b[0;34m(self, chunk_bytes_and_specs)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m bb_codec, chunk_spec_batch \u001b[38;5;129;01min\u001b[39;00m bb_codecs_with_spec[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 172\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m bb_codec\u001b[38;5;241m.\u001b[39mdecode(\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mzip\u001b[39m(chunk_bytes_batch, chunk_spec_batch, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 174\u001b[0m )\n\u001b[1;32m 176\u001b[0m ab_codec, chunk_spec_batch \u001b[38;5;241m=\u001b[39m ab_codec_with_spec\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:130\u001b[0m, in \u001b[0;36mBaseCodec.decode\u001b[0;34m(self, chunks_and_specs)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decodes a batch of chunks.\u001b[39;00m\n\u001b[1;32m 119\u001b[0m \u001b[38;5;124;03mChunks can be None in which case they are ignored by the codec.\u001b[39;00m\n\u001b[1;32m 120\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124;03mIterable[CodecInput | None]\u001b[39;00m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m _batching_helper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decode_single, chunks_and_specs)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:408\u001b[0m, in \u001b[0;36m_batching_helper\u001b[0;34m(func, batch_info)\u001b[0m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_batching_helper\u001b[39m(\n\u001b[1;32m 405\u001b[0m func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]],\n\u001b[1;32m 406\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[CodecInput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, ArraySpec]],\n\u001b[1;32m 407\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]:\n\u001b[0;32m--> 408\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 409\u001b[0m \u001b[38;5;28mlist\u001b[39m(batch_info),\n\u001b[1;32m 410\u001b[0m _noop_for_none(func),\n\u001b[1;32m 411\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 412\u001b[0m )\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:421\u001b[0m, in \u001b[0;36m_noop_for_none..wrap\u001b[0;34m(chunk, chunk_spec)\u001b[0m\n\u001b[1;32m 420\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 421\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(chunk, chunk_spec)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zarr3.py:125\u001b[0m, in \u001b[0;36m_NumcodecsBytesBytesCodec._decode_single\u001b[0;34m(self, chunk_bytes, chunk_spec)\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_decode_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, chunk_bytes: Buffer, chunk_spec: ArraySpec) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Buffer:\n\u001b[0;32m--> 125\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mto_thread(\n\u001b[1;32m 126\u001b[0m as_numpy_array_wrapper,\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_codec\u001b[38;5;241m.\u001b[39mdecode,\n\u001b[1;32m 128\u001b[0m chunk_bytes,\n\u001b[1;32m 129\u001b[0m chunk_spec\u001b[38;5;241m.\u001b[39mprototype,\n\u001b[1;32m 130\u001b[0m )\n", - "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/asyncio/threads.py:25\u001b[0m, in \u001b[0;36mto_thread\u001b[0;34m(func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m func_call \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mpartial(ctx\u001b[38;5;241m.\u001b[39mrun, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m loop\u001b[38;5;241m.\u001b[39mrun_in_executor(\u001b[38;5;28;01mNone\u001b[39;00m, func_call)\n", - "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/buffer/cpu.py:213\u001b[0m, in \u001b[0;36mas_numpy_array_wrapper\u001b[0;34m(func, buf, prototype)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Converts the input of `func` to a numpy array and the output back to `Buffer`.\u001b[39;00m\n\u001b[1;32m 192\u001b[0m \n\u001b[1;32m 193\u001b[0m \u001b[38;5;124;03mThis function is useful when calling a `func` that only support host memory such\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[38;5;124;03m The result of `func` converted to a `Buffer`\u001b[39;00m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m prototype\u001b[38;5;241m.\u001b[39mbuffer\u001b[38;5;241m.\u001b[39mfrom_bytes(\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_numpy_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zlib.py:37\u001b[0m, in \u001b[0;36mZlib.decode\u001b[0;34m(self, buf, out)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# do decompression\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m dec \u001b[38;5;241m=\u001b[39m \u001b[43m_zlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecompress\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;66;03m# handle destination - Python standard library zlib module does not\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;66;03m# support direct decompression into buffer, so we have to copy into\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# out if given\u001b[39;00m\n", - "\u001b[0;31merror\u001b[0m: Error -3 while decompressing data: incorrect header check", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/conventions.py:453\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 452\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 453\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m \u001b[43mdecode_cf_variable\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 454\u001b[0m \u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 456\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 459\u001b[0m \u001b[43m \u001b[49m\u001b[43mstack_char_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstack_char_dim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 460\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 461\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 462\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/conventions.py:294\u001b[0m, in \u001b[0;36mdecode_cf_variable\u001b[0;34m(name, var, concat_characters, mask_and_scale, decode_times, decode_endianness, stack_char_dim, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_times:\n\u001b[0;32m--> 294\u001b[0m var \u001b[38;5;241m=\u001b[39m \u001b[43mtimes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCFDatetimeCoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 296\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_endianness \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39misnative:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/coding/times.py:1005\u001b[0m, in \u001b[0;36mCFDatetimeCoder.decode\u001b[0;34m(self, variable, name)\u001b[0m\n\u001b[1;32m 1004\u001b[0m calendar \u001b[38;5;241m=\u001b[39m pop_to(attrs, encoding, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalendar\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1005\u001b[0m dtype \u001b[38;5;241m=\u001b[39m \u001b[43m_decode_cf_datetime_dtype\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munits\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcalendar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1006\u001b[0m transform \u001b[38;5;241m=\u001b[39m partial(\n\u001b[1;32m 1007\u001b[0m decode_cf_datetime,\n\u001b[1;32m 1008\u001b[0m units\u001b[38;5;241m=\u001b[39munits,\n\u001b[1;32m 1009\u001b[0m calendar\u001b[38;5;241m=\u001b[39mcalendar,\n\u001b[1;32m 1010\u001b[0m use_cftime\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_cftime,\n\u001b[1;32m 1011\u001b[0m )\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/coding/times.py:214\u001b[0m, in \u001b[0;36m_decode_cf_datetime_dtype\u001b[0;34m(data, units, calendar, use_cftime)\u001b[0m\n\u001b[1;32m 212\u001b[0m values \u001b[38;5;241m=\u001b[39m indexing\u001b[38;5;241m.\u001b[39mImplicitToExplicitIndexingAdapter(indexing\u001b[38;5;241m.\u001b[39mas_indexable(data))\n\u001b[1;32m 213\u001b[0m example_value \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate(\n\u001b[0;32m--> 214\u001b[0m [\u001b[43mfirst_n_items\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m], last_item(values) \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m 215\u001b[0m )\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/formatting.py:97\u001b[0m, in \u001b[0;36mfirst_n_items\u001b[0;34m(array, n_desired)\u001b[0m\n\u001b[1;32m 96\u001b[0m array \u001b[38;5;241m=\u001b[39m array\u001b[38;5;241m.\u001b[39m_data\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mravel(\u001b[43mto_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43marray\u001b[49m\u001b[43m)\u001b[49m)[:n_desired]\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/namedarray/pycompat.py:138\u001b[0m, in \u001b[0;36mto_duck_array\u001b[0;34m(data, **kwargs)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 138\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(data)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/indexing.py:578\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.__array__\u001b[0;34m(self, dtype, copy)\u001b[0m\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m Version(np\u001b[38;5;241m.\u001b[39m__version__) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m Version(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.0.0\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 578\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, dtype\u001b[38;5;241m=\u001b[39mdtype, copy\u001b[38;5;241m=\u001b[39mcopy)\n\u001b[1;32m 579\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/indexing.py:583\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 583\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/indexing.py:657\u001b[0m, in \u001b[0;36mLazilyIndexedArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 655\u001b[0m \u001b[38;5;66;03m# If the array is not an ExplicitlyIndexedNDArrayMixin,\u001b[39;00m\n\u001b[1;32m 656\u001b[0m \u001b[38;5;66;03m# it may wrap a BackendArray so use its __getitem__\u001b[39;00m\n\u001b[0;32m--> 657\u001b[0m array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 659\u001b[0m \u001b[38;5;66;03m# self.array[self.key] is now a numpy array when\u001b[39;00m\n\u001b[1;32m 660\u001b[0m \u001b[38;5;66;03m# self.array is a BackendArray subclass\u001b[39;00m\n\u001b[1;32m 661\u001b[0m \u001b[38;5;66;03m# and self.key is BasicIndexer((slice(None, None, None),))\u001b[39;00m\n\u001b[1;32m 662\u001b[0m \u001b[38;5;66;03m# so we need the explicit check for ExplicitlyIndexed\u001b[39;00m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/zarr.py:226\u001b[0m, in \u001b[0;36mZarrArrayWrapper.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 225\u001b[0m method \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oindex\n\u001b[0;32m--> 226\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexplicit_indexing_adapter\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 227\u001b[0m \u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mIndexingSupport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mVECTORIZED\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\n\u001b[1;32m 228\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/indexing.py:1018\u001b[0m, in \u001b[0;36mexplicit_indexing_adapter\u001b[0;34m(key, shape, indexing_support, raw_indexing_method)\u001b[0m\n\u001b[1;32m 1017\u001b[0m raw_key, numpy_indices \u001b[38;5;241m=\u001b[39m decompose_indexer(key, shape, indexing_support)\n\u001b[0;32m-> 1018\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mraw_indexing_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_key\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtuple\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1019\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m numpy_indices\u001b[38;5;241m.\u001b[39mtuple:\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;66;03m# index the loaded np.ndarray\u001b[39;00m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/zarr.py:216\u001b[0m, in \u001b[0;36mZarrArrayWrapper._getitem\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_getitem\u001b[39m(\u001b[38;5;28mself\u001b[39m, key):\n\u001b[0;32m--> 216\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_array\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/array.py:1660\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 1659\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_pure_orthogonal_indexing(pure_selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim):\n\u001b[0;32m-> 1660\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_orthogonal_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1661\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/_compat.py:43\u001b[0m, in \u001b[0;36m_deprecate_positional_args.._inner_deprecate_positional_args..inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m extra_args \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# extra_args > 0\u001b[39;00m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/array.py:2102\u001b[0m, in \u001b[0;36mArray.get_orthogonal_selection\u001b[0;34m(self, selection, out, fields, prototype)\u001b[0m\n\u001b[1;32m 2101\u001b[0m indexer \u001b[38;5;241m=\u001b[39m OrthogonalIndexer(selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mchunk_grid)\n\u001b[0;32m-> 2102\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2103\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_async_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2104\u001b[0m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprototype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprototype\u001b[49m\n\u001b[1;32m 2105\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2106\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/sync.py:141\u001b[0m, in \u001b[0;36msync\u001b[0;34m(coro, loop, timeout)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 141\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/sync.py:100\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 100\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/array.py:961\u001b[0m, in \u001b[0;36mAsyncArray._get_selection\u001b[0;34m(self, indexer, prototype, out, fields)\u001b[0m\n\u001b[1;32m 959\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m product(indexer\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 960\u001b[0m \u001b[38;5;66;03m# reading chunks and decoding them\u001b[39;00m\n\u001b[0;32m--> 961\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcodec_pipeline\u001b[38;5;241m.\u001b[39mread(\n\u001b[1;32m 962\u001b[0m [\n\u001b[1;32m 963\u001b[0m (\n\u001b[1;32m 964\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_path \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mencode_chunk_key(chunk_coords),\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget_chunk_spec(chunk_coords, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morder, prototype\u001b[38;5;241m=\u001b[39mprototype),\n\u001b[1;32m 966\u001b[0m chunk_selection,\n\u001b[1;32m 967\u001b[0m out_selection,\n\u001b[1;32m 968\u001b[0m )\n\u001b[1;32m 969\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_coords, chunk_selection, out_selection \u001b[38;5;129;01min\u001b[39;00m indexer\n\u001b[1;32m 970\u001b[0m ],\n\u001b[1;32m 971\u001b[0m out_buffer,\n\u001b[1;32m 972\u001b[0m drop_axes\u001b[38;5;241m=\u001b[39mindexer\u001b[38;5;241m.\u001b[39mdrop_axes,\n\u001b[1;32m 973\u001b[0m )\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_buffer\u001b[38;5;241m.\u001b[39mas_ndarray_like()\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/codecs/pipeline.py:440\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread\u001b[39m(\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 436\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]],\n\u001b[1;32m 437\u001b[0m out: NDBuffer,\n\u001b[1;32m 438\u001b[0m drop_axes: \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (),\n\u001b[1;32m 439\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 440\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 441\u001b[0m [\n\u001b[1;32m 442\u001b[0m (single_batch_info, out, drop_axes)\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m single_batch_info \u001b[38;5;129;01min\u001b[39;00m batched(batch_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m 444\u001b[0m ],\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_batch,\n\u001b[1;32m 446\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 447\u001b[0m )\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/codecs/pipeline.py:270\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read_batch\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 262\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 263\u001b[0m [\n\u001b[1;32m 264\u001b[0m (byte_getter, array_spec\u001b[38;5;241m.\u001b[39mprototype)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 268\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 269\u001b[0m )\n\u001b[0;32m--> 270\u001b[0m chunk_array_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode_batch(\n\u001b[1;32m 271\u001b[0m [\n\u001b[1;32m 272\u001b[0m (chunk_bytes, chunk_spec)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_bytes, (_, chunk_spec, _, _) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 274\u001b[0m chunk_bytes_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 275\u001b[0m )\n\u001b[1;32m 276\u001b[0m ],\n\u001b[1;32m 277\u001b[0m )\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_array, (_, chunk_spec, chunk_selection, out_selection) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 279\u001b[0m chunk_array_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 280\u001b[0m ):\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/codecs/pipeline.py:172\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.decode_batch\u001b[0;34m(self, chunk_bytes_and_specs)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m bb_codec, chunk_spec_batch \u001b[38;5;129;01min\u001b[39;00m bb_codecs_with_spec[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 172\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m bb_codec\u001b[38;5;241m.\u001b[39mdecode(\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mzip\u001b[39m(chunk_bytes_batch, chunk_spec_batch, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 174\u001b[0m )\n\u001b[1;32m 176\u001b[0m ab_codec, chunk_spec_batch \u001b[38;5;241m=\u001b[39m ab_codec_with_spec\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/abc/codec.py:130\u001b[0m, in \u001b[0;36mBaseCodec.decode\u001b[0;34m(self, chunks_and_specs)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decodes a batch of chunks.\u001b[39;00m\n\u001b[1;32m 119\u001b[0m \u001b[38;5;124;03mChunks can be None in which case they are ignored by the codec.\u001b[39;00m\n\u001b[1;32m 120\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124;03mIterable[CodecInput | None]\u001b[39;00m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m _batching_helper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decode_single, chunks_and_specs)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/abc/codec.py:408\u001b[0m, in \u001b[0;36m_batching_helper\u001b[0;34m(func, batch_info)\u001b[0m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_batching_helper\u001b[39m(\n\u001b[1;32m 405\u001b[0m func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]],\n\u001b[1;32m 406\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[CodecInput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, ArraySpec]],\n\u001b[1;32m 407\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]:\n\u001b[0;32m--> 408\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 409\u001b[0m \u001b[38;5;28mlist\u001b[39m(batch_info),\n\u001b[1;32m 410\u001b[0m _noop_for_none(func),\n\u001b[1;32m 411\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 412\u001b[0m )\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/abc/codec.py:421\u001b[0m, in \u001b[0;36m_noop_for_none..wrap\u001b[0;34m(chunk, chunk_spec)\u001b[0m\n\u001b[1;32m 420\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 421\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(chunk, chunk_spec)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/numcodecs/zarr3.py:125\u001b[0m, in \u001b[0;36m_NumcodecsBytesBytesCodec._decode_single\u001b[0;34m(self, chunk_bytes, chunk_spec)\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_decode_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, chunk_bytes: Buffer, chunk_spec: ArraySpec) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Buffer:\n\u001b[0;32m--> 125\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mto_thread(\n\u001b[1;32m 126\u001b[0m as_numpy_array_wrapper,\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_codec\u001b[38;5;241m.\u001b[39mdecode,\n\u001b[1;32m 128\u001b[0m chunk_bytes,\n\u001b[1;32m 129\u001b[0m chunk_spec\u001b[38;5;241m.\u001b[39mprototype,\n\u001b[1;32m 130\u001b[0m )\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/asyncio/threads.py:25\u001b[0m, in \u001b[0;36mto_thread\u001b[0;34m(func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m func_call \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mpartial(ctx\u001b[38;5;241m.\u001b[39mrun, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m loop\u001b[38;5;241m.\u001b[39mrun_in_executor(\u001b[38;5;28;01mNone\u001b[39;00m, func_call)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/buffer/cpu.py:213\u001b[0m, in \u001b[0;36mas_numpy_array_wrapper\u001b[0;34m(func, buf, prototype)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Converts the input of `func` to a numpy array and the output back to `Buffer`.\u001b[39;00m\n\u001b[1;32m 192\u001b[0m \n\u001b[1;32m 193\u001b[0m \u001b[38;5;124;03mThis function is useful when calling a `func` that only support host memory such\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[38;5;124;03m The result of `func` converted to a `Buffer`\u001b[39;00m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m prototype\u001b[38;5;241m.\u001b[39mbuffer\u001b[38;5;241m.\u001b[39mfrom_bytes(\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_numpy_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/numcodecs/zlib.py:37\u001b[0m, in \u001b[0;36mZlib.decode\u001b[0;34m(self, buf, out)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# do decompression\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m dec \u001b[38;5;241m=\u001b[39m \u001b[43m_zlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecompress\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;66;03m# handle destination - Python standard library zlib module does not\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;66;03m# support direct decompression into buffer, so we have to copy into\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# out if given\u001b[39;00m\n", + "\u001b[0;31merror\u001b[0m: Error -3 while decompressing data: unknown compression method", "\nThe above exception was the direct cause of the following exception:\n", "\u001b[0;31merror\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[26], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_zarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mread_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconsolidated\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mzarr_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1346\u001b[0m, in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, zarr_version, zarr_format, use_zarr_fill_value_as_mask, chunked_array_type, from_array_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1335\u001b[0m backend_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 1336\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msynchronizer\u001b[39m\u001b[38;5;124m\"\u001b[39m: synchronizer,\n\u001b[1;32m 1337\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconsolidated\u001b[39m\u001b[38;5;124m\"\u001b[39m: consolidated,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1343\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_format\u001b[39m\u001b[38;5;124m\"\u001b[39m: zarr_format,\n\u001b[1;32m 1344\u001b[0m }\n\u001b[1;32m 1345\u001b[0m \u001b[38;5;66;03m#import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[0;32m-> 1346\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1347\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1348\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1349\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_cf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_cf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1350\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1351\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1352\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1353\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1354\u001b[0m \u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mzarr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1355\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1356\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1357\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked_array_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked_array_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1358\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1359\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1360\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1361\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1362\u001b[0m \u001b[43m \u001b[49m\u001b[43mzarr_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzarr_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1363\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1364\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1365\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/api.py:671\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 659\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 660\u001b[0m decode_cf,\n\u001b[1;32m 661\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 667\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 668\u001b[0m )\n\u001b[1;32m 670\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 671\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 677\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 678\u001b[0m backend_ds,\n\u001b[1;32m 679\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 689\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 690\u001b[0m )\n\u001b[1;32m 691\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1436\u001b[0m, in \u001b[0;36mZarrBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask)\u001b[0m\n\u001b[1;32m 1434\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 1435\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n\u001b[0;32m-> 1436\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mstore_entrypoint\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1439\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1440\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1441\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1442\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1443\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1444\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1445\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/store.py:46\u001b[0m, in \u001b[0;36mStoreBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mload()\n\u001b[1;32m 44\u001b[0m encoding \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mget_encoding()\n\u001b[0;32m---> 46\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs, coord_names \u001b[38;5;241m=\u001b[39m \u001b[43mconventions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_cf_variables\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 47\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mvars\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 56\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 58\u001b[0m ds \u001b[38;5;241m=\u001b[39m Dataset(\u001b[38;5;28mvars\u001b[39m, attrs\u001b[38;5;241m=\u001b[39mattrs)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:462\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 451\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m decode_cf_variable(\n\u001b[1;32m 452\u001b[0m k,\n\u001b[1;32m 453\u001b[0m v,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 459\u001b[0m decode_timedelta\u001b[38;5;241m=\u001b[39m_item_or_default(decode_timedelta, k, \u001b[38;5;28;01mNone\u001b[39;00m),\n\u001b[1;32m 460\u001b[0m )\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 462\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(e)(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to decode variable \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_coords \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcoordinates\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mall\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 464\u001b[0m var_attrs \u001b[38;5;241m=\u001b[39m new_vars[k]\u001b[38;5;241m.\u001b[39mattrs\n", - "\u001b[0;31merror\u001b[0m: Failed to decode variable 'time': Error -3 while decompressing data: incorrect header check" + "Cell \u001b[0;32mIn[21], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_zarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mread_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconsolidated\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mzarr_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/zarr.py:1436\u001b[0m, in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, zarr_version, zarr_format, use_zarr_fill_value_as_mask, chunked_array_type, from_array_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1422\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 1423\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mopen_zarr() got unexpected keyword arguments \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(kwargs\u001b[38;5;241m.\u001b[39mkeys())\n\u001b[1;32m 1424\u001b[0m )\n\u001b[1;32m 1426\u001b[0m backend_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 1427\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msynchronizer\u001b[39m\u001b[38;5;124m\"\u001b[39m: synchronizer,\n\u001b[1;32m 1428\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconsolidated\u001b[39m\u001b[38;5;124m\"\u001b[39m: consolidated,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1433\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_format\u001b[39m\u001b[38;5;124m\"\u001b[39m: zarr_format,\n\u001b[1;32m 1434\u001b[0m }\n\u001b[0;32m-> 1436\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1439\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_cf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_cf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1440\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1441\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1442\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1443\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1444\u001b[0m \u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mzarr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1445\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1446\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1447\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked_array_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked_array_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1448\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1449\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1450\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1451\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1452\u001b[0m \u001b[43m \u001b[49m\u001b[43mzarr_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzarr_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1453\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1454\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1455\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/api.py:670\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 658\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 659\u001b[0m decode_cf,\n\u001b[1;32m 660\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 666\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 667\u001b[0m )\n\u001b[1;32m 669\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 670\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 671\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 676\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 677\u001b[0m backend_ds,\n\u001b[1;32m 678\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 688\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 689\u001b[0m )\n\u001b[1;32m 690\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/zarr.py:1524\u001b[0m, in \u001b[0;36mZarrBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask)\u001b[0m\n\u001b[1;32m 1522\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n\u001b[0;32m-> 1524\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mstore_entrypoint\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1525\u001b[0m \u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1526\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1527\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1528\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1530\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1531\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1532\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1533\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1534\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/store.py:47\u001b[0m, in \u001b[0;36mStoreBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mload()\n\u001b[1;32m 45\u001b[0m encoding \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mget_encoding()\n\u001b[0;32m---> 47\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs, coord_names \u001b[38;5;241m=\u001b[39m \u001b[43mconventions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_cf_variables\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mvars\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 56\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m ds \u001b[38;5;241m=\u001b[39m Dataset(\u001b[38;5;28mvars\u001b[39m, attrs\u001b[38;5;241m=\u001b[39mattrs)\n\u001b[1;32m 60\u001b[0m ds \u001b[38;5;241m=\u001b[39m ds\u001b[38;5;241m.\u001b[39mset_coords(coord_names\u001b[38;5;241m.\u001b[39mintersection(\u001b[38;5;28mvars\u001b[39m))\n", + "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/conventions.py:464\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 453\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m decode_cf_variable(\n\u001b[1;32m 454\u001b[0m k,\n\u001b[1;32m 455\u001b[0m v,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 461\u001b[0m decode_timedelta\u001b[38;5;241m=\u001b[39m_item_or_default(decode_timedelta, k, \u001b[38;5;28;01mNone\u001b[39;00m),\n\u001b[1;32m 462\u001b[0m )\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 464\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(e)(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to decode variable \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 465\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_coords \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcoordinates\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mall\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 466\u001b[0m var_attrs \u001b[38;5;241m=\u001b[39m new_vars[k]\u001b[38;5;241m.\u001b[39mattrs\n", + "\u001b[0;31merror\u001b[0m: Failed to decode variable 'time': Error -3 while decompressing data: unknown compression method" ] } ], @@ -1220,9 +1246,9 @@ ], "metadata": { "kernelspec": { - "display_name": "virtualizarr", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "venv" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1234,7 +1260,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 1af74fe9..6c61137f 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -433,10 +433,13 @@ def test_append_virtual_ref_without_encoding( storage=icechunk_storage, mode="a" ) dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") + icechunk_filestore_append.commit("appended data") + dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") + icechunk_filestore_append.commit("appended data again") array = open_zarr(icechunk_filestore_append, consolidated=False, zarr_format=3) expected_ds = open_dataset(simple_netcdf4) - expected_array = concat([expected_ds, expected_ds], dim="x") + expected_array = concat([expected_ds, expected_ds, expected_ds], dim="x") xrt.assert_identical(array, expected_array) ## When appending to a virtual ref with encoding, it succeeds From f5976d11f334afefb391092c522a191ff654455f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 25 Nov 2024 11:13:45 -0800 Subject: [PATCH 65/90] Fix multiple appends --- virtualizarr/writers/icechunk.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 39c9f1b4..33733f84 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -135,10 +135,11 @@ def num_chunks( def resize_array( arr: "Array", + manifest_array: "ManifestArray", append_axis: int, ) -> None: new_shape = list(arr.shape) - new_shape[append_axis] += arr.shape[append_axis] + new_shape[append_axis] += manifest_array.shape[append_axis] arr.resize(tuple(new_shape)) @@ -199,6 +200,7 @@ def write_virtual_variable_to_icechunk( # resize the array resize_array( group[name], + manifest_array=ma, append_axis=append_axis, ) else: From f903291cdb2074a97bb8e72094afd9bd19289e41 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 25 Nov 2024 11:21:02 -0800 Subject: [PATCH 66/90] Fix test error message --- virtualizarr/tests/test_writers/test_icechunk.py | 2 +- virtualizarr/writers/icechunk.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 6c61137f..efe2bcdb 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -707,7 +707,7 @@ def test_append_dim_not_in_dims_raises_error( ) with pytest.raises( ValueError, - match="append_dim='z' does not match any existing dataset dimensions", + match="append_dim z does not match any existing dataset dimensions", ): dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="z") diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 33733f84..4401a7db 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -69,7 +69,7 @@ def dataset_to_icechunk( ) if append_dim not in ds.dims: raise ValueError( - f"append_dim {append_dim} not found in dataset dimensions {ds.dims}" + f"append_dim {append_dim} does not match any existing dataset dimensions" ) root_group = Group.open(store=store, zarr_format=3) else: From c10962680dcf2d3648eef10d4a08f60e91e3734b Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 25 Nov 2024 17:53:26 -0800 Subject: [PATCH 67/90] Add new cell to notebook to display original time chunk --- noaa-cdr-sst.ipynb | 237 ++++++++++++++++++++++++--------------------- 1 file changed, 128 insertions(+), 109 deletions(-) diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb index a01bff91..c700fa9c 100644 --- a/noaa-cdr-sst.ipynb +++ b/noaa-cdr-sst.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "d09bbff3-4e96-4490-b837-14b78b64df35", "metadata": {}, "outputs": [], @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", "metadata": {}, "outputs": [], @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", "metadata": {}, "outputs": [], @@ -48,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "06bbec92-3974-4859-8bda-353afc7800b9", "metadata": {}, "outputs": [], @@ -63,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", "metadata": {}, "outputs": [], @@ -79,12 +79,12 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "id": "abefd6fa-386a-4e07-a7c8-219d3730eeeb", "metadata": {}, "outputs": [], "source": [ - "#!rm -rf ./noaa-cdr-icechunk/" + "!rm -rf ./noaa-cdr-icechunk/" ] }, { @@ -129,7 +129,7 @@ { "data": { "text/plain": [ - "'MZWAPPV1HD75JFQFCZ2G'" + "'VEA6RE6TYKDAN90Y42EG'" ] }, "execution_count": 10, @@ -144,6 +144,27 @@ { "cell_type": "code", "execution_count": 11, + "id": "9387e1ff-46c1-45fd-9796-0457538209a7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'x^cx\\xd3\\xe2\\x06\\x00\\x04\\x16\\x01\\xb7'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "await store._store.get(\"time/c/0\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, "id": "b6271bd1-bc0b-4901-9901-91aabe508cf7", "metadata": {}, "outputs": [ @@ -180,14 +201,14 @@ " --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n", "}\n", "\n", - "html[theme=\"dark\"],\n", - "html[data-theme=\"dark\"],\n", - "body[data-theme=\"dark\"],\n", + "html[theme=dark],\n", + "html[data-theme=dark],\n", + "body[data-theme=dark],\n", "body.vscode-dark {\n", " --xr-font-color0: rgba(255, 255, 255, 1);\n", " --xr-font-color2: rgba(255, 255, 255, 0.54);\n", " --xr-font-color3: rgba(255, 255, 255, 0.38);\n", - " --xr-border-color: #1f1f1f;\n", + " --xr-border-color: #1F1F1F;\n", " --xr-disabled-color: #515151;\n", " --xr-background-color: #111111;\n", " --xr-background-color-row-even: #111111;\n", @@ -242,7 +263,6 @@ ".xr-section-item input {\n", " display: inline-block;\n", " opacity: 0;\n", - " height: 0;\n", "}\n", "\n", ".xr-section-item input + label {\n", @@ -279,7 +299,7 @@ "\n", ".xr-section-summary-in + label:before {\n", " display: inline-block;\n", - " content: \"►\";\n", + " content: '►';\n", " font-size: 11px;\n", " width: 15px;\n", " text-align: center;\n", @@ -290,7 +310,7 @@ "}\n", "\n", ".xr-section-summary-in:checked + label:before {\n", - " content: \"▼\";\n", + " content: '▼';\n", "}\n", "\n", ".xr-section-summary-in:checked + label > span {\n", @@ -362,15 +382,15 @@ "}\n", "\n", ".xr-dim-list:before {\n", - " content: \"(\";\n", + " content: '(';\n", "}\n", "\n", ".xr-dim-list:after {\n", - " content: \")\";\n", + " content: ')';\n", "}\n", "\n", ".xr-dim-list li:not(:last-child):after {\n", - " content: \",\";\n", + " content: ',';\n", " padding-right: 5px;\n", "}\n", "\n", @@ -521,15 +541,15 @@ " fill: currentColor;\n", "}\n", "
    <xarray.Dataset> Size: 66MB\n",
    -       "Dimensions:  (lon: 1440, time: 2, zlev: 1, lat: 720)\n",
    +       "Dimensions:  (lat: 720, time: 2, zlev: 1, lon: 1440)\n",
            "Coordinates:\n",
    +       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
            "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
            "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
    -       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
            "  * zlev     (zlev) float32 4B 0.0\n",
            "Data variables:\n",
    -       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
            "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
            "    anom     (time, zlev, lat, lon) float64 17MB ...\n",
            "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
            "Attributes: (12/37)\n",
    @@ -545,32 +565,32 @@
            "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
            "    time_coverage_end:          2024-08-01T23:59:59Z\n",
            "    time_coverage_start:        2024-08-01T00:00:00Z\n",
    -       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
  • Conventions :
    CF-1.6, ACDD-1.3
    cdm_data_type :
    Grid
    comment :
    Data was converted from NetCDF-3 to NetCDF-4 format with metadata updates in November 2017.
    creator_email :
    oisst-help@noaa.gov
    creator_url :
    https://www.ncei.noaa.gov/
    date_created :
    2024-08-16T09:12:00Z
    date_modified :
    2024-08-16T09:12:00Z
    geospatial_lat_max :
    90.0
    geospatial_lat_min :
    -90.0
    geospatial_lat_resolution :
    0.25
    geospatial_lat_units :
    degrees_north
    geospatial_lon_max :
    360.0
    geospatial_lon_min :
    0.0
    geospatial_lon_resolution :
    0.25
    geospatial_lon_units :
    degrees_east
    history :
    Final file created using preliminary as first guess, and 3 days of AVHRR data. Preliminary uses only 1 day of AVHRR data.
    id :
    oisst-avhrr-v02r01.20240801.nc
    institution :
    NOAA/National Centers for Environmental Information
    instrument :
    Earth Remote Sensing Instruments > Passive Remote Sensing > Spectrometers/Radiometers > Imaging Spectrometers/Radiometers > AVHRR > Advanced Very High Resolution Radiometer
    instrument_vocabulary :
    Global Change Master Directory (GCMD) Instrument Keywords
    keywords :
    Earth Science > Oceans > Ocean Temperature > Sea Surface Temperature
    keywords_vocabulary :
    Global Change Master Directory (GCMD) Earth Science Keywords
    metadata_link :
    https://doi.org/10.25921/RE9P-PT57
    naming_authority :
    gov.noaa.ncei
    ncei_template_version :
    NCEI_NetCDF_Grid_Template_v2.0
    platform :
    Ships, buoys, Argo floats, MetOp-A, MetOp-B
    platform_vocabulary :
    Global Change Master Directory (GCMD) Platform Keywords
    processing_level :
    NOAA Level 4
    product_version :
    Version v02r01
    references :
    Reynolds, et al.(2007) Daily High-Resolution-Blended Analyses for Sea Surface Temperature (available at https://doi.org/10.1175/2007JCLI1824.1). Banzon, et al.(2016) A long-term record of blended satellite and in situ sea-surface temperature for climate monitoring, modeling and environmental studies (available at https://doi.org/10.5194/essd-8-165-2016). Huang et al. (2020) Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version v02r01, submitted.Climatology is based on 1971-2000 OI.v2 SST. Satellite data: Pathfinder AVHRR SST, Navy AVHRR SST, and NOAA ACSPO SST. Ice data: NCEP Ice and GSFC Ice.
    sensor :
    Thermometer, AVHRR
    source :
    ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfinder_AVHRR, Navy_AVHRR, NOAA_ACSP
    standard_name_vocabulary :
    CF Standard Name Table (v40, 25 January 2017)
    summary :
    NOAAs 1/4-degree Daily Optimum Interpolation Sea Surface Temperature (OISST) (sometimes referred to as Reynolds SST, which however also refers to earlier products at different resolution), currently available as version v02r01, is created by interpolating and extrapolating SST observations from different sources, resulting in a smoothed complete field. The sources of data are satellite (AVHRR) and in situ platforms (i.e., ships and buoys), and the specific datasets employed may change over time. At the marginal ice zone, sea ice concentrations are used to generate proxy SSTs. A preliminary version of this file is produced in near-real time (1-day latency), and then replaced with a final version after 2 weeks. Note that this is the AVHRR-ONLY DOISST, available from Oct 1981, but there is a companion DOISST product that includes microwave satellite data, available from June 2002
    time_coverage_end :
    2024-08-01T23:59:59Z
    time_coverage_start :
    2024-08-01T00:00:00Z
    title :
    NOAA/NCEI 1/4 Degree Daily Optimum Interpolation Sea Surface Temperature (OISST) Analysis, Version 2.1 - Final
  • " ], "text/plain": [ " Size: 66MB\n", - "Dimensions: (lon: 1440, time: 2, zlev: 1, lat: 720)\n", + "Dimensions: (lat: 720, time: 2, zlev: 1, lon: 1440)\n", "Coordinates:\n", + " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", - " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", " * zlev (zlev) float32 4B 0.0\n", "Data variables:\n", - " err (time, zlev, lat, lon) float64 17MB ...\n", " sst (time, zlev, lat, lon) float64 17MB ...\n", + " err (time, zlev, lat, lon) float64 17MB ...\n", " anom (time, zlev, lat, lon) float64 17MB ...\n", " ice (time, zlev, lat, lon) float64 17MB ...\n", "Attributes: (12/37)\n", @@ -589,7 +609,7 @@ " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." ] }, - "execution_count": 11, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -609,7 +629,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "190c25f9-e000-4b17-83eb-cf551141dfea", "metadata": {}, "outputs": [], @@ -624,7 +644,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "af330082-207a-4f08-aefe-fc15aa8b2eb3", "metadata": {}, "outputs": [], @@ -640,7 +660,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "id": "103b44d2-124a-4de5-8074-e997fd5a1698", "metadata": {}, "outputs": [ @@ -677,14 +697,14 @@ " --xr-background-color-row-odd: var(--jp-layout-color2, #eeeeee);\n", "}\n", "\n", - "html[theme=\"dark\"],\n", - "html[data-theme=\"dark\"],\n", - "body[data-theme=\"dark\"],\n", + "html[theme=dark],\n", + "html[data-theme=dark],\n", + "body[data-theme=dark],\n", "body.vscode-dark {\n", " --xr-font-color0: rgba(255, 255, 255, 1);\n", " --xr-font-color2: rgba(255, 255, 255, 0.54);\n", " --xr-font-color3: rgba(255, 255, 255, 0.38);\n", - " --xr-border-color: #1f1f1f;\n", + " --xr-border-color: #1F1F1F;\n", " --xr-disabled-color: #515151;\n", " --xr-background-color: #111111;\n", " --xr-background-color-row-even: #111111;\n", @@ -739,7 +759,6 @@ ".xr-section-item input {\n", " display: inline-block;\n", " opacity: 0;\n", - " height: 0;\n", "}\n", "\n", ".xr-section-item input + label {\n", @@ -776,7 +795,7 @@ "\n", ".xr-section-summary-in + label:before {\n", " display: inline-block;\n", - " content: \"►\";\n", + " content: '►';\n", " font-size: 11px;\n", " width: 15px;\n", " text-align: center;\n", @@ -787,7 +806,7 @@ "}\n", "\n", ".xr-section-summary-in:checked + label:before {\n", - " content: \"▼\";\n", + " content: '▼';\n", "}\n", "\n", ".xr-section-summary-in:checked + label > span {\n", @@ -859,15 +878,15 @@ "}\n", "\n", ".xr-dim-list:before {\n", - " content: \"(\";\n", + " content: '(';\n", "}\n", "\n", ".xr-dim-list:after {\n", - " content: \")\";\n", + " content: ')';\n", "}\n", "\n", ".xr-dim-list li:not(:last-child):after {\n", - " content: \",\";\n", + " content: ',';\n", " padding-right: 5px;\n", "}\n", "\n", @@ -1021,14 +1040,14 @@ "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", "Coordinates:\n", " zlev (zlev) float32 4B ManifestArray<shape=(1,), dtype=float32, chunk...\n", - " lat (lat) float32 3kB ManifestArray<shape=(720,), dtype=float32, chu...\n", " time (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n", + " lat (lat) float32 3kB ManifestArray<shape=(720,), dtype=float32, chu...\n", " lon (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n", "Data variables:\n", - " err (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n", " ice (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n", - " sst (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n", " anom (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n", + " err (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n", + " sst (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n", "Attributes: (12/37)\n", " Conventions: CF-1.6, ACDD-1.3\n", " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...\n", @@ -1042,21 +1061,21 @@ " metadata_link: https://doi.org/10.25921/RE9P-PT57\n", " ncei_template_version: NCEI_NetCDF_Grid_Template_v2.0\n", " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", - " sensor: Thermometer, AVHRR" + " sensor: Thermometer, AVHRR" ], "text/plain": [ " Size: 17MB\n", "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", "Coordinates:\n", " zlev (zlev) float32 4B ManifestArray\\x00\\x00\\xc0>\\x00\\x00 ?\\x00\\x00`?\\x00\\x00\\x90?\\x00\\x00\\xb0?\\x00\\x00\\xd0?\\x00\\x00\\xf0?\\x00\\x00\\x08@\\x00\\x00\\x18@\\x00\\x00(@\\x00\\x008@\\x00\\x00H@\\x00\\x00X@\\x00\\x00h@\\x00\\x00x@\\x00\\x00\\x84@\\x00\\x00\\x8c@\\x00\\x00\\x94@\\x00\\x00\\x9c@\\x00\\x00\\xa4@\\x00\\x00\\xac@\\x00\\x00\\xb4@\\x00\\x00\\xbc@\\x00\\x00\\xc4@\\x00\\x00\\xcc@\\x00\\x00\\xd4@\\x00\\x00\\xdc@\\x00\\x00\\xe4@\\x00\\x00\\xec@\\x00\\x00\\xf4@\\x00\\x00\\xfc@\\x00\\x00\\x02A\\x00\\x00\\x06A\\x00\\x00\\nA\\x00\\x00\\x0eA\\x00\\x00\\x12A\\x00\\x00\\x16A\\x00\\x00\\x1aA\\x00\\x00\\x1eA\\x00\\x00\"A\\x00\\x00&A\\x00\\x00*A\\x00\\x00.A\\x00\\x002A\\x00\\x006A\\x00\\x00:A\\x00\\x00>A\\x00\\x00BA\\x00\\x00FA\\x00\\x00JA\\x00\\x00NA\\x00\\x00RA\\x00\\x00VA\\x00\\x00ZA\\x00\\x00^A\\x00\\x00bA\\x00\\x00fA\\x00\\x00jA\\x00\\x00nA\\x00\\x00rA\\x00\\x00vA\\x00\\x00zA\\x00\\x00~A\\x00\\x00\\x81A\\x00\\x00\\x83A\\x00\\x00\\x85A\\x00\\x00\\x87A\\x00\\x00\\x89A\\x00\\x00\\x8bA\\x00\\x00\\x8dA\\x00\\x00\\x8fA\\x00\\x00\\x91A\\x00\\x00\\x93A\\x00\\x00\\x95A\\x00\\x00\\x97A\\x00\\x00\\x99A\\x00\\x00\\x9bA\\x00\\x00\\x9dA\\x00\\x00\\x9fA\\x00\\x00\\xa1A\\x00\\x00\\xa3A\\x00\\x00\\xa5A\\x00\\x00\\xa7A\\x00\\x00\\xa9A\\x00\\x00\\xabA\\x00\\x00\\xadA\\x00\\x00\\xafA\\x00\\x00\\xb1A\\x00\\x00\\xb3A\\x00\\x00\\xb5A\\x00\\x00\\xb7A\\x00\\x00\\xb9A\\x00\\x00\\xbbA\\x00\\x00\\xbdA\\x00\\x00\\xbfA\\x00\\x00\\xc1A\\x00\\x00\\xc3A\\x00\\x00\\xc5A\\x00\\x00\\xc7A\\x00\\x00\\xc9A\\x00\\x00\\xcbA\\x00\\x00\\xcdA\\x00\\x00\\xcfA\\x00\\x00\\xd1A\\x00\\x00\\xd3A\\x00\\x00\\xd5A\\x00\\x00\\xd7A\\x00\\x00\\xd9A\\x00\\x00\\xdbA\\x00\\x00\\xddA\\x00\\x00\\xdfA\\x00\\x00\\xe1A\\x00\\x00\\xe3A\\x00\\x00\\xe5A\\x00\\x00\\xe7A\\x00\\x00\\xe9A\\x00\\x00\\xebA\\x00\\x00\\xedA\\x00\\x00\\xefA\\x00\\x00\\xf1A\\x00\\x00\\xf3A\\x00\\x00\\xf5A\\x00\\x00\\xf7A\\x00\\x00\\xf9A\\x00\\x00\\xfbA\\x00\\x00\\xfdA\\x00\\x00\\xffA\\x00\\x80\\x00B\\x00\\x80\\x01B\\x00\\x80\\x02B\\x00\\x80\\x03B\\x00\\x80\\x04B\\x00\\x80\\x05B\\x00\\x80\\x06B\\x00\\x80\\x07B\\x00\\x80\\x08B\\x00\\x80\\tB\\x00\\x80\\nB\\x00\\x80\\x0bB\\x00\\x80\\x0cB\\x00\\x80\\rB\\x00\\x80\\x0eB\\x00\\x80\\x0fB\\x00\\x80\\x10B\\x00\\x80\\x11B\\x00\\x80\\x12B\\x00\\x80\\x13B\\x00\\x80\\x14B\\x00\\x80\\x15B\\x00\\x80\\x16B\\x00\\x80\\x17B\\x00\\x80\\x18B\\x00\\x80\\x19B\\x00\\x80\\x1aB\\x00\\x80\\x1bB\\x00\\x80\\x1cB\\x00\\x80\\x1dB\\x00\\x80\\x1eB\\x00\\x80\\x1fB\\x00\\x80 B\\x00\\x80!B\\x00\\x80\"B\\x00\\x80#B\\x00\\x80$B\\x00\\x80%B\\x00\\x80&B\\x00\\x80\\'B\\x00\\x80(B\\x00\\x80)B\\x00\\x80*B\\x00\\x80+B\\x00\\x80,B\\x00\\x80-B\\x00\\x80.B\\x00\\x80/B\\x00\\x800B\\x00\\x801B\\x00\\x802B\\x00\\x803B\\x00\\x804B\\x00\\x805B\\x00\\x806B\\x00\\x807B\\x00\\x808B\\x00\\x809B\\x00\\x80:B\\x00\\x80;B\\x00\\x80B\\x00\\x80?B\\x00\\x80@B\\x00\\x80AB\\x00\\x80BB\\x00\\x80CB\\x00\\x80DB\\x00\\x80EB\\x00\\x80FB\\x00\\x80GB\\x00\\x80HB\\x00\\x80IB\\x00\\x80JB\\x00\\x80KB\\x00\\x80LB\\x00\\x80MB\\x00\\x80NB\\x00\\x80OB\\x00\\x80PB\\x00\\x80QB\\x00\\x80RB\\x00\\x80SB\\x00\\x80TB\\x00\\x80UB\\x00\\x80VB\\x00\\x80WB\\x00\\x80XB\\x00\\x80YB\\x00\\x80ZB\\x00\\x80[B\\x00\\x80\\\\B\\x00\\x80]B\\x00\\x80^B\\x00\\x80_B\\x00\\x80`B\\x00\\x80aB\\x00\\x80bB\\x00\\x80cB\\x00\\x80dB\\x00\\x80eB\\x00\\x80fB\\x00\\x80gB\\x00\\x80hB\\x00\\x80iB\\x00\\x80jB\\x00\\x80kB\\x00\\x80lB\\x00\\x80mB\\x00\\x80nB\\x00\\x80oB\\x00\\x80pB\\x00\\x80qB\\x00\\x80rB\\x00\\x80sB\\x00\\x80tB\\x00\\x80uB\\x00\\x80vB\\x00\\x80wB\\x00\\x80xB\\x00\\x80yB\\x00\\x80zB\\x00\\x80{B\\x00\\x80|B\\x00\\x80}B\\x00\\x80~B\\x00\\x80\\x7fB\\x00@\\x80B\\x00\\xc0\\x80B\\x00@\\x81B\\x00\\xc0\\x81B\\x00@\\x82B\\x00\\xc0\\x82B\\x00@\\x83B\\x00\\xc0\\x83B\\x00@\\x84B\\x00\\xc0\\x84B\\x00@\\x85B\\x00\\xc0\\x85B\\x00@\\x86B\\x00\\xc0\\x86B\\x00@\\x87B\\x00\\xc0\\x87B\\x00@\\x88B\\x00\\xc0\\x88B\\x00@\\x89B\\x00\\xc0\\x89B\\x00@\\x8aB\\x00\\xc0\\x8aB\\x00@\\x8bB\\x00\\xc0\\x8bB\\x00@\\x8cB\\x00\\xc0\\x8cB\\x00@\\x8dB\\x00\\xc0\\x8dB\\x00@\\x8eB\\x00\\xc0\\x8eB\\x00@\\x8fB\\x00\\xc0\\x8fB\\x00@\\x90B\\x00\\xc0\\x90B\\x00@\\x91B\\x00\\xc0\\x91B\\x00@\\x92B\\x00\\xc0\\x92B\\x00@\\x93B\\x00\\xc0\\x93B\\x00@\\x94B\\x00\\xc0\\x94B\\x00@\\x95B\\x00\\xc0\\x95B\\x00@\\x96B\\x00\\xc0\\x96B\\x00@\\x97B\\x00\\xc0\\x97B\\x00@\\x98B\\x00\\xc0\\x98B\\x00@\\x99B\\x00\\xc0\\x99B\\x00@\\x9aB\\x00\\xc0\\x9aB\\x00@\\x9bB\\x00\\xc0\\x9bB\\x00@\\x9cB\\x00\\xc0\\x9cB\\x00@\\x9dB\\x00\\xc0\\x9dB\\x00@\\x9eB\\x00\\xc0\\x9eB\\x00@\\x9fB\\x00\\xc0\\x9fB\\x00@\\xa0B\\x00\\xc0\\xa0B\\x00@\\xa1B\\x00\\xc0\\xa1B\\x00@\\xa2B\\x00\\xc0\\xa2B\\x00@\\xa3B\\x00\\xc0\\xa3B\\x00@\\xa4B\\x00\\xc0\\xa4B\\x00@\\xa5B\\x00\\xc0\\xa5B\\x00@\\xa6B\\x00\\xc0\\xa6B\\x00@\\xa7B\\x00\\xc0\\xa7B\\x00@\\xa8B\\x00\\xc0\\xa8B\\x00@\\xa9B\\x00\\xc0\\xa9B\\x00@\\xaaB\\x00\\xc0\\xaaB\\x00@\\xabB\\x00\\xc0\\xabB\\x00@\\xacB\\x00\\xc0\\xacB\\x00@\\xadB\\x00\\xc0\\xadB\\x00@\\xaeB\\x00\\xc0\\xaeB\\x00@\\xafB\\x00\\xc0\\xafB\\x00@\\xb0B\\x00\\xc0\\xb0B\\x00@\\xb1B\\x00\\xc0\\xb1B\\x00@\\xb2B\\x00\\xc0\\xb2B\\x00@\\xb3B\\x00\\xc0\\xb3B\\x00@\\xb4B\\x00\\xc0\\xb4B\\x00@\\xb5B\\x00\\xc0\\xb5B\\x00@\\xb6B\\x00\\xc0\\xb6B\\x00@\\xb7B\\x00\\xc0\\xb7B\\x00@\\xb8B\\x00\\xc0\\xb8B\\x00@\\xb9B\\x00\\xc0\\xb9B\\x00@\\xbaB\\x00\\xc0\\xbaB\\x00@\\xbbB\\x00\\xc0\\xbbB\\x00@\\xbcB\\x00\\xc0\\xbcB\\x00@\\xbdB\\x00\\xc0\\xbdB\\x00@\\xbeB\\x00\\xc0\\xbeB\\x00@\\xbfB\\x00\\xc0\\xbfB\\x00@\\xc0B\\x00\\xc0\\xc0B\\x00@\\xc1B\\x00\\xc0\\xc1B\\x00@\\xc2B\\x00\\xc0\\xc2B\\x00@\\xc3B\\x00\\xc0\\xc3B\\x00@\\xc4B\\x00\\xc0\\xc4B\\x00@\\xc5B\\x00\\xc0\\xc5B\\x00@\\xc6B\\x00\\xc0\\xc6B\\x00@\\xc7B\\x00\\xc0\\xc7B\\x00@\\xc8B\\x00\\xc0\\xc8B\\x00@\\xc9B\\x00\\xc0\\xc9B\\x00@\\xcaB\\x00\\xc0\\xcaB\\x00@\\xcbB\\x00\\xc0\\xcbB\\x00@\\xccB\\x00\\xc0\\xccB\\x00@\\xcdB\\x00\\xc0\\xcdB\\x00@\\xceB\\x00\\xc0\\xceB\\x00@\\xcfB\\x00\\xc0\\xcfB\\x00@\\xd0B\\x00\\xc0\\xd0B\\x00@\\xd1B\\x00\\xc0\\xd1B\\x00@\\xd2B\\x00\\xc0\\xd2B\\x00@\\xd3B\\x00\\xc0\\xd3B\\x00@\\xd4B\\x00\\xc0\\xd4B\\x00@\\xd5B\\x00\\xc0\\xd5B\\x00@\\xd6B\\x00\\xc0\\xd6B\\x00@\\xd7B\\x00\\xc0\\xd7B\\x00@\\xd8B\\x00\\xc0\\xd8B\\x00@\\xd9B\\x00\\xc0\\xd9B\\x00@\\xdaB\\x00\\xc0\\xdaB\\x00@\\xdbB\\x00\\xc0\\xdbB\\x00@\\xdcB\\x00\\xc0\\xdcB\\x00@\\xddB\\x00\\xc0\\xddB\\x00@\\xdeB\\x00\\xc0\\xdeB\\x00@\\xdfB\\x00\\xc0\\xdfB\\x00@\\xe0B\\x00\\xc0\\xe0B\\x00@\\xe1B\\x00\\xc0\\xe1B\\x00@\\xe2B\\x00\\xc0\\xe2B\\x00@\\xe3B\\x00\\xc0\\xe3B\\x00@\\xe4B\\x00\\xc0\\xe4B\\x00@\\xe5B\\x00\\xc0\\xe5B\\x00@\\xe6B\\x00\\xc0\\xe6B\\x00@\\xe7B\\x00\\xc0\\xe7B\\x00@\\xe8B\\x00\\xc0\\xe8B\\x00@\\xe9B\\x00\\xc0\\xe9B\\x00@\\xeaB\\x00\\xc0\\xeaB\\x00@\\xebB\\x00\\xc0\\xebB\\x00@\\xecB\\x00\\xc0\\xecB\\x00@\\xedB\\x00\\xc0\\xedB\\x00@\\xeeB\\x00\\xc0\\xeeB\\x00@\\xefB\\x00\\xc0\\xefB\\x00@\\xf0B\\x00\\xc0\\xf0B\\x00@\\xf1B\\x00\\xc0\\xf1B\\x00@\\xf2B\\x00\\xc0\\xf2B\\x00@\\xf3B\\x00\\xc0\\xf3B\\x00@\\xf4B\\x00\\xc0\\xf4B\\x00@\\xf5B\\x00\\xc0\\xf5B\\x00@\\xf6B\\x00\\xc0\\xf6B\\x00@\\xf7B\\x00\\xc0\\xf7B\\x00@\\xf8B\\x00\\xc0\\xf8B\\x00@\\xf9B\\x00\\xc0\\xf9B\\x00@\\xfaB\\x00\\xc0\\xfaB\\x00@\\xfbB\\x00\\xc0\\xfbB\\x00@\\xfcB\\x00\\xc0\\xfcB\\x00@\\xfdB\\x00\\xc0\\xfdB\\x00@\\xfeB\\x00\\xc0\\xfeB\\x00@\\xffB\\x00\\xc0\\xffB\\x00 \\x00C\\x00`\\x00C\\x00\\xa0\\x00C\\x00\\xe0\\x00C\\x00 \\x01C\\x00`\\x01C\\x00\\xa0\\x01C\\x00\\xe0\\x01C\\x00 \\x02C\\x00`\\x02C\\x00\\xa0\\x02C\\x00\\xe0\\x02C\\x00 \\x03C\\x00`\\x03C\\x00\\xa0\\x03C\\x00\\xe0\\x03C\\x00 \\x04C\\x00`\\x04C\\x00\\xa0\\x04C\\x00\\xe0\\x04C\\x00 \\x05C\\x00`\\x05C\\x00\\xa0\\x05C\\x00\\xe0\\x05C\\x00 \\x06C\\x00`\\x06C\\x00\\xa0\\x06C\\x00\\xe0\\x06C\\x00 \\x07C\\x00`\\x07C\\x00\\xa0\\x07C\\x00\\xe0\\x07C\\x00 \\x08C\\x00`\\x08C\\x00\\xa0\\x08C\\x00\\xe0\\x08C\\x00 \\tC\\x00`\\tC\\x00\\xa0\\tC\\x00\\xe0\\tC\\x00 \\nC\\x00`\\nC\\x00\\xa0\\nC\\x00\\xe0\\nC\\x00 \\x0bC\\x00`\\x0bC\\x00\\xa0\\x0bC\\x00\\xe0\\x0bC\\x00 \\x0cC\\x00`\\x0cC\\x00\\xa0\\x0cC\\x00\\xe0\\x0cC\\x00 \\rC\\x00`\\rC\\x00\\xa0\\rC\\x00\\xe0\\rC\\x00 \\x0eC\\x00`\\x0eC\\x00\\xa0\\x0eC\\x00\\xe0\\x0eC\\x00 \\x0fC\\x00`\\x0fC\\x00\\xa0\\x0fC\\x00\\xe0\\x0fC\\x00 \\x10C\\x00`\\x10C\\x00\\xa0\\x10C\\x00\\xe0\\x10C\\x00 \\x11C\\x00`\\x11C\\x00\\xa0\\x11C\\x00\\xe0\\x11C\\x00 \\x12C\\x00`\\x12C\\x00\\xa0\\x12C\\x00\\xe0\\x12C\\x00 \\x13C\\x00`\\x13C\\x00\\xa0\\x13C\\x00\\xe0\\x13C\\x00 \\x14C\\x00`\\x14C\\x00\\xa0\\x14C\\x00\\xe0\\x14C\\x00 \\x15C\\x00`\\x15C\\x00\\xa0\\x15C\\x00\\xe0\\x15C\\x00 \\x16C\\x00`\\x16C\\x00\\xa0\\x16C\\x00\\xe0\\x16C\\x00 \\x17C\\x00`\\x17C\\x00\\xa0\\x17C\\x00\\xe0\\x17C\\x00 \\x18C\\x00`\\x18C\\x00\\xa0\\x18C\\x00\\xe0\\x18C\\x00 \\x19C\\x00`\\x19C\\x00\\xa0\\x19C\\x00\\xe0\\x19C\\x00 \\x1aC\\x00`\\x1aC\\x00\\xa0\\x1aC\\x00\\xe0\\x1aC\\x00 \\x1bC\\x00`\\x1bC\\x00\\xa0\\x1bC\\x00\\xe0\\x1bC\\x00 \\x1cC\\x00`\\x1cC\\x00\\xa0\\x1cC\\x00\\xe0\\x1cC\\x00 \\x1dC\\x00`\\x1dC\\x00\\xa0\\x1dC\\x00\\xe0\\x1dC\\x00 \\x1eC\\x00`\\x1eC\\x00\\xa0\\x1eC\\x00\\xe0\\x1eC\\x00 \\x1fC\\x00`\\x1fC\\x00\\xa0\\x1fC\\x00\\xe0\\x1fC\\x00 C\\x00` C\\x00\\xa0 C\\x00\\xe0 C\\x00 !C\\x00`!C\\x00\\xa0!C\\x00\\xe0!C\\x00 \"C\\x00`\"C\\x00\\xa0\"C\\x00\\xe0\"C\\x00 #C\\x00`#C\\x00\\xa0#C\\x00\\xe0#C\\x00 $C\\x00`$C\\x00\\xa0$C\\x00\\xe0$C\\x00 %C\\x00`%C\\x00\\xa0%C\\x00\\xe0%C\\x00 &C\\x00`&C\\x00\\xa0&C\\x00\\xe0&C\\x00 \\'C\\x00`\\'C\\x00\\xa0\\'C\\x00\\xe0\\'C\\x00 (C\\x00`(C\\x00\\xa0(C\\x00\\xe0(C\\x00 )C\\x00`)C\\x00\\xa0)C\\x00\\xe0)C\\x00 *C\\x00`*C\\x00\\xa0*C\\x00\\xe0*C\\x00 +C\\x00`+C\\x00\\xa0+C\\x00\\xe0+C\\x00 ,C\\x00`,C\\x00\\xa0,C\\x00\\xe0,C\\x00 -C\\x00`-C\\x00\\xa0-C\\x00\\xe0-C\\x00 .C\\x00`.C\\x00\\xa0.C\\x00\\xe0.C\\x00 /C\\x00`/C\\x00\\xa0/C\\x00\\xe0/C\\x00 0C\\x00`0C\\x00\\xa00C\\x00\\xe00C\\x00 1C\\x00`1C\\x00\\xa01C\\x00\\xe01C\\x00 2C\\x00`2C\\x00\\xa02C\\x00\\xe02C\\x00 3C\\x00`3C\\x00\\xa03C\\x00\\xe03C\\x00 4C\\x00`4C\\x00\\xa04C\\x00\\xe04C\\x00 5C\\x00`5C\\x00\\xa05C\\x00\\xe05C\\x00 6C\\x00`6C\\x00\\xa06C\\x00\\xe06C\\x00 7C\\x00`7C\\x00\\xa07C\\x00\\xe07C\\x00 8C\\x00`8C\\x00\\xa08C\\x00\\xe08C\\x00 9C\\x00`9C\\x00\\xa09C\\x00\\xe09C\\x00 :C\\x00`:C\\x00\\xa0:C\\x00\\xe0:C\\x00 ;C\\x00`;C\\x00\\xa0;C\\x00\\xe0;C\\x00 C\\x00`>C\\x00\\xa0>C\\x00\\xe0>C\\x00 ?C\\x00`?C\\x00\\xa0?C\\x00\\xe0?C\\x00 @C\\x00`@C\\x00\\xa0@C\\x00\\xe0@C\\x00 AC\\x00`AC\\x00\\xa0AC\\x00\\xe0AC\\x00 BC\\x00`BC\\x00\\xa0BC\\x00\\xe0BC\\x00 CC\\x00`CC\\x00\\xa0CC\\x00\\xe0CC\\x00 DC\\x00`DC\\x00\\xa0DC\\x00\\xe0DC\\x00 EC\\x00`EC\\x00\\xa0EC\\x00\\xe0EC\\x00 FC\\x00`FC\\x00\\xa0FC\\x00\\xe0FC\\x00 GC\\x00`GC\\x00\\xa0GC\\x00\\xe0GC\\x00 HC\\x00`HC\\x00\\xa0HC\\x00\\xe0HC\\x00 IC\\x00`IC\\x00\\xa0IC\\x00\\xe0IC\\x00 JC\\x00`JC\\x00\\xa0JC\\x00\\xe0JC\\x00 KC\\x00`KC\\x00\\xa0KC\\x00\\xe0KC\\x00 LC\\x00`LC\\x00\\xa0LC\\x00\\xe0LC\\x00 MC\\x00`MC\\x00\\xa0MC\\x00\\xe0MC\\x00 NC\\x00`NC\\x00\\xa0NC\\x00\\xe0NC\\x00 OC\\x00`OC\\x00\\xa0OC\\x00\\xe0OC\\x00 PC\\x00`PC\\x00\\xa0PC\\x00\\xe0PC\\x00 QC\\x00`QC\\x00\\xa0QC\\x00\\xe0QC\\x00 RC\\x00`RC\\x00\\xa0RC\\x00\\xe0RC\\x00 SC\\x00`SC\\x00\\xa0SC\\x00\\xe0SC\\x00 TC\\x00`TC\\x00\\xa0TC\\x00\\xe0TC\\x00 UC\\x00`UC\\x00\\xa0UC\\x00\\xe0UC\\x00 VC\\x00`VC\\x00\\xa0VC\\x00\\xe0VC\\x00 WC\\x00`WC\\x00\\xa0WC\\x00\\xe0WC\\x00 XC\\x00`XC\\x00\\xa0XC\\x00\\xe0XC\\x00 YC\\x00`YC\\x00\\xa0YC\\x00\\xe0YC\\x00 ZC\\x00`ZC\\x00\\xa0ZC\\x00\\xe0ZC\\x00 [C\\x00`[C\\x00\\xa0[C\\x00\\xe0[C\\x00 \\\\C\\x00`\\\\C\\x00\\xa0\\\\C\\x00\\xe0\\\\C\\x00 ]C\\x00`]C\\x00\\xa0]C\\x00\\xe0]C\\x00 ^C\\x00`^C\\x00\\xa0^C\\x00\\xe0^C\\x00 _C\\x00`_C\\x00\\xa0_C\\x00\\xe0_C\\x00 `C\\x00``C\\x00\\xa0`C\\x00\\xe0`C\\x00 aC\\x00`aC\\x00\\xa0aC\\x00\\xe0aC\\x00 bC\\x00`bC\\x00\\xa0bC\\x00\\xe0bC\\x00 cC\\x00`cC\\x00\\xa0cC\\x00\\xe0cC\\x00 dC\\x00`dC\\x00\\xa0dC\\x00\\xe0dC\\x00 eC\\x00`eC\\x00\\xa0eC\\x00\\xe0eC\\x00 fC\\x00`fC\\x00\\xa0fC\\x00\\xe0fC\\x00 gC\\x00`gC\\x00\\xa0gC\\x00\\xe0gC\\x00 hC\\x00`hC\\x00\\xa0hC\\x00\\xe0hC\\x00 iC\\x00`iC\\x00\\xa0iC\\x00\\xe0iC\\x00 jC\\x00`jC\\x00\\xa0jC\\x00\\xe0jC\\x00 kC\\x00`kC\\x00\\xa0kC\\x00\\xe0kC\\x00 lC\\x00`lC\\x00\\xa0lC\\x00\\xe0lC\\x00 mC\\x00`mC\\x00\\xa0mC\\x00\\xe0mC\\x00 nC\\x00`nC\\x00\\xa0nC\\x00\\xe0nC\\x00 oC\\x00`oC\\x00\\xa0oC\\x00\\xe0oC\\x00 pC\\x00`pC\\x00\\xa0pC\\x00\\xe0pC\\x00 qC\\x00`qC\\x00\\xa0qC\\x00\\xe0qC\\x00 rC\\x00`rC\\x00\\xa0rC\\x00\\xe0rC\\x00 sC\\x00`sC\\x00\\xa0sC\\x00\\xe0sC\\x00 tC\\x00`tC\\x00\\xa0tC\\x00\\xe0tC\\x00 uC\\x00`uC\\x00\\xa0uC\\x00\\xe0uC\\x00 vC\\x00`vC\\x00\\xa0vC\\x00\\xe0vC\\x00 wC\\x00`wC\\x00\\xa0wC\\x00\\xe0wC\\x00 xC\\x00`xC\\x00\\xa0xC\\x00\\xe0xC\\x00 yC\\x00`yC\\x00\\xa0yC\\x00\\xe0yC\\x00 zC\\x00`zC\\x00\\xa0zC\\x00\\xe0zC\\x00 {C\\x00`{C\\x00\\xa0{C\\x00\\xe0{C\\x00 |C\\x00`|C\\x00\\xa0|C\\x00\\xe0|C\\x00 }C\\x00`}C\\x00\\xa0}C\\x00\\xe0}C\\x00 ~C\\x00`~C\\x00\\xa0~C\\x00\\xe0~C\\x00 \\x7fC\\x00`\\x7fC\\x00\\xa0\\x7fC\\x00\\xe0\\x7fC\\x00\\x10\\x80C\\x000\\x80C\\x00P\\x80C\\x00p\\x80C\\x00\\x90\\x80C\\x00\\xb0\\x80C\\x00\\xd0\\x80C\\x00\\xf0\\x80C\\x00\\x10\\x81C\\x000\\x81C\\x00P\\x81C\\x00p\\x81C\\x00\\x90\\x81C\\x00\\xb0\\x81C\\x00\\xd0\\x81C\\x00\\xf0\\x81C\\x00\\x10\\x82C\\x000\\x82C\\x00P\\x82C\\x00p\\x82C\\x00\\x90\\x82C\\x00\\xb0\\x82C\\x00\\xd0\\x82C\\x00\\xf0\\x82C\\x00\\x10\\x83C\\x000\\x83C\\x00P\\x83C\\x00p\\x83C\\x00\\x90\\x83C\\x00\\xb0\\x83C\\x00\\xd0\\x83C\\x00\\xf0\\x83C\\x00\\x10\\x84C\\x000\\x84C\\x00P\\x84C\\x00p\\x84C\\x00\\x90\\x84C\\x00\\xb0\\x84C\\x00\\xd0\\x84C\\x00\\xf0\\x84C\\x00\\x10\\x85C\\x000\\x85C\\x00P\\x85C\\x00p\\x85C\\x00\\x90\\x85C\\x00\\xb0\\x85C\\x00\\xd0\\x85C\\x00\\xf0\\x85C\\x00\\x10\\x86C\\x000\\x86C\\x00P\\x86C\\x00p\\x86C\\x00\\x90\\x86C\\x00\\xb0\\x86C\\x00\\xd0\\x86C\\x00\\xf0\\x86C\\x00\\x10\\x87C\\x000\\x87C\\x00P\\x87C\\x00p\\x87C\\x00\\x90\\x87C\\x00\\xb0\\x87C\\x00\\xd0\\x87C\\x00\\xf0\\x87C\\x00\\x10\\x88C\\x000\\x88C\\x00P\\x88C\\x00p\\x88C\\x00\\x90\\x88C\\x00\\xb0\\x88C\\x00\\xd0\\x88C\\x00\\xf0\\x88C\\x00\\x10\\x89C\\x000\\x89C\\x00P\\x89C\\x00p\\x89C\\x00\\x90\\x89C\\x00\\xb0\\x89C\\x00\\xd0\\x89C\\x00\\xf0\\x89C\\x00\\x10\\x8aC\\x000\\x8aC\\x00P\\x8aC\\x00p\\x8aC\\x00\\x90\\x8aC\\x00\\xb0\\x8aC\\x00\\xd0\\x8aC\\x00\\xf0\\x8aC\\x00\\x10\\x8bC\\x000\\x8bC\\x00P\\x8bC\\x00p\\x8bC\\x00\\x90\\x8bC\\x00\\xb0\\x8bC\\x00\\xd0\\x8bC\\x00\\xf0\\x8bC\\x00\\x10\\x8cC\\x000\\x8cC\\x00P\\x8cC\\x00p\\x8cC\\x00\\x90\\x8cC\\x00\\xb0\\x8cC\\x00\\xd0\\x8cC\\x00\\xf0\\x8cC\\x00\\x10\\x8dC\\x000\\x8dC\\x00P\\x8dC\\x00p\\x8dC\\x00\\x90\\x8dC\\x00\\xb0\\x8dC\\x00\\xd0\\x8dC\\x00\\xf0\\x8dC\\x00\\x10\\x8eC\\x000\\x8eC\\x00P\\x8eC\\x00p\\x8eC\\x00\\x90\\x8eC\\x00\\xb0\\x8eC\\x00\\xd0\\x8eC\\x00\\xf0\\x8eC\\x00\\x10\\x8fC\\x000\\x8fC\\x00P\\x8fC\\x00p\\x8fC\\x00\\x90\\x8fC\\x00\\xb0\\x8fC\\x00\\xd0\\x8fC\\x00\\xf0\\x8fC\\x00\\x10\\x90C\\x000\\x90C\\x00P\\x90C\\x00p\\x90C\\x00\\x90\\x90C\\x00\\xb0\\x90C\\x00\\xd0\\x90C\\x00\\xf0\\x90C\\x00\\x10\\x91C\\x000\\x91C\\x00P\\x91C\\x00p\\x91C\\x00\\x90\\x91C\\x00\\xb0\\x91C\\x00\\xd0\\x91C\\x00\\xf0\\x91C\\x00\\x10\\x92C\\x000\\x92C\\x00P\\x92C\\x00p\\x92C\\x00\\x90\\x92C\\x00\\xb0\\x92C\\x00\\xd0\\x92C\\x00\\xf0\\x92C\\x00\\x10\\x93C\\x000\\x93C\\x00P\\x93C\\x00p\\x93C\\x00\\x90\\x93C\\x00\\xb0\\x93C\\x00\\xd0\\x93C\\x00\\xf0\\x93C\\x00\\x10\\x94C\\x000\\x94C\\x00P\\x94C\\x00p\\x94C\\x00\\x90\\x94C\\x00\\xb0\\x94C\\x00\\xd0\\x94C\\x00\\xf0\\x94C\\x00\\x10\\x95C\\x000\\x95C\\x00P\\x95C\\x00p\\x95C\\x00\\x90\\x95C\\x00\\xb0\\x95C\\x00\\xd0\\x95C\\x00\\xf0\\x95C\\x00\\x10\\x96C\\x000\\x96C\\x00P\\x96C\\x00p\\x96C\\x00\\x90\\x96C\\x00\\xb0\\x96C\\x00\\xd0\\x96C\\x00\\xf0\\x96C\\x00\\x10\\x97C\\x000\\x97C\\x00P\\x97C\\x00p\\x97C\\x00\\x90\\x97C\\x00\\xb0\\x97C\\x00\\xd0\\x97C\\x00\\xf0\\x97C\\x00\\x10\\x98C\\x000\\x98C\\x00P\\x98C\\x00p\\x98C\\x00\\x90\\x98C\\x00\\xb0\\x98C\\x00\\xd0\\x98C\\x00\\xf0\\x98C\\x00\\x10\\x99C\\x000\\x99C\\x00P\\x99C\\x00p\\x99C\\x00\\x90\\x99C\\x00\\xb0\\x99C\\x00\\xd0\\x99C\\x00\\xf0\\x99C\\x00\\x10\\x9aC\\x000\\x9aC\\x00P\\x9aC\\x00p\\x9aC\\x00\\x90\\x9aC\\x00\\xb0\\x9aC\\x00\\xd0\\x9aC\\x00\\xf0\\x9aC\\x00\\x10\\x9bC\\x000\\x9bC\\x00P\\x9bC\\x00p\\x9bC\\x00\\x90\\x9bC\\x00\\xb0\\x9bC\\x00\\xd0\\x9bC\\x00\\xf0\\x9bC\\x00\\x10\\x9cC\\x000\\x9cC\\x00P\\x9cC\\x00p\\x9cC\\x00\\x90\\x9cC\\x00\\xb0\\x9cC\\x00\\xd0\\x9cC\\x00\\xf0\\x9cC\\x00\\x10\\x9dC\\x000\\x9dC\\x00P\\x9dC\\x00p\\x9dC\\x00\\x90\\x9dC\\x00\\xb0\\x9dC\\x00\\xd0\\x9dC\\x00\\xf0\\x9dC\\x00\\x10\\x9eC\\x000\\x9eC\\x00P\\x9eC\\x00p\\x9eC\\x00\\x90\\x9eC\\x00\\xb0\\x9eC\\x00\\xd0\\x9eC\\x00\\xf0\\x9eC\\x00\\x10\\x9fC\\x000\\x9fC\\x00P\\x9fC\\x00p\\x9fC\\x00\\x90\\x9fC\\x00\\xb0\\x9fC\\x00\\xd0\\x9fC\\x00\\xf0\\x9fC\\x00\\x10\\xa0C\\x000\\xa0C\\x00P\\xa0C\\x00p\\xa0C\\x00\\x90\\xa0C\\x00\\xb0\\xa0C\\x00\\xd0\\xa0C\\x00\\xf0\\xa0C\\x00\\x10\\xa1C\\x000\\xa1C\\x00P\\xa1C\\x00p\\xa1C\\x00\\x90\\xa1C\\x00\\xb0\\xa1C\\x00\\xd0\\xa1C\\x00\\xf0\\xa1C\\x00\\x10\\xa2C\\x000\\xa2C\\x00P\\xa2C\\x00p\\xa2C\\x00\\x90\\xa2C\\x00\\xb0\\xa2C\\x00\\xd0\\xa2C\\x00\\xf0\\xa2C\\x00\\x10\\xa3C\\x000\\xa3C\\x00P\\xa3C\\x00p\\xa3C\\x00\\x90\\xa3C\\x00\\xb0\\xa3C\\x00\\xd0\\xa3C\\x00\\xf0\\xa3C\\x00\\x10\\xa4C\\x000\\xa4C\\x00P\\xa4C\\x00p\\xa4C\\x00\\x90\\xa4C\\x00\\xb0\\xa4C\\x00\\xd0\\xa4C\\x00\\xf0\\xa4C\\x00\\x10\\xa5C\\x000\\xa5C\\x00P\\xa5C\\x00p\\xa5C\\x00\\x90\\xa5C\\x00\\xb0\\xa5C\\x00\\xd0\\xa5C\\x00\\xf0\\xa5C\\x00\\x10\\xa6C\\x000\\xa6C\\x00P\\xa6C\\x00p\\xa6C\\x00\\x90\\xa6C\\x00\\xb0\\xa6C\\x00\\xd0\\xa6C\\x00\\xf0\\xa6C\\x00\\x10\\xa7C\\x000\\xa7C\\x00P\\xa7C\\x00p\\xa7C\\x00\\x90\\xa7C\\x00\\xb0\\xa7C\\x00\\xd0\\xa7C\\x00\\xf0\\xa7C\\x00\\x10\\xa8C\\x000\\xa8C\\x00P\\xa8C\\x00p\\xa8C\\x00\\x90\\xa8C\\x00\\xb0\\xa8C\\x00\\xd0\\xa8C\\x00\\xf0\\xa8C\\x00\\x10\\xa9C\\x000\\xa9C\\x00P\\xa9C\\x00p\\xa9C\\x00\\x90\\xa9C\\x00\\xb0\\xa9C\\x00\\xd0\\xa9C\\x00\\xf0\\xa9C\\x00\\x10\\xaaC\\x000\\xaaC\\x00P\\xaaC\\x00p\\xaaC\\x00\\x90\\xaaC\\x00\\xb0\\xaaC\\x00\\xd0\\xaaC\\x00\\xf0\\xaaC\\x00\\x10\\xabC\\x000\\xabC\\x00P\\xabC\\x00p\\xabC\\x00\\x90\\xabC\\x00\\xb0\\xabC\\x00\\xd0\\xabC\\x00\\xf0\\xabC\\x00\\x10\\xacC\\x000\\xacC\\x00P\\xacC\\x00p\\xacC\\x00\\x90\\xacC\\x00\\xb0\\xacC\\x00\\xd0\\xacC\\x00\\xf0\\xacC\\x00\\x10\\xadC\\x000\\xadC\\x00P\\xadC\\x00p\\xadC\\x00\\x90\\xadC\\x00\\xb0\\xadC\\x00\\xd0\\xadC\\x00\\xf0\\xadC\\x00\\x10\\xaeC\\x000\\xaeC\\x00P\\xaeC\\x00p\\xaeC\\x00\\x90\\xaeC\\x00\\xb0\\xaeC\\x00\\xd0\\xaeC\\x00\\xf0\\xaeC\\x00\\x10\\xafC\\x000\\xafC\\x00P\\xafC\\x00p\\xafC\\x00\\x90\\xafC\\x00\\xb0\\xafC\\x00\\xd0\\xafC\\x00\\xf0\\xafC\\x00\\x10\\xb0C\\x000\\xb0C\\x00P\\xb0C\\x00p\\xb0C\\x00\\x90\\xb0C\\x00\\xb0\\xb0C\\x00\\xd0\\xb0C\\x00\\xf0\\xb0C\\x00\\x10\\xb1C\\x000\\xb1C\\x00P\\xb1C\\x00p\\xb1C\\x00\\x90\\xb1C\\x00\\xb0\\xb1C\\x00\\xd0\\xb1C\\x00\\xf0\\xb1C\\x00\\x10\\xb2C\\x000\\xb2C\\x00P\\xb2C\\x00p\\xb2C\\x00\\x90\\xb2C\\x00\\xb0\\xb2C\\x00\\xd0\\xb2C\\x00\\xf0\\xb2C\\x00\\x10\\xb3C\\x000\\xb3C\\x00P\\xb3C\\x00p\\xb3C\\x00\\x90\\xb3C\\x00\\xb0\\xb3C\\x00\\xd0\\xb3C\\x00\\xf0\\xb3C'" ] }, - "execution_count": 20, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -1181,7 +1200,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "id": "47a53027-dbae-48aa-85d2-dcbc04e01e61", "metadata": {}, "outputs": [ @@ -1192,48 +1211,48 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31merror\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/conventions.py:453\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 452\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 453\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m \u001b[43mdecode_cf_variable\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 454\u001b[0m \u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 456\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 459\u001b[0m \u001b[43m \u001b[49m\u001b[43mstack_char_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstack_char_dim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 460\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 461\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 462\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/conventions.py:294\u001b[0m, in \u001b[0;36mdecode_cf_variable\u001b[0;34m(name, var, concat_characters, mask_and_scale, decode_times, decode_endianness, stack_char_dim, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_times:\n\u001b[0;32m--> 294\u001b[0m var \u001b[38;5;241m=\u001b[39m \u001b[43mtimes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCFDatetimeCoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 296\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_endianness \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39misnative:\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/coding/times.py:1005\u001b[0m, in \u001b[0;36mCFDatetimeCoder.decode\u001b[0;34m(self, variable, name)\u001b[0m\n\u001b[1;32m 1004\u001b[0m calendar \u001b[38;5;241m=\u001b[39m pop_to(attrs, encoding, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalendar\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1005\u001b[0m dtype \u001b[38;5;241m=\u001b[39m \u001b[43m_decode_cf_datetime_dtype\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munits\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcalendar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1006\u001b[0m transform \u001b[38;5;241m=\u001b[39m partial(\n\u001b[1;32m 1007\u001b[0m decode_cf_datetime,\n\u001b[1;32m 1008\u001b[0m units\u001b[38;5;241m=\u001b[39munits,\n\u001b[1;32m 1009\u001b[0m calendar\u001b[38;5;241m=\u001b[39mcalendar,\n\u001b[1;32m 1010\u001b[0m use_cftime\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_cftime,\n\u001b[1;32m 1011\u001b[0m )\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/coding/times.py:214\u001b[0m, in \u001b[0;36m_decode_cf_datetime_dtype\u001b[0;34m(data, units, calendar, use_cftime)\u001b[0m\n\u001b[1;32m 212\u001b[0m values \u001b[38;5;241m=\u001b[39m indexing\u001b[38;5;241m.\u001b[39mImplicitToExplicitIndexingAdapter(indexing\u001b[38;5;241m.\u001b[39mas_indexable(data))\n\u001b[1;32m 213\u001b[0m example_value \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate(\n\u001b[0;32m--> 214\u001b[0m [\u001b[43mfirst_n_items\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m], last_item(values) \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m 215\u001b[0m )\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/formatting.py:97\u001b[0m, in \u001b[0;36mfirst_n_items\u001b[0;34m(array, n_desired)\u001b[0m\n\u001b[1;32m 96\u001b[0m array \u001b[38;5;241m=\u001b[39m array\u001b[38;5;241m.\u001b[39m_data\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mravel(\u001b[43mto_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43marray\u001b[49m\u001b[43m)\u001b[49m)[:n_desired]\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/namedarray/pycompat.py:138\u001b[0m, in \u001b[0;36mto_duck_array\u001b[0;34m(data, **kwargs)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 138\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(data)\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/indexing.py:578\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.__array__\u001b[0;34m(self, dtype, copy)\u001b[0m\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m Version(np\u001b[38;5;241m.\u001b[39m__version__) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m Version(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.0.0\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 578\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, dtype\u001b[38;5;241m=\u001b[39mdtype, copy\u001b[38;5;241m=\u001b[39mcopy)\n\u001b[1;32m 579\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/indexing.py:583\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 583\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/indexing.py:657\u001b[0m, in \u001b[0;36mLazilyIndexedArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 655\u001b[0m \u001b[38;5;66;03m# If the array is not an ExplicitlyIndexedNDArrayMixin,\u001b[39;00m\n\u001b[1;32m 656\u001b[0m \u001b[38;5;66;03m# it may wrap a BackendArray so use its __getitem__\u001b[39;00m\n\u001b[0;32m--> 657\u001b[0m array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 659\u001b[0m \u001b[38;5;66;03m# self.array[self.key] is now a numpy array when\u001b[39;00m\n\u001b[1;32m 660\u001b[0m \u001b[38;5;66;03m# self.array is a BackendArray subclass\u001b[39;00m\n\u001b[1;32m 661\u001b[0m \u001b[38;5;66;03m# and self.key is BasicIndexer((slice(None, None, None),))\u001b[39;00m\n\u001b[1;32m 662\u001b[0m \u001b[38;5;66;03m# so we need the explicit check for ExplicitlyIndexed\u001b[39;00m\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/zarr.py:226\u001b[0m, in \u001b[0;36mZarrArrayWrapper.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 225\u001b[0m method \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oindex\n\u001b[0;32m--> 226\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexplicit_indexing_adapter\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 227\u001b[0m \u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mIndexingSupport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mVECTORIZED\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\n\u001b[1;32m 228\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/core/indexing.py:1018\u001b[0m, in \u001b[0;36mexplicit_indexing_adapter\u001b[0;34m(key, shape, indexing_support, raw_indexing_method)\u001b[0m\n\u001b[1;32m 1017\u001b[0m raw_key, numpy_indices \u001b[38;5;241m=\u001b[39m decompose_indexer(key, shape, indexing_support)\n\u001b[0;32m-> 1018\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mraw_indexing_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_key\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtuple\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1019\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m numpy_indices\u001b[38;5;241m.\u001b[39mtuple:\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;66;03m# index the loaded np.ndarray\u001b[39;00m\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/zarr.py:216\u001b[0m, in \u001b[0;36mZarrArrayWrapper._getitem\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_getitem\u001b[39m(\u001b[38;5;28mself\u001b[39m, key):\n\u001b[0;32m--> 216\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_array\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/array.py:1660\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 1659\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_pure_orthogonal_indexing(pure_selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim):\n\u001b[0;32m-> 1660\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_orthogonal_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1661\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/_compat.py:43\u001b[0m, in \u001b[0;36m_deprecate_positional_args.._inner_deprecate_positional_args..inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m extra_args \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# extra_args > 0\u001b[39;00m\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/array.py:2102\u001b[0m, in \u001b[0;36mArray.get_orthogonal_selection\u001b[0;34m(self, selection, out, fields, prototype)\u001b[0m\n\u001b[1;32m 2101\u001b[0m indexer \u001b[38;5;241m=\u001b[39m OrthogonalIndexer(selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mchunk_grid)\n\u001b[0;32m-> 2102\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2103\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_async_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2104\u001b[0m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprototype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprototype\u001b[49m\n\u001b[1;32m 2105\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2106\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/sync.py:141\u001b[0m, in \u001b[0;36msync\u001b[0;34m(coro, loop, timeout)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 141\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/sync.py:100\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 100\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/array.py:961\u001b[0m, in \u001b[0;36mAsyncArray._get_selection\u001b[0;34m(self, indexer, prototype, out, fields)\u001b[0m\n\u001b[1;32m 959\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m product(indexer\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 960\u001b[0m \u001b[38;5;66;03m# reading chunks and decoding them\u001b[39;00m\n\u001b[0;32m--> 961\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcodec_pipeline\u001b[38;5;241m.\u001b[39mread(\n\u001b[1;32m 962\u001b[0m [\n\u001b[1;32m 963\u001b[0m (\n\u001b[1;32m 964\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_path \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mencode_chunk_key(chunk_coords),\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget_chunk_spec(chunk_coords, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morder, prototype\u001b[38;5;241m=\u001b[39mprototype),\n\u001b[1;32m 966\u001b[0m chunk_selection,\n\u001b[1;32m 967\u001b[0m out_selection,\n\u001b[1;32m 968\u001b[0m )\n\u001b[1;32m 969\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_coords, chunk_selection, out_selection \u001b[38;5;129;01min\u001b[39;00m indexer\n\u001b[1;32m 970\u001b[0m ],\n\u001b[1;32m 971\u001b[0m out_buffer,\n\u001b[1;32m 972\u001b[0m drop_axes\u001b[38;5;241m=\u001b[39mindexer\u001b[38;5;241m.\u001b[39mdrop_axes,\n\u001b[1;32m 973\u001b[0m )\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_buffer\u001b[38;5;241m.\u001b[39mas_ndarray_like()\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/codecs/pipeline.py:440\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread\u001b[39m(\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 436\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]],\n\u001b[1;32m 437\u001b[0m out: NDBuffer,\n\u001b[1;32m 438\u001b[0m drop_axes: \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (),\n\u001b[1;32m 439\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 440\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 441\u001b[0m [\n\u001b[1;32m 442\u001b[0m (single_batch_info, out, drop_axes)\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m single_batch_info \u001b[38;5;129;01min\u001b[39;00m batched(batch_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m 444\u001b[0m ],\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_batch,\n\u001b[1;32m 446\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 447\u001b[0m )\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/codecs/pipeline.py:270\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read_batch\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 262\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 263\u001b[0m [\n\u001b[1;32m 264\u001b[0m (byte_getter, array_spec\u001b[38;5;241m.\u001b[39mprototype)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 268\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 269\u001b[0m )\n\u001b[0;32m--> 270\u001b[0m chunk_array_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode_batch(\n\u001b[1;32m 271\u001b[0m [\n\u001b[1;32m 272\u001b[0m (chunk_bytes, chunk_spec)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_bytes, (_, chunk_spec, _, _) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 274\u001b[0m chunk_bytes_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 275\u001b[0m )\n\u001b[1;32m 276\u001b[0m ],\n\u001b[1;32m 277\u001b[0m )\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_array, (_, chunk_spec, chunk_selection, out_selection) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 279\u001b[0m chunk_array_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 280\u001b[0m ):\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/codecs/pipeline.py:172\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.decode_batch\u001b[0;34m(self, chunk_bytes_and_specs)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m bb_codec, chunk_spec_batch \u001b[38;5;129;01min\u001b[39;00m bb_codecs_with_spec[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 172\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m bb_codec\u001b[38;5;241m.\u001b[39mdecode(\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mzip\u001b[39m(chunk_bytes_batch, chunk_spec_batch, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 174\u001b[0m )\n\u001b[1;32m 176\u001b[0m ab_codec, chunk_spec_batch \u001b[38;5;241m=\u001b[39m ab_codec_with_spec\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/abc/codec.py:130\u001b[0m, in \u001b[0;36mBaseCodec.decode\u001b[0;34m(self, chunks_and_specs)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decodes a batch of chunks.\u001b[39;00m\n\u001b[1;32m 119\u001b[0m \u001b[38;5;124;03mChunks can be None in which case they are ignored by the codec.\u001b[39;00m\n\u001b[1;32m 120\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124;03mIterable[CodecInput | None]\u001b[39;00m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m _batching_helper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decode_single, chunks_and_specs)\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/abc/codec.py:408\u001b[0m, in \u001b[0;36m_batching_helper\u001b[0;34m(func, batch_info)\u001b[0m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_batching_helper\u001b[39m(\n\u001b[1;32m 405\u001b[0m func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]],\n\u001b[1;32m 406\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[CodecInput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, ArraySpec]],\n\u001b[1;32m 407\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]:\n\u001b[0;32m--> 408\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 409\u001b[0m \u001b[38;5;28mlist\u001b[39m(batch_info),\n\u001b[1;32m 410\u001b[0m _noop_for_none(func),\n\u001b[1;32m 411\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 412\u001b[0m )\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/abc/codec.py:421\u001b[0m, in \u001b[0;36m_noop_for_none..wrap\u001b[0;34m(chunk, chunk_spec)\u001b[0m\n\u001b[1;32m 420\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 421\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(chunk, chunk_spec)\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/numcodecs/zarr3.py:125\u001b[0m, in \u001b[0;36m_NumcodecsBytesBytesCodec._decode_single\u001b[0;34m(self, chunk_bytes, chunk_spec)\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_decode_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, chunk_bytes: Buffer, chunk_spec: ArraySpec) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Buffer:\n\u001b[0;32m--> 125\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mto_thread(\n\u001b[1;32m 126\u001b[0m as_numpy_array_wrapper,\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_codec\u001b[38;5;241m.\u001b[39mdecode,\n\u001b[1;32m 128\u001b[0m chunk_bytes,\n\u001b[1;32m 129\u001b[0m chunk_spec\u001b[38;5;241m.\u001b[39mprototype,\n\u001b[1;32m 130\u001b[0m )\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/asyncio/threads.py:25\u001b[0m, in \u001b[0;36mto_thread\u001b[0;34m(func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m func_call \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mpartial(ctx\u001b[38;5;241m.\u001b[39mrun, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m loop\u001b[38;5;241m.\u001b[39mrun_in_executor(\u001b[38;5;28;01mNone\u001b[39;00m, func_call)\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/zarr/core/buffer/cpu.py:213\u001b[0m, in \u001b[0;36mas_numpy_array_wrapper\u001b[0;34m(func, buf, prototype)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Converts the input of `func` to a numpy array and the output back to `Buffer`.\u001b[39;00m\n\u001b[1;32m 192\u001b[0m \n\u001b[1;32m 193\u001b[0m \u001b[38;5;124;03mThis function is useful when calling a `func` that only support host memory such\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[38;5;124;03m The result of `func` converted to a `Buffer`\u001b[39;00m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m prototype\u001b[38;5;241m.\u001b[39mbuffer\u001b[38;5;241m.\u001b[39mfrom_bytes(\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_numpy_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/numcodecs/zlib.py:37\u001b[0m, in \u001b[0;36mZlib.decode\u001b[0;34m(self, buf, out)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# do decompression\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m dec \u001b[38;5;241m=\u001b[39m \u001b[43m_zlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecompress\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;66;03m# handle destination - Python standard library zlib module does not\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;66;03m# support direct decompression into buffer, so we have to copy into\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# out if given\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:451\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 450\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 451\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m \u001b[43mdecode_cf_variable\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 452\u001b[0m \u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 453\u001b[0m \u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 454\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 456\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mstack_char_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstack_char_dim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 459\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 460\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:292\u001b[0m, in \u001b[0;36mdecode_cf_variable\u001b[0;34m(name, var, concat_characters, mask_and_scale, decode_times, decode_endianness, stack_char_dim, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_times:\n\u001b[0;32m--> 292\u001b[0m var \u001b[38;5;241m=\u001b[39m \u001b[43mtimes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCFDatetimeCoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 294\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_endianness \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39misnative:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/coding/times.py:1001\u001b[0m, in \u001b[0;36mCFDatetimeCoder.decode\u001b[0;34m(self, variable, name)\u001b[0m\n\u001b[1;32m 1000\u001b[0m calendar \u001b[38;5;241m=\u001b[39m pop_to(attrs, encoding, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalendar\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1001\u001b[0m dtype \u001b[38;5;241m=\u001b[39m \u001b[43m_decode_cf_datetime_dtype\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munits\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcalendar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1002\u001b[0m transform \u001b[38;5;241m=\u001b[39m partial(\n\u001b[1;32m 1003\u001b[0m decode_cf_datetime,\n\u001b[1;32m 1004\u001b[0m units\u001b[38;5;241m=\u001b[39munits,\n\u001b[1;32m 1005\u001b[0m calendar\u001b[38;5;241m=\u001b[39mcalendar,\n\u001b[1;32m 1006\u001b[0m use_cftime\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_cftime,\n\u001b[1;32m 1007\u001b[0m )\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/coding/times.py:214\u001b[0m, in \u001b[0;36m_decode_cf_datetime_dtype\u001b[0;34m(data, units, calendar, use_cftime)\u001b[0m\n\u001b[1;32m 212\u001b[0m values \u001b[38;5;241m=\u001b[39m indexing\u001b[38;5;241m.\u001b[39mImplicitToExplicitIndexingAdapter(indexing\u001b[38;5;241m.\u001b[39mas_indexable(data))\n\u001b[1;32m 213\u001b[0m example_value \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate(\n\u001b[0;32m--> 214\u001b[0m [\u001b[43mfirst_n_items\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m], last_item(values) \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m 215\u001b[0m )\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/formatting.py:97\u001b[0m, in \u001b[0;36mfirst_n_items\u001b[0;34m(array, n_desired)\u001b[0m\n\u001b[1;32m 96\u001b[0m array \u001b[38;5;241m=\u001b[39m array\u001b[38;5;241m.\u001b[39m_data\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mravel(\u001b[43mto_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43marray\u001b[49m\u001b[43m)\u001b[49m)[:n_desired]\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/namedarray/pycompat.py:138\u001b[0m, in \u001b[0;36mto_duck_array\u001b[0;34m(data, **kwargs)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 138\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43masarray\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:578\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.__array__\u001b[0;34m(self, dtype, copy)\u001b[0m\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m Version(np\u001b[38;5;241m.\u001b[39m__version__) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m Version(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.0.0\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 578\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, dtype\u001b[38;5;241m=\u001b[39mdtype, copy\u001b[38;5;241m=\u001b[39mcopy)\n\u001b[1;32m 579\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:583\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 583\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:657\u001b[0m, in \u001b[0;36mLazilyIndexedArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 655\u001b[0m \u001b[38;5;66;03m# If the array is not an ExplicitlyIndexedNDArrayMixin,\u001b[39;00m\n\u001b[1;32m 656\u001b[0m \u001b[38;5;66;03m# it may wrap a BackendArray so use its __getitem__\u001b[39;00m\n\u001b[0;32m--> 657\u001b[0m array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 659\u001b[0m \u001b[38;5;66;03m# self.array[self.key] is now a numpy array when\u001b[39;00m\n\u001b[1;32m 660\u001b[0m \u001b[38;5;66;03m# self.array is a BackendArray subclass\u001b[39;00m\n\u001b[1;32m 661\u001b[0m \u001b[38;5;66;03m# and self.key is BasicIndexer((slice(None, None, None),))\u001b[39;00m\n\u001b[1;32m 662\u001b[0m \u001b[38;5;66;03m# so we need the explicit check for ExplicitlyIndexed\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:175\u001b[0m, in \u001b[0;36mZarrArrayWrapper.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 174\u001b[0m method \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oindex\n\u001b[0;32m--> 175\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexplicit_indexing_adapter\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 176\u001b[0m \u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mIndexingSupport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mVECTORIZED\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\n\u001b[1;32m 177\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:1018\u001b[0m, in \u001b[0;36mexplicit_indexing_adapter\u001b[0;34m(key, shape, indexing_support, raw_indexing_method)\u001b[0m\n\u001b[1;32m 1017\u001b[0m raw_key, numpy_indices \u001b[38;5;241m=\u001b[39m decompose_indexer(key, shape, indexing_support)\n\u001b[0;32m-> 1018\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mraw_indexing_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_key\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtuple\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1019\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m numpy_indices\u001b[38;5;241m.\u001b[39mtuple:\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;66;03m# index the loaded np.ndarray\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:158\u001b[0m, in \u001b[0;36mZarrArrayWrapper._getitem\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_getitem\u001b[39m(\u001b[38;5;28mself\u001b[39m, key):\n\u001b[1;32m 156\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 157\u001b[0m \u001b[38;5;66;03m# try:\u001b[39;00m\n\u001b[0;32m--> 158\u001b[0m item \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_array\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;66;03m# except Exception as e:\u001b[39;00m\n\u001b[1;32m 160\u001b[0m \u001b[38;5;66;03m# import traceback\u001b[39;00m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;66;03m# print(f\"An error occurred: {e}\")\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[38;5;66;03m# print(\"Stack trace:\")\u001b[39;00m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;66;03m# traceback.print_exc() # Prints the full traceback \u001b[39;00m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:1688\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 1687\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_pure_orthogonal_indexing(pure_selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim):\n\u001b[0;32m-> 1688\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_orthogonal_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1689\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/_compat.py:43\u001b[0m, in \u001b[0;36m_deprecate_positional_args.._inner_deprecate_positional_args..inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m extra_args \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# extra_args > 0\u001b[39;00m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:2130\u001b[0m, in \u001b[0;36mArray.get_orthogonal_selection\u001b[0;34m(self, selection, out, fields, prototype)\u001b[0m\n\u001b[1;32m 2129\u001b[0m indexer \u001b[38;5;241m=\u001b[39m OrthogonalIndexer(selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mchunk_grid)\n\u001b[0;32m-> 2130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2131\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_async_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2132\u001b[0m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprototype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprototype\u001b[49m\n\u001b[1;32m 2133\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2134\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:141\u001b[0m, in \u001b[0;36msync\u001b[0;34m(coro, loop, timeout)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 141\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:100\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 100\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:961\u001b[0m, in \u001b[0;36mAsyncArray._get_selection\u001b[0;34m(self, indexer, prototype, out, fields)\u001b[0m\n\u001b[1;32m 959\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m product(indexer\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 960\u001b[0m \u001b[38;5;66;03m# reading chunks and decoding them\u001b[39;00m\n\u001b[0;32m--> 961\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcodec_pipeline\u001b[38;5;241m.\u001b[39mread(\n\u001b[1;32m 962\u001b[0m [\n\u001b[1;32m 963\u001b[0m (\n\u001b[1;32m 964\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_path \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mencode_chunk_key(chunk_coords),\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget_chunk_spec(chunk_coords, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morder, prototype\u001b[38;5;241m=\u001b[39mprototype),\n\u001b[1;32m 966\u001b[0m chunk_selection,\n\u001b[1;32m 967\u001b[0m out_selection,\n\u001b[1;32m 968\u001b[0m )\n\u001b[1;32m 969\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_coords, chunk_selection, out_selection \u001b[38;5;129;01min\u001b[39;00m indexer\n\u001b[1;32m 970\u001b[0m ],\n\u001b[1;32m 971\u001b[0m out_buffer,\n\u001b[1;32m 972\u001b[0m drop_axes\u001b[38;5;241m=\u001b[39mindexer\u001b[38;5;241m.\u001b[39mdrop_axes,\n\u001b[1;32m 973\u001b[0m )\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_buffer\u001b[38;5;241m.\u001b[39mas_ndarray_like()\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:440\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread\u001b[39m(\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 436\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]],\n\u001b[1;32m 437\u001b[0m out: NDBuffer,\n\u001b[1;32m 438\u001b[0m drop_axes: \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (),\n\u001b[1;32m 439\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 440\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 441\u001b[0m [\n\u001b[1;32m 442\u001b[0m (single_batch_info, out, drop_axes)\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m single_batch_info \u001b[38;5;129;01min\u001b[39;00m batched(batch_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m 444\u001b[0m ],\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_batch,\n\u001b[1;32m 446\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 447\u001b[0m )\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:270\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read_batch\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 262\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 263\u001b[0m [\n\u001b[1;32m 264\u001b[0m (byte_getter, array_spec\u001b[38;5;241m.\u001b[39mprototype)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 268\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 269\u001b[0m )\n\u001b[0;32m--> 270\u001b[0m chunk_array_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode_batch(\n\u001b[1;32m 271\u001b[0m [\n\u001b[1;32m 272\u001b[0m (chunk_bytes, chunk_spec)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_bytes, (_, chunk_spec, _, _) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 274\u001b[0m chunk_bytes_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 275\u001b[0m )\n\u001b[1;32m 276\u001b[0m ],\n\u001b[1;32m 277\u001b[0m )\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_array, (_, chunk_spec, chunk_selection, out_selection) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 279\u001b[0m chunk_array_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 280\u001b[0m ):\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:172\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.decode_batch\u001b[0;34m(self, chunk_bytes_and_specs)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m bb_codec, chunk_spec_batch \u001b[38;5;129;01min\u001b[39;00m bb_codecs_with_spec[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 172\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m bb_codec\u001b[38;5;241m.\u001b[39mdecode(\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mzip\u001b[39m(chunk_bytes_batch, chunk_spec_batch, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 174\u001b[0m )\n\u001b[1;32m 176\u001b[0m ab_codec, chunk_spec_batch \u001b[38;5;241m=\u001b[39m ab_codec_with_spec\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:130\u001b[0m, in \u001b[0;36mBaseCodec.decode\u001b[0;34m(self, chunks_and_specs)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decodes a batch of chunks.\u001b[39;00m\n\u001b[1;32m 119\u001b[0m \u001b[38;5;124;03mChunks can be None in which case they are ignored by the codec.\u001b[39;00m\n\u001b[1;32m 120\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124;03mIterable[CodecInput | None]\u001b[39;00m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m _batching_helper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decode_single, chunks_and_specs)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:408\u001b[0m, in \u001b[0;36m_batching_helper\u001b[0;34m(func, batch_info)\u001b[0m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_batching_helper\u001b[39m(\n\u001b[1;32m 405\u001b[0m func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]],\n\u001b[1;32m 406\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[CodecInput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, ArraySpec]],\n\u001b[1;32m 407\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]:\n\u001b[0;32m--> 408\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 409\u001b[0m \u001b[38;5;28mlist\u001b[39m(batch_info),\n\u001b[1;32m 410\u001b[0m _noop_for_none(func),\n\u001b[1;32m 411\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 412\u001b[0m )\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:421\u001b[0m, in \u001b[0;36m_noop_for_none..wrap\u001b[0;34m(chunk, chunk_spec)\u001b[0m\n\u001b[1;32m 420\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 421\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(chunk, chunk_spec)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zarr3.py:125\u001b[0m, in \u001b[0;36m_NumcodecsBytesBytesCodec._decode_single\u001b[0;34m(self, chunk_bytes, chunk_spec)\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_decode_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, chunk_bytes: Buffer, chunk_spec: ArraySpec) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Buffer:\n\u001b[0;32m--> 125\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mto_thread(\n\u001b[1;32m 126\u001b[0m as_numpy_array_wrapper,\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_codec\u001b[38;5;241m.\u001b[39mdecode,\n\u001b[1;32m 128\u001b[0m chunk_bytes,\n\u001b[1;32m 129\u001b[0m chunk_spec\u001b[38;5;241m.\u001b[39mprototype,\n\u001b[1;32m 130\u001b[0m )\n", + "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/asyncio/threads.py:25\u001b[0m, in \u001b[0;36mto_thread\u001b[0;34m(func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m func_call \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mpartial(ctx\u001b[38;5;241m.\u001b[39mrun, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m loop\u001b[38;5;241m.\u001b[39mrun_in_executor(\u001b[38;5;28;01mNone\u001b[39;00m, func_call)\n", + "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/buffer/cpu.py:213\u001b[0m, in \u001b[0;36mas_numpy_array_wrapper\u001b[0;34m(func, buf, prototype)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Converts the input of `func` to a numpy array and the output back to `Buffer`.\u001b[39;00m\n\u001b[1;32m 192\u001b[0m \n\u001b[1;32m 193\u001b[0m \u001b[38;5;124;03mThis function is useful when calling a `func` that only support host memory such\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[38;5;124;03m The result of `func` converted to a `Buffer`\u001b[39;00m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m prototype\u001b[38;5;241m.\u001b[39mbuffer\u001b[38;5;241m.\u001b[39mfrom_bytes(\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_numpy_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zlib.py:37\u001b[0m, in \u001b[0;36mZlib.decode\u001b[0;34m(self, buf, out)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# do decompression\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m dec \u001b[38;5;241m=\u001b[39m \u001b[43m_zlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecompress\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;66;03m# handle destination - Python standard library zlib module does not\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;66;03m# support direct decompression into buffer, so we have to copy into\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# out if given\u001b[39;00m\n", "\u001b[0;31merror\u001b[0m: Error -3 while decompressing data: unknown compression method", "\nThe above exception was the direct cause of the following exception:\n", "\u001b[0;31merror\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[21], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_zarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mread_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconsolidated\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mzarr_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/zarr.py:1436\u001b[0m, in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, zarr_version, zarr_format, use_zarr_fill_value_as_mask, chunked_array_type, from_array_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1422\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m 1423\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mopen_zarr() got unexpected keyword arguments \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(kwargs\u001b[38;5;241m.\u001b[39mkeys())\n\u001b[1;32m 1424\u001b[0m )\n\u001b[1;32m 1426\u001b[0m backend_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 1427\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msynchronizer\u001b[39m\u001b[38;5;124m\"\u001b[39m: synchronizer,\n\u001b[1;32m 1428\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconsolidated\u001b[39m\u001b[38;5;124m\"\u001b[39m: consolidated,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1433\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_format\u001b[39m\u001b[38;5;124m\"\u001b[39m: zarr_format,\n\u001b[1;32m 1434\u001b[0m }\n\u001b[0;32m-> 1436\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1439\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_cf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_cf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1440\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1441\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1442\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1443\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1444\u001b[0m \u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mzarr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1445\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1446\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1447\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked_array_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked_array_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1448\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1449\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1450\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1451\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1452\u001b[0m \u001b[43m \u001b[49m\u001b[43mzarr_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzarr_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1453\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1454\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1455\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/api.py:670\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 658\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 659\u001b[0m decode_cf,\n\u001b[1;32m 660\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 666\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 667\u001b[0m )\n\u001b[1;32m 669\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 670\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 671\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 676\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 677\u001b[0m backend_ds,\n\u001b[1;32m 678\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 688\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 689\u001b[0m )\n\u001b[1;32m 690\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/zarr.py:1524\u001b[0m, in \u001b[0;36mZarrBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask)\u001b[0m\n\u001b[1;32m 1522\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 1523\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n\u001b[0;32m-> 1524\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mstore_entrypoint\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1525\u001b[0m \u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1526\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1527\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1528\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1529\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1530\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1531\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1532\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1533\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1534\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/backends/store.py:47\u001b[0m, in \u001b[0;36mStoreBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mload()\n\u001b[1;32m 45\u001b[0m encoding \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mget_encoding()\n\u001b[0;32m---> 47\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs, coord_names \u001b[38;5;241m=\u001b[39m \u001b[43mconventions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_cf_variables\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mvars\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 56\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m ds \u001b[38;5;241m=\u001b[39m Dataset(\u001b[38;5;28mvars\u001b[39m, attrs\u001b[38;5;241m=\u001b[39mattrs)\n\u001b[1;32m 60\u001b[0m ds \u001b[38;5;241m=\u001b[39m ds\u001b[38;5;241m.\u001b[39mset_coords(coord_names\u001b[38;5;241m.\u001b[39mintersection(\u001b[38;5;28mvars\u001b[39m))\n", - "File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.11/site-packages/xarray/conventions.py:464\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 453\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m decode_cf_variable(\n\u001b[1;32m 454\u001b[0m k,\n\u001b[1;32m 455\u001b[0m v,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 461\u001b[0m decode_timedelta\u001b[38;5;241m=\u001b[39m_item_or_default(decode_timedelta, k, \u001b[38;5;28;01mNone\u001b[39;00m),\n\u001b[1;32m 462\u001b[0m )\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 464\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(e)(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to decode variable \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 465\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_coords \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcoordinates\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mall\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 466\u001b[0m var_attrs \u001b[38;5;241m=\u001b[39m new_vars[k]\u001b[38;5;241m.\u001b[39mattrs\n", + "Cell \u001b[0;32mIn[22], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_zarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mread_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconsolidated\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mzarr_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1346\u001b[0m, in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, zarr_version, zarr_format, use_zarr_fill_value_as_mask, chunked_array_type, from_array_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1335\u001b[0m backend_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 1336\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msynchronizer\u001b[39m\u001b[38;5;124m\"\u001b[39m: synchronizer,\n\u001b[1;32m 1337\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconsolidated\u001b[39m\u001b[38;5;124m\"\u001b[39m: consolidated,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1343\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_format\u001b[39m\u001b[38;5;124m\"\u001b[39m: zarr_format,\n\u001b[1;32m 1344\u001b[0m }\n\u001b[1;32m 1345\u001b[0m \u001b[38;5;66;03m#import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[0;32m-> 1346\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1347\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1348\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1349\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_cf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_cf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1350\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1351\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1352\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1353\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1354\u001b[0m \u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mzarr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1355\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1356\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1357\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked_array_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked_array_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1358\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1359\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1360\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1361\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1362\u001b[0m \u001b[43m \u001b[49m\u001b[43mzarr_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzarr_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1363\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1364\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1365\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/api.py:671\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 659\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 660\u001b[0m decode_cf,\n\u001b[1;32m 661\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 667\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 668\u001b[0m )\n\u001b[1;32m 670\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 671\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 677\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 678\u001b[0m backend_ds,\n\u001b[1;32m 679\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 689\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 690\u001b[0m )\n\u001b[1;32m 691\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1436\u001b[0m, in \u001b[0;36mZarrBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask)\u001b[0m\n\u001b[1;32m 1434\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 1435\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n\u001b[0;32m-> 1436\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mstore_entrypoint\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1439\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1440\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1441\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1442\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1443\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1444\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1445\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/store.py:46\u001b[0m, in \u001b[0;36mStoreBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mload()\n\u001b[1;32m 44\u001b[0m encoding \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mget_encoding()\n\u001b[0;32m---> 46\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs, coord_names \u001b[38;5;241m=\u001b[39m \u001b[43mconventions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_cf_variables\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 47\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mvars\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 56\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 58\u001b[0m ds \u001b[38;5;241m=\u001b[39m Dataset(\u001b[38;5;28mvars\u001b[39m, attrs\u001b[38;5;241m=\u001b[39mattrs)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:462\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 451\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m decode_cf_variable(\n\u001b[1;32m 452\u001b[0m k,\n\u001b[1;32m 453\u001b[0m v,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 459\u001b[0m decode_timedelta\u001b[38;5;241m=\u001b[39m_item_or_default(decode_timedelta, k, \u001b[38;5;28;01mNone\u001b[39;00m),\n\u001b[1;32m 460\u001b[0m )\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 462\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(e)(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to decode variable \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_coords \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcoordinates\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mall\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 464\u001b[0m var_attrs \u001b[38;5;241m=\u001b[39m new_vars[k]\u001b[38;5;241m.\u001b[39mattrs\n", "\u001b[0;31merror\u001b[0m: Failed to decode variable 'time': Error -3 while decompressing data: unknown compression method" ] } @@ -1246,9 +1265,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "virtualizarr", "language": "python", - "name": "python3" + "name": "venv" }, "language_info": { "codemirror_mode": { @@ -1260,7 +1279,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.12.7" } }, "nbformat": 4, From dd9c381c3deed8d731b3ddda88fd58babfc536ea Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 26 Nov 2024 10:19:27 -0800 Subject: [PATCH 68/90] Upgrade icechunk to 1.0.0a5 --- pyproject.toml | 2 +- .../tests/test_writers/test_icechunk.py | 47 ++++--------------- virtualizarr/writers/icechunk.py | 9 +--- 3 files changed, 11 insertions(+), 47 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 649d0b4a..2478daea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ hdf_reader = [ "numcodecs" ] icechunk = [ - "icechunk==0.1.0a4", + "icechunk==0.1.0a5", ] test = [ "codecov", diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index efe2bcdb..d10eec21 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -430,7 +430,7 @@ def test_append_virtual_ref_without_encoding( # Append the same dataset to the same store icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" + storage=icechunk_storage, read_only=False ) dataset_to_icechunk(vds, icechunk_filestore_append, append_dim="x") icechunk_filestore_append.commit("appended data") @@ -486,7 +486,7 @@ def test_append_virtual_ref_with_encoding( # Append the same dataset to the same store icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" + storage=icechunk_storage, read_only=False ) dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") new_ds = open_zarr(icechunk_filestore_append, consolidated=False, zarr_format=3) @@ -500,7 +500,7 @@ def test_append_virtual_ref_with_encoding( # but encoded it is [185.16000366210935, 322.1000061035156] xrt.assert_equal(new_ds, expected_ds) - ## When appending to a virtual ref with compression, it succeeds + # When appending to a virtual ref with compression, it succeeds @pytest.mark.parametrize("zarr_format", [2, 3]) def test_append_with_compression_succeeds( self, @@ -555,7 +555,7 @@ def test_append_with_compression_succeeds( # Append another dataset with compatible compression icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" + storage=icechunk_storage, read_only=False ) dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") icechunk_filestore_append.commit("appended data") @@ -587,7 +587,7 @@ def test_append_with_different_chunking_fails( file_uri=simple_netcdf4, chunk_shape=(1, 1) ) icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" + storage=icechunk_storage, read_only=False ) with pytest.raises( ValueError, match="Cannot concatenate arrays with inconsistent chunk shapes" @@ -617,7 +617,7 @@ def test_append_with_different_encoding_fails( # Try to append with different encoding, expect failure icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" + storage=icechunk_storage, read_only=False ) with pytest.raises( ValueError, @@ -649,42 +649,11 @@ def test_dimensions_do_not_align( # Attempt to append dataset with different length in non-append dimension, expect failure icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" + storage=icechunk_storage, read_only=False ) with pytest.raises(ValueError, match="Cannot concatenate arrays with shapes"): dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="y") - def test_no_append_dim_in_append_mode_it_fails( - self, icechunk_storage: "StorageConfig", simple_netcdf4: str - ): - """ - Test case to validate that appending without specifying an append dimension fails. - This test is expected to fail until appropriate handling for missing append dimension is implemented. - """ - from icechunk import IcechunkStore - - # Generate a virtual dataset - vds = gen_virtual_dataset( - file_uri=simple_netcdf4, shape=(5, 4), chunk_shape=(5, 4) - ) - - # Create the icechunk store and commit the initial virtual dataset - icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) - dataset_to_icechunk(vds, icechunk_filestore) - icechunk_filestore.commit("initial commit") - - # Attempt to append the same dataset without specifying an append dimension - icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" - ) - - with pytest.raises( - ValueError, - match="append_dim must be provided when opening store in append mode", - ): - # This should raise a ValueError because append_dim is not provided - dataset_to_icechunk(vds, icechunk_filestore_append) - def test_append_dim_not_in_dims_raises_error( self, icechunk_storage: "StorageConfig", simple_netcdf4: str ): @@ -703,7 +672,7 @@ def test_append_dim_not_in_dims_raises_error( # Attempt to append using a non-existent append_dim "z" icechunk_filestore_append = IcechunkStore.open_existing( - storage=icechunk_storage, mode="a" + storage=icechunk_storage, read_only=False ) with pytest.raises( ValueError, diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 4401a7db..a097bbb1 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -62,11 +62,7 @@ def dataset_to_icechunk( # TODO only supports writing to the root group currently # TODO pass zarr_format kwarg? - if store.mode.str == "a": - if append_dim is None: - raise ValueError( - "append_dim must be provided when opening store in append mode" - ) + if append_dim is not None: if append_dim not in ds.dims: raise ValueError( f"append_dim {append_dim} does not match any existing dataset dimensions" @@ -176,11 +172,10 @@ def write_virtual_variable_to_icechunk( ma = cast(ManifestArray, var.data) zarray = ma.zarray - mode = store.mode.str dims: list[str] = cast(list[str], list(var.dims)) existing_num_chunks = 0 - if mode == "a" and append_dim in dims: + if append_dim and append_dim in dims: # TODO: MRP - zarr, or icechunk zarr, array assignment to a variable doesn't work to point to the same object # for example, if you resize an array, it resizes the array but not the bound variable. if not isinstance(group[name], Array): From 7c1fcfaabd8c78589abbfff90b10b2a013c9ad7f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 26 Nov 2024 10:37:26 -0800 Subject: [PATCH 69/90] Upgrade icechunk in upstream.yml --- ci/upstream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/upstream.yml b/ci/upstream.yml index 46af1474..bc685f9a 100644 --- a/ci/upstream.yml +++ b/ci/upstream.yml @@ -28,6 +28,6 @@ dependencies: - fsspec - pip - pip: - - icechunk==0.1.0a4 # Installs zarr v3 as dependency + - icechunk==0.1.0a5 # Installs zarr v3 as dependency # - git+https://github.com/fsspec/kerchunk@main # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516) - imagecodecs-numcodecs==2024.6.1 From 1bb2ad0718e33bf1aaa0a4bcf1843d86b36c4839 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 26 Nov 2024 10:38:45 -0800 Subject: [PATCH 70/90] Updated notebook with kechunk comment an upgraded icechunk version --- noaa-cdr-sst.ipynb | 164 ++++++++++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 68 deletions(-) diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb index c700fa9c..84241ac9 100644 --- a/noaa-cdr-sst.ipynb +++ b/noaa-cdr-sst.ipynb @@ -2,12 +2,13 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "d09bbff3-4e96-4490-b837-14b78b64df35", "metadata": {}, "outputs": [], "source": [ "# !pip install -e \".[icechunk]\"\n", + "# NOTE: a change has to be made https://github.com/mpiannucci/kerchunk/blob/v3/kerchunk/utils.py#L55 from mode='w' to read_only=False\n", "# !pip install git+https://github.com/mpiannucci/kerchunk@v3\n", "# !pip install fsspec s3fs" ] @@ -15,6 +16,33 @@ { "cell_type": "code", "execution_count": 2, + "id": "3055eff4-9e22-4a95-a7fd-96933f381183", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: icechunk\n", + "Version: 0.1.0a5\n", + "Summary: Transactional storage engine for Zarr designed for use on cloud object storage\n", + "Home-page: https://github.com/earth-mover/icechunk\n", + "Author: Earthmover PBC\n", + "Author-email: \n", + "License: Apache-2.0\n", + "Location: /Users/aimeebarciauskas/github/virtualizarr/venv/lib/python3.12/site-packages\n", + "Requires: zarr\n", + "Required-by: \n" + ] + } + ], + "source": [ + "!pip show icechunk" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", "metadata": {}, "outputs": [], @@ -32,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", "metadata": {}, "outputs": [], @@ -48,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "06bbec92-3974-4859-8bda-353afc7800b9", "metadata": {}, "outputs": [], @@ -63,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", "metadata": {}, "outputs": [], @@ -79,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "abefd6fa-386a-4e07-a7c8-219d3730eeeb", "metadata": {}, "outputs": [], @@ -89,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "79a4228a-0e17-4b07-9144-f24fe06db832", "metadata": {}, "outputs": [], @@ -102,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "5fd0c082-8d5e-46a8-a994-fee80baa4ecc", "metadata": {}, "outputs": [], @@ -112,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "55ebbc5f-add2-4de8-81f6-5aaf64d9e2b6", "metadata": {}, "outputs": [], @@ -122,17 +150,17 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "cae7e34f-d6dd-42aa-9e7a-f0d5420ba0b9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'VEA6RE6TYKDAN90Y42EG'" + "'JXPBY5KT843ZBSVQ2KNG'" ] }, - "execution_count": 10, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -143,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "id": "9387e1ff-46c1-45fd-9796-0457538209a7", "metadata": {}, "outputs": [ @@ -153,7 +181,7 @@ "b'x^cx\\xd3\\xe2\\x06\\x00\\x04\\x16\\x01\\xb7'" ] }, - "execution_count": 11, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -164,7 +192,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "b6271bd1-bc0b-4901-9901-91aabe508cf7", "metadata": {}, "outputs": [ @@ -541,17 +569,17 @@ " fill: currentColor;\n", "}\n", "
    <xarray.Dataset> Size: 66MB\n",
    -       "Dimensions:  (lat: 720, time: 2, zlev: 1, lon: 1440)\n",
    +       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
            "Coordinates:\n",
    +       "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
            "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
            "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
    -       "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
            "  * zlev     (zlev) float32 4B 0.0\n",
            "Data variables:\n",
    -       "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
    -       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
            "    anom     (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
            "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
            "Attributes: (12/37)\n",
            "    Conventions:                CF-1.6, ACDD-1.3\n",
            "    cdm_data_type:              Grid\n",
    @@ -565,34 +593,34 @@
            "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
            "    time_coverage_end:          2024-08-01T23:59:59Z\n",
            "    time_coverage_start:        2024-08-01T00:00:00Z\n",
    -       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
  • Conventions :
    CF-1.6, ACDD-1.3
    cdm_data_type :
    Grid
    comment :
    Data was converted from NetCDF-3 to NetCDF-4 format with metadata updates in November 2017.
    creator_email :
    oisst-help@noaa.gov
    creator_url :
    https://www.ncei.noaa.gov/
    date_created :
    2024-08-16T09:12:00Z
    date_modified :
    2024-08-16T09:12:00Z
    geospatial_lat_max :
    90.0
    geospatial_lat_min :
    -90.0
    geospatial_lat_resolution :
    0.25
    geospatial_lat_units :
    degrees_north
    geospatial_lon_max :
    360.0
    geospatial_lon_min :
    0.0
    geospatial_lon_resolution :
    0.25
    geospatial_lon_units :
    degrees_east
    history :
    Final file created using preliminary as first guess, and 3 days of AVHRR data. Preliminary uses only 1 day of AVHRR data.
    id :
    oisst-avhrr-v02r01.20240801.nc
    institution :
    NOAA/National Centers for Environmental Information
    instrument :
    Earth Remote Sensing Instruments > Passive Remote Sensing > Spectrometers/Radiometers > Imaging Spectrometers/Radiometers > AVHRR > Advanced Very High Resolution Radiometer
    instrument_vocabulary :
    Global Change Master Directory (GCMD) Instrument Keywords
    keywords :
    Earth Science > Oceans > Ocean Temperature > Sea Surface Temperature
    keywords_vocabulary :
    Global Change Master Directory (GCMD) Earth Science Keywords
    metadata_link :
    https://doi.org/10.25921/RE9P-PT57
    naming_authority :
    gov.noaa.ncei
    ncei_template_version :
    NCEI_NetCDF_Grid_Template_v2.0
    platform :
    Ships, buoys, Argo floats, MetOp-A, MetOp-B
    platform_vocabulary :
    Global Change Master Directory (GCMD) Platform Keywords
    processing_level :
    NOAA Level 4
    product_version :
    Version v02r01
    references :
    Reynolds, et al.(2007) Daily High-Resolution-Blended Analyses for Sea Surface Temperature (available at https://doi.org/10.1175/2007JCLI1824.1). Banzon, et al.(2016) A long-term record of blended satellite and in situ sea-surface temperature for climate monitoring, modeling and environmental studies (available at https://doi.org/10.5194/essd-8-165-2016). Huang et al. (2020) Improvements of the Daily Optimum Interpolation Sea Surface Temperature (DOISST) Version v02r01, submitted.Climatology is based on 1971-2000 OI.v2 SST. Satellite data: Pathfinder AVHRR SST, Navy AVHRR SST, and NOAA ACSPO SST. Ice data: NCEP Ice and GSFC Ice.
    sensor :
    Thermometer, AVHRR
    source :
    ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfinder_AVHRR, Navy_AVHRR, NOAA_ACSP
    standard_name_vocabulary :
    CF Standard Name Table (v40, 25 January 2017)
    summary :
    NOAAs 1/4-degree Daily Optimum Interpolation Sea Surface Temperature (OISST) (sometimes referred to as Reynolds SST, which however also refers to earlier products at different resolution), currently available as version v02r01, is created by interpolating and extrapolating SST observations from different sources, resulting in a smoothed complete field. The sources of data are satellite (AVHRR) and in situ platforms (i.e., ships and buoys), and the specific datasets employed may change over time. At the marginal ice zone, sea ice concentrations are used to generate proxy SSTs. A preliminary version of this file is produced in near-real time (1-day latency), and then replaced with a final version after 2 weeks. Note that this is the AVHRR-ONLY DOISST, available from Oct 1981, but there is a companion DOISST product that includes microwave satellite data, available from June 2002
    time_coverage_end :
    2024-08-01T23:59:59Z
    time_coverage_start :
    2024-08-01T00:00:00Z
    title :
    NOAA/NCEI 1/4 Degree Daily Optimum Interpolation Sea Surface Temperature (OISST) Analysis, Version 2.1 - Final
  • " ], "text/plain": [ " Size: 66MB\n", - "Dimensions: (lat: 720, time: 2, zlev: 1, lon: 1440)\n", + "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", "Coordinates:\n", + " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", - " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", " * zlev (zlev) float32 4B 0.0\n", "Data variables:\n", - " sst (time, zlev, lat, lon) float64 17MB ...\n", - " err (time, zlev, lat, lon) float64 17MB ...\n", " anom (time, zlev, lat, lon) float64 17MB ...\n", + " err (time, zlev, lat, lon) float64 17MB ...\n", " ice (time, zlev, lat, lon) float64 17MB ...\n", + " sst (time, zlev, lat, lon) float64 17MB ...\n", "Attributes: (12/37)\n", " Conventions: CF-1.6, ACDD-1.3\n", " cdm_data_type: Grid\n", @@ -609,7 +637,7 @@ " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -629,7 +657,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "190c25f9-e000-4b17-83eb-cf551141dfea", "metadata": {}, "outputs": [], @@ -644,7 +672,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "id": "af330082-207a-4f08-aefe-fc15aa8b2eb3", "metadata": {}, "outputs": [], @@ -660,7 +688,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "id": "103b44d2-124a-4de5-8074-e997fd5a1698", "metadata": {}, "outputs": [ @@ -1039,15 +1067,15 @@ "
    <xarray.Dataset> Size: 17MB\n",
            "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
            "Coordinates:\n",
    +       "    lon      (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n",
            "    zlev     (zlev) float32 4B ManifestArray<shape=(1,), dtype=float32, chunk...\n",
    -       "    time     (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n",
            "    lat      (lat) float32 3kB ManifestArray<shape=(720,), dtype=float32, chu...\n",
    -       "    lon      (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n",
    +       "    time     (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n",
            "Data variables:\n",
    -       "    ice      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
            "    anom     (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
            "    err      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
            "    sst      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    +       "    ice      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
            "Attributes: (12/37)\n",
            "    Conventions:                CF-1.6, ACDD-1.3\n",
            "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...\n",
    @@ -1061,21 +1089,21 @@
            "    metadata_link:              https://doi.org/10.25921/RE9P-PT57\n",
            "    ncei_template_version:      NCEI_NetCDF_Grid_Template_v2.0\n",
            "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    -       "    sensor:                     Thermometer, AVHRR
    " + " sensor: Thermometer, AVHRR" ], "text/plain": [ " Size: 17MB\n", "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", "Coordinates:\n", + " lon (lon) float32 6kB ManifestArray\\x00\\x00\\xc0>\\x00\\x00 ?\\x00\\x00`?\\x00\\x00\\x90?\\x00\\x00\\xb0?\\x00\\x00\\xd0?\\x00\\x00\\xf0?\\x00\\x00\\x08@\\x00\\x00\\x18@\\x00\\x00(@\\x00\\x008@\\x00\\x00H@\\x00\\x00X@\\x00\\x00h@\\x00\\x00x@\\x00\\x00\\x84@\\x00\\x00\\x8c@\\x00\\x00\\x94@\\x00\\x00\\x9c@\\x00\\x00\\xa4@\\x00\\x00\\xac@\\x00\\x00\\xb4@\\x00\\x00\\xbc@\\x00\\x00\\xc4@\\x00\\x00\\xcc@\\x00\\x00\\xd4@\\x00\\x00\\xdc@\\x00\\x00\\xe4@\\x00\\x00\\xec@\\x00\\x00\\xf4@\\x00\\x00\\xfc@\\x00\\x00\\x02A\\x00\\x00\\x06A\\x00\\x00\\nA\\x00\\x00\\x0eA\\x00\\x00\\x12A\\x00\\x00\\x16A\\x00\\x00\\x1aA\\x00\\x00\\x1eA\\x00\\x00\"A\\x00\\x00&A\\x00\\x00*A\\x00\\x00.A\\x00\\x002A\\x00\\x006A\\x00\\x00:A\\x00\\x00>A\\x00\\x00BA\\x00\\x00FA\\x00\\x00JA\\x00\\x00NA\\x00\\x00RA\\x00\\x00VA\\x00\\x00ZA\\x00\\x00^A\\x00\\x00bA\\x00\\x00fA\\x00\\x00jA\\x00\\x00nA\\x00\\x00rA\\x00\\x00vA\\x00\\x00zA\\x00\\x00~A\\x00\\x00\\x81A\\x00\\x00\\x83A\\x00\\x00\\x85A\\x00\\x00\\x87A\\x00\\x00\\x89A\\x00\\x00\\x8bA\\x00\\x00\\x8dA\\x00\\x00\\x8fA\\x00\\x00\\x91A\\x00\\x00\\x93A\\x00\\x00\\x95A\\x00\\x00\\x97A\\x00\\x00\\x99A\\x00\\x00\\x9bA\\x00\\x00\\x9dA\\x00\\x00\\x9fA\\x00\\x00\\xa1A\\x00\\x00\\xa3A\\x00\\x00\\xa5A\\x00\\x00\\xa7A\\x00\\x00\\xa9A\\x00\\x00\\xabA\\x00\\x00\\xadA\\x00\\x00\\xafA\\x00\\x00\\xb1A\\x00\\x00\\xb3A\\x00\\x00\\xb5A\\x00\\x00\\xb7A\\x00\\x00\\xb9A\\x00\\x00\\xbbA\\x00\\x00\\xbdA\\x00\\x00\\xbfA\\x00\\x00\\xc1A\\x00\\x00\\xc3A\\x00\\x00\\xc5A\\x00\\x00\\xc7A\\x00\\x00\\xc9A\\x00\\x00\\xcbA\\x00\\x00\\xcdA\\x00\\x00\\xcfA\\x00\\x00\\xd1A\\x00\\x00\\xd3A\\x00\\x00\\xd5A\\x00\\x00\\xd7A\\x00\\x00\\xd9A\\x00\\x00\\xdbA\\x00\\x00\\xddA\\x00\\x00\\xdfA\\x00\\x00\\xe1A\\x00\\x00\\xe3A\\x00\\x00\\xe5A\\x00\\x00\\xe7A\\x00\\x00\\xe9A\\x00\\x00\\xebA\\x00\\x00\\xedA\\x00\\x00\\xefA\\x00\\x00\\xf1A\\x00\\x00\\xf3A\\x00\\x00\\xf5A\\x00\\x00\\xf7A\\x00\\x00\\xf9A\\x00\\x00\\xfbA\\x00\\x00\\xfdA\\x00\\x00\\xffA\\x00\\x80\\x00B\\x00\\x80\\x01B\\x00\\x80\\x02B\\x00\\x80\\x03B\\x00\\x80\\x04B\\x00\\x80\\x05B\\x00\\x80\\x06B\\x00\\x80\\x07B\\x00\\x80\\x08B\\x00\\x80\\tB\\x00\\x80\\nB\\x00\\x80\\x0bB\\x00\\x80\\x0cB\\x00\\x80\\rB\\x00\\x80\\x0eB\\x00\\x80\\x0fB\\x00\\x80\\x10B\\x00\\x80\\x11B\\x00\\x80\\x12B\\x00\\x80\\x13B\\x00\\x80\\x14B\\x00\\x80\\x15B\\x00\\x80\\x16B\\x00\\x80\\x17B\\x00\\x80\\x18B\\x00\\x80\\x19B\\x00\\x80\\x1aB\\x00\\x80\\x1bB\\x00\\x80\\x1cB\\x00\\x80\\x1dB\\x00\\x80\\x1eB\\x00\\x80\\x1fB\\x00\\x80 B\\x00\\x80!B\\x00\\x80\"B\\x00\\x80#B\\x00\\x80$B\\x00\\x80%B\\x00\\x80&B\\x00\\x80\\'B\\x00\\x80(B\\x00\\x80)B\\x00\\x80*B\\x00\\x80+B\\x00\\x80,B\\x00\\x80-B\\x00\\x80.B\\x00\\x80/B\\x00\\x800B\\x00\\x801B\\x00\\x802B\\x00\\x803B\\x00\\x804B\\x00\\x805B\\x00\\x806B\\x00\\x807B\\x00\\x808B\\x00\\x809B\\x00\\x80:B\\x00\\x80;B\\x00\\x80B\\x00\\x80?B\\x00\\x80@B\\x00\\x80AB\\x00\\x80BB\\x00\\x80CB\\x00\\x80DB\\x00\\x80EB\\x00\\x80FB\\x00\\x80GB\\x00\\x80HB\\x00\\x80IB\\x00\\x80JB\\x00\\x80KB\\x00\\x80LB\\x00\\x80MB\\x00\\x80NB\\x00\\x80OB\\x00\\x80PB\\x00\\x80QB\\x00\\x80RB\\x00\\x80SB\\x00\\x80TB\\x00\\x80UB\\x00\\x80VB\\x00\\x80WB\\x00\\x80XB\\x00\\x80YB\\x00\\x80ZB\\x00\\x80[B\\x00\\x80\\\\B\\x00\\x80]B\\x00\\x80^B\\x00\\x80_B\\x00\\x80`B\\x00\\x80aB\\x00\\x80bB\\x00\\x80cB\\x00\\x80dB\\x00\\x80eB\\x00\\x80fB\\x00\\x80gB\\x00\\x80hB\\x00\\x80iB\\x00\\x80jB\\x00\\x80kB\\x00\\x80lB\\x00\\x80mB\\x00\\x80nB\\x00\\x80oB\\x00\\x80pB\\x00\\x80qB\\x00\\x80rB\\x00\\x80sB\\x00\\x80tB\\x00\\x80uB\\x00\\x80vB\\x00\\x80wB\\x00\\x80xB\\x00\\x80yB\\x00\\x80zB\\x00\\x80{B\\x00\\x80|B\\x00\\x80}B\\x00\\x80~B\\x00\\x80\\x7fB\\x00@\\x80B\\x00\\xc0\\x80B\\x00@\\x81B\\x00\\xc0\\x81B\\x00@\\x82B\\x00\\xc0\\x82B\\x00@\\x83B\\x00\\xc0\\x83B\\x00@\\x84B\\x00\\xc0\\x84B\\x00@\\x85B\\x00\\xc0\\x85B\\x00@\\x86B\\x00\\xc0\\x86B\\x00@\\x87B\\x00\\xc0\\x87B\\x00@\\x88B\\x00\\xc0\\x88B\\x00@\\x89B\\x00\\xc0\\x89B\\x00@\\x8aB\\x00\\xc0\\x8aB\\x00@\\x8bB\\x00\\xc0\\x8bB\\x00@\\x8cB\\x00\\xc0\\x8cB\\x00@\\x8dB\\x00\\xc0\\x8dB\\x00@\\x8eB\\x00\\xc0\\x8eB\\x00@\\x8fB\\x00\\xc0\\x8fB\\x00@\\x90B\\x00\\xc0\\x90B\\x00@\\x91B\\x00\\xc0\\x91B\\x00@\\x92B\\x00\\xc0\\x92B\\x00@\\x93B\\x00\\xc0\\x93B\\x00@\\x94B\\x00\\xc0\\x94B\\x00@\\x95B\\x00\\xc0\\x95B\\x00@\\x96B\\x00\\xc0\\x96B\\x00@\\x97B\\x00\\xc0\\x97B\\x00@\\x98B\\x00\\xc0\\x98B\\x00@\\x99B\\x00\\xc0\\x99B\\x00@\\x9aB\\x00\\xc0\\x9aB\\x00@\\x9bB\\x00\\xc0\\x9bB\\x00@\\x9cB\\x00\\xc0\\x9cB\\x00@\\x9dB\\x00\\xc0\\x9dB\\x00@\\x9eB\\x00\\xc0\\x9eB\\x00@\\x9fB\\x00\\xc0\\x9fB\\x00@\\xa0B\\x00\\xc0\\xa0B\\x00@\\xa1B\\x00\\xc0\\xa1B\\x00@\\xa2B\\x00\\xc0\\xa2B\\x00@\\xa3B\\x00\\xc0\\xa3B\\x00@\\xa4B\\x00\\xc0\\xa4B\\x00@\\xa5B\\x00\\xc0\\xa5B\\x00@\\xa6B\\x00\\xc0\\xa6B\\x00@\\xa7B\\x00\\xc0\\xa7B\\x00@\\xa8B\\x00\\xc0\\xa8B\\x00@\\xa9B\\x00\\xc0\\xa9B\\x00@\\xaaB\\x00\\xc0\\xaaB\\x00@\\xabB\\x00\\xc0\\xabB\\x00@\\xacB\\x00\\xc0\\xacB\\x00@\\xadB\\x00\\xc0\\xadB\\x00@\\xaeB\\x00\\xc0\\xaeB\\x00@\\xafB\\x00\\xc0\\xafB\\x00@\\xb0B\\x00\\xc0\\xb0B\\x00@\\xb1B\\x00\\xc0\\xb1B\\x00@\\xb2B\\x00\\xc0\\xb2B\\x00@\\xb3B\\x00\\xc0\\xb3B\\x00@\\xb4B\\x00\\xc0\\xb4B\\x00@\\xb5B\\x00\\xc0\\xb5B\\x00@\\xb6B\\x00\\xc0\\xb6B\\x00@\\xb7B\\x00\\xc0\\xb7B\\x00@\\xb8B\\x00\\xc0\\xb8B\\x00@\\xb9B\\x00\\xc0\\xb9B\\x00@\\xbaB\\x00\\xc0\\xbaB\\x00@\\xbbB\\x00\\xc0\\xbbB\\x00@\\xbcB\\x00\\xc0\\xbcB\\x00@\\xbdB\\x00\\xc0\\xbdB\\x00@\\xbeB\\x00\\xc0\\xbeB\\x00@\\xbfB\\x00\\xc0\\xbfB\\x00@\\xc0B\\x00\\xc0\\xc0B\\x00@\\xc1B\\x00\\xc0\\xc1B\\x00@\\xc2B\\x00\\xc0\\xc2B\\x00@\\xc3B\\x00\\xc0\\xc3B\\x00@\\xc4B\\x00\\xc0\\xc4B\\x00@\\xc5B\\x00\\xc0\\xc5B\\x00@\\xc6B\\x00\\xc0\\xc6B\\x00@\\xc7B\\x00\\xc0\\xc7B\\x00@\\xc8B\\x00\\xc0\\xc8B\\x00@\\xc9B\\x00\\xc0\\xc9B\\x00@\\xcaB\\x00\\xc0\\xcaB\\x00@\\xcbB\\x00\\xc0\\xcbB\\x00@\\xccB\\x00\\xc0\\xccB\\x00@\\xcdB\\x00\\xc0\\xcdB\\x00@\\xceB\\x00\\xc0\\xceB\\x00@\\xcfB\\x00\\xc0\\xcfB\\x00@\\xd0B\\x00\\xc0\\xd0B\\x00@\\xd1B\\x00\\xc0\\xd1B\\x00@\\xd2B\\x00\\xc0\\xd2B\\x00@\\xd3B\\x00\\xc0\\xd3B\\x00@\\xd4B\\x00\\xc0\\xd4B\\x00@\\xd5B\\x00\\xc0\\xd5B\\x00@\\xd6B\\x00\\xc0\\xd6B\\x00@\\xd7B\\x00\\xc0\\xd7B\\x00@\\xd8B\\x00\\xc0\\xd8B\\x00@\\xd9B\\x00\\xc0\\xd9B\\x00@\\xdaB\\x00\\xc0\\xdaB\\x00@\\xdbB\\x00\\xc0\\xdbB\\x00@\\xdcB\\x00\\xc0\\xdcB\\x00@\\xddB\\x00\\xc0\\xddB\\x00@\\xdeB\\x00\\xc0\\xdeB\\x00@\\xdfB\\x00\\xc0\\xdfB\\x00@\\xe0B\\x00\\xc0\\xe0B\\x00@\\xe1B\\x00\\xc0\\xe1B\\x00@\\xe2B\\x00\\xc0\\xe2B\\x00@\\xe3B\\x00\\xc0\\xe3B\\x00@\\xe4B\\x00\\xc0\\xe4B\\x00@\\xe5B\\x00\\xc0\\xe5B\\x00@\\xe6B\\x00\\xc0\\xe6B\\x00@\\xe7B\\x00\\xc0\\xe7B\\x00@\\xe8B\\x00\\xc0\\xe8B\\x00@\\xe9B\\x00\\xc0\\xe9B\\x00@\\xeaB\\x00\\xc0\\xeaB\\x00@\\xebB\\x00\\xc0\\xebB\\x00@\\xecB\\x00\\xc0\\xecB\\x00@\\xedB\\x00\\xc0\\xedB\\x00@\\xeeB\\x00\\xc0\\xeeB\\x00@\\xefB\\x00\\xc0\\xefB\\x00@\\xf0B\\x00\\xc0\\xf0B\\x00@\\xf1B\\x00\\xc0\\xf1B\\x00@\\xf2B\\x00\\xc0\\xf2B\\x00@\\xf3B\\x00\\xc0\\xf3B\\x00@\\xf4B\\x00\\xc0\\xf4B\\x00@\\xf5B\\x00\\xc0\\xf5B\\x00@\\xf6B\\x00\\xc0\\xf6B\\x00@\\xf7B\\x00\\xc0\\xf7B\\x00@\\xf8B\\x00\\xc0\\xf8B\\x00@\\xf9B\\x00\\xc0\\xf9B\\x00@\\xfaB\\x00\\xc0\\xfaB\\x00@\\xfbB\\x00\\xc0\\xfbB\\x00@\\xfcB\\x00\\xc0\\xfcB\\x00@\\xfdB\\x00\\xc0\\xfdB\\x00@\\xfeB\\x00\\xc0\\xfeB\\x00@\\xffB\\x00\\xc0\\xffB\\x00 \\x00C\\x00`\\x00C\\x00\\xa0\\x00C\\x00\\xe0\\x00C\\x00 \\x01C\\x00`\\x01C\\x00\\xa0\\x01C\\x00\\xe0\\x01C\\x00 \\x02C\\x00`\\x02C\\x00\\xa0\\x02C\\x00\\xe0\\x02C\\x00 \\x03C\\x00`\\x03C\\x00\\xa0\\x03C\\x00\\xe0\\x03C\\x00 \\x04C\\x00`\\x04C\\x00\\xa0\\x04C\\x00\\xe0\\x04C\\x00 \\x05C\\x00`\\x05C\\x00\\xa0\\x05C\\x00\\xe0\\x05C\\x00 \\x06C\\x00`\\x06C\\x00\\xa0\\x06C\\x00\\xe0\\x06C\\x00 \\x07C\\x00`\\x07C\\x00\\xa0\\x07C\\x00\\xe0\\x07C\\x00 \\x08C\\x00`\\x08C\\x00\\xa0\\x08C\\x00\\xe0\\x08C\\x00 \\tC\\x00`\\tC\\x00\\xa0\\tC\\x00\\xe0\\tC\\x00 \\nC\\x00`\\nC\\x00\\xa0\\nC\\x00\\xe0\\nC\\x00 \\x0bC\\x00`\\x0bC\\x00\\xa0\\x0bC\\x00\\xe0\\x0bC\\x00 \\x0cC\\x00`\\x0cC\\x00\\xa0\\x0cC\\x00\\xe0\\x0cC\\x00 \\rC\\x00`\\rC\\x00\\xa0\\rC\\x00\\xe0\\rC\\x00 \\x0eC\\x00`\\x0eC\\x00\\xa0\\x0eC\\x00\\xe0\\x0eC\\x00 \\x0fC\\x00`\\x0fC\\x00\\xa0\\x0fC\\x00\\xe0\\x0fC\\x00 \\x10C\\x00`\\x10C\\x00\\xa0\\x10C\\x00\\xe0\\x10C\\x00 \\x11C\\x00`\\x11C\\x00\\xa0\\x11C\\x00\\xe0\\x11C\\x00 \\x12C\\x00`\\x12C\\x00\\xa0\\x12C\\x00\\xe0\\x12C\\x00 \\x13C\\x00`\\x13C\\x00\\xa0\\x13C\\x00\\xe0\\x13C\\x00 \\x14C\\x00`\\x14C\\x00\\xa0\\x14C\\x00\\xe0\\x14C\\x00 \\x15C\\x00`\\x15C\\x00\\xa0\\x15C\\x00\\xe0\\x15C\\x00 \\x16C\\x00`\\x16C\\x00\\xa0\\x16C\\x00\\xe0\\x16C\\x00 \\x17C\\x00`\\x17C\\x00\\xa0\\x17C\\x00\\xe0\\x17C\\x00 \\x18C\\x00`\\x18C\\x00\\xa0\\x18C\\x00\\xe0\\x18C\\x00 \\x19C\\x00`\\x19C\\x00\\xa0\\x19C\\x00\\xe0\\x19C\\x00 \\x1aC\\x00`\\x1aC\\x00\\xa0\\x1aC\\x00\\xe0\\x1aC\\x00 \\x1bC\\x00`\\x1bC\\x00\\xa0\\x1bC\\x00\\xe0\\x1bC\\x00 \\x1cC\\x00`\\x1cC\\x00\\xa0\\x1cC\\x00\\xe0\\x1cC\\x00 \\x1dC\\x00`\\x1dC\\x00\\xa0\\x1dC\\x00\\xe0\\x1dC\\x00 \\x1eC\\x00`\\x1eC\\x00\\xa0\\x1eC\\x00\\xe0\\x1eC\\x00 \\x1fC\\x00`\\x1fC\\x00\\xa0\\x1fC\\x00\\xe0\\x1fC\\x00 C\\x00` C\\x00\\xa0 C\\x00\\xe0 C\\x00 !C\\x00`!C\\x00\\xa0!C\\x00\\xe0!C\\x00 \"C\\x00`\"C\\x00\\xa0\"C\\x00\\xe0\"C\\x00 #C\\x00`#C\\x00\\xa0#C\\x00\\xe0#C\\x00 $C\\x00`$C\\x00\\xa0$C\\x00\\xe0$C\\x00 %C\\x00`%C\\x00\\xa0%C\\x00\\xe0%C\\x00 &C\\x00`&C\\x00\\xa0&C\\x00\\xe0&C\\x00 \\'C\\x00`\\'C\\x00\\xa0\\'C\\x00\\xe0\\'C\\x00 (C\\x00`(C\\x00\\xa0(C\\x00\\xe0(C\\x00 )C\\x00`)C\\x00\\xa0)C\\x00\\xe0)C\\x00 *C\\x00`*C\\x00\\xa0*C\\x00\\xe0*C\\x00 +C\\x00`+C\\x00\\xa0+C\\x00\\xe0+C\\x00 ,C\\x00`,C\\x00\\xa0,C\\x00\\xe0,C\\x00 -C\\x00`-C\\x00\\xa0-C\\x00\\xe0-C\\x00 .C\\x00`.C\\x00\\xa0.C\\x00\\xe0.C\\x00 /C\\x00`/C\\x00\\xa0/C\\x00\\xe0/C\\x00 0C\\x00`0C\\x00\\xa00C\\x00\\xe00C\\x00 1C\\x00`1C\\x00\\xa01C\\x00\\xe01C\\x00 2C\\x00`2C\\x00\\xa02C\\x00\\xe02C\\x00 3C\\x00`3C\\x00\\xa03C\\x00\\xe03C\\x00 4C\\x00`4C\\x00\\xa04C\\x00\\xe04C\\x00 5C\\x00`5C\\x00\\xa05C\\x00\\xe05C\\x00 6C\\x00`6C\\x00\\xa06C\\x00\\xe06C\\x00 7C\\x00`7C\\x00\\xa07C\\x00\\xe07C\\x00 8C\\x00`8C\\x00\\xa08C\\x00\\xe08C\\x00 9C\\x00`9C\\x00\\xa09C\\x00\\xe09C\\x00 :C\\x00`:C\\x00\\xa0:C\\x00\\xe0:C\\x00 ;C\\x00`;C\\x00\\xa0;C\\x00\\xe0;C\\x00 C\\x00`>C\\x00\\xa0>C\\x00\\xe0>C\\x00 ?C\\x00`?C\\x00\\xa0?C\\x00\\xe0?C\\x00 @C\\x00`@C\\x00\\xa0@C\\x00\\xe0@C\\x00 AC\\x00`AC\\x00\\xa0AC\\x00\\xe0AC\\x00 BC\\x00`BC\\x00\\xa0BC\\x00\\xe0BC\\x00 CC\\x00`CC\\x00\\xa0CC\\x00\\xe0CC\\x00 DC\\x00`DC\\x00\\xa0DC\\x00\\xe0DC\\x00 EC\\x00`EC\\x00\\xa0EC\\x00\\xe0EC\\x00 FC\\x00`FC\\x00\\xa0FC\\x00\\xe0FC\\x00 GC\\x00`GC\\x00\\xa0GC\\x00\\xe0GC\\x00 HC\\x00`HC\\x00\\xa0HC\\x00\\xe0HC\\x00 IC\\x00`IC\\x00\\xa0IC\\x00\\xe0IC\\x00 JC\\x00`JC\\x00\\xa0JC\\x00\\xe0JC\\x00 KC\\x00`KC\\x00\\xa0KC\\x00\\xe0KC\\x00 LC\\x00`LC\\x00\\xa0LC\\x00\\xe0LC\\x00 MC\\x00`MC\\x00\\xa0MC\\x00\\xe0MC\\x00 NC\\x00`NC\\x00\\xa0NC\\x00\\xe0NC\\x00 OC\\x00`OC\\x00\\xa0OC\\x00\\xe0OC\\x00 PC\\x00`PC\\x00\\xa0PC\\x00\\xe0PC\\x00 QC\\x00`QC\\x00\\xa0QC\\x00\\xe0QC\\x00 RC\\x00`RC\\x00\\xa0RC\\x00\\xe0RC\\x00 SC\\x00`SC\\x00\\xa0SC\\x00\\xe0SC\\x00 TC\\x00`TC\\x00\\xa0TC\\x00\\xe0TC\\x00 UC\\x00`UC\\x00\\xa0UC\\x00\\xe0UC\\x00 VC\\x00`VC\\x00\\xa0VC\\x00\\xe0VC\\x00 WC\\x00`WC\\x00\\xa0WC\\x00\\xe0WC\\x00 XC\\x00`XC\\x00\\xa0XC\\x00\\xe0XC\\x00 YC\\x00`YC\\x00\\xa0YC\\x00\\xe0YC\\x00 ZC\\x00`ZC\\x00\\xa0ZC\\x00\\xe0ZC\\x00 [C\\x00`[C\\x00\\xa0[C\\x00\\xe0[C\\x00 \\\\C\\x00`\\\\C\\x00\\xa0\\\\C\\x00\\xe0\\\\C\\x00 ]C\\x00`]C\\x00\\xa0]C\\x00\\xe0]C\\x00 ^C\\x00`^C\\x00\\xa0^C\\x00\\xe0^C\\x00 _C\\x00`_C\\x00\\xa0_C\\x00\\xe0_C\\x00 `C\\x00``C\\x00\\xa0`C\\x00\\xe0`C\\x00 aC\\x00`aC\\x00\\xa0aC\\x00\\xe0aC\\x00 bC\\x00`bC\\x00\\xa0bC\\x00\\xe0bC\\x00 cC\\x00`cC\\x00\\xa0cC\\x00\\xe0cC\\x00 dC\\x00`dC\\x00\\xa0dC\\x00\\xe0dC\\x00 eC\\x00`eC\\x00\\xa0eC\\x00\\xe0eC\\x00 fC\\x00`fC\\x00\\xa0fC\\x00\\xe0fC\\x00 gC\\x00`gC\\x00\\xa0gC\\x00\\xe0gC\\x00 hC\\x00`hC\\x00\\xa0hC\\x00\\xe0hC\\x00 iC\\x00`iC\\x00\\xa0iC\\x00\\xe0iC\\x00 jC\\x00`jC\\x00\\xa0jC\\x00\\xe0jC\\x00 kC\\x00`kC\\x00\\xa0kC\\x00\\xe0kC\\x00 lC\\x00`lC\\x00\\xa0lC\\x00\\xe0lC\\x00 mC\\x00`mC\\x00\\xa0mC\\x00\\xe0mC\\x00 nC\\x00`nC\\x00\\xa0nC\\x00\\xe0nC\\x00 oC\\x00`oC\\x00\\xa0oC\\x00\\xe0oC\\x00 pC\\x00`pC\\x00\\xa0pC\\x00\\xe0pC\\x00 qC\\x00`qC\\x00\\xa0qC\\x00\\xe0qC\\x00 rC\\x00`rC\\x00\\xa0rC\\x00\\xe0rC\\x00 sC\\x00`sC\\x00\\xa0sC\\x00\\xe0sC\\x00 tC\\x00`tC\\x00\\xa0tC\\x00\\xe0tC\\x00 uC\\x00`uC\\x00\\xa0uC\\x00\\xe0uC\\x00 vC\\x00`vC\\x00\\xa0vC\\x00\\xe0vC\\x00 wC\\x00`wC\\x00\\xa0wC\\x00\\xe0wC\\x00 xC\\x00`xC\\x00\\xa0xC\\x00\\xe0xC\\x00 yC\\x00`yC\\x00\\xa0yC\\x00\\xe0yC\\x00 zC\\x00`zC\\x00\\xa0zC\\x00\\xe0zC\\x00 {C\\x00`{C\\x00\\xa0{C\\x00\\xe0{C\\x00 |C\\x00`|C\\x00\\xa0|C\\x00\\xe0|C\\x00 }C\\x00`}C\\x00\\xa0}C\\x00\\xe0}C\\x00 ~C\\x00`~C\\x00\\xa0~C\\x00\\xe0~C\\x00 \\x7fC\\x00`\\x7fC\\x00\\xa0\\x7fC\\x00\\xe0\\x7fC\\x00\\x10\\x80C\\x000\\x80C\\x00P\\x80C\\x00p\\x80C\\x00\\x90\\x80C\\x00\\xb0\\x80C\\x00\\xd0\\x80C\\x00\\xf0\\x80C\\x00\\x10\\x81C\\x000\\x81C\\x00P\\x81C\\x00p\\x81C\\x00\\x90\\x81C\\x00\\xb0\\x81C\\x00\\xd0\\x81C\\x00\\xf0\\x81C\\x00\\x10\\x82C\\x000\\x82C\\x00P\\x82C\\x00p\\x82C\\x00\\x90\\x82C\\x00\\xb0\\x82C\\x00\\xd0\\x82C\\x00\\xf0\\x82C\\x00\\x10\\x83C\\x000\\x83C\\x00P\\x83C\\x00p\\x83C\\x00\\x90\\x83C\\x00\\xb0\\x83C\\x00\\xd0\\x83C\\x00\\xf0\\x83C\\x00\\x10\\x84C\\x000\\x84C\\x00P\\x84C\\x00p\\x84C\\x00\\x90\\x84C\\x00\\xb0\\x84C\\x00\\xd0\\x84C\\x00\\xf0\\x84C\\x00\\x10\\x85C\\x000\\x85C\\x00P\\x85C\\x00p\\x85C\\x00\\x90\\x85C\\x00\\xb0\\x85C\\x00\\xd0\\x85C\\x00\\xf0\\x85C\\x00\\x10\\x86C\\x000\\x86C\\x00P\\x86C\\x00p\\x86C\\x00\\x90\\x86C\\x00\\xb0\\x86C\\x00\\xd0\\x86C\\x00\\xf0\\x86C\\x00\\x10\\x87C\\x000\\x87C\\x00P\\x87C\\x00p\\x87C\\x00\\x90\\x87C\\x00\\xb0\\x87C\\x00\\xd0\\x87C\\x00\\xf0\\x87C\\x00\\x10\\x88C\\x000\\x88C\\x00P\\x88C\\x00p\\x88C\\x00\\x90\\x88C\\x00\\xb0\\x88C\\x00\\xd0\\x88C\\x00\\xf0\\x88C\\x00\\x10\\x89C\\x000\\x89C\\x00P\\x89C\\x00p\\x89C\\x00\\x90\\x89C\\x00\\xb0\\x89C\\x00\\xd0\\x89C\\x00\\xf0\\x89C\\x00\\x10\\x8aC\\x000\\x8aC\\x00P\\x8aC\\x00p\\x8aC\\x00\\x90\\x8aC\\x00\\xb0\\x8aC\\x00\\xd0\\x8aC\\x00\\xf0\\x8aC\\x00\\x10\\x8bC\\x000\\x8bC\\x00P\\x8bC\\x00p\\x8bC\\x00\\x90\\x8bC\\x00\\xb0\\x8bC\\x00\\xd0\\x8bC\\x00\\xf0\\x8bC\\x00\\x10\\x8cC\\x000\\x8cC\\x00P\\x8cC\\x00p\\x8cC\\x00\\x90\\x8cC\\x00\\xb0\\x8cC\\x00\\xd0\\x8cC\\x00\\xf0\\x8cC\\x00\\x10\\x8dC\\x000\\x8dC\\x00P\\x8dC\\x00p\\x8dC\\x00\\x90\\x8dC\\x00\\xb0\\x8dC\\x00\\xd0\\x8dC\\x00\\xf0\\x8dC\\x00\\x10\\x8eC\\x000\\x8eC\\x00P\\x8eC\\x00p\\x8eC\\x00\\x90\\x8eC\\x00\\xb0\\x8eC\\x00\\xd0\\x8eC\\x00\\xf0\\x8eC\\x00\\x10\\x8fC\\x000\\x8fC\\x00P\\x8fC\\x00p\\x8fC\\x00\\x90\\x8fC\\x00\\xb0\\x8fC\\x00\\xd0\\x8fC\\x00\\xf0\\x8fC\\x00\\x10\\x90C\\x000\\x90C\\x00P\\x90C\\x00p\\x90C\\x00\\x90\\x90C\\x00\\xb0\\x90C\\x00\\xd0\\x90C\\x00\\xf0\\x90C\\x00\\x10\\x91C\\x000\\x91C\\x00P\\x91C\\x00p\\x91C\\x00\\x90\\x91C\\x00\\xb0\\x91C\\x00\\xd0\\x91C\\x00\\xf0\\x91C\\x00\\x10\\x92C\\x000\\x92C\\x00P\\x92C\\x00p\\x92C\\x00\\x90\\x92C\\x00\\xb0\\x92C\\x00\\xd0\\x92C\\x00\\xf0\\x92C\\x00\\x10\\x93C\\x000\\x93C\\x00P\\x93C\\x00p\\x93C\\x00\\x90\\x93C\\x00\\xb0\\x93C\\x00\\xd0\\x93C\\x00\\xf0\\x93C\\x00\\x10\\x94C\\x000\\x94C\\x00P\\x94C\\x00p\\x94C\\x00\\x90\\x94C\\x00\\xb0\\x94C\\x00\\xd0\\x94C\\x00\\xf0\\x94C\\x00\\x10\\x95C\\x000\\x95C\\x00P\\x95C\\x00p\\x95C\\x00\\x90\\x95C\\x00\\xb0\\x95C\\x00\\xd0\\x95C\\x00\\xf0\\x95C\\x00\\x10\\x96C\\x000\\x96C\\x00P\\x96C\\x00p\\x96C\\x00\\x90\\x96C\\x00\\xb0\\x96C\\x00\\xd0\\x96C\\x00\\xf0\\x96C\\x00\\x10\\x97C\\x000\\x97C\\x00P\\x97C\\x00p\\x97C\\x00\\x90\\x97C\\x00\\xb0\\x97C\\x00\\xd0\\x97C\\x00\\xf0\\x97C\\x00\\x10\\x98C\\x000\\x98C\\x00P\\x98C\\x00p\\x98C\\x00\\x90\\x98C\\x00\\xb0\\x98C\\x00\\xd0\\x98C\\x00\\xf0\\x98C\\x00\\x10\\x99C\\x000\\x99C\\x00P\\x99C\\x00p\\x99C\\x00\\x90\\x99C\\x00\\xb0\\x99C\\x00\\xd0\\x99C\\x00\\xf0\\x99C\\x00\\x10\\x9aC\\x000\\x9aC\\x00P\\x9aC\\x00p\\x9aC\\x00\\x90\\x9aC\\x00\\xb0\\x9aC\\x00\\xd0\\x9aC\\x00\\xf0\\x9aC\\x00\\x10\\x9bC\\x000\\x9bC\\x00P\\x9bC\\x00p\\x9bC\\x00\\x90\\x9bC\\x00\\xb0\\x9bC\\x00\\xd0\\x9bC\\x00\\xf0\\x9bC\\x00\\x10\\x9cC\\x000\\x9cC\\x00P\\x9cC\\x00p\\x9cC\\x00\\x90\\x9cC\\x00\\xb0\\x9cC\\x00\\xd0\\x9cC\\x00\\xf0\\x9cC\\x00\\x10\\x9dC\\x000\\x9dC\\x00P\\x9dC\\x00p\\x9dC\\x00\\x90\\x9dC\\x00\\xb0\\x9dC\\x00\\xd0\\x9dC\\x00\\xf0\\x9dC\\x00\\x10\\x9eC\\x000\\x9eC\\x00P\\x9eC\\x00p\\x9eC\\x00\\x90\\x9eC\\x00\\xb0\\x9eC\\x00\\xd0\\x9eC\\x00\\xf0\\x9eC\\x00\\x10\\x9fC\\x000\\x9fC\\x00P\\x9fC\\x00p\\x9fC\\x00\\x90\\x9fC\\x00\\xb0\\x9fC\\x00\\xd0\\x9fC\\x00\\xf0\\x9fC\\x00\\x10\\xa0C\\x000\\xa0C\\x00P\\xa0C\\x00p\\xa0C\\x00\\x90\\xa0C\\x00\\xb0\\xa0C\\x00\\xd0\\xa0C\\x00\\xf0\\xa0C\\x00\\x10\\xa1C\\x000\\xa1C\\x00P\\xa1C\\x00p\\xa1C\\x00\\x90\\xa1C\\x00\\xb0\\xa1C\\x00\\xd0\\xa1C\\x00\\xf0\\xa1C\\x00\\x10\\xa2C\\x000\\xa2C\\x00P\\xa2C\\x00p\\xa2C\\x00\\x90\\xa2C\\x00\\xb0\\xa2C\\x00\\xd0\\xa2C\\x00\\xf0\\xa2C\\x00\\x10\\xa3C\\x000\\xa3C\\x00P\\xa3C\\x00p\\xa3C\\x00\\x90\\xa3C\\x00\\xb0\\xa3C\\x00\\xd0\\xa3C\\x00\\xf0\\xa3C\\x00\\x10\\xa4C\\x000\\xa4C\\x00P\\xa4C\\x00p\\xa4C\\x00\\x90\\xa4C\\x00\\xb0\\xa4C\\x00\\xd0\\xa4C\\x00\\xf0\\xa4C\\x00\\x10\\xa5C\\x000\\xa5C\\x00P\\xa5C\\x00p\\xa5C\\x00\\x90\\xa5C\\x00\\xb0\\xa5C\\x00\\xd0\\xa5C\\x00\\xf0\\xa5C\\x00\\x10\\xa6C\\x000\\xa6C\\x00P\\xa6C\\x00p\\xa6C\\x00\\x90\\xa6C\\x00\\xb0\\xa6C\\x00\\xd0\\xa6C\\x00\\xf0\\xa6C\\x00\\x10\\xa7C\\x000\\xa7C\\x00P\\xa7C\\x00p\\xa7C\\x00\\x90\\xa7C\\x00\\xb0\\xa7C\\x00\\xd0\\xa7C\\x00\\xf0\\xa7C\\x00\\x10\\xa8C\\x000\\xa8C\\x00P\\xa8C\\x00p\\xa8C\\x00\\x90\\xa8C\\x00\\xb0\\xa8C\\x00\\xd0\\xa8C\\x00\\xf0\\xa8C\\x00\\x10\\xa9C\\x000\\xa9C\\x00P\\xa9C\\x00p\\xa9C\\x00\\x90\\xa9C\\x00\\xb0\\xa9C\\x00\\xd0\\xa9C\\x00\\xf0\\xa9C\\x00\\x10\\xaaC\\x000\\xaaC\\x00P\\xaaC\\x00p\\xaaC\\x00\\x90\\xaaC\\x00\\xb0\\xaaC\\x00\\xd0\\xaaC\\x00\\xf0\\xaaC\\x00\\x10\\xabC\\x000\\xabC\\x00P\\xabC\\x00p\\xabC\\x00\\x90\\xabC\\x00\\xb0\\xabC\\x00\\xd0\\xabC\\x00\\xf0\\xabC\\x00\\x10\\xacC\\x000\\xacC\\x00P\\xacC\\x00p\\xacC\\x00\\x90\\xacC\\x00\\xb0\\xacC\\x00\\xd0\\xacC\\x00\\xf0\\xacC\\x00\\x10\\xadC\\x000\\xadC\\x00P\\xadC\\x00p\\xadC\\x00\\x90\\xadC\\x00\\xb0\\xadC\\x00\\xd0\\xadC\\x00\\xf0\\xadC\\x00\\x10\\xaeC\\x000\\xaeC\\x00P\\xaeC\\x00p\\xaeC\\x00\\x90\\xaeC\\x00\\xb0\\xaeC\\x00\\xd0\\xaeC\\x00\\xf0\\xaeC\\x00\\x10\\xafC\\x000\\xafC\\x00P\\xafC\\x00p\\xafC\\x00\\x90\\xafC\\x00\\xb0\\xafC\\x00\\xd0\\xafC\\x00\\xf0\\xafC\\x00\\x10\\xb0C\\x000\\xb0C\\x00P\\xb0C\\x00p\\xb0C\\x00\\x90\\xb0C\\x00\\xb0\\xb0C\\x00\\xd0\\xb0C\\x00\\xf0\\xb0C\\x00\\x10\\xb1C\\x000\\xb1C\\x00P\\xb1C\\x00p\\xb1C\\x00\\x90\\xb1C\\x00\\xb0\\xb1C\\x00\\xd0\\xb1C\\x00\\xf0\\xb1C\\x00\\x10\\xb2C\\x000\\xb2C\\x00P\\xb2C\\x00p\\xb2C\\x00\\x90\\xb2C\\x00\\xb0\\xb2C\\x00\\xd0\\xb2C\\x00\\xf0\\xb2C\\x00\\x10\\xb3C\\x000\\xb3C\\x00P\\xb3C\\x00p\\xb3C\\x00\\x90\\xb3C\\x00\\xb0\\xb3C\\x00\\xd0\\xb3C\\x00\\xf0\\xb3C'" ] }, - "execution_count": 21, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -1200,7 +1228,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "id": "47a53027-dbae-48aa-85d2-dcbc04e01e61", "metadata": {}, "outputs": [ @@ -1223,22 +1251,22 @@ "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:175\u001b[0m, in \u001b[0;36mZarrArrayWrapper.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 174\u001b[0m method \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oindex\n\u001b[0;32m--> 175\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexplicit_indexing_adapter\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 176\u001b[0m \u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mIndexingSupport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mVECTORIZED\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\n\u001b[1;32m 177\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:1018\u001b[0m, in \u001b[0;36mexplicit_indexing_adapter\u001b[0;34m(key, shape, indexing_support, raw_indexing_method)\u001b[0m\n\u001b[1;32m 1017\u001b[0m raw_key, numpy_indices \u001b[38;5;241m=\u001b[39m decompose_indexer(key, shape, indexing_support)\n\u001b[0;32m-> 1018\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mraw_indexing_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_key\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtuple\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1019\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m numpy_indices\u001b[38;5;241m.\u001b[39mtuple:\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;66;03m# index the loaded np.ndarray\u001b[39;00m\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:158\u001b[0m, in \u001b[0;36mZarrArrayWrapper._getitem\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_getitem\u001b[39m(\u001b[38;5;28mself\u001b[39m, key):\n\u001b[1;32m 156\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 157\u001b[0m \u001b[38;5;66;03m# try:\u001b[39;00m\n\u001b[0;32m--> 158\u001b[0m item \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_array\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;66;03m# except Exception as e:\u001b[39;00m\n\u001b[1;32m 160\u001b[0m \u001b[38;5;66;03m# import traceback\u001b[39;00m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;66;03m# print(f\"An error occurred: {e}\")\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[38;5;66;03m# print(\"Stack trace:\")\u001b[39;00m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;66;03m# traceback.print_exc() # Prints the full traceback \u001b[39;00m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:1688\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 1687\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_pure_orthogonal_indexing(pure_selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim):\n\u001b[0;32m-> 1688\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_orthogonal_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1689\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:1953\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 1952\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_pure_orthogonal_indexing(pure_selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim):\n\u001b[0;32m-> 1953\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_orthogonal_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1954\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/_compat.py:43\u001b[0m, in \u001b[0;36m_deprecate_positional_args.._inner_deprecate_positional_args..inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m extra_args \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# extra_args > 0\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:2130\u001b[0m, in \u001b[0;36mArray.get_orthogonal_selection\u001b[0;34m(self, selection, out, fields, prototype)\u001b[0m\n\u001b[1;32m 2129\u001b[0m indexer \u001b[38;5;241m=\u001b[39m OrthogonalIndexer(selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mchunk_grid)\n\u001b[0;32m-> 2130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2131\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_async_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2132\u001b[0m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprototype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprototype\u001b[49m\n\u001b[1;32m 2133\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2134\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:2395\u001b[0m, in \u001b[0;36mArray.get_orthogonal_selection\u001b[0;34m(self, selection, out, fields, prototype)\u001b[0m\n\u001b[1;32m 2394\u001b[0m indexer \u001b[38;5;241m=\u001b[39m OrthogonalIndexer(selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mchunk_grid)\n\u001b[0;32m-> 2395\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2396\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_async_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2397\u001b[0m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprototype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprototype\u001b[49m\n\u001b[1;32m 2398\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2399\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:141\u001b[0m, in \u001b[0;36msync\u001b[0;34m(coro, loop, timeout)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 141\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:100\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 100\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:961\u001b[0m, in \u001b[0;36mAsyncArray._get_selection\u001b[0;34m(self, indexer, prototype, out, fields)\u001b[0m\n\u001b[1;32m 959\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m product(indexer\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 960\u001b[0m \u001b[38;5;66;03m# reading chunks and decoding them\u001b[39;00m\n\u001b[0;32m--> 961\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcodec_pipeline\u001b[38;5;241m.\u001b[39mread(\n\u001b[1;32m 962\u001b[0m [\n\u001b[1;32m 963\u001b[0m (\n\u001b[1;32m 964\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_path \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mencode_chunk_key(chunk_coords),\n\u001b[1;32m 965\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget_chunk_spec(chunk_coords, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morder, prototype\u001b[38;5;241m=\u001b[39mprototype),\n\u001b[1;32m 966\u001b[0m chunk_selection,\n\u001b[1;32m 967\u001b[0m out_selection,\n\u001b[1;32m 968\u001b[0m )\n\u001b[1;32m 969\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_coords, chunk_selection, out_selection \u001b[38;5;129;01min\u001b[39;00m indexer\n\u001b[1;32m 970\u001b[0m ],\n\u001b[1;32m 971\u001b[0m out_buffer,\n\u001b[1;32m 972\u001b[0m drop_axes\u001b[38;5;241m=\u001b[39mindexer\u001b[38;5;241m.\u001b[39mdrop_axes,\n\u001b[1;32m 973\u001b[0m )\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_buffer\u001b[38;5;241m.\u001b[39mas_ndarray_like()\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:440\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread\u001b[39m(\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 436\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]],\n\u001b[1;32m 437\u001b[0m out: NDBuffer,\n\u001b[1;32m 438\u001b[0m drop_axes: \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (),\n\u001b[1;32m 439\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 440\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 441\u001b[0m [\n\u001b[1;32m 442\u001b[0m (single_batch_info, out, drop_axes)\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m single_batch_info \u001b[38;5;129;01min\u001b[39;00m batched(batch_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m 444\u001b[0m ],\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_batch,\n\u001b[1;32m 446\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 447\u001b[0m )\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:270\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read_batch\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 262\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 263\u001b[0m [\n\u001b[1;32m 264\u001b[0m (byte_getter, array_spec\u001b[38;5;241m.\u001b[39mprototype)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 268\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 269\u001b[0m )\n\u001b[0;32m--> 270\u001b[0m chunk_array_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode_batch(\n\u001b[1;32m 271\u001b[0m [\n\u001b[1;32m 272\u001b[0m (chunk_bytes, chunk_spec)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_bytes, (_, chunk_spec, _, _) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 274\u001b[0m chunk_bytes_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 275\u001b[0m )\n\u001b[1;32m 276\u001b[0m ],\n\u001b[1;32m 277\u001b[0m )\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_array, (_, chunk_spec, chunk_selection, out_selection) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 279\u001b[0m chunk_array_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 280\u001b[0m ):\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/codecs/pipeline.py:172\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.decode_batch\u001b[0;34m(self, chunk_bytes_and_specs)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m bb_codec, chunk_spec_batch \u001b[38;5;129;01min\u001b[39;00m bb_codecs_with_spec[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 172\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m bb_codec\u001b[38;5;241m.\u001b[39mdecode(\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mzip\u001b[39m(chunk_bytes_batch, chunk_spec_batch, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 174\u001b[0m )\n\u001b[1;32m 176\u001b[0m ab_codec, chunk_spec_batch \u001b[38;5;241m=\u001b[39m ab_codec_with_spec\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:130\u001b[0m, in \u001b[0;36mBaseCodec.decode\u001b[0;34m(self, chunks_and_specs)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decodes a batch of chunks.\u001b[39;00m\n\u001b[1;32m 119\u001b[0m \u001b[38;5;124;03mChunks can be None in which case they are ignored by the codec.\u001b[39;00m\n\u001b[1;32m 120\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124;03mIterable[CodecInput | None]\u001b[39;00m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m _batching_helper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decode_single, chunks_and_specs)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:408\u001b[0m, in \u001b[0;36m_batching_helper\u001b[0;34m(func, batch_info)\u001b[0m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_batching_helper\u001b[39m(\n\u001b[1;32m 405\u001b[0m func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]],\n\u001b[1;32m 406\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[CodecInput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, ArraySpec]],\n\u001b[1;32m 407\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]:\n\u001b[0;32m--> 408\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 409\u001b[0m \u001b[38;5;28mlist\u001b[39m(batch_info),\n\u001b[1;32m 410\u001b[0m _noop_for_none(func),\n\u001b[1;32m 411\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 412\u001b[0m )\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:64\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 64\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:62\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:421\u001b[0m, in \u001b[0;36m_noop_for_none..wrap\u001b[0;34m(chunk, chunk_spec)\u001b[0m\n\u001b[1;32m 420\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 421\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(chunk, chunk_spec)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:1009\u001b[0m, in \u001b[0;36mAsyncArray._get_selection\u001b[0;34m(self, indexer, prototype, out, fields)\u001b[0m\n\u001b[1;32m 1007\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m product(indexer\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 1008\u001b[0m \u001b[38;5;66;03m# reading chunks and decoding them\u001b[39;00m\n\u001b[0;32m-> 1009\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcodec_pipeline\u001b[38;5;241m.\u001b[39mread(\n\u001b[1;32m 1010\u001b[0m [\n\u001b[1;32m 1011\u001b[0m (\n\u001b[1;32m 1012\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_path \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mencode_chunk_key(chunk_coords),\n\u001b[1;32m 1013\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget_chunk_spec(chunk_coords, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morder, prototype\u001b[38;5;241m=\u001b[39mprototype),\n\u001b[1;32m 1014\u001b[0m chunk_selection,\n\u001b[1;32m 1015\u001b[0m out_selection,\n\u001b[1;32m 1016\u001b[0m )\n\u001b[1;32m 1017\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_coords, chunk_selection, out_selection \u001b[38;5;129;01min\u001b[39;00m indexer\n\u001b[1;32m 1018\u001b[0m ],\n\u001b[1;32m 1019\u001b[0m out_buffer,\n\u001b[1;32m 1020\u001b[0m drop_axes\u001b[38;5;241m=\u001b[39mindexer\u001b[38;5;241m.\u001b[39mdrop_axes,\n\u001b[1;32m 1021\u001b[0m )\n\u001b[1;32m 1022\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_buffer\u001b[38;5;241m.\u001b[39mas_ndarray_like()\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py:440\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread\u001b[39m(\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 436\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]],\n\u001b[1;32m 437\u001b[0m out: NDBuffer,\n\u001b[1;32m 438\u001b[0m drop_axes: \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (),\n\u001b[1;32m 439\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 440\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 441\u001b[0m [\n\u001b[1;32m 442\u001b[0m (single_batch_info, out, drop_axes)\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m single_batch_info \u001b[38;5;129;01min\u001b[39;00m batched(batch_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m 444\u001b[0m ],\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_batch,\n\u001b[1;32m 446\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 447\u001b[0m )\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:67\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 67\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:65\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 65\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py:270\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read_batch\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 262\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 263\u001b[0m [\n\u001b[1;32m 264\u001b[0m (byte_getter, array_spec\u001b[38;5;241m.\u001b[39mprototype)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 268\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 269\u001b[0m )\n\u001b[0;32m--> 270\u001b[0m chunk_array_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode_batch(\n\u001b[1;32m 271\u001b[0m [\n\u001b[1;32m 272\u001b[0m (chunk_bytes, chunk_spec)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_bytes, (_, chunk_spec, _, _) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 274\u001b[0m chunk_bytes_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 275\u001b[0m )\n\u001b[1;32m 276\u001b[0m ],\n\u001b[1;32m 277\u001b[0m )\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_array, (_, chunk_spec, chunk_selection, out_selection) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 279\u001b[0m chunk_array_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 280\u001b[0m ):\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py:172\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.decode_batch\u001b[0;34m(self, chunk_bytes_and_specs)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m bb_codec, chunk_spec_batch \u001b[38;5;129;01min\u001b[39;00m bb_codecs_with_spec[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 172\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m bb_codec\u001b[38;5;241m.\u001b[39mdecode(\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mzip\u001b[39m(chunk_bytes_batch, chunk_spec_batch, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 174\u001b[0m )\n\u001b[1;32m 176\u001b[0m ab_codec, chunk_spec_batch \u001b[38;5;241m=\u001b[39m ab_codec_with_spec\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:129\u001b[0m, in \u001b[0;36mBaseCodec.decode\u001b[0;34m(self, chunks_and_specs)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decodes a batch of chunks.\u001b[39;00m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;124;03mChunks can be None in which case they are ignored by the codec.\u001b[39;00m\n\u001b[1;32m 119\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[38;5;124;03mIterable[CodecInput | None]\u001b[39;00m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 129\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m _batching_helper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decode_single, chunks_and_specs)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:407\u001b[0m, in \u001b[0;36m_batching_helper\u001b[0;34m(func, batch_info)\u001b[0m\n\u001b[1;32m 403\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_batching_helper\u001b[39m(\n\u001b[1;32m 404\u001b[0m func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]],\n\u001b[1;32m 405\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[CodecInput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, ArraySpec]],\n\u001b[1;32m 406\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]:\n\u001b[0;32m--> 407\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 408\u001b[0m \u001b[38;5;28mlist\u001b[39m(batch_info),\n\u001b[1;32m 409\u001b[0m _noop_for_none(func),\n\u001b[1;32m 410\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 411\u001b[0m )\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:67\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 67\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:65\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 65\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", + "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:420\u001b[0m, in \u001b[0;36m_noop_for_none..wrap\u001b[0;34m(chunk, chunk_spec)\u001b[0m\n\u001b[1;32m 419\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 420\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(chunk, chunk_spec)\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zarr3.py:125\u001b[0m, in \u001b[0;36m_NumcodecsBytesBytesCodec._decode_single\u001b[0;34m(self, chunk_bytes, chunk_spec)\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_decode_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, chunk_bytes: Buffer, chunk_spec: ArraySpec) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Buffer:\n\u001b[0;32m--> 125\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mto_thread(\n\u001b[1;32m 126\u001b[0m as_numpy_array_wrapper,\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_codec\u001b[38;5;241m.\u001b[39mdecode,\n\u001b[1;32m 128\u001b[0m chunk_bytes,\n\u001b[1;32m 129\u001b[0m chunk_spec\u001b[38;5;241m.\u001b[39mprototype,\n\u001b[1;32m 130\u001b[0m )\n", "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/asyncio/threads.py:25\u001b[0m, in \u001b[0;36mto_thread\u001b[0;34m(func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m func_call \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mpartial(ctx\u001b[38;5;241m.\u001b[39mrun, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m loop\u001b[38;5;241m.\u001b[39mrun_in_executor(\u001b[38;5;28;01mNone\u001b[39;00m, func_call)\n", "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n", @@ -1247,7 +1275,7 @@ "\u001b[0;31merror\u001b[0m: Error -3 while decompressing data: unknown compression method", "\nThe above exception was the direct cause of the following exception:\n", "\u001b[0;31merror\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[22], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_zarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mread_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconsolidated\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mzarr_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", + "Cell \u001b[0;32mIn[23], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_zarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mread_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconsolidated\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mzarr_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1346\u001b[0m, in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, zarr_version, zarr_format, use_zarr_fill_value_as_mask, chunked_array_type, from_array_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1335\u001b[0m backend_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 1336\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msynchronizer\u001b[39m\u001b[38;5;124m\"\u001b[39m: synchronizer,\n\u001b[1;32m 1337\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconsolidated\u001b[39m\u001b[38;5;124m\"\u001b[39m: consolidated,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1343\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_format\u001b[39m\u001b[38;5;124m\"\u001b[39m: zarr_format,\n\u001b[1;32m 1344\u001b[0m }\n\u001b[1;32m 1345\u001b[0m \u001b[38;5;66;03m#import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[0;32m-> 1346\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1347\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1348\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1349\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_cf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_cf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1350\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1351\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1352\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1353\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1354\u001b[0m \u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mzarr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1355\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1356\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1357\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked_array_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked_array_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1358\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1359\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1360\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1361\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1362\u001b[0m \u001b[43m \u001b[49m\u001b[43mzarr_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzarr_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1363\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1364\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1365\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/api.py:671\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 659\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 660\u001b[0m decode_cf,\n\u001b[1;32m 661\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 667\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 668\u001b[0m )\n\u001b[1;32m 670\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 671\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 677\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 678\u001b[0m backend_ds,\n\u001b[1;32m 679\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 689\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 690\u001b[0m )\n\u001b[1;32m 691\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1436\u001b[0m, in \u001b[0;36mZarrBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask)\u001b[0m\n\u001b[1;32m 1434\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 1435\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n\u001b[0;32m-> 1436\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mstore_entrypoint\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1439\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1440\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1441\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1442\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1443\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1444\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1445\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", @@ -1265,7 +1293,7 @@ ], "metadata": { "kernelspec": { - "display_name": "virtualizarr", + "display_name": "venv", "language": "python", "name": "venv" }, From 64f2478083244ad3c3e6d5cbc0b31e768a7aefb3 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 14:20:17 -0800 Subject: [PATCH 71/90] Modify test so it fails without updated icechunk --- .../tests/test_writers/test_icechunk.py | 127 +++++++++++++++--- 1 file changed, 108 insertions(+), 19 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index d10eec21..e9adc1ef 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -14,6 +14,7 @@ from zarr.core.metadata import ArrayV3Metadata # type: ignore from virtualizarr.manifests import ChunkManifest, ManifestArray +from virtualizarr.readers.common import separate_coords from virtualizarr.writers.icechunk import dataset_to_icechunk, generate_chunk_key from virtualizarr.zarr import ZArray @@ -339,12 +340,12 @@ def generate_chunk_manifest( netcdf4_file: str, shape: Tuple[int, ...], chunks: Tuple[int, ...], - base_offset=6144, + offset=6144, length=48, ) -> ChunkManifest: chunk_dict = {} num_chunks = [shape[i] // chunks[i] for i in range(len(shape))] - offset = base_offset + offset = offset # Generate all possible chunk indices using Cartesian product for chunk_indices in product(*[range(n) for n in num_chunks]): @@ -359,7 +360,7 @@ def generate_chunk_manifest( return ChunkManifest(chunk_dict) -def gen_virtual_dataset( +def gen_virtual_variable( file_uri: str, shape: tuple[int, ...] = (3, 4), chunk_shape: tuple[int, ...] = (3, 4), @@ -368,17 +369,17 @@ def gen_virtual_dataset( filters: Optional[list[dict[Any, Any]]] = None, fill_value: Optional[str] = None, encoding: Optional[dict] = None, - variable_name: str = "foo", - base_offset: int = 6144, + offset: int = 6144, length: int = 48, - dims: Optional[list[str]] = None, + dims: list[str] = [], zarr_format: Literal[2, 3] = 2, -): + attrs: dict[str, Any] = {}, +) -> tuple[Variable, Dataset]: manifest = generate_chunk_manifest( file_uri, shape=shape, chunks=chunk_shape, - base_offset=base_offset, + offset=offset, length=length, ) zarray = ZArray( @@ -391,17 +392,51 @@ def gen_virtual_dataset( zarr_format=zarr_format, ) ma = ManifestArray(chunkmanifest=manifest, zarray=zarray) + return Variable( + data=ma, + dims=dims, + encoding=encoding, + attrs=attrs, + ) + + +def gen_virtual_dataset( + file_uri: str, + shape: tuple[int, ...] = (3, 4), + chunk_shape: tuple[int, ...] = (3, 4), + dtype: np.dtype = np.dtype("int32"), + compressor: Optional[dict] = None, + filters: Optional[list[dict[Any, Any]]] = None, + fill_value: Optional[str] = None, + encoding: Optional[dict] = None, + variable_name: str = "foo", + offset: int = 6144, + length: int = 48, + dims: Optional[list[str]] = None, + zarr_format: Literal[2, 3] = 2, + coords: Optional[dict[str, Variable]] = None, +) -> Dataset: ds = open_dataset(file_uri) ds_dims: list[str] = cast(list[str], list(ds.dims)) dims = dims or ds_dims - var = Variable( - data=ma, - dims=dims, + var = gen_virtual_variable( + file_uri, + shape=shape, + chunk_shape=chunk_shape, + dtype=dtype, + compressor=compressor, + filters=filters, + fill_value=fill_value, encoding=encoding, + offset=offset, + length=length, + dims=dims, + zarr_format=zarr_format, attrs=ds[variable_name].attrs, ) return Dataset( {variable_name: var}, + coords=coords, attrs=ds.attrs, ) @@ -443,7 +478,8 @@ def test_append_virtual_ref_without_encoding( xrt.assert_identical(array, expected_array) ## When appending to a virtual ref with encoding, it succeeds - def test_append_virtual_ref_with_encoding( + @pytest.mark.asyncio + async def test_append_virtual_ref_with_encoding( self, icechunk_storage: "StorageConfig", netcdf4_files_factory: Callable ): import xarray.testing as xrt @@ -452,6 +488,52 @@ def test_append_virtual_ref_with_encoding( scale_factor = 0.01 encoding = {"air": {"scale_factor": scale_factor}} filepath1, filepath2 = netcdf4_files_factory(encoding=encoding) + + lon_manifest = gen_virtual_variable( + filepath1, + shape=(53,), + chunk_shape=(53,), + dtype=np.dtype("float32"), + offset=5279, + length=212, + dims=["lon"], + ) + lat_manifest = gen_virtual_variable( + filepath1, + shape=(25,), + chunk_shape=(25,), + dtype=np.dtype("float32"), + offset=5179, + length=100, + dims=["lat"], + ) + time_attrs = { + "standard_name": "time", + "long_name": "Time", + "units": "hours since 1800-01-01", + "calendar": "standard", + } + time_manifest1, time_manifest2 = [ + gen_virtual_variable( + filepath, + shape=(1460,), + chunk_shape=(1460,), + dtype=np.dtype("float32"), + offset=15495515, + length=5840, + dims=["time"], + attrs=time_attrs, + ) + for filepath in [filepath1, filepath2] + ] + [[_, coords1], [_, coords2]] = [ + separate_coords( + vars={"time": time_manifest, "lat": lat_manifest, "lon": lon_manifest}, + indexes={}, + coord_names=[], + ) + for time_manifest in [time_manifest1, time_manifest2] + ] vds1, vds2 = ( gen_virtual_dataset( file_uri=filepath1, @@ -461,8 +543,9 @@ def test_append_virtual_ref_with_encoding( dtype=np.dtype("float64"), variable_name="air", encoding={"scale_factor": scale_factor}, - base_offset=15419, + offset=15419, length=15476000, + coords=coords1, ), gen_virtual_dataset( file_uri=filepath2, @@ -472,8 +555,9 @@ def test_append_virtual_ref_with_encoding( dtype=np.dtype("float64"), variable_name="air", encoding={"scale_factor": scale_factor}, - base_offset=15419, + offset=15419, length=15476000, + coords=coords2, ), ) @@ -483,18 +567,23 @@ def test_append_virtual_ref_with_encoding( icechunk_filestore.commit( "test commit" ) # need to commit it in order to append to it in the next lines + new_ds = open_zarr(icechunk_filestore, consolidated=False, zarr_format=3) + first_time_chunk_before_append = await icechunk_filestore._store.get("time/c/0") # Append the same dataset to the same store icechunk_filestore_append = IcechunkStore.open_existing( storage=icechunk_storage, read_only=False ) dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") + icechunk_filestore_append.commit("appended data") + # chunks = [chunk async for chunk in icechunk_filestore_append._store.list_dir('/')] + assert ( + await icechunk_filestore_append._store.get("time/c/0") + ) == first_time_chunk_before_append new_ds = open_zarr(icechunk_filestore_append, consolidated=False, zarr_format=3) expected_ds1, expected_ds2 = open_dataset(filepath1), open_dataset(filepath2) - expected_ds = concat([expected_ds1, expected_ds2], dim="time").drop_vars( - ["lon", "lat", "time"], errors="ignore" - ) + expected_ds = concat([expected_ds1, expected_ds2], dim="time") # Because we encode attributes, attributes may differ, for example # actual_range for expected_ds.air is array([185.16, 322.1 ], dtype=float32) # but encoded it is [185.16000366210935, 322.1000061035156] @@ -530,7 +619,7 @@ def test_append_with_compression_succeeds( dims=["time", "lat", "lon"], dtype=np.dtype("float64"), variable_name="air", - base_offset=18043, + offset=18043, length=3936114, zarr_format=zarr_format, ), @@ -542,7 +631,7 @@ def test_append_with_compression_succeeds( dims=["time", "lat", "lon"], dtype=np.dtype("float64"), variable_name="air", - base_offset=18043, + offset=18043, length=3938672, zarr_format=zarr_format, ), From 13de8d35d0a208cf92e8219dd6a3683f30bd2622 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 14:20:42 -0800 Subject: [PATCH 72/90] Update icechunk dependency --- ci/upstream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/upstream.yml b/ci/upstream.yml index bc685f9a..440bff7d 100644 --- a/ci/upstream.yml +++ b/ci/upstream.yml @@ -28,6 +28,6 @@ dependencies: - fsspec - pip - pip: - - icechunk==0.1.0a5 # Installs zarr v3 as dependency + - icechunk==0.1.0a6 # Installs zarr v3 as dependency # - git+https://github.com/fsspec/kerchunk@main # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516) - imagecodecs-numcodecs==2024.6.1 From 4a65e5a7682ea12ece4602e7547a0220e7c66e9c Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 14:25:27 -0800 Subject: [PATCH 73/90] Fix mypy errors --- virtualizarr/tests/test_writers/test_icechunk.py | 6 +++--- virtualizarr/writers/icechunk.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index e9adc1ef..eede9ac2 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -8,7 +8,7 @@ import numpy as np import numpy.testing as npt -from xarray import Dataset, concat, open_dataset, open_zarr +from xarray import Coordinates, Dataset, concat, open_dataset, open_zarr from xarray.core.variable import Variable from zarr import Array, Group, group # type: ignore from zarr.core.metadata import ArrayV3Metadata # type: ignore @@ -374,7 +374,7 @@ def gen_virtual_variable( dims: list[str] = [], zarr_format: Literal[2, 3] = 2, attrs: dict[str, Any] = {}, -) -> tuple[Variable, Dataset]: +) -> Variable: manifest = generate_chunk_manifest( file_uri, shape=shape, @@ -414,7 +414,7 @@ def gen_virtual_dataset( length: int = 48, dims: Optional[list[str]] = None, zarr_format: Literal[2, 3] = 2, - coords: Optional[dict[str, Variable]] = None, + coords: Optional[Coordinates] = None, ) -> Dataset: ds = open_dataset(file_uri) ds_dims: list[str] = cast(list[str], list(ds.dims)) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index a097bbb1..4258cb16 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -183,7 +183,7 @@ def write_virtual_variable_to_icechunk( append_axis = get_axis(dims, append_dim) # check if arrays can be concatenated - check_compatible_arrays(ma, group[name], append_axis) + check_compatible_arrays(ma, group[name], append_axis) # type: ignore[arg-type] check_compatible_encodings(var.encoding, group[name].attrs) # determine number of existing chunks along the append axis @@ -194,7 +194,7 @@ def write_virtual_variable_to_icechunk( # resize the array resize_array( - group[name], + group[name], # type: ignore[arg-type] manifest_array=ma, append_axis=append_axis, ) From 64e52775b66e106a6681a2c9502c04ff7fbc2cf6 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 14:26:09 -0800 Subject: [PATCH 74/90] update icechunk version in pyproject --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2478daea..34e479f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ hdf_reader = [ "numcodecs" ] icechunk = [ - "icechunk==0.1.0a5", + "icechunk==0.1.0a6", ] test = [ "codecov", From 9d2f7f8151dfd44cbc0c4c4bb9c921d939b1972d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2024 22:28:05 +0000 Subject: [PATCH 75/90] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- virtualizarr/writers/icechunk.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/virtualizarr/writers/icechunk.py b/virtualizarr/writers/icechunk.py index 1de06723..03c5be57 100644 --- a/virtualizarr/writers/icechunk.py +++ b/virtualizarr/writers/icechunk.py @@ -22,7 +22,9 @@ from zarr import Array, Group # type: ignore -def dataset_to_icechunk(ds: Dataset, store: "IcechunkStore", append_dim: Optional[str] = None) -> None: +def dataset_to_icechunk( + ds: Dataset, store: "IcechunkStore", append_dim: Optional[str] = None +) -> None: """ Write an xarray dataset whose variables wrap ManifestArrays to an Icechunk store. From 53045c3083b9d0d07596ca5e8b7aef4042e1323d Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 15:02:01 -0800 Subject: [PATCH 76/90] Remove obsolete comment --- virtualizarr/tests/test_writers/test_icechunk.py | 1 - 1 file changed, 1 deletion(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index eede9ac2..0eadf3c4 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -576,7 +576,6 @@ async def test_append_virtual_ref_with_encoding( ) dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") icechunk_filestore_append.commit("appended data") - # chunks = [chunk async for chunk in icechunk_filestore_append._store.list_dir('/')] assert ( await icechunk_filestore_append._store.get("time/c/0") ) == first_time_chunk_before_append From b21a0e52cc5b71db9b3ad85166fffc288cbbaa19 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 16:16:20 -0800 Subject: [PATCH 77/90] Use icechunk 0.1.0a7 --- ci/upstream.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/upstream.yml b/ci/upstream.yml index 440bff7d..172d9271 100644 --- a/ci/upstream.yml +++ b/ci/upstream.yml @@ -28,6 +28,6 @@ dependencies: - fsspec - pip - pip: - - icechunk==0.1.0a6 # Installs zarr v3 as dependency + - icechunk==0.1.0a7 # Installs zarr v3 as dependency # - git+https://github.com/fsspec/kerchunk@main # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516) - imagecodecs-numcodecs==2024.6.1 diff --git a/pyproject.toml b/pyproject.toml index f2935e7e..2856e304 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ hdf_reader = [ "numcodecs" ] icechunk = [ - "icechunk==0.1.0a6", + "icechunk==0.1.0a7", ] test = [ "codecov", From 208e83e6249ed5362f8bab1b4b857b42255e8bfc Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 16:30:47 -0800 Subject: [PATCH 78/90] Updated notebook --- noaa-cdr-sst.ipynb | 1090 ++++---------------------------------------- 1 file changed, 79 insertions(+), 1011 deletions(-) diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb index 84241ac9..937969bf 100644 --- a/noaa-cdr-sst.ipynb +++ b/noaa-cdr-sst.ipynb @@ -8,14 +8,13 @@ "outputs": [], "source": [ "# !pip install -e \".[icechunk]\"\n", - "# NOTE: a change has to be made https://github.com/mpiannucci/kerchunk/blob/v3/kerchunk/utils.py#L55 from mode='w' to read_only=False\n", "# !pip install git+https://github.com/mpiannucci/kerchunk@v3\n", "# !pip install fsspec s3fs" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "3055eff4-9e22-4a95-a7fd-96933f381183", "metadata": {}, "outputs": [ @@ -24,7 +23,7 @@ "output_type": "stream", "text": [ "Name: icechunk\n", - "Version: 0.1.0a5\n", + "Version: 0.1.0a7\n", "Summary: Transactional storage engine for Zarr designed for use on cloud object storage\n", "Home-page: https://github.com/earth-mover/icechunk\n", "Author: Earthmover PBC\n", @@ -42,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", "metadata": {}, "outputs": [], @@ -60,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", "metadata": {}, "outputs": [], @@ -76,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "06bbec92-3974-4859-8bda-353afc7800b9", "metadata": {}, "outputs": [], @@ -91,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", "metadata": {}, "outputs": [], @@ -107,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 18, "id": "abefd6fa-386a-4e07-a7c8-219d3730eeeb", "metadata": {}, "outputs": [], @@ -117,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 19, "id": "79a4228a-0e17-4b07-9144-f24fe06db832", "metadata": {}, "outputs": [], @@ -130,60 +129,94 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 20, "id": "5fd0c082-8d5e-46a8-a994-fee80baa4ecc", "metadata": {}, "outputs": [], "source": [ - "store = IcechunkStore.create(storage=storage_config, config=virtual_ref_store_config)" + "store = await IcechunkStore.create(\n", + " storage=storage_config, config=virtual_ref_store_config, read_only=False\n", + ")" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 21, "id": "55ebbc5f-add2-4de8-81f6-5aaf64d9e2b6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", + " store.set_virtual_ref(\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", + " store.set_virtual_ref(\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", + " store.set_virtual_ref(\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", + " store.set_virtual_ref(\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", + " store.set_virtual_ref(\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", + " store.set_virtual_ref(\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", + " store.set_virtual_ref(\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", + " store.set_virtual_ref(\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + } + ], "source": [ "virtual_ds.virtualize.to_icechunk(store)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 16, "id": "cae7e34f-d6dd-42aa-9e7a-f0d5420ba0b9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'JXPBY5KT843ZBSVQ2KNG'" + "'PZMXKYCPJQRRFXV33Q7G'" ] }, - "execution_count": 11, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "store.commit(\"first 2 days of 202408 data\")" + "await store.commit(\"first 2 days of 202408 data\")" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 17, "id": "9387e1ff-46c1-45fd-9796-0457538209a7", "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "b'x^cx\\xd3\\xe2\\x06\\x00\\x04\\x16\\x01\\xb7'" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" + "ename": "KeyNotFound", + "evalue": "time/c/0", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyNotFound\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[17], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m store\u001b[38;5;241m.\u001b[39m_store\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtime/c/0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mKeyNotFound\u001b[0m: time/c/0" + ] } ], "source": [ @@ -192,456 +225,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "b6271bd1-bc0b-4901-9901-91aabe508cf7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset> Size: 66MB\n",
    -       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
    -       "Coordinates:\n",
    -       "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
    -       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
    -       "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
    -       "  * zlev     (zlev) float32 4B 0.0\n",
    -       "Data variables:\n",
    -       "    anom     (time, zlev, lat, lon) float64 17MB ...\n",
    -       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
    -       "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
    -       "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
    -       "Attributes: (12/37)\n",
    -       "    Conventions:                CF-1.6, ACDD-1.3\n",
    -       "    cdm_data_type:              Grid\n",
    -       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    -       "    creator_email:              oisst-help@noaa.gov\n",
    -       "    creator_url:                https://www.ncei.noaa.gov/\n",
    -       "    date_created:               2024-08-16T09:12:00Z\n",
    -       "    ...                         ...\n",
    -       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    -       "    standard_name_vocabulary:   CF Standard Name Table (v40, 25 January 2017)\n",
    -       "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
    -       "    time_coverage_end:          2024-08-01T23:59:59Z\n",
    -       "    time_coverage_start:        2024-08-01T00:00:00Z\n",
    -       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
    " - ], - "text/plain": [ - " Size: 66MB\n", - "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", - " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", - " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", - " * zlev (zlev) float32 4B 0.0\n", - "Data variables:\n", - " anom (time, zlev, lat, lon) float64 17MB ...\n", - " err (time, zlev, lat, lon) float64 17MB ...\n", - " ice (time, zlev, lat, lon) float64 17MB ...\n", - " sst (time, zlev, lat, lon) float64 17MB ...\n", - "Attributes: (12/37)\n", - " Conventions: CF-1.6, ACDD-1.3\n", - " cdm_data_type: Grid\n", - " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", - " creator_email: oisst-help@noaa.gov\n", - " creator_url: https://www.ncei.noaa.gov/\n", - " date_created: 2024-08-16T09:12:00Z\n", - " ... ...\n", - " source: ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n", - " standard_name_vocabulary: CF Standard Name Table (v40, 25 January 2017)\n", - " summary: NOAAs 1/4-degree Daily Optimum Interpolation ...\n", - " time_coverage_end: 2024-08-01T23:59:59Z\n", - " time_coverage_start: 2024-08-01T00:00:00Z\n", - " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ds = xr.open_zarr(store, consolidated=False, zarr_format=3)\n", "ds" @@ -657,7 +244,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "190c25f9-e000-4b17-83eb-cf551141dfea", "metadata": {}, "outputs": [], @@ -672,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "af330082-207a-4f08-aefe-fc15aa8b2eb3", "metadata": {}, "outputs": [], @@ -688,450 +275,17 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "103b44d2-124a-4de5-8074-e997fd5a1698", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset> Size: 17MB\n",
    -       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
    -       "Coordinates:\n",
    -       "    lon      (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n",
    -       "    zlev     (zlev) float32 4B ManifestArray<shape=(1,), dtype=float32, chunk...\n",
    -       "    lat      (lat) float32 3kB ManifestArray<shape=(720,), dtype=float32, chu...\n",
    -       "    time     (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n",
    -       "Data variables:\n",
    -       "    anom     (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "    err      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "    sst      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "    ice      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "Attributes: (12/37)\n",
    -       "    Conventions:                CF-1.6, ACDD-1.3\n",
    -       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...\n",
    -       "    references:                 Reynolds, et al.(2007) Daily High-Resolution-...\n",
    -       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    -       "    id:                         oisst-avhrr-v02r01.20240803.nc\n",
    -       "    naming_authority:           gov.noaa.ncei\n",
    -       "    ...                         ...\n",
    -       "    time_coverage_start:        2024-08-03T00:00:00Z\n",
    -       "    time_coverage_end:          2024-08-03T23:59:59Z\n",
    -       "    metadata_link:              https://doi.org/10.25921/RE9P-PT57\n",
    -       "    ncei_template_version:      NCEI_NetCDF_Grid_Template_v2.0\n",
    -       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    -       "    sensor:                     Thermometer, AVHRR
    " - ], - "text/plain": [ - " Size: 17MB\n", - "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", - "Coordinates:\n", - " lon (lon) float32 6kB ManifestArray\\x00\\x00\\xc0>\\x00\\x00 ?\\x00\\x00`?\\x00\\x00\\x90?\\x00\\x00\\xb0?\\x00\\x00\\xd0?\\x00\\x00\\xf0?\\x00\\x00\\x08@\\x00\\x00\\x18@\\x00\\x00(@\\x00\\x008@\\x00\\x00H@\\x00\\x00X@\\x00\\x00h@\\x00\\x00x@\\x00\\x00\\x84@\\x00\\x00\\x8c@\\x00\\x00\\x94@\\x00\\x00\\x9c@\\x00\\x00\\xa4@\\x00\\x00\\xac@\\x00\\x00\\xb4@\\x00\\x00\\xbc@\\x00\\x00\\xc4@\\x00\\x00\\xcc@\\x00\\x00\\xd4@\\x00\\x00\\xdc@\\x00\\x00\\xe4@\\x00\\x00\\xec@\\x00\\x00\\xf4@\\x00\\x00\\xfc@\\x00\\x00\\x02A\\x00\\x00\\x06A\\x00\\x00\\nA\\x00\\x00\\x0eA\\x00\\x00\\x12A\\x00\\x00\\x16A\\x00\\x00\\x1aA\\x00\\x00\\x1eA\\x00\\x00\"A\\x00\\x00&A\\x00\\x00*A\\x00\\x00.A\\x00\\x002A\\x00\\x006A\\x00\\x00:A\\x00\\x00>A\\x00\\x00BA\\x00\\x00FA\\x00\\x00JA\\x00\\x00NA\\x00\\x00RA\\x00\\x00VA\\x00\\x00ZA\\x00\\x00^A\\x00\\x00bA\\x00\\x00fA\\x00\\x00jA\\x00\\x00nA\\x00\\x00rA\\x00\\x00vA\\x00\\x00zA\\x00\\x00~A\\x00\\x00\\x81A\\x00\\x00\\x83A\\x00\\x00\\x85A\\x00\\x00\\x87A\\x00\\x00\\x89A\\x00\\x00\\x8bA\\x00\\x00\\x8dA\\x00\\x00\\x8fA\\x00\\x00\\x91A\\x00\\x00\\x93A\\x00\\x00\\x95A\\x00\\x00\\x97A\\x00\\x00\\x99A\\x00\\x00\\x9bA\\x00\\x00\\x9dA\\x00\\x00\\x9fA\\x00\\x00\\xa1A\\x00\\x00\\xa3A\\x00\\x00\\xa5A\\x00\\x00\\xa7A\\x00\\x00\\xa9A\\x00\\x00\\xabA\\x00\\x00\\xadA\\x00\\x00\\xafA\\x00\\x00\\xb1A\\x00\\x00\\xb3A\\x00\\x00\\xb5A\\x00\\x00\\xb7A\\x00\\x00\\xb9A\\x00\\x00\\xbbA\\x00\\x00\\xbdA\\x00\\x00\\xbfA\\x00\\x00\\xc1A\\x00\\x00\\xc3A\\x00\\x00\\xc5A\\x00\\x00\\xc7A\\x00\\x00\\xc9A\\x00\\x00\\xcbA\\x00\\x00\\xcdA\\x00\\x00\\xcfA\\x00\\x00\\xd1A\\x00\\x00\\xd3A\\x00\\x00\\xd5A\\x00\\x00\\xd7A\\x00\\x00\\xd9A\\x00\\x00\\xdbA\\x00\\x00\\xddA\\x00\\x00\\xdfA\\x00\\x00\\xe1A\\x00\\x00\\xe3A\\x00\\x00\\xe5A\\x00\\x00\\xe7A\\x00\\x00\\xe9A\\x00\\x00\\xebA\\x00\\x00\\xedA\\x00\\x00\\xefA\\x00\\x00\\xf1A\\x00\\x00\\xf3A\\x00\\x00\\xf5A\\x00\\x00\\xf7A\\x00\\x00\\xf9A\\x00\\x00\\xfbA\\x00\\x00\\xfdA\\x00\\x00\\xffA\\x00\\x80\\x00B\\x00\\x80\\x01B\\x00\\x80\\x02B\\x00\\x80\\x03B\\x00\\x80\\x04B\\x00\\x80\\x05B\\x00\\x80\\x06B\\x00\\x80\\x07B\\x00\\x80\\x08B\\x00\\x80\\tB\\x00\\x80\\nB\\x00\\x80\\x0bB\\x00\\x80\\x0cB\\x00\\x80\\rB\\x00\\x80\\x0eB\\x00\\x80\\x0fB\\x00\\x80\\x10B\\x00\\x80\\x11B\\x00\\x80\\x12B\\x00\\x80\\x13B\\x00\\x80\\x14B\\x00\\x80\\x15B\\x00\\x80\\x16B\\x00\\x80\\x17B\\x00\\x80\\x18B\\x00\\x80\\x19B\\x00\\x80\\x1aB\\x00\\x80\\x1bB\\x00\\x80\\x1cB\\x00\\x80\\x1dB\\x00\\x80\\x1eB\\x00\\x80\\x1fB\\x00\\x80 B\\x00\\x80!B\\x00\\x80\"B\\x00\\x80#B\\x00\\x80$B\\x00\\x80%B\\x00\\x80&B\\x00\\x80\\'B\\x00\\x80(B\\x00\\x80)B\\x00\\x80*B\\x00\\x80+B\\x00\\x80,B\\x00\\x80-B\\x00\\x80.B\\x00\\x80/B\\x00\\x800B\\x00\\x801B\\x00\\x802B\\x00\\x803B\\x00\\x804B\\x00\\x805B\\x00\\x806B\\x00\\x807B\\x00\\x808B\\x00\\x809B\\x00\\x80:B\\x00\\x80;B\\x00\\x80B\\x00\\x80?B\\x00\\x80@B\\x00\\x80AB\\x00\\x80BB\\x00\\x80CB\\x00\\x80DB\\x00\\x80EB\\x00\\x80FB\\x00\\x80GB\\x00\\x80HB\\x00\\x80IB\\x00\\x80JB\\x00\\x80KB\\x00\\x80LB\\x00\\x80MB\\x00\\x80NB\\x00\\x80OB\\x00\\x80PB\\x00\\x80QB\\x00\\x80RB\\x00\\x80SB\\x00\\x80TB\\x00\\x80UB\\x00\\x80VB\\x00\\x80WB\\x00\\x80XB\\x00\\x80YB\\x00\\x80ZB\\x00\\x80[B\\x00\\x80\\\\B\\x00\\x80]B\\x00\\x80^B\\x00\\x80_B\\x00\\x80`B\\x00\\x80aB\\x00\\x80bB\\x00\\x80cB\\x00\\x80dB\\x00\\x80eB\\x00\\x80fB\\x00\\x80gB\\x00\\x80hB\\x00\\x80iB\\x00\\x80jB\\x00\\x80kB\\x00\\x80lB\\x00\\x80mB\\x00\\x80nB\\x00\\x80oB\\x00\\x80pB\\x00\\x80qB\\x00\\x80rB\\x00\\x80sB\\x00\\x80tB\\x00\\x80uB\\x00\\x80vB\\x00\\x80wB\\x00\\x80xB\\x00\\x80yB\\x00\\x80zB\\x00\\x80{B\\x00\\x80|B\\x00\\x80}B\\x00\\x80~B\\x00\\x80\\x7fB\\x00@\\x80B\\x00\\xc0\\x80B\\x00@\\x81B\\x00\\xc0\\x81B\\x00@\\x82B\\x00\\xc0\\x82B\\x00@\\x83B\\x00\\xc0\\x83B\\x00@\\x84B\\x00\\xc0\\x84B\\x00@\\x85B\\x00\\xc0\\x85B\\x00@\\x86B\\x00\\xc0\\x86B\\x00@\\x87B\\x00\\xc0\\x87B\\x00@\\x88B\\x00\\xc0\\x88B\\x00@\\x89B\\x00\\xc0\\x89B\\x00@\\x8aB\\x00\\xc0\\x8aB\\x00@\\x8bB\\x00\\xc0\\x8bB\\x00@\\x8cB\\x00\\xc0\\x8cB\\x00@\\x8dB\\x00\\xc0\\x8dB\\x00@\\x8eB\\x00\\xc0\\x8eB\\x00@\\x8fB\\x00\\xc0\\x8fB\\x00@\\x90B\\x00\\xc0\\x90B\\x00@\\x91B\\x00\\xc0\\x91B\\x00@\\x92B\\x00\\xc0\\x92B\\x00@\\x93B\\x00\\xc0\\x93B\\x00@\\x94B\\x00\\xc0\\x94B\\x00@\\x95B\\x00\\xc0\\x95B\\x00@\\x96B\\x00\\xc0\\x96B\\x00@\\x97B\\x00\\xc0\\x97B\\x00@\\x98B\\x00\\xc0\\x98B\\x00@\\x99B\\x00\\xc0\\x99B\\x00@\\x9aB\\x00\\xc0\\x9aB\\x00@\\x9bB\\x00\\xc0\\x9bB\\x00@\\x9cB\\x00\\xc0\\x9cB\\x00@\\x9dB\\x00\\xc0\\x9dB\\x00@\\x9eB\\x00\\xc0\\x9eB\\x00@\\x9fB\\x00\\xc0\\x9fB\\x00@\\xa0B\\x00\\xc0\\xa0B\\x00@\\xa1B\\x00\\xc0\\xa1B\\x00@\\xa2B\\x00\\xc0\\xa2B\\x00@\\xa3B\\x00\\xc0\\xa3B\\x00@\\xa4B\\x00\\xc0\\xa4B\\x00@\\xa5B\\x00\\xc0\\xa5B\\x00@\\xa6B\\x00\\xc0\\xa6B\\x00@\\xa7B\\x00\\xc0\\xa7B\\x00@\\xa8B\\x00\\xc0\\xa8B\\x00@\\xa9B\\x00\\xc0\\xa9B\\x00@\\xaaB\\x00\\xc0\\xaaB\\x00@\\xabB\\x00\\xc0\\xabB\\x00@\\xacB\\x00\\xc0\\xacB\\x00@\\xadB\\x00\\xc0\\xadB\\x00@\\xaeB\\x00\\xc0\\xaeB\\x00@\\xafB\\x00\\xc0\\xafB\\x00@\\xb0B\\x00\\xc0\\xb0B\\x00@\\xb1B\\x00\\xc0\\xb1B\\x00@\\xb2B\\x00\\xc0\\xb2B\\x00@\\xb3B\\x00\\xc0\\xb3B\\x00@\\xb4B\\x00\\xc0\\xb4B\\x00@\\xb5B\\x00\\xc0\\xb5B\\x00@\\xb6B\\x00\\xc0\\xb6B\\x00@\\xb7B\\x00\\xc0\\xb7B\\x00@\\xb8B\\x00\\xc0\\xb8B\\x00@\\xb9B\\x00\\xc0\\xb9B\\x00@\\xbaB\\x00\\xc0\\xbaB\\x00@\\xbbB\\x00\\xc0\\xbbB\\x00@\\xbcB\\x00\\xc0\\xbcB\\x00@\\xbdB\\x00\\xc0\\xbdB\\x00@\\xbeB\\x00\\xc0\\xbeB\\x00@\\xbfB\\x00\\xc0\\xbfB\\x00@\\xc0B\\x00\\xc0\\xc0B\\x00@\\xc1B\\x00\\xc0\\xc1B\\x00@\\xc2B\\x00\\xc0\\xc2B\\x00@\\xc3B\\x00\\xc0\\xc3B\\x00@\\xc4B\\x00\\xc0\\xc4B\\x00@\\xc5B\\x00\\xc0\\xc5B\\x00@\\xc6B\\x00\\xc0\\xc6B\\x00@\\xc7B\\x00\\xc0\\xc7B\\x00@\\xc8B\\x00\\xc0\\xc8B\\x00@\\xc9B\\x00\\xc0\\xc9B\\x00@\\xcaB\\x00\\xc0\\xcaB\\x00@\\xcbB\\x00\\xc0\\xcbB\\x00@\\xccB\\x00\\xc0\\xccB\\x00@\\xcdB\\x00\\xc0\\xcdB\\x00@\\xceB\\x00\\xc0\\xceB\\x00@\\xcfB\\x00\\xc0\\xcfB\\x00@\\xd0B\\x00\\xc0\\xd0B\\x00@\\xd1B\\x00\\xc0\\xd1B\\x00@\\xd2B\\x00\\xc0\\xd2B\\x00@\\xd3B\\x00\\xc0\\xd3B\\x00@\\xd4B\\x00\\xc0\\xd4B\\x00@\\xd5B\\x00\\xc0\\xd5B\\x00@\\xd6B\\x00\\xc0\\xd6B\\x00@\\xd7B\\x00\\xc0\\xd7B\\x00@\\xd8B\\x00\\xc0\\xd8B\\x00@\\xd9B\\x00\\xc0\\xd9B\\x00@\\xdaB\\x00\\xc0\\xdaB\\x00@\\xdbB\\x00\\xc0\\xdbB\\x00@\\xdcB\\x00\\xc0\\xdcB\\x00@\\xddB\\x00\\xc0\\xddB\\x00@\\xdeB\\x00\\xc0\\xdeB\\x00@\\xdfB\\x00\\xc0\\xdfB\\x00@\\xe0B\\x00\\xc0\\xe0B\\x00@\\xe1B\\x00\\xc0\\xe1B\\x00@\\xe2B\\x00\\xc0\\xe2B\\x00@\\xe3B\\x00\\xc0\\xe3B\\x00@\\xe4B\\x00\\xc0\\xe4B\\x00@\\xe5B\\x00\\xc0\\xe5B\\x00@\\xe6B\\x00\\xc0\\xe6B\\x00@\\xe7B\\x00\\xc0\\xe7B\\x00@\\xe8B\\x00\\xc0\\xe8B\\x00@\\xe9B\\x00\\xc0\\xe9B\\x00@\\xeaB\\x00\\xc0\\xeaB\\x00@\\xebB\\x00\\xc0\\xebB\\x00@\\xecB\\x00\\xc0\\xecB\\x00@\\xedB\\x00\\xc0\\xedB\\x00@\\xeeB\\x00\\xc0\\xeeB\\x00@\\xefB\\x00\\xc0\\xefB\\x00@\\xf0B\\x00\\xc0\\xf0B\\x00@\\xf1B\\x00\\xc0\\xf1B\\x00@\\xf2B\\x00\\xc0\\xf2B\\x00@\\xf3B\\x00\\xc0\\xf3B\\x00@\\xf4B\\x00\\xc0\\xf4B\\x00@\\xf5B\\x00\\xc0\\xf5B\\x00@\\xf6B\\x00\\xc0\\xf6B\\x00@\\xf7B\\x00\\xc0\\xf7B\\x00@\\xf8B\\x00\\xc0\\xf8B\\x00@\\xf9B\\x00\\xc0\\xf9B\\x00@\\xfaB\\x00\\xc0\\xfaB\\x00@\\xfbB\\x00\\xc0\\xfbB\\x00@\\xfcB\\x00\\xc0\\xfcB\\x00@\\xfdB\\x00\\xc0\\xfdB\\x00@\\xfeB\\x00\\xc0\\xfeB\\x00@\\xffB\\x00\\xc0\\xffB\\x00 \\x00C\\x00`\\x00C\\x00\\xa0\\x00C\\x00\\xe0\\x00C\\x00 \\x01C\\x00`\\x01C\\x00\\xa0\\x01C\\x00\\xe0\\x01C\\x00 \\x02C\\x00`\\x02C\\x00\\xa0\\x02C\\x00\\xe0\\x02C\\x00 \\x03C\\x00`\\x03C\\x00\\xa0\\x03C\\x00\\xe0\\x03C\\x00 \\x04C\\x00`\\x04C\\x00\\xa0\\x04C\\x00\\xe0\\x04C\\x00 \\x05C\\x00`\\x05C\\x00\\xa0\\x05C\\x00\\xe0\\x05C\\x00 \\x06C\\x00`\\x06C\\x00\\xa0\\x06C\\x00\\xe0\\x06C\\x00 \\x07C\\x00`\\x07C\\x00\\xa0\\x07C\\x00\\xe0\\x07C\\x00 \\x08C\\x00`\\x08C\\x00\\xa0\\x08C\\x00\\xe0\\x08C\\x00 \\tC\\x00`\\tC\\x00\\xa0\\tC\\x00\\xe0\\tC\\x00 \\nC\\x00`\\nC\\x00\\xa0\\nC\\x00\\xe0\\nC\\x00 \\x0bC\\x00`\\x0bC\\x00\\xa0\\x0bC\\x00\\xe0\\x0bC\\x00 \\x0cC\\x00`\\x0cC\\x00\\xa0\\x0cC\\x00\\xe0\\x0cC\\x00 \\rC\\x00`\\rC\\x00\\xa0\\rC\\x00\\xe0\\rC\\x00 \\x0eC\\x00`\\x0eC\\x00\\xa0\\x0eC\\x00\\xe0\\x0eC\\x00 \\x0fC\\x00`\\x0fC\\x00\\xa0\\x0fC\\x00\\xe0\\x0fC\\x00 \\x10C\\x00`\\x10C\\x00\\xa0\\x10C\\x00\\xe0\\x10C\\x00 \\x11C\\x00`\\x11C\\x00\\xa0\\x11C\\x00\\xe0\\x11C\\x00 \\x12C\\x00`\\x12C\\x00\\xa0\\x12C\\x00\\xe0\\x12C\\x00 \\x13C\\x00`\\x13C\\x00\\xa0\\x13C\\x00\\xe0\\x13C\\x00 \\x14C\\x00`\\x14C\\x00\\xa0\\x14C\\x00\\xe0\\x14C\\x00 \\x15C\\x00`\\x15C\\x00\\xa0\\x15C\\x00\\xe0\\x15C\\x00 \\x16C\\x00`\\x16C\\x00\\xa0\\x16C\\x00\\xe0\\x16C\\x00 \\x17C\\x00`\\x17C\\x00\\xa0\\x17C\\x00\\xe0\\x17C\\x00 \\x18C\\x00`\\x18C\\x00\\xa0\\x18C\\x00\\xe0\\x18C\\x00 \\x19C\\x00`\\x19C\\x00\\xa0\\x19C\\x00\\xe0\\x19C\\x00 \\x1aC\\x00`\\x1aC\\x00\\xa0\\x1aC\\x00\\xe0\\x1aC\\x00 \\x1bC\\x00`\\x1bC\\x00\\xa0\\x1bC\\x00\\xe0\\x1bC\\x00 \\x1cC\\x00`\\x1cC\\x00\\xa0\\x1cC\\x00\\xe0\\x1cC\\x00 \\x1dC\\x00`\\x1dC\\x00\\xa0\\x1dC\\x00\\xe0\\x1dC\\x00 \\x1eC\\x00`\\x1eC\\x00\\xa0\\x1eC\\x00\\xe0\\x1eC\\x00 \\x1fC\\x00`\\x1fC\\x00\\xa0\\x1fC\\x00\\xe0\\x1fC\\x00 C\\x00` C\\x00\\xa0 C\\x00\\xe0 C\\x00 !C\\x00`!C\\x00\\xa0!C\\x00\\xe0!C\\x00 \"C\\x00`\"C\\x00\\xa0\"C\\x00\\xe0\"C\\x00 #C\\x00`#C\\x00\\xa0#C\\x00\\xe0#C\\x00 $C\\x00`$C\\x00\\xa0$C\\x00\\xe0$C\\x00 %C\\x00`%C\\x00\\xa0%C\\x00\\xe0%C\\x00 &C\\x00`&C\\x00\\xa0&C\\x00\\xe0&C\\x00 \\'C\\x00`\\'C\\x00\\xa0\\'C\\x00\\xe0\\'C\\x00 (C\\x00`(C\\x00\\xa0(C\\x00\\xe0(C\\x00 )C\\x00`)C\\x00\\xa0)C\\x00\\xe0)C\\x00 *C\\x00`*C\\x00\\xa0*C\\x00\\xe0*C\\x00 +C\\x00`+C\\x00\\xa0+C\\x00\\xe0+C\\x00 ,C\\x00`,C\\x00\\xa0,C\\x00\\xe0,C\\x00 -C\\x00`-C\\x00\\xa0-C\\x00\\xe0-C\\x00 .C\\x00`.C\\x00\\xa0.C\\x00\\xe0.C\\x00 /C\\x00`/C\\x00\\xa0/C\\x00\\xe0/C\\x00 0C\\x00`0C\\x00\\xa00C\\x00\\xe00C\\x00 1C\\x00`1C\\x00\\xa01C\\x00\\xe01C\\x00 2C\\x00`2C\\x00\\xa02C\\x00\\xe02C\\x00 3C\\x00`3C\\x00\\xa03C\\x00\\xe03C\\x00 4C\\x00`4C\\x00\\xa04C\\x00\\xe04C\\x00 5C\\x00`5C\\x00\\xa05C\\x00\\xe05C\\x00 6C\\x00`6C\\x00\\xa06C\\x00\\xe06C\\x00 7C\\x00`7C\\x00\\xa07C\\x00\\xe07C\\x00 8C\\x00`8C\\x00\\xa08C\\x00\\xe08C\\x00 9C\\x00`9C\\x00\\xa09C\\x00\\xe09C\\x00 :C\\x00`:C\\x00\\xa0:C\\x00\\xe0:C\\x00 ;C\\x00`;C\\x00\\xa0;C\\x00\\xe0;C\\x00 C\\x00`>C\\x00\\xa0>C\\x00\\xe0>C\\x00 ?C\\x00`?C\\x00\\xa0?C\\x00\\xe0?C\\x00 @C\\x00`@C\\x00\\xa0@C\\x00\\xe0@C\\x00 AC\\x00`AC\\x00\\xa0AC\\x00\\xe0AC\\x00 BC\\x00`BC\\x00\\xa0BC\\x00\\xe0BC\\x00 CC\\x00`CC\\x00\\xa0CC\\x00\\xe0CC\\x00 DC\\x00`DC\\x00\\xa0DC\\x00\\xe0DC\\x00 EC\\x00`EC\\x00\\xa0EC\\x00\\xe0EC\\x00 FC\\x00`FC\\x00\\xa0FC\\x00\\xe0FC\\x00 GC\\x00`GC\\x00\\xa0GC\\x00\\xe0GC\\x00 HC\\x00`HC\\x00\\xa0HC\\x00\\xe0HC\\x00 IC\\x00`IC\\x00\\xa0IC\\x00\\xe0IC\\x00 JC\\x00`JC\\x00\\xa0JC\\x00\\xe0JC\\x00 KC\\x00`KC\\x00\\xa0KC\\x00\\xe0KC\\x00 LC\\x00`LC\\x00\\xa0LC\\x00\\xe0LC\\x00 MC\\x00`MC\\x00\\xa0MC\\x00\\xe0MC\\x00 NC\\x00`NC\\x00\\xa0NC\\x00\\xe0NC\\x00 OC\\x00`OC\\x00\\xa0OC\\x00\\xe0OC\\x00 PC\\x00`PC\\x00\\xa0PC\\x00\\xe0PC\\x00 QC\\x00`QC\\x00\\xa0QC\\x00\\xe0QC\\x00 RC\\x00`RC\\x00\\xa0RC\\x00\\xe0RC\\x00 SC\\x00`SC\\x00\\xa0SC\\x00\\xe0SC\\x00 TC\\x00`TC\\x00\\xa0TC\\x00\\xe0TC\\x00 UC\\x00`UC\\x00\\xa0UC\\x00\\xe0UC\\x00 VC\\x00`VC\\x00\\xa0VC\\x00\\xe0VC\\x00 WC\\x00`WC\\x00\\xa0WC\\x00\\xe0WC\\x00 XC\\x00`XC\\x00\\xa0XC\\x00\\xe0XC\\x00 YC\\x00`YC\\x00\\xa0YC\\x00\\xe0YC\\x00 ZC\\x00`ZC\\x00\\xa0ZC\\x00\\xe0ZC\\x00 [C\\x00`[C\\x00\\xa0[C\\x00\\xe0[C\\x00 \\\\C\\x00`\\\\C\\x00\\xa0\\\\C\\x00\\xe0\\\\C\\x00 ]C\\x00`]C\\x00\\xa0]C\\x00\\xe0]C\\x00 ^C\\x00`^C\\x00\\xa0^C\\x00\\xe0^C\\x00 _C\\x00`_C\\x00\\xa0_C\\x00\\xe0_C\\x00 `C\\x00``C\\x00\\xa0`C\\x00\\xe0`C\\x00 aC\\x00`aC\\x00\\xa0aC\\x00\\xe0aC\\x00 bC\\x00`bC\\x00\\xa0bC\\x00\\xe0bC\\x00 cC\\x00`cC\\x00\\xa0cC\\x00\\xe0cC\\x00 dC\\x00`dC\\x00\\xa0dC\\x00\\xe0dC\\x00 eC\\x00`eC\\x00\\xa0eC\\x00\\xe0eC\\x00 fC\\x00`fC\\x00\\xa0fC\\x00\\xe0fC\\x00 gC\\x00`gC\\x00\\xa0gC\\x00\\xe0gC\\x00 hC\\x00`hC\\x00\\xa0hC\\x00\\xe0hC\\x00 iC\\x00`iC\\x00\\xa0iC\\x00\\xe0iC\\x00 jC\\x00`jC\\x00\\xa0jC\\x00\\xe0jC\\x00 kC\\x00`kC\\x00\\xa0kC\\x00\\xe0kC\\x00 lC\\x00`lC\\x00\\xa0lC\\x00\\xe0lC\\x00 mC\\x00`mC\\x00\\xa0mC\\x00\\xe0mC\\x00 nC\\x00`nC\\x00\\xa0nC\\x00\\xe0nC\\x00 oC\\x00`oC\\x00\\xa0oC\\x00\\xe0oC\\x00 pC\\x00`pC\\x00\\xa0pC\\x00\\xe0pC\\x00 qC\\x00`qC\\x00\\xa0qC\\x00\\xe0qC\\x00 rC\\x00`rC\\x00\\xa0rC\\x00\\xe0rC\\x00 sC\\x00`sC\\x00\\xa0sC\\x00\\xe0sC\\x00 tC\\x00`tC\\x00\\xa0tC\\x00\\xe0tC\\x00 uC\\x00`uC\\x00\\xa0uC\\x00\\xe0uC\\x00 vC\\x00`vC\\x00\\xa0vC\\x00\\xe0vC\\x00 wC\\x00`wC\\x00\\xa0wC\\x00\\xe0wC\\x00 xC\\x00`xC\\x00\\xa0xC\\x00\\xe0xC\\x00 yC\\x00`yC\\x00\\xa0yC\\x00\\xe0yC\\x00 zC\\x00`zC\\x00\\xa0zC\\x00\\xe0zC\\x00 {C\\x00`{C\\x00\\xa0{C\\x00\\xe0{C\\x00 |C\\x00`|C\\x00\\xa0|C\\x00\\xe0|C\\x00 }C\\x00`}C\\x00\\xa0}C\\x00\\xe0}C\\x00 ~C\\x00`~C\\x00\\xa0~C\\x00\\xe0~C\\x00 \\x7fC\\x00`\\x7fC\\x00\\xa0\\x7fC\\x00\\xe0\\x7fC\\x00\\x10\\x80C\\x000\\x80C\\x00P\\x80C\\x00p\\x80C\\x00\\x90\\x80C\\x00\\xb0\\x80C\\x00\\xd0\\x80C\\x00\\xf0\\x80C\\x00\\x10\\x81C\\x000\\x81C\\x00P\\x81C\\x00p\\x81C\\x00\\x90\\x81C\\x00\\xb0\\x81C\\x00\\xd0\\x81C\\x00\\xf0\\x81C\\x00\\x10\\x82C\\x000\\x82C\\x00P\\x82C\\x00p\\x82C\\x00\\x90\\x82C\\x00\\xb0\\x82C\\x00\\xd0\\x82C\\x00\\xf0\\x82C\\x00\\x10\\x83C\\x000\\x83C\\x00P\\x83C\\x00p\\x83C\\x00\\x90\\x83C\\x00\\xb0\\x83C\\x00\\xd0\\x83C\\x00\\xf0\\x83C\\x00\\x10\\x84C\\x000\\x84C\\x00P\\x84C\\x00p\\x84C\\x00\\x90\\x84C\\x00\\xb0\\x84C\\x00\\xd0\\x84C\\x00\\xf0\\x84C\\x00\\x10\\x85C\\x000\\x85C\\x00P\\x85C\\x00p\\x85C\\x00\\x90\\x85C\\x00\\xb0\\x85C\\x00\\xd0\\x85C\\x00\\xf0\\x85C\\x00\\x10\\x86C\\x000\\x86C\\x00P\\x86C\\x00p\\x86C\\x00\\x90\\x86C\\x00\\xb0\\x86C\\x00\\xd0\\x86C\\x00\\xf0\\x86C\\x00\\x10\\x87C\\x000\\x87C\\x00P\\x87C\\x00p\\x87C\\x00\\x90\\x87C\\x00\\xb0\\x87C\\x00\\xd0\\x87C\\x00\\xf0\\x87C\\x00\\x10\\x88C\\x000\\x88C\\x00P\\x88C\\x00p\\x88C\\x00\\x90\\x88C\\x00\\xb0\\x88C\\x00\\xd0\\x88C\\x00\\xf0\\x88C\\x00\\x10\\x89C\\x000\\x89C\\x00P\\x89C\\x00p\\x89C\\x00\\x90\\x89C\\x00\\xb0\\x89C\\x00\\xd0\\x89C\\x00\\xf0\\x89C\\x00\\x10\\x8aC\\x000\\x8aC\\x00P\\x8aC\\x00p\\x8aC\\x00\\x90\\x8aC\\x00\\xb0\\x8aC\\x00\\xd0\\x8aC\\x00\\xf0\\x8aC\\x00\\x10\\x8bC\\x000\\x8bC\\x00P\\x8bC\\x00p\\x8bC\\x00\\x90\\x8bC\\x00\\xb0\\x8bC\\x00\\xd0\\x8bC\\x00\\xf0\\x8bC\\x00\\x10\\x8cC\\x000\\x8cC\\x00P\\x8cC\\x00p\\x8cC\\x00\\x90\\x8cC\\x00\\xb0\\x8cC\\x00\\xd0\\x8cC\\x00\\xf0\\x8cC\\x00\\x10\\x8dC\\x000\\x8dC\\x00P\\x8dC\\x00p\\x8dC\\x00\\x90\\x8dC\\x00\\xb0\\x8dC\\x00\\xd0\\x8dC\\x00\\xf0\\x8dC\\x00\\x10\\x8eC\\x000\\x8eC\\x00P\\x8eC\\x00p\\x8eC\\x00\\x90\\x8eC\\x00\\xb0\\x8eC\\x00\\xd0\\x8eC\\x00\\xf0\\x8eC\\x00\\x10\\x8fC\\x000\\x8fC\\x00P\\x8fC\\x00p\\x8fC\\x00\\x90\\x8fC\\x00\\xb0\\x8fC\\x00\\xd0\\x8fC\\x00\\xf0\\x8fC\\x00\\x10\\x90C\\x000\\x90C\\x00P\\x90C\\x00p\\x90C\\x00\\x90\\x90C\\x00\\xb0\\x90C\\x00\\xd0\\x90C\\x00\\xf0\\x90C\\x00\\x10\\x91C\\x000\\x91C\\x00P\\x91C\\x00p\\x91C\\x00\\x90\\x91C\\x00\\xb0\\x91C\\x00\\xd0\\x91C\\x00\\xf0\\x91C\\x00\\x10\\x92C\\x000\\x92C\\x00P\\x92C\\x00p\\x92C\\x00\\x90\\x92C\\x00\\xb0\\x92C\\x00\\xd0\\x92C\\x00\\xf0\\x92C\\x00\\x10\\x93C\\x000\\x93C\\x00P\\x93C\\x00p\\x93C\\x00\\x90\\x93C\\x00\\xb0\\x93C\\x00\\xd0\\x93C\\x00\\xf0\\x93C\\x00\\x10\\x94C\\x000\\x94C\\x00P\\x94C\\x00p\\x94C\\x00\\x90\\x94C\\x00\\xb0\\x94C\\x00\\xd0\\x94C\\x00\\xf0\\x94C\\x00\\x10\\x95C\\x000\\x95C\\x00P\\x95C\\x00p\\x95C\\x00\\x90\\x95C\\x00\\xb0\\x95C\\x00\\xd0\\x95C\\x00\\xf0\\x95C\\x00\\x10\\x96C\\x000\\x96C\\x00P\\x96C\\x00p\\x96C\\x00\\x90\\x96C\\x00\\xb0\\x96C\\x00\\xd0\\x96C\\x00\\xf0\\x96C\\x00\\x10\\x97C\\x000\\x97C\\x00P\\x97C\\x00p\\x97C\\x00\\x90\\x97C\\x00\\xb0\\x97C\\x00\\xd0\\x97C\\x00\\xf0\\x97C\\x00\\x10\\x98C\\x000\\x98C\\x00P\\x98C\\x00p\\x98C\\x00\\x90\\x98C\\x00\\xb0\\x98C\\x00\\xd0\\x98C\\x00\\xf0\\x98C\\x00\\x10\\x99C\\x000\\x99C\\x00P\\x99C\\x00p\\x99C\\x00\\x90\\x99C\\x00\\xb0\\x99C\\x00\\xd0\\x99C\\x00\\xf0\\x99C\\x00\\x10\\x9aC\\x000\\x9aC\\x00P\\x9aC\\x00p\\x9aC\\x00\\x90\\x9aC\\x00\\xb0\\x9aC\\x00\\xd0\\x9aC\\x00\\xf0\\x9aC\\x00\\x10\\x9bC\\x000\\x9bC\\x00P\\x9bC\\x00p\\x9bC\\x00\\x90\\x9bC\\x00\\xb0\\x9bC\\x00\\xd0\\x9bC\\x00\\xf0\\x9bC\\x00\\x10\\x9cC\\x000\\x9cC\\x00P\\x9cC\\x00p\\x9cC\\x00\\x90\\x9cC\\x00\\xb0\\x9cC\\x00\\xd0\\x9cC\\x00\\xf0\\x9cC\\x00\\x10\\x9dC\\x000\\x9dC\\x00P\\x9dC\\x00p\\x9dC\\x00\\x90\\x9dC\\x00\\xb0\\x9dC\\x00\\xd0\\x9dC\\x00\\xf0\\x9dC\\x00\\x10\\x9eC\\x000\\x9eC\\x00P\\x9eC\\x00p\\x9eC\\x00\\x90\\x9eC\\x00\\xb0\\x9eC\\x00\\xd0\\x9eC\\x00\\xf0\\x9eC\\x00\\x10\\x9fC\\x000\\x9fC\\x00P\\x9fC\\x00p\\x9fC\\x00\\x90\\x9fC\\x00\\xb0\\x9fC\\x00\\xd0\\x9fC\\x00\\xf0\\x9fC\\x00\\x10\\xa0C\\x000\\xa0C\\x00P\\xa0C\\x00p\\xa0C\\x00\\x90\\xa0C\\x00\\xb0\\xa0C\\x00\\xd0\\xa0C\\x00\\xf0\\xa0C\\x00\\x10\\xa1C\\x000\\xa1C\\x00P\\xa1C\\x00p\\xa1C\\x00\\x90\\xa1C\\x00\\xb0\\xa1C\\x00\\xd0\\xa1C\\x00\\xf0\\xa1C\\x00\\x10\\xa2C\\x000\\xa2C\\x00P\\xa2C\\x00p\\xa2C\\x00\\x90\\xa2C\\x00\\xb0\\xa2C\\x00\\xd0\\xa2C\\x00\\xf0\\xa2C\\x00\\x10\\xa3C\\x000\\xa3C\\x00P\\xa3C\\x00p\\xa3C\\x00\\x90\\xa3C\\x00\\xb0\\xa3C\\x00\\xd0\\xa3C\\x00\\xf0\\xa3C\\x00\\x10\\xa4C\\x000\\xa4C\\x00P\\xa4C\\x00p\\xa4C\\x00\\x90\\xa4C\\x00\\xb0\\xa4C\\x00\\xd0\\xa4C\\x00\\xf0\\xa4C\\x00\\x10\\xa5C\\x000\\xa5C\\x00P\\xa5C\\x00p\\xa5C\\x00\\x90\\xa5C\\x00\\xb0\\xa5C\\x00\\xd0\\xa5C\\x00\\xf0\\xa5C\\x00\\x10\\xa6C\\x000\\xa6C\\x00P\\xa6C\\x00p\\xa6C\\x00\\x90\\xa6C\\x00\\xb0\\xa6C\\x00\\xd0\\xa6C\\x00\\xf0\\xa6C\\x00\\x10\\xa7C\\x000\\xa7C\\x00P\\xa7C\\x00p\\xa7C\\x00\\x90\\xa7C\\x00\\xb0\\xa7C\\x00\\xd0\\xa7C\\x00\\xf0\\xa7C\\x00\\x10\\xa8C\\x000\\xa8C\\x00P\\xa8C\\x00p\\xa8C\\x00\\x90\\xa8C\\x00\\xb0\\xa8C\\x00\\xd0\\xa8C\\x00\\xf0\\xa8C\\x00\\x10\\xa9C\\x000\\xa9C\\x00P\\xa9C\\x00p\\xa9C\\x00\\x90\\xa9C\\x00\\xb0\\xa9C\\x00\\xd0\\xa9C\\x00\\xf0\\xa9C\\x00\\x10\\xaaC\\x000\\xaaC\\x00P\\xaaC\\x00p\\xaaC\\x00\\x90\\xaaC\\x00\\xb0\\xaaC\\x00\\xd0\\xaaC\\x00\\xf0\\xaaC\\x00\\x10\\xabC\\x000\\xabC\\x00P\\xabC\\x00p\\xabC\\x00\\x90\\xabC\\x00\\xb0\\xabC\\x00\\xd0\\xabC\\x00\\xf0\\xabC\\x00\\x10\\xacC\\x000\\xacC\\x00P\\xacC\\x00p\\xacC\\x00\\x90\\xacC\\x00\\xb0\\xacC\\x00\\xd0\\xacC\\x00\\xf0\\xacC\\x00\\x10\\xadC\\x000\\xadC\\x00P\\xadC\\x00p\\xadC\\x00\\x90\\xadC\\x00\\xb0\\xadC\\x00\\xd0\\xadC\\x00\\xf0\\xadC\\x00\\x10\\xaeC\\x000\\xaeC\\x00P\\xaeC\\x00p\\xaeC\\x00\\x90\\xaeC\\x00\\xb0\\xaeC\\x00\\xd0\\xaeC\\x00\\xf0\\xaeC\\x00\\x10\\xafC\\x000\\xafC\\x00P\\xafC\\x00p\\xafC\\x00\\x90\\xafC\\x00\\xb0\\xafC\\x00\\xd0\\xafC\\x00\\xf0\\xafC\\x00\\x10\\xb0C\\x000\\xb0C\\x00P\\xb0C\\x00p\\xb0C\\x00\\x90\\xb0C\\x00\\xb0\\xb0C\\x00\\xd0\\xb0C\\x00\\xf0\\xb0C\\x00\\x10\\xb1C\\x000\\xb1C\\x00P\\xb1C\\x00p\\xb1C\\x00\\x90\\xb1C\\x00\\xb0\\xb1C\\x00\\xd0\\xb1C\\x00\\xf0\\xb1C\\x00\\x10\\xb2C\\x000\\xb2C\\x00P\\xb2C\\x00p\\xb2C\\x00\\x90\\xb2C\\x00\\xb0\\xb2C\\x00\\xd0\\xb2C\\x00\\xf0\\xb2C\\x00\\x10\\xb3C\\x000\\xb3C\\x00P\\xb3C\\x00p\\xb3C\\x00\\x90\\xb3C\\x00\\xb0\\xb3C\\x00\\xd0\\xb3C\\x00\\xf0\\xb3C'" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "await read_store._store.get(\"time/c/0\")" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "47a53027-dbae-48aa-85d2-dcbc04e01e61", "metadata": {}, - "outputs": [ - { - "ename": "error", - "evalue": "Failed to decode variable 'time': Error -3 while decompressing data: unknown compression method", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31merror\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:451\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 450\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 451\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m \u001b[43mdecode_cf_variable\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 452\u001b[0m \u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 453\u001b[0m \u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 454\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 455\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 456\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 457\u001b[0m \u001b[43m \u001b[49m\u001b[43mstack_char_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstack_char_dim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 458\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 459\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_item_or_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 460\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:292\u001b[0m, in \u001b[0;36mdecode_cf_variable\u001b[0;34m(name, var, concat_characters, mask_and_scale, decode_times, decode_endianness, stack_char_dim, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_times:\n\u001b[0;32m--> 292\u001b[0m var \u001b[38;5;241m=\u001b[39m \u001b[43mtimes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mCFDatetimeCoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 294\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_endianness \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39misnative:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/coding/times.py:1001\u001b[0m, in \u001b[0;36mCFDatetimeCoder.decode\u001b[0;34m(self, variable, name)\u001b[0m\n\u001b[1;32m 1000\u001b[0m calendar \u001b[38;5;241m=\u001b[39m pop_to(attrs, encoding, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcalendar\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1001\u001b[0m dtype \u001b[38;5;241m=\u001b[39m \u001b[43m_decode_cf_datetime_dtype\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munits\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcalendar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1002\u001b[0m transform \u001b[38;5;241m=\u001b[39m partial(\n\u001b[1;32m 1003\u001b[0m decode_cf_datetime,\n\u001b[1;32m 1004\u001b[0m units\u001b[38;5;241m=\u001b[39munits,\n\u001b[1;32m 1005\u001b[0m calendar\u001b[38;5;241m=\u001b[39mcalendar,\n\u001b[1;32m 1006\u001b[0m use_cftime\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_cftime,\n\u001b[1;32m 1007\u001b[0m )\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/coding/times.py:214\u001b[0m, in \u001b[0;36m_decode_cf_datetime_dtype\u001b[0;34m(data, units, calendar, use_cftime)\u001b[0m\n\u001b[1;32m 212\u001b[0m values \u001b[38;5;241m=\u001b[39m indexing\u001b[38;5;241m.\u001b[39mImplicitToExplicitIndexingAdapter(indexing\u001b[38;5;241m.\u001b[39mas_indexable(data))\n\u001b[1;32m 213\u001b[0m example_value \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate(\n\u001b[0;32m--> 214\u001b[0m [\u001b[43mfirst_n_items\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m], last_item(values) \u001b[38;5;129;01mor\u001b[39;00m [\u001b[38;5;241m0\u001b[39m]]\n\u001b[1;32m 215\u001b[0m )\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/formatting.py:97\u001b[0m, in \u001b[0;36mfirst_n_items\u001b[0;34m(array, n_desired)\u001b[0m\n\u001b[1;32m 96\u001b[0m array \u001b[38;5;241m=\u001b[39m array\u001b[38;5;241m.\u001b[39m_data\n\u001b[0;32m---> 97\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39mravel(\u001b[43mto_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43marray\u001b[49m\u001b[43m)\u001b[49m)[:n_desired]\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/namedarray/pycompat.py:138\u001b[0m, in \u001b[0;36mto_duck_array\u001b[0;34m(data, **kwargs)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 138\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43masarray\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:578\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.__array__\u001b[0;34m(self, dtype, copy)\u001b[0m\n\u001b[1;32m 577\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m Version(np\u001b[38;5;241m.\u001b[39m__version__) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m Version(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.0.0\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 578\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m np\u001b[38;5;241m.\u001b[39masarray(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, dtype\u001b[38;5;241m=\u001b[39mdtype, copy\u001b[38;5;241m=\u001b[39mcopy)\n\u001b[1;32m 579\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:583\u001b[0m, in \u001b[0;36mImplicitToExplicitIndexingAdapter.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_duck_array\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 583\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_duck_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:657\u001b[0m, in \u001b[0;36mLazilyIndexedArray.get_duck_array\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 654\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 655\u001b[0m \u001b[38;5;66;03m# If the array is not an ExplicitlyIndexedNDArrayMixin,\u001b[39;00m\n\u001b[1;32m 656\u001b[0m \u001b[38;5;66;03m# it may wrap a BackendArray so use its __getitem__\u001b[39;00m\n\u001b[0;32m--> 657\u001b[0m array \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 659\u001b[0m \u001b[38;5;66;03m# self.array[self.key] is now a numpy array when\u001b[39;00m\n\u001b[1;32m 660\u001b[0m \u001b[38;5;66;03m# self.array is a BackendArray subclass\u001b[39;00m\n\u001b[1;32m 661\u001b[0m \u001b[38;5;66;03m# and self.key is BasicIndexer((slice(None, None, None),))\u001b[39;00m\n\u001b[1;32m 662\u001b[0m \u001b[38;5;66;03m# so we need the explicit check for ExplicitlyIndexed\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:175\u001b[0m, in \u001b[0;36mZarrArrayWrapper.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 174\u001b[0m method \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oindex\n\u001b[0;32m--> 175\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexplicit_indexing_adapter\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 176\u001b[0m \u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marray\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexing\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mIndexingSupport\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mVECTORIZED\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\n\u001b[1;32m 177\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/core/indexing.py:1018\u001b[0m, in \u001b[0;36mexplicit_indexing_adapter\u001b[0;34m(key, shape, indexing_support, raw_indexing_method)\u001b[0m\n\u001b[1;32m 1017\u001b[0m raw_key, numpy_indices \u001b[38;5;241m=\u001b[39m decompose_indexer(key, shape, indexing_support)\n\u001b[0;32m-> 1018\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mraw_indexing_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_key\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtuple\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1019\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m numpy_indices\u001b[38;5;241m.\u001b[39mtuple:\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;66;03m# index the loaded np.ndarray\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:158\u001b[0m, in \u001b[0;36mZarrArrayWrapper._getitem\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_getitem\u001b[39m(\u001b[38;5;28mself\u001b[39m, key):\n\u001b[1;32m 156\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 157\u001b[0m \u001b[38;5;66;03m# try:\u001b[39;00m\n\u001b[0;32m--> 158\u001b[0m item \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_array\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;66;03m# except Exception as e:\u001b[39;00m\n\u001b[1;32m 160\u001b[0m \u001b[38;5;66;03m# import traceback\u001b[39;00m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;66;03m# print(f\"An error occurred: {e}\")\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[38;5;66;03m# print(\"Stack trace:\")\u001b[39;00m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;66;03m# traceback.print_exc() # Prints the full traceback \u001b[39;00m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:1953\u001b[0m, in \u001b[0;36mArray.__getitem__\u001b[0;34m(self, selection)\u001b[0m\n\u001b[1;32m 1952\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_pure_orthogonal_indexing(pure_selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim):\n\u001b[0;32m-> 1953\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_orthogonal_selection\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpure_selection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1954\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/_compat.py:43\u001b[0m, in \u001b[0;36m_deprecate_positional_args.._inner_deprecate_positional_args..inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m extra_args \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;66;03m# extra_args > 0\u001b[39;00m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:2395\u001b[0m, in \u001b[0;36mArray.get_orthogonal_selection\u001b[0;34m(self, selection, out, fields, prototype)\u001b[0m\n\u001b[1;32m 2394\u001b[0m indexer \u001b[38;5;241m=\u001b[39m OrthogonalIndexer(selection, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mchunk_grid)\n\u001b[0;32m-> 2395\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2396\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_async_array\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_selection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2397\u001b[0m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfields\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfields\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprototype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprototype\u001b[49m\n\u001b[1;32m 2398\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2399\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:141\u001b[0m, in \u001b[0;36msync\u001b[0;34m(coro, loop, timeout)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m--> 141\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/sync.py:100\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 100\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/array.py:1009\u001b[0m, in \u001b[0;36mAsyncArray._get_selection\u001b[0;34m(self, indexer, prototype, out, fields)\u001b[0m\n\u001b[1;32m 1007\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m product(indexer\u001b[38;5;241m.\u001b[39mshape) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 1008\u001b[0m \u001b[38;5;66;03m# reading chunks and decoding them\u001b[39;00m\n\u001b[0;32m-> 1009\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcodec_pipeline\u001b[38;5;241m.\u001b[39mread(\n\u001b[1;32m 1010\u001b[0m [\n\u001b[1;32m 1011\u001b[0m (\n\u001b[1;32m 1012\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore_path \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mencode_chunk_key(chunk_coords),\n\u001b[1;32m 1013\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget_chunk_spec(chunk_coords, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39morder, prototype\u001b[38;5;241m=\u001b[39mprototype),\n\u001b[1;32m 1014\u001b[0m chunk_selection,\n\u001b[1;32m 1015\u001b[0m out_selection,\n\u001b[1;32m 1016\u001b[0m )\n\u001b[1;32m 1017\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_coords, chunk_selection, out_selection \u001b[38;5;129;01min\u001b[39;00m indexer\n\u001b[1;32m 1018\u001b[0m ],\n\u001b[1;32m 1019\u001b[0m out_buffer,\n\u001b[1;32m 1020\u001b[0m drop_axes\u001b[38;5;241m=\u001b[39mindexer\u001b[38;5;241m.\u001b[39mdrop_axes,\n\u001b[1;32m 1021\u001b[0m )\n\u001b[1;32m 1022\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m out_buffer\u001b[38;5;241m.\u001b[39mas_ndarray_like()\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py:440\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 434\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mread\u001b[39m(\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 436\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[ByteGetter, ArraySpec, SelectorTuple, SelectorTuple]],\n\u001b[1;32m 437\u001b[0m out: NDBuffer,\n\u001b[1;32m 438\u001b[0m drop_axes: \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mint\u001b[39m, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m=\u001b[39m (),\n\u001b[1;32m 439\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 440\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 441\u001b[0m [\n\u001b[1;32m 442\u001b[0m (single_batch_info, out, drop_axes)\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m single_batch_info \u001b[38;5;129;01min\u001b[39;00m batched(batch_info, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m 444\u001b[0m ],\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mread_batch,\n\u001b[1;32m 446\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 447\u001b[0m )\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:67\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 67\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:65\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 65\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py:270\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.read_batch\u001b[0;34m(self, batch_info, out, drop_axes)\u001b[0m\n\u001b[1;32m 262\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 263\u001b[0m [\n\u001b[1;32m 264\u001b[0m (byte_getter, array_spec\u001b[38;5;241m.\u001b[39mprototype)\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 268\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 269\u001b[0m )\n\u001b[0;32m--> 270\u001b[0m chunk_array_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecode_batch(\n\u001b[1;32m 271\u001b[0m [\n\u001b[1;32m 272\u001b[0m (chunk_bytes, chunk_spec)\n\u001b[1;32m 273\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_bytes, (_, chunk_spec, _, _) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 274\u001b[0m chunk_bytes_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 275\u001b[0m )\n\u001b[1;32m 276\u001b[0m ],\n\u001b[1;32m 277\u001b[0m )\n\u001b[1;32m 278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk_array, (_, chunk_spec, chunk_selection, out_selection) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\n\u001b[1;32m 279\u001b[0m chunk_array_batch, batch_info, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 280\u001b[0m ):\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py:172\u001b[0m, in \u001b[0;36mBatchedCodecPipeline.decode_batch\u001b[0;34m(self, chunk_bytes_and_specs)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m bb_codec, chunk_spec_batch \u001b[38;5;129;01min\u001b[39;00m bb_codecs_with_spec[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 172\u001b[0m chunk_bytes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m bb_codec\u001b[38;5;241m.\u001b[39mdecode(\n\u001b[1;32m 173\u001b[0m \u001b[38;5;28mzip\u001b[39m(chunk_bytes_batch, chunk_spec_batch, strict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m 174\u001b[0m )\n\u001b[1;32m 176\u001b[0m ab_codec, chunk_spec_batch \u001b[38;5;241m=\u001b[39m ab_codec_with_spec\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:129\u001b[0m, in \u001b[0;36mBaseCodec.decode\u001b[0;34m(self, chunks_and_specs)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Decodes a batch of chunks.\u001b[39;00m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;124;03mChunks can be None in which case they are ignored by the codec.\u001b[39;00m\n\u001b[1;32m 119\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[38;5;124;03mIterable[CodecInput | None]\u001b[39;00m\n\u001b[1;32m 128\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 129\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m _batching_helper(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_decode_single, chunks_and_specs)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:407\u001b[0m, in \u001b[0;36m_batching_helper\u001b[0;34m(func, batch_info)\u001b[0m\n\u001b[1;32m 403\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_batching_helper\u001b[39m(\n\u001b[1;32m 404\u001b[0m func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]],\n\u001b[1;32m 405\u001b[0m batch_info: Iterable[\u001b[38;5;28mtuple\u001b[39m[CodecInput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, ArraySpec]],\n\u001b[1;32m 406\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mlist\u001b[39m[CodecOutput \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m]:\n\u001b[0;32m--> 407\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m concurrent_map(\n\u001b[1;32m 408\u001b[0m \u001b[38;5;28mlist\u001b[39m(batch_info),\n\u001b[1;32m 409\u001b[0m _noop_for_none(func),\n\u001b[1;32m 410\u001b[0m config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masync.concurrency\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[1;32m 411\u001b[0m )\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:67\u001b[0m, in \u001b[0;36mconcurrent_map\u001b[0;34m(items, func, limit)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n\u001b[0;32m---> 67\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mgather(\u001b[38;5;241m*\u001b[39m[asyncio\u001b[38;5;241m.\u001b[39mensure_future(run(item)) \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m items])\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/common.py:65\u001b[0m, in \u001b[0;36mconcurrent_map..run\u001b[0;34m(item)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m sem:\n\u001b[0;32m---> 65\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(\u001b[38;5;241m*\u001b[39mitem)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/abc/codec.py:420\u001b[0m, in \u001b[0;36m_noop_for_none..wrap\u001b[0;34m(chunk, chunk_spec)\u001b[0m\n\u001b[1;32m 419\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 420\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m func(chunk, chunk_spec)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zarr3.py:125\u001b[0m, in \u001b[0;36m_NumcodecsBytesBytesCodec._decode_single\u001b[0;34m(self, chunk_bytes, chunk_spec)\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_decode_single\u001b[39m(\u001b[38;5;28mself\u001b[39m, chunk_bytes: Buffer, chunk_spec: ArraySpec) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Buffer:\n\u001b[0;32m--> 125\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mto_thread(\n\u001b[1;32m 126\u001b[0m as_numpy_array_wrapper,\n\u001b[1;32m 127\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_codec\u001b[38;5;241m.\u001b[39mdecode,\n\u001b[1;32m 128\u001b[0m chunk_bytes,\n\u001b[1;32m 129\u001b[0m chunk_spec\u001b[38;5;241m.\u001b[39mprototype,\n\u001b[1;32m 130\u001b[0m )\n", - "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/asyncio/threads.py:25\u001b[0m, in \u001b[0;36mto_thread\u001b[0;34m(func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m func_call \u001b[38;5;241m=\u001b[39m functools\u001b[38;5;241m.\u001b[39mpartial(ctx\u001b[38;5;241m.\u001b[39mrun, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m---> 25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m loop\u001b[38;5;241m.\u001b[39mrun_in_executor(\u001b[38;5;28;01mNone\u001b[39;00m, func_call)\n", - "File \u001b[0;32m/usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/zarr/core/buffer/cpu.py:213\u001b[0m, in \u001b[0;36mas_numpy_array_wrapper\u001b[0;34m(func, buf, prototype)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Converts the input of `func` to a numpy array and the output back to `Buffer`.\u001b[39;00m\n\u001b[1;32m 192\u001b[0m \n\u001b[1;32m 193\u001b[0m \u001b[38;5;124;03mThis function is useful when calling a `func` that only support host memory such\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[38;5;124;03m The result of `func` converted to a `Buffer`\u001b[39;00m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 213\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m prototype\u001b[38;5;241m.\u001b[39mbuffer\u001b[38;5;241m.\u001b[39mfrom_bytes(\u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_numpy_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/numcodecs/zlib.py:37\u001b[0m, in \u001b[0;36mZlib.decode\u001b[0;34m(self, buf, out)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# do decompression\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m dec \u001b[38;5;241m=\u001b[39m \u001b[43m_zlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecompress\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;66;03m# handle destination - Python standard library zlib module does not\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;66;03m# support direct decompression into buffer, so we have to copy into\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# out if given\u001b[39;00m\n", - "\u001b[0;31merror\u001b[0m: Error -3 while decompressing data: unknown compression method", - "\nThe above exception was the direct cause of the following exception:\n", - "\u001b[0;31merror\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[23], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mxr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_zarr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mread_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconsolidated\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mzarr_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m ds\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1346\u001b[0m, in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, zarr_version, zarr_format, use_zarr_fill_value_as_mask, chunked_array_type, from_array_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 1335\u001b[0m backend_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 1336\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msynchronizer\u001b[39m\u001b[38;5;124m\"\u001b[39m: synchronizer,\n\u001b[1;32m 1337\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconsolidated\u001b[39m\u001b[38;5;124m\"\u001b[39m: consolidated,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1343\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_format\u001b[39m\u001b[38;5;124m\"\u001b[39m: zarr_format,\n\u001b[1;32m 1344\u001b[0m }\n\u001b[1;32m 1345\u001b[0m \u001b[38;5;66;03m#import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[0;32m-> 1346\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1347\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1348\u001b[0m \u001b[43m \u001b[49m\u001b[43mgroup\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1349\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_cf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_cf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1350\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1351\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1352\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1353\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1354\u001b[0m \u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mzarr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1355\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1356\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1357\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked_array_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked_array_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1358\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_array_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1359\u001b[0m \u001b[43m \u001b[49m\u001b[43mbackend_kwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbackend_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1360\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1361\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1362\u001b[0m \u001b[43m \u001b[49m\u001b[43mzarr_version\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzarr_version\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1363\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_zarr_fill_value_as_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1364\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1365\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/api.py:671\u001b[0m, in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)\u001b[0m\n\u001b[1;32m 659\u001b[0m decoders \u001b[38;5;241m=\u001b[39m _resolve_decoders_kwargs(\n\u001b[1;32m 660\u001b[0m decode_cf,\n\u001b[1;32m 661\u001b[0m open_backend_dataset_parameters\u001b[38;5;241m=\u001b[39mbackend\u001b[38;5;241m.\u001b[39mopen_dataset_parameters,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 667\u001b[0m decode_coords\u001b[38;5;241m=\u001b[39mdecode_coords,\n\u001b[1;32m 668\u001b[0m )\n\u001b[1;32m 670\u001b[0m overwrite_encoded_chunks \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverwrite_encoded_chunks\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m--> 671\u001b[0m backend_ds \u001b[38;5;241m=\u001b[39m \u001b[43mbackend\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename_or_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdecoders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 677\u001b[0m ds \u001b[38;5;241m=\u001b[39m _dataset_from_backend_dataset(\n\u001b[1;32m 678\u001b[0m backend_ds,\n\u001b[1;32m 679\u001b[0m filename_or_obj,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 689\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m 690\u001b[0m )\n\u001b[1;32m 691\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/zarr.py:1436\u001b[0m, in \u001b[0;36mZarrBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, chunk_store, storage_options, stacklevel, zarr_version, zarr_format, store, engine, use_zarr_fill_value_as_mask)\u001b[0m\n\u001b[1;32m 1434\u001b[0m store_entrypoint \u001b[38;5;241m=\u001b[39m StoreBackendEntrypoint()\n\u001b[1;32m 1435\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m close_on_error(store):\n\u001b[0;32m-> 1436\u001b[0m ds \u001b[38;5;241m=\u001b[39m \u001b[43mstore_entrypoint\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_dataset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1439\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1440\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1441\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1442\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1443\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1444\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1445\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ds\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/backends/store.py:46\u001b[0m, in \u001b[0;36mStoreBackendEntrypoint.open_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mload()\n\u001b[1;32m 44\u001b[0m encoding \u001b[38;5;241m=\u001b[39m filename_or_obj\u001b[38;5;241m.\u001b[39mget_encoding()\n\u001b[0;32m---> 46\u001b[0m \u001b[38;5;28mvars\u001b[39m, attrs, coord_names \u001b[38;5;241m=\u001b[39m \u001b[43mconventions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_cf_variables\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 47\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mvars\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 48\u001b[0m \u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 49\u001b[0m \u001b[43m \u001b[49m\u001b[43mmask_and_scale\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmask_and_scale\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 50\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_times\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_times\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 51\u001b[0m \u001b[43m \u001b[49m\u001b[43mconcat_characters\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcat_characters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 52\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_coords\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_coords\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 53\u001b[0m \u001b[43m \u001b[49m\u001b[43mdrop_variables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdrop_variables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 54\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_cftime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cftime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 55\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_timedelta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_timedelta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 56\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# import pdb; pdb.set_trace()\u001b[39;00m\n\u001b[1;32m 58\u001b[0m ds \u001b[38;5;241m=\u001b[39m Dataset(\u001b[38;5;28mvars\u001b[39m, attrs\u001b[38;5;241m=\u001b[39mattrs)\n", - "File \u001b[0;32m~/github/virtualizarr/venv/lib/python3.12/site-packages/xarray/conventions.py:462\u001b[0m, in \u001b[0;36mdecode_cf_variables\u001b[0;34m(variables, attributes, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)\u001b[0m\n\u001b[1;32m 451\u001b[0m new_vars[k] \u001b[38;5;241m=\u001b[39m decode_cf_variable(\n\u001b[1;32m 452\u001b[0m k,\n\u001b[1;32m 453\u001b[0m v,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 459\u001b[0m decode_timedelta\u001b[38;5;241m=\u001b[39m_item_or_default(decode_timedelta, k, \u001b[38;5;28;01mNone\u001b[39;00m),\n\u001b[1;32m 460\u001b[0m )\n\u001b[1;32m 461\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 462\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(e)(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to decode variable \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 463\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m decode_coords \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcoordinates\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mall\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 464\u001b[0m var_attrs \u001b[38;5;241m=\u001b[39m new_vars[k]\u001b[38;5;241m.\u001b[39mattrs\n", - "\u001b[0;31merror\u001b[0m: Failed to decode variable 'time': Error -3 while decompressing data: unknown compression method" - ] - } - ], + "outputs": [], "source": [ "ds = xr.open_zarr(read_store, consolidated=False, zarr_format=3)\n", "ds" @@ -1293,9 +361,9 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "virtualizarr", "language": "python", - "name": "venv" + "name": "virtualizarr" }, "language_info": { "codemirror_mode": { @@ -1307,7 +375,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.11.10" } }, "nbformat": 4, From 79e0c1b688f2988842a10e47d54a261746553f8f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 19:18:14 -0800 Subject: [PATCH 79/90] Updated notebook --- noaa-cdr-sst.ipynb | 100 +++++++++++++++------------------------------ 1 file changed, 34 insertions(+), 66 deletions(-) diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb index 937969bf..a94bfcba 100644 --- a/noaa-cdr-sst.ipynb +++ b/noaa-cdr-sst.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "d09bbff3-4e96-4490-b837-14b78b64df35", "metadata": {}, "outputs": [], @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "3055eff4-9e22-4a95-a7fd-96933f381183", "metadata": {}, "outputs": [ @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", "metadata": {}, "outputs": [], @@ -59,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", "metadata": {}, "outputs": [], @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "06bbec92-3974-4859-8bda-353afc7800b9", "metadata": {}, "outputs": [], @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", "metadata": {}, "outputs": [], @@ -106,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 7, "id": "abefd6fa-386a-4e07-a7c8-219d3730eeeb", "metadata": {}, "outputs": [], @@ -116,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 8, "id": "79a4228a-0e17-4b07-9144-f24fe06db832", "metadata": {}, "outputs": [], @@ -129,50 +129,33 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 9, "id": "5fd0c082-8d5e-46a8-a994-fee80baa4ecc", "metadata": {}, "outputs": [], "source": [ - "store = await IcechunkStore.create(\n", + "store = IcechunkStore.create(\n", " storage=storage_config, config=virtual_ref_store_config, read_only=False\n", ")" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 10, "id": "55ebbc5f-add2-4de8-81f6-5aaf64d9e2b6", "metadata": {}, "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", - " store.set_virtual_ref(\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", - " store.set_virtual_ref(\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", - " store.set_virtual_ref(\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", - " store.set_virtual_ref(\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", - " store.set_virtual_ref(\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", - " store.set_virtual_ref(\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", - " store.set_virtual_ref(\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/aimeebarciauskas/github/virtualizarr/virtualizarr/writers/icechunk.py:272: RuntimeWarning: coroutine 'IcechunkStore.set_virtual_ref' was never awaited\n", - " store.set_virtual_ref(\n", - "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + "ename": "TypeError", + "evalue": "expected type IcechunkStore, but got type ", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mvirtual_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvirtualize\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_icechunk\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstore\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/virtualizarr/accessor.py:58\u001b[0m, in \u001b[0;36mVirtualiZarrDatasetAccessor.to_icechunk\u001b[0;34m(self, store, append_dim)\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;124;03mWrite an xarray dataset to an Icechunk store.\u001b[39;00m\n\u001b[1;32m 46\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[38;5;124;03mappend_dim: str, optional\u001b[39;00m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mvirtualizarr\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwriters\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01micechunk\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m dataset_to_icechunk\n\u001b[0;32m---> 58\u001b[0m \u001b[43mdataset_to_icechunk\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mappend_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mappend_dim\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/github/virtualizarr/virtualizarr/writers/icechunk.py:47\u001b[0m, in \u001b[0;36mdataset_to_icechunk\u001b[0;34m(ds, store, append_dim)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[1;32m 43\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124micechunk\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m and \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mzarr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m version 3 libraries are required to use this function\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 44\u001b[0m )\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(store, IcechunkStore):\n\u001b[0;32m---> 47\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexpected type IcechunkStore, but got type \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(store)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m store\u001b[38;5;241m.\u001b[39msupports_writes:\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msupplied store does not support writes\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mTypeError\u001b[0m: expected type IcechunkStore, but got type " ] } ], @@ -182,43 +165,20 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "cae7e34f-d6dd-42aa-9e7a-f0d5420ba0b9", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'PZMXKYCPJQRRFXV33Q7G'" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "await store.commit(\"first 2 days of 202408 data\")" + "store.commit(\"first 2 days of 202408 data\")" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "9387e1ff-46c1-45fd-9796-0457538209a7", "metadata": {}, - "outputs": [ - { - "ename": "KeyNotFound", - "evalue": "time/c/0", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyNotFound\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[17], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m store\u001b[38;5;241m.\u001b[39m_store\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtime/c/0\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "\u001b[0;31mKeyNotFound\u001b[0m: time/c/0" - ] - } - ], + "outputs": [], "source": [ "await store._store.get(\"time/c/0\")" ] @@ -357,6 +317,14 @@ "ds = xr.open_zarr(read_store, consolidated=False, zarr_format=3)\n", "ds" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41808f96", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From e38823c20134858029188260bb834669a202b13e Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 19:19:59 -0800 Subject: [PATCH 80/90] print store --- noaa-cdr-sst.ipynb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb index a94bfcba..af96ecc3 100644 --- a/noaa-cdr-sst.ipynb +++ b/noaa-cdr-sst.ipynb @@ -139,6 +139,27 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": 11, + "id": "53a74fb9-006b-4d2b-9157-7090af6c9e09", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store" + ] + }, { "cell_type": "code", "execution_count": 10, From ad17b83a3dda119ad32a59862066b059c0b2b3d2 Mon Sep 17 00:00:00 2001 From: Matthew Iannucci Date: Wed, 4 Dec 2024 22:34:42 -0500 Subject: [PATCH 81/90] Update notebook (#327) Co-authored-by: Aimee Barciauskas --- noaa-cdr-sst.ipynb | 1445 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 1403 insertions(+), 42 deletions(-) diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb index af96ecc3..a72deb04 100644 --- a/noaa-cdr-sst.ipynb +++ b/noaa-cdr-sst.ipynb @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", "metadata": {}, "outputs": [], @@ -59,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", "metadata": {}, "outputs": [], @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "id": "06bbec92-3974-4859-8bda-353afc7800b9", "metadata": {}, "outputs": [], @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", "metadata": {}, "outputs": [], @@ -106,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "id": "abefd6fa-386a-4e07-a7c8-219d3730eeeb", "metadata": {}, "outputs": [], @@ -116,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "id": "79a4228a-0e17-4b07-9144-f24fe06db832", "metadata": {}, "outputs": [], @@ -129,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "id": "5fd0c082-8d5e-46a8-a994-fee80baa4ecc", "metadata": {}, "outputs": [], @@ -148,36 +148,33 @@ { "data": { "text/plain": [ - "" + "'JKZCDPMTJ3ETZFY2KXKG'" ] }, - "execution_count": 11, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "store" + "store.commit(\"first 2 days of 202408 data\")" ] }, { "cell_type": "code", "execution_count": 10, - "id": "55ebbc5f-add2-4de8-81f6-5aaf64d9e2b6", + "id": "9387e1ff-46c1-45fd-9796-0457538209a7", "metadata": {}, "outputs": [ { - "ename": "TypeError", - "evalue": "expected type IcechunkStore, but got type ", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mvirtual_ds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvirtualize\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_icechunk\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstore\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/virtualizarr/accessor.py:58\u001b[0m, in \u001b[0;36mVirtualiZarrDatasetAccessor.to_icechunk\u001b[0;34m(self, store, append_dim)\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;124;03mWrite an xarray dataset to an Icechunk store.\u001b[39;00m\n\u001b[1;32m 46\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[38;5;124;03mappend_dim: str, optional\u001b[39;00m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mvirtualizarr\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwriters\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01micechunk\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m dataset_to_icechunk\n\u001b[0;32m---> 58\u001b[0m \u001b[43mdataset_to_icechunk\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mappend_dim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mappend_dim\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/github/virtualizarr/virtualizarr/writers/icechunk.py:47\u001b[0m, in \u001b[0;36mdataset_to_icechunk\u001b[0;34m(ds, store, append_dim)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[1;32m 43\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124micechunk\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m and \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mzarr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m version 3 libraries are required to use this function\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 44\u001b[0m )\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(store, IcechunkStore):\n\u001b[0;32m---> 47\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexpected type IcechunkStore, but got type \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(store)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m store\u001b[38;5;241m.\u001b[39msupports_writes:\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msupplied store does not support writes\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "\u001b[0;31mTypeError\u001b[0m: expected type IcechunkStore, but got type " - ] + "data": { + "text/plain": [ + "b'x^cx\\xd3\\xe2\\x06\\x00\\x04\\x16\\x01\\xb7'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -206,10 +203,457 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "b6271bd1-bc0b-4901-9901-91aabe508cf7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 66MB\n",
    +       "Dimensions:  (time: 2, lon: 1440, zlev: 1, lat: 720)\n",
    +       "Coordinates:\n",
    +       "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
    +       "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
    +       "  * zlev     (zlev) float32 4B 0.0\n",
    +       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
    +       "Data variables:\n",
    +       "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    anom     (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
    +       "Attributes: (12/37)\n",
    +       "    Conventions:                CF-1.6, ACDD-1.3\n",
    +       "    cdm_data_type:              Grid\n",
    +       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    +       "    creator_email:              oisst-help@noaa.gov\n",
    +       "    creator_url:                https://www.ncei.noaa.gov/\n",
    +       "    date_created:               2024-08-16T09:12:00Z\n",
    +       "    ...                         ...\n",
    +       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    +       "    standard_name_vocabulary:   CF Standard Name Table (v40, 25 January 2017)\n",
    +       "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
    +       "    time_coverage_end:          2024-08-01T23:59:59Z\n",
    +       "    time_coverage_start:        2024-08-01T00:00:00Z\n",
    +       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
    " + ], + "text/plain": [ + " Size: 66MB\n", + "Dimensions: (time: 2, lon: 1440, zlev: 1, lat: 720)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", + " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", + " * zlev (zlev) float32 4B 0.0\n", + " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", + "Data variables:\n", + " ice (time, zlev, lat, lon) float64 17MB ...\n", + " sst (time, zlev, lat, lon) float64 17MB ...\n", + " anom (time, zlev, lat, lon) float64 17MB ...\n", + " err (time, zlev, lat, lon) float64 17MB ...\n", + "Attributes: (12/37)\n", + " Conventions: CF-1.6, ACDD-1.3\n", + " cdm_data_type: Grid\n", + " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", + " creator_email: oisst-help@noaa.gov\n", + " creator_url: https://www.ncei.noaa.gov/\n", + " date_created: 2024-08-16T09:12:00Z\n", + " ... ...\n", + " source: ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n", + " standard_name_vocabulary: CF Standard Name Table (v40, 25 January 2017)\n", + " summary: NOAAs 1/4-degree Daily Optimum Interpolation ...\n", + " time_coverage_end: 2024-08-01T23:59:59Z\n", + " time_coverage_start: 2024-08-01T00:00:00Z\n", + " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ds = xr.open_zarr(store, consolidated=False, zarr_format=3)\n", "ds" @@ -225,7 +669,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "190c25f9-e000-4b17-83eb-cf551141dfea", "metadata": {}, "outputs": [], @@ -240,7 +684,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "af330082-207a-4f08-aefe-fc15aa8b2eb3", "metadata": {}, "outputs": [], @@ -256,17 +700,451 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "103b44d2-124a-4de5-8074-e997fd5a1698", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 17MB\n",
    +       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
    +       "Coordinates:\n",
    +       "    lat      (lat) float32 3kB ManifestArray<shape=(720,), dtype=float32, chu...\n",
    +       "    lon      (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n",
    +       "    time     (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n",
    +       "    zlev     (zlev) float32 4B ManifestArray<shape=(1,), dtype=float32, chunk...\n",
    +       "Data variables:\n",
    +       "    anom     (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    +       "    ice      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    +       "    sst      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    +       "    err      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    +       "Attributes: (12/37)\n",
    +       "    Conventions:                CF-1.6, ACDD-1.3\n",
    +       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...\n",
    +       "    references:                 Reynolds, et al.(2007) Daily High-Resolution-...\n",
    +       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    +       "    id:                         oisst-avhrr-v02r01.20240803.nc\n",
    +       "    naming_authority:           gov.noaa.ncei\n",
    +       "    ...                         ...\n",
    +       "    time_coverage_start:        2024-08-03T00:00:00Z\n",
    +       "    time_coverage_end:          2024-08-03T23:59:59Z\n",
    +       "    metadata_link:              https://doi.org/10.25921/RE9P-PT57\n",
    +       "    ncei_template_version:      NCEI_NetCDF_Grid_Template_v2.0\n",
    +       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    +       "    sensor:                     Thermometer, AVHRR
    " + ], + "text/plain": [ + " Size: 17MB\n", + "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", + "Coordinates:\n", + " lat (lat) float32 3kB ManifestArray\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 133MB\n",
    +       "Dimensions:  (lat: 720, time: 4, zlev: 1, lon: 1440)\n",
    +       "Coordinates:\n",
    +       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
    +       "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
    +       "  * zlev     (zlev) float32 4B 0.0\n",
    +       "  * time     (time) datetime64[ns] 32B 2024-08-01T12:00:00 ... 2024-08-04T12:...\n",
    +       "Data variables:\n",
    +       "    ice      (time, zlev, lat, lon) float64 33MB ...\n",
    +       "    anom     (time, zlev, lat, lon) float64 33MB ...\n",
    +       "    err      (time, zlev, lat, lon) float64 33MB ...\n",
    +       "    sst      (time, zlev, lat, lon) float64 33MB ...\n",
    +       "Attributes: (12/37)\n",
    +       "    Conventions:                CF-1.6, ACDD-1.3\n",
    +       "    cdm_data_type:              Grid\n",
    +       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    +       "    creator_email:              oisst-help@noaa.gov\n",
    +       "    creator_url:                https://www.ncei.noaa.gov/\n",
    +       "    date_created:               2024-08-18T09:12:00Z\n",
    +       "    ...                         ...\n",
    +       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    +       "    standard_name_vocabulary:   CF Standard Name Table (v40, 25 January 2017)\n",
    +       "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
    +       "    time_coverage_end:          2024-08-03T23:59:59Z\n",
    +       "    time_coverage_start:        2024-08-03T00:00:00Z\n",
    +       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
    " + ], + "text/plain": [ + " Size: 133MB\n", + "Dimensions: (lat: 720, time: 4, zlev: 1, lon: 1440)\n", + "Coordinates:\n", + " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", + " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", + " * zlev (zlev) float32 4B 0.0\n", + " * time (time) datetime64[ns] 32B 2024-08-01T12:00:00 ... 2024-08-04T12:...\n", + "Data variables:\n", + " ice (time, zlev, lat, lon) float64 33MB ...\n", + " anom (time, zlev, lat, lon) float64 33MB ...\n", + " err (time, zlev, lat, lon) float64 33MB ...\n", + " sst (time, zlev, lat, lon) float64 33MB ...\n", + "Attributes: (12/37)\n", + " Conventions: CF-1.6, ACDD-1.3\n", + " cdm_data_type: Grid\n", + " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", + " creator_email: oisst-help@noaa.gov\n", + " creator_url: https://www.ncei.noaa.gov/\n", + " date_created: 2024-08-18T09:12:00Z\n", + " ... ...\n", + " source: ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n", + " standard_name_vocabulary: CF Standard Name Table (v40, 25 January 2017)\n", + " summary: NOAAs 1/4-degree Daily Optimum Interpolation ...\n", + " time_coverage_end: 2024-08-03T23:59:59Z\n", + " time_coverage_start: 2024-08-03T00:00:00Z\n", + " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ds = xr.open_zarr(read_store, consolidated=False, zarr_format=3)\n", "ds" @@ -350,9 +1711,9 @@ ], "metadata": { "kernelspec": { - "display_name": "virtualizarr", + "display_name": ".venv", "language": "python", - "name": "virtualizarr" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -364,7 +1725,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.10" + "version": "3.11.9" } }, "nbformat": 4, From 8b9a8305408c8549ccb927820e0642d3cfe61c49 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 19:45:28 -0800 Subject: [PATCH 82/90] Add append to examples --- .gitignore | 2 +- examples/append/noaa-cdr-sst.ipynb | 1279 ++++++++++++++++++++ noaa-cdr-sst.ipynb | 1733 ---------------------------- 3 files changed, 1280 insertions(+), 1734 deletions(-) create mode 100644 examples/append/noaa-cdr-sst.ipynb delete mode 100644 noaa-cdr-sst.ipynb diff --git a/.gitignore b/.gitignore index d6720a7a..2e785e23 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,4 @@ cython_debug/ #.idea/ virtualizarr/_version.py docs/generated/ -examples/ +#examples/ diff --git a/examples/append/noaa-cdr-sst.ipynb b/examples/append/noaa-cdr-sst.ipynb new file mode 100644 index 00000000..0f80b169 --- /dev/null +++ b/examples/append/noaa-cdr-sst.ipynb @@ -0,0 +1,1279 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "120903d6-ea52-4a1e-83d2-4d434ad2cb98", + "metadata": {}, + "source": [ + "# Appending to an Icechunk Store with Virtual References\n", + "\n", + "This notebook demonstrates how to append to an icechunk store.\n", + "\n", + "Please ensure the correct dependencies are installed before starting." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "d09bbff3-4e96-4490-b837-14b78b64df35", + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install -e \".[icechunk]\"\n", + "# !pip install git+https://github.com/mpiannucci/kerchunk@v3\n", + "# !pip install fsspec s3fs" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3055eff4-9e22-4a95-a7fd-96933f381183", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: icechunk\n", + "Version: 0.1.0a7\n", + "Summary: Transactional storage engine for Zarr designed for use on cloud object storage\n", + "Home-page: https://github.com/earth-mover/icechunk\n", + "Author: Earthmover PBC\n", + "Author-email: \n", + "License: Apache-2.0\n", + "Location: /Users/aimeebarciauskas/github/virtualizarr/venv/lib/python3.12/site-packages\n", + "Requires: zarr\n", + "Required-by: \n" + ] + } + ], + "source": [ + "!pip show icechunk" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "import fsspec\n", + "import xarray as xr\n", + "from icechunk import IcechunkStore, StorageConfig, StoreConfig, VirtualRefConfig\n", + "\n", + "from virtualizarr import open_virtual_dataset\n", + "\n", + "warnings.filterwarnings(\"ignore\", category=UserWarning)" + ] + }, + { + "cell_type": "markdown", + "id": "0df547e4-456d-44c1-b190-606f0b9e056e", + "metadata": {}, + "source": [ + "# Before you start\n", + "\n", + "Identify the dataset you will be using and create a list of files to generate a virtual icechunk datastore with." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", + "metadata": {}, + "outputs": [], + "source": [ + "fs = fsspec.filesystem(\"s3\", anon=True)\n", + "\n", + "oisst_files = fs.glob(\n", + " \"s3://noaa-cdr-sea-surface-temp-optimum-interpolation-pds/data/v2.1/avhrr/202408/oisst-avhrr-v02r01.*.nc\"\n", + ")\n", + "\n", + "oisst_files = sorted([\"s3://\" + f for f in oisst_files])" + ] + }, + { + "cell_type": "markdown", + "id": "73ceb93b-b0ac-48b2-928a-84da0d2019ac", + "metadata": {}, + "source": [ + "## Create virtual datasets with VirtualiZarr's `open_virtual_dataset`" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "06bbec92-3974-4859-8bda-353afc7800b9", + "metadata": {}, + "outputs": [], + "source": [ + "so = dict(anon=True, default_fill_cache=False, default_cache_type=\"none\")\n", + "\n", + "virtual_datasets = [\n", + " open_virtual_dataset(url, indexes={}, reader_options={\"storage_options\": so})\n", + " for url in oisst_files[0:2]\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_ds = xr.concat(\n", + " virtual_datasets,\n", + " dim=\"time\",\n", + " coords=\"minimal\",\n", + " compat=\"override\",\n", + " combine_attrs=\"override\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "abefd6fa-386a-4e07-a7c8-219d3730eeeb", + "metadata": {}, + "outputs": [], + "source": [ + "# Clean up the store if running this notebook multiple times.\n", + "#!rm -rf ./noaa-cdr-icechunk/" + ] + }, + { + "cell_type": "markdown", + "id": "05f41a0b-6292-419d-a9d3-d8ddf8c0c15b", + "metadata": {}, + "source": [ + "## Initialize the Icechunk Store" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "79a4228a-0e17-4b07-9144-f24fe06db832", + "metadata": {}, + "outputs": [], + "source": [ + "storage_config = StorageConfig.filesystem(\"./noaa-cdr-icechunk\")\n", + "virtual_ref_store_config = StoreConfig(\n", + " virtual_ref_config=VirtualRefConfig.s3_anonymous(region=\"us-east-1\"),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "5fd0c082-8d5e-46a8-a994-fee80baa4ecc", + "metadata": {}, + "outputs": [], + "source": [ + "store = IcechunkStore.create(\n", + " storage=storage_config, config=virtual_ref_store_config, read_only=False\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "749193c1-38b9-4400-a08f-f0a675d30f06", + "metadata": {}, + "source": [ + "## Write the virtual datasets to the icechunk store and commit" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9387e1ff-46c1-45fd-9796-0457538209a7", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_ds.virtualize.to_icechunk(store)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "53a74fb9-006b-4d2b-9157-7090af6c9e09", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'R1BP6057NW5A1ZANMBDG'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.commit(\"first 2 days of 202408 data\")" + ] + }, + { + "cell_type": "markdown", + "id": "8becd176-1c7d-4c74-a3f1-1b9f55b445a2", + "metadata": {}, + "source": [ + "## Check your work!" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "b6271bd1-bc0b-4901-9901-91aabe508cf7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 66MB\n",
    +       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
    +       "Coordinates:\n",
    +       "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
    +       "  * zlev     (zlev) float32 4B 0.0\n",
    +       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
    +       "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
    +       "Data variables:\n",
    +       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    anom     (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
    +       "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
    +       "Attributes: (12/37)\n",
    +       "    Conventions:                CF-1.6, ACDD-1.3\n",
    +       "    cdm_data_type:              Grid\n",
    +       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    +       "    creator_email:              oisst-help@noaa.gov\n",
    +       "    creator_url:                https://www.ncei.noaa.gov/\n",
    +       "    date_created:               2024-08-16T09:12:00Z\n",
    +       "    ...                         ...\n",
    +       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    +       "    standard_name_vocabulary:   CF Standard Name Table (v40, 25 January 2017)\n",
    +       "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
    +       "    time_coverage_end:          2024-08-01T23:59:59Z\n",
    +       "    time_coverage_start:        2024-08-01T00:00:00Z\n",
    +       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
    " + ], + "text/plain": [ + " Size: 66MB\n", + "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", + " * zlev (zlev) float32 4B 0.0\n", + " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", + " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", + "Data variables:\n", + " err (time, zlev, lat, lon) float64 17MB ...\n", + " anom (time, zlev, lat, lon) float64 17MB ...\n", + " ice (time, zlev, lat, lon) float64 17MB ...\n", + " sst (time, zlev, lat, lon) float64 17MB ...\n", + "Attributes: (12/37)\n", + " Conventions: CF-1.6, ACDD-1.3\n", + " cdm_data_type: Grid\n", + " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", + " creator_email: oisst-help@noaa.gov\n", + " creator_url: https://www.ncei.noaa.gov/\n", + " date_created: 2024-08-16T09:12:00Z\n", + " ... ...\n", + " source: ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n", + " standard_name_vocabulary: CF Standard Name Table (v40, 25 January 2017)\n", + " summary: NOAAs 1/4-degree Daily Optimum Interpolation ...\n", + " time_coverage_end: 2024-08-01T23:59:59Z\n", + " time_coverage_start: 2024-08-01T00:00:00Z\n", + " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds = xr.open_zarr(store, consolidated=False, zarr_format=3)\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "id": "23dd5a13-9c0e-4132-9073-474c0af65920", + "metadata": {}, + "source": [ + "# Append\n", + "\n", + "That was all nothing new! Basically a repeat of what is in the [icechunk docs](https://icechunk.io/icechunk-python/virtual/). Here we follow the same steps to create a virtual dataset, but we add an `append_dim` argument to the `to_icechunk` function." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "190c25f9-e000-4b17-83eb-cf551141dfea", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_datasets_a = [\n", + " open_virtual_dataset(\n", + " url, indexes={}, reader_options={\"storage_options\": {\"anon\": True}}\n", + " )\n", + " for url in oisst_files[2:4]\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "af330082-207a-4f08-aefe-fc15aa8b2eb3", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_ds_a = xr.concat(\n", + " virtual_datasets_a,\n", + " dim=\"time\",\n", + " coords=\"minimal\",\n", + " compat=\"override\",\n", + " combine_attrs=\"override\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "b6137cc1-b996-4e60-8c12-01eb19930da6", + "metadata": {}, + "outputs": [], + "source": [ + "append_store = IcechunkStore.open_existing(\n", + " storage=storage_config, config=virtual_ref_store_config, read_only=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "a465be46-bb81-4e36-b1b6-67c3b8e4b9ec", + "metadata": {}, + "outputs": [], + "source": [ + "virtual_ds_a.virtualize.to_icechunk(append_store, append_dim=\"time\")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "e9908d2f-664b-4256-b9d4-842df2e512c3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'0HE5RZ869HTG8RZESHCG'" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "append_store.commit(\"wrote 2 more days of data\")" + ] + }, + { + "cell_type": "markdown", + "id": "e1384e99-c284-4942-a49b-7799802728b0", + "metadata": {}, + "source": [ + "# Check that it worked!" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "477094aa-2094-46e2-ae78-516fc2a51690", + "metadata": {}, + "outputs": [], + "source": [ + "read_store = IcechunkStore.open_existing(\n", + " storage=storage_config, config=virtual_ref_store_config, read_only=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "47a53027-dbae-48aa-85d2-dcbc04e01e61", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
    <xarray.Dataset> Size: 133MB\n",
    +       "Dimensions:  (time: 4, zlev: 1, lat: 720, lon: 1440)\n",
    +       "Coordinates:\n",
    +       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
    +       "  * time     (time) datetime64[ns] 32B 2024-08-01T12:00:00 ... 2024-08-04T12:...\n",
    +       "  * zlev     (zlev) float32 4B 0.0\n",
    +       "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
    +       "Data variables:\n",
    +       "    anom     (time, zlev, lat, lon) float64 33MB ...\n",
    +       "    ice      (time, zlev, lat, lon) float64 33MB ...\n",
    +       "    err      (time, zlev, lat, lon) float64 33MB ...\n",
    +       "    sst      (time, zlev, lat, lon) float64 33MB ...\n",
    +       "Attributes: (12/37)\n",
    +       "    Conventions:                CF-1.6, ACDD-1.3\n",
    +       "    cdm_data_type:              Grid\n",
    +       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    +       "    creator_email:              oisst-help@noaa.gov\n",
    +       "    creator_url:                https://www.ncei.noaa.gov/\n",
    +       "    date_created:               2024-08-18T09:12:00Z\n",
    +       "    ...                         ...\n",
    +       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    +       "    standard_name_vocabulary:   CF Standard Name Table (v40, 25 January 2017)\n",
    +       "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
    +       "    time_coverage_end:          2024-08-03T23:59:59Z\n",
    +       "    time_coverage_start:        2024-08-03T00:00:00Z\n",
    +       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
    " + ], + "text/plain": [ + " Size: 133MB\n", + "Dimensions: (time: 4, zlev: 1, lat: 720, lon: 1440)\n", + "Coordinates:\n", + " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", + " * time (time) datetime64[ns] 32B 2024-08-01T12:00:00 ... 2024-08-04T12:...\n", + " * zlev (zlev) float32 4B 0.0\n", + " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", + "Data variables:\n", + " anom (time, zlev, lat, lon) float64 33MB ...\n", + " ice (time, zlev, lat, lon) float64 33MB ...\n", + " err (time, zlev, lat, lon) float64 33MB ...\n", + " sst (time, zlev, lat, lon) float64 33MB ...\n", + "Attributes: (12/37)\n", + " Conventions: CF-1.6, ACDD-1.3\n", + " cdm_data_type: Grid\n", + " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", + " creator_email: oisst-help@noaa.gov\n", + " creator_url: https://www.ncei.noaa.gov/\n", + " date_created: 2024-08-18T09:12:00Z\n", + " ... ...\n", + " source: ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n", + " standard_name_vocabulary: CF Standard Name Table (v40, 25 January 2017)\n", + " summary: NOAAs 1/4-degree Daily Optimum Interpolation ...\n", + " time_coverage_end: 2024-08-03T23:59:59Z\n", + " time_coverage_start: 2024-08-03T00:00:00Z\n", + " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds = xr.open_zarr(read_store, consolidated=False, zarr_format=3)\n", + "ds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41808f96", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "virtualizarr", + "language": "python", + "name": "venv" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/noaa-cdr-sst.ipynb b/noaa-cdr-sst.ipynb deleted file mode 100644 index a72deb04..00000000 --- a/noaa-cdr-sst.ipynb +++ /dev/null @@ -1,1733 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "d09bbff3-4e96-4490-b837-14b78b64df35", - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install -e \".[icechunk]\"\n", - "# !pip install git+https://github.com/mpiannucci/kerchunk@v3\n", - "# !pip install fsspec s3fs" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "3055eff4-9e22-4a95-a7fd-96933f381183", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Name: icechunk\n", - "Version: 0.1.0a7\n", - "Summary: Transactional storage engine for Zarr designed for use on cloud object storage\n", - "Home-page: https://github.com/earth-mover/icechunk\n", - "Author: Earthmover PBC\n", - "Author-email: \n", - "License: Apache-2.0\n", - "Location: /Users/aimeebarciauskas/github/virtualizarr/venv/lib/python3.12/site-packages\n", - "Requires: zarr\n", - "Required-by: \n" - ] - } - ], - "source": [ - "!pip show icechunk" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "2f69f0bb-316b-452c-b1ba-4d7ef4afcf67", - "metadata": {}, - "outputs": [], - "source": [ - "import warnings\n", - "\n", - "import fsspec\n", - "import xarray as xr\n", - "from icechunk import IcechunkStore, StorageConfig, StoreConfig, VirtualRefConfig\n", - "\n", - "from virtualizarr import open_virtual_dataset\n", - "\n", - "warnings.filterwarnings(\"ignore\", category=UserWarning)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "1532c33b-804f-49fa-9fa9-0eb42ea87e26", - "metadata": {}, - "outputs": [], - "source": [ - "fs = fsspec.filesystem(\"s3\", anon=True)\n", - "\n", - "oisst_files = fs.glob(\n", - " \"s3://noaa-cdr-sea-surface-temp-optimum-interpolation-pds/data/v2.1/avhrr/202408/oisst-avhrr-v02r01.*.nc\"\n", - ")\n", - "\n", - "oisst_files = sorted([\"s3://\" + f for f in oisst_files])" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "06bbec92-3974-4859-8bda-353afc7800b9", - "metadata": {}, - "outputs": [], - "source": [ - "so = dict(anon=True, default_fill_cache=False, default_cache_type=\"none\")\n", - "\n", - "virtual_datasets = [\n", - " open_virtual_dataset(url, indexes={}, reader_options={\"storage_options\": so})\n", - " for url in oisst_files[0:2]\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "77fb94c8-870f-4c9e-8421-ac9c17402122", - "metadata": {}, - "outputs": [], - "source": [ - "virtual_ds = xr.concat(\n", - " virtual_datasets,\n", - " dim=\"time\",\n", - " coords=\"minimal\",\n", - " compat=\"override\",\n", - " combine_attrs=\"override\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "abefd6fa-386a-4e07-a7c8-219d3730eeeb", - "metadata": {}, - "outputs": [], - "source": [ - "!rm -rf ./noaa-cdr-icechunk/" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "79a4228a-0e17-4b07-9144-f24fe06db832", - "metadata": {}, - "outputs": [], - "source": [ - "storage_config = StorageConfig.filesystem(\"./noaa-cdr-icechunk\")\n", - "virtual_ref_store_config = StoreConfig(\n", - " virtual_ref_config=VirtualRefConfig.s3_anonymous(region=\"us-east-1\"),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "5fd0c082-8d5e-46a8-a994-fee80baa4ecc", - "metadata": {}, - "outputs": [], - "source": [ - "store = IcechunkStore.create(\n", - " storage=storage_config, config=virtual_ref_store_config, read_only=False\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "53a74fb9-006b-4d2b-9157-7090af6c9e09", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'JKZCDPMTJ3ETZFY2KXKG'" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "store.commit(\"first 2 days of 202408 data\")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "9387e1ff-46c1-45fd-9796-0457538209a7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "b'x^cx\\xd3\\xe2\\x06\\x00\\x04\\x16\\x01\\xb7'" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "virtual_ds.virtualize.to_icechunk(store)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cae7e34f-d6dd-42aa-9e7a-f0d5420ba0b9", - "metadata": {}, - "outputs": [], - "source": [ - "store.commit(\"first 2 days of 202408 data\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9387e1ff-46c1-45fd-9796-0457538209a7", - "metadata": {}, - "outputs": [], - "source": [ - "await store._store.get(\"time/c/0\")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "b6271bd1-bc0b-4901-9901-91aabe508cf7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset> Size: 66MB\n",
    -       "Dimensions:  (time: 2, lon: 1440, zlev: 1, lat: 720)\n",
    -       "Coordinates:\n",
    -       "  * time     (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n",
    -       "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
    -       "  * zlev     (zlev) float32 4B 0.0\n",
    -       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
    -       "Data variables:\n",
    -       "    ice      (time, zlev, lat, lon) float64 17MB ...\n",
    -       "    sst      (time, zlev, lat, lon) float64 17MB ...\n",
    -       "    anom     (time, zlev, lat, lon) float64 17MB ...\n",
    -       "    err      (time, zlev, lat, lon) float64 17MB ...\n",
    -       "Attributes: (12/37)\n",
    -       "    Conventions:                CF-1.6, ACDD-1.3\n",
    -       "    cdm_data_type:              Grid\n",
    -       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    -       "    creator_email:              oisst-help@noaa.gov\n",
    -       "    creator_url:                https://www.ncei.noaa.gov/\n",
    -       "    date_created:               2024-08-16T09:12:00Z\n",
    -       "    ...                         ...\n",
    -       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    -       "    standard_name_vocabulary:   CF Standard Name Table (v40, 25 January 2017)\n",
    -       "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
    -       "    time_coverage_end:          2024-08-01T23:59:59Z\n",
    -       "    time_coverage_start:        2024-08-01T00:00:00Z\n",
    -       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
    " - ], - "text/plain": [ - " Size: 66MB\n", - "Dimensions: (time: 2, lon: 1440, zlev: 1, lat: 720)\n", - "Coordinates:\n", - " * time (time) datetime64[ns] 16B 2024-08-01T12:00:00 2024-08-02T12:00:00\n", - " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", - " * zlev (zlev) float32 4B 0.0\n", - " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", - "Data variables:\n", - " ice (time, zlev, lat, lon) float64 17MB ...\n", - " sst (time, zlev, lat, lon) float64 17MB ...\n", - " anom (time, zlev, lat, lon) float64 17MB ...\n", - " err (time, zlev, lat, lon) float64 17MB ...\n", - "Attributes: (12/37)\n", - " Conventions: CF-1.6, ACDD-1.3\n", - " cdm_data_type: Grid\n", - " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", - " creator_email: oisst-help@noaa.gov\n", - " creator_url: https://www.ncei.noaa.gov/\n", - " date_created: 2024-08-16T09:12:00Z\n", - " ... ...\n", - " source: ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n", - " standard_name_vocabulary: CF Standard Name Table (v40, 25 January 2017)\n", - " summary: NOAAs 1/4-degree Daily Optimum Interpolation ...\n", - " time_coverage_end: 2024-08-01T23:59:59Z\n", - " time_coverage_start: 2024-08-01T00:00:00Z\n", - " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds = xr.open_zarr(store, consolidated=False, zarr_format=3)\n", - "ds" - ] - }, - { - "cell_type": "markdown", - "id": "23dd5a13-9c0e-4132-9073-474c0af65920", - "metadata": {}, - "source": [ - "## Append" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "190c25f9-e000-4b17-83eb-cf551141dfea", - "metadata": {}, - "outputs": [], - "source": [ - "virtual_datasets_a = [\n", - " open_virtual_dataset(\n", - " url, indexes={}, reader_options={\"storage_options\": {\"anon\": True}}\n", - " )\n", - " for url in oisst_files[2:4]\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "af330082-207a-4f08-aefe-fc15aa8b2eb3", - "metadata": {}, - "outputs": [], - "source": [ - "virtual_ds_a = xr.concat(\n", - " virtual_datasets_a,\n", - " dim=\"time\",\n", - " coords=\"minimal\",\n", - " compat=\"override\",\n", - " combine_attrs=\"override\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "103b44d2-124a-4de5-8074-e997fd5a1698", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset> Size: 17MB\n",
    -       "Dimensions:  (time: 2, zlev: 1, lat: 720, lon: 1440)\n",
    -       "Coordinates:\n",
    -       "    lat      (lat) float32 3kB ManifestArray<shape=(720,), dtype=float32, chu...\n",
    -       "    lon      (lon) float32 6kB ManifestArray<shape=(1440,), dtype=float32, ch...\n",
    -       "    time     (time) float32 8B ManifestArray<shape=(2,), dtype=float32, chunk...\n",
    -       "    zlev     (zlev) float32 4B ManifestArray<shape=(1,), dtype=float32, chunk...\n",
    -       "Data variables:\n",
    -       "    anom     (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "    ice      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "    sst      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "    err      (time, zlev, lat, lon) int16 4MB ManifestArray<shape=(2, 1, 720,...\n",
    -       "Attributes: (12/37)\n",
    -       "    Conventions:                CF-1.6, ACDD-1.3\n",
    -       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...\n",
    -       "    references:                 Reynolds, et al.(2007) Daily High-Resolution-...\n",
    -       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    -       "    id:                         oisst-avhrr-v02r01.20240803.nc\n",
    -       "    naming_authority:           gov.noaa.ncei\n",
    -       "    ...                         ...\n",
    -       "    time_coverage_start:        2024-08-03T00:00:00Z\n",
    -       "    time_coverage_end:          2024-08-03T23:59:59Z\n",
    -       "    metadata_link:              https://doi.org/10.25921/RE9P-PT57\n",
    -       "    ncei_template_version:      NCEI_NetCDF_Grid_Template_v2.0\n",
    -       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    -       "    sensor:                     Thermometer, AVHRR
    " - ], - "text/plain": [ - " Size: 17MB\n", - "Dimensions: (time: 2, zlev: 1, lat: 720, lon: 1440)\n", - "Coordinates:\n", - " lat (lat) float32 3kB ManifestArray\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
    <xarray.Dataset> Size: 133MB\n",
    -       "Dimensions:  (lat: 720, time: 4, zlev: 1, lon: 1440)\n",
    -       "Coordinates:\n",
    -       "  * lat      (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n",
    -       "  * lon      (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n",
    -       "  * zlev     (zlev) float32 4B 0.0\n",
    -       "  * time     (time) datetime64[ns] 32B 2024-08-01T12:00:00 ... 2024-08-04T12:...\n",
    -       "Data variables:\n",
    -       "    ice      (time, zlev, lat, lon) float64 33MB ...\n",
    -       "    anom     (time, zlev, lat, lon) float64 33MB ...\n",
    -       "    err      (time, zlev, lat, lon) float64 33MB ...\n",
    -       "    sst      (time, zlev, lat, lon) float64 33MB ...\n",
    -       "Attributes: (12/37)\n",
    -       "    Conventions:                CF-1.6, ACDD-1.3\n",
    -       "    cdm_data_type:              Grid\n",
    -       "    comment:                    Data was converted from NetCDF-3 to NetCDF-4 ...\n",
    -       "    creator_email:              oisst-help@noaa.gov\n",
    -       "    creator_url:                https://www.ncei.noaa.gov/\n",
    -       "    date_created:               2024-08-18T09:12:00Z\n",
    -       "    ...                         ...\n",
    -       "    source:                     ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n",
    -       "    standard_name_vocabulary:   CF Standard Name Table (v40, 25 January 2017)\n",
    -       "    summary:                    NOAAs 1/4-degree Daily Optimum Interpolation ...\n",
    -       "    time_coverage_end:          2024-08-03T23:59:59Z\n",
    -       "    time_coverage_start:        2024-08-03T00:00:00Z\n",
    -       "    title:                      NOAA/NCEI 1/4 Degree Daily Optimum Interpolat...
    " - ], - "text/plain": [ - " Size: 133MB\n", - "Dimensions: (lat: 720, time: 4, zlev: 1, lon: 1440)\n", - "Coordinates:\n", - " * lat (lat) float32 3kB -89.88 -89.62 -89.38 -89.12 ... 89.38 89.62 89.88\n", - " * lon (lon) float32 6kB 0.125 0.375 0.625 0.875 ... 359.4 359.6 359.9\n", - " * zlev (zlev) float32 4B 0.0\n", - " * time (time) datetime64[ns] 32B 2024-08-01T12:00:00 ... 2024-08-04T12:...\n", - "Data variables:\n", - " ice (time, zlev, lat, lon) float64 33MB ...\n", - " anom (time, zlev, lat, lon) float64 33MB ...\n", - " err (time, zlev, lat, lon) float64 33MB ...\n", - " sst (time, zlev, lat, lon) float64 33MB ...\n", - "Attributes: (12/37)\n", - " Conventions: CF-1.6, ACDD-1.3\n", - " cdm_data_type: Grid\n", - " comment: Data was converted from NetCDF-3 to NetCDF-4 ...\n", - " creator_email: oisst-help@noaa.gov\n", - " creator_url: https://www.ncei.noaa.gov/\n", - " date_created: 2024-08-18T09:12:00Z\n", - " ... ...\n", - " source: ICOADS, NCEP_GTS, GSFC_ICE, NCEP_ICE, Pathfin...\n", - " standard_name_vocabulary: CF Standard Name Table (v40, 25 January 2017)\n", - " summary: NOAAs 1/4-degree Daily Optimum Interpolation ...\n", - " time_coverage_end: 2024-08-03T23:59:59Z\n", - " time_coverage_start: 2024-08-03T00:00:00Z\n", - " title: NOAA/NCEI 1/4 Degree Daily Optimum Interpolat..." - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds = xr.open_zarr(read_store, consolidated=False, zarr_format=3)\n", - "ds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41808f96", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 3f9f58ce6b13d34a5c48100abb7ace502ca207be Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 19:51:42 -0800 Subject: [PATCH 83/90] Add to releases.rst --- docs/releases.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/releases.rst b/docs/releases.rst index 35b7c488..e42e7662 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -11,6 +11,7 @@ New Features - Add a ``virtual_backend_kwargs`` keyword argument to file readers and to ``open_virtual_dataset``, to allow reader-specific options to be passed down. (:pull:`315`) By `Tom Nicholas `_. +- Added append functionality to `to_icechunk` (:pull:`272`) By `Aimee Barciauskas `_. Breaking changes ~~~~~~~~~~~~~~~~ From 84963598a61d12e2426c5b4b0d01b1602ecda786 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 4 Dec 2024 19:53:51 -0800 Subject: [PATCH 84/90] Revert change to .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2e785e23..d6720a7a 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,4 @@ cython_debug/ #.idea/ virtualizarr/_version.py docs/generated/ -#examples/ +examples/ From 491b701a29491a946d97a0033313f9ee5e7d1ed6 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 5 Dec 2024 08:25:22 -0800 Subject: [PATCH 85/90] Update ci/upstream.yml Co-authored-by: Tom Nicholas --- ci/upstream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/upstream.yml b/ci/upstream.yml index 172d9271..feee5044 100644 --- a/ci/upstream.yml +++ b/ci/upstream.yml @@ -28,6 +28,6 @@ dependencies: - fsspec - pip - pip: - - icechunk==0.1.0a7 # Installs zarr v3 as dependency + - icechunk>=0.1.0a7 # Installs zarr v3 as dependency # - git+https://github.com/fsspec/kerchunk@main # kerchunk is currently incompatible with zarr-python v3 (https://github.com/fsspec/kerchunk/pull/516) - imagecodecs-numcodecs==2024.6.1 From 94ef469592f6a7b1c9582e2dfbc08aa70225ca5e Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 5 Dec 2024 08:25:44 -0800 Subject: [PATCH 86/90] Update pyproject.toml Co-authored-by: Tom Nicholas --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2856e304..ecda9a16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ hdf_reader = [ "numcodecs" ] icechunk = [ - "icechunk==0.1.0a7", + "icechunk>=0.1.0a7", ] test = [ "codecov", From fad188b4d372ec8eee67aeb3debca3fa9b7f05e1 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 5 Dec 2024 08:25:54 -0800 Subject: [PATCH 87/90] Update virtualizarr/tests/test_writers/test_icechunk.py Co-authored-by: Tom Nicholas --- virtualizarr/tests/test_writers/test_icechunk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 0eadf3c4..4c1eb765 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -338,8 +338,8 @@ def icechunk_storage(tmpdir) -> "StorageConfig": def generate_chunk_manifest( netcdf4_file: str, - shape: Tuple[int, ...], - chunks: Tuple[int, ...], + shape: tuple[int, ...], + chunks: tuple[int, ...], offset=6144, length=48, ) -> ChunkManifest: From 8df67e9d914aad06d31d17b2f67fd855adc3855e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:25:59 +0000 Subject: [PATCH 88/90] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- virtualizarr/tests/test_writers/test_icechunk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 4c1eb765..fb1757ce 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -1,6 +1,6 @@ from itertools import product from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Tuple, cast +from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, cast import pytest From 258c92fb4d5661014856c18c130bfa27f5c72602 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 5 Dec 2024 08:26:37 -0800 Subject: [PATCH 89/90] Update virtualizarr/accessor.py Co-authored-by: Tom Nicholas --- virtualizarr/accessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualizarr/accessor.py b/virtualizarr/accessor.py index d9b9d6a2..969d21af 100644 --- a/virtualizarr/accessor.py +++ b/virtualizarr/accessor.py @@ -46,7 +46,7 @@ def to_icechunk( Any variables backed by ManifestArray objects will be be written as virtual references, any other variables will be loaded into memory before their binary chunk data is written into the store. - If `append_dim` is provided, the virtual dataset will be appended on the append_dim axis to the existing IcechunkStore. + If `append_dim` is provided, the virtual dataset will be appended to the existing IcechunkStore along the `append_dim` dimension. Parameters ---------- From 84a4d01e6022edb7a8e1cb965a28bdb2fc6d18af Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 5 Dec 2024 08:58:49 -0800 Subject: [PATCH 90/90] Separate out multiple arrays test --- .../tests/test_writers/test_icechunk.py | 76 ++++++++++++++++--- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/virtualizarr/tests/test_writers/test_icechunk.py b/virtualizarr/tests/test_writers/test_icechunk.py index 0eadf3c4..135f5deb 100644 --- a/virtualizarr/tests/test_writers/test_icechunk.py +++ b/virtualizarr/tests/test_writers/test_icechunk.py @@ -477,9 +477,7 @@ def test_append_virtual_ref_without_encoding( expected_array = concat([expected_ds, expected_ds, expected_ds], dim="x") xrt.assert_identical(array, expected_array) - ## When appending to a virtual ref with encoding, it succeeds - @pytest.mark.asyncio - async def test_append_virtual_ref_with_encoding( + def test_append_virtual_ref_with_encoding( self, icechunk_storage: "StorageConfig", netcdf4_files_factory: Callable ): import xarray.testing as xrt @@ -489,6 +487,67 @@ async def test_append_virtual_ref_with_encoding( encoding = {"air": {"scale_factor": scale_factor}} filepath1, filepath2 = netcdf4_files_factory(encoding=encoding) + vds1, vds2 = ( + gen_virtual_dataset( + file_uri=filepath1, + shape=(1460, 25, 53), + chunk_shape=(1460, 25, 53), + dims=["time", "lat", "lon"], + dtype=np.dtype("float64"), + variable_name="air", + encoding={"scale_factor": scale_factor}, + offset=15419, + length=15476000, + ), + gen_virtual_dataset( + file_uri=filepath2, + shape=(1460, 25, 53), + chunk_shape=(1460, 25, 53), + dims=["time", "lat", "lon"], + dtype=np.dtype("float64"), + variable_name="air", + encoding={"scale_factor": scale_factor}, + offset=15419, + length=15476000, + ), + ) + + # create the icechunk store and commit the first virtual dataset + icechunk_filestore = IcechunkStore.create(storage=icechunk_storage) + dataset_to_icechunk(vds1, icechunk_filestore) + icechunk_filestore.commit( + "test commit" + ) # need to commit it in order to append to it in the next lines + + # Append the same dataset to the same store + icechunk_filestore_append = IcechunkStore.open_existing( + storage=icechunk_storage, read_only=False + ) + dataset_to_icechunk(vds2, icechunk_filestore_append, append_dim="time") + icechunk_filestore_append.commit("appended data") + new_ds = open_zarr(icechunk_filestore_append, consolidated=False, zarr_format=3) + + expected_ds1, expected_ds2 = open_dataset(filepath1), open_dataset(filepath2) + expected_ds = concat([expected_ds1, expected_ds2], dim="time").drop_vars( + ["time", "lat", "lon"], errors="ignore" + ) + # Because we encode attributes, attributes may differ, for example + # actual_range for expected_ds.air is array([185.16, 322.1 ], dtype=float32) + # but encoded it is [185.16000366210935, 322.1000061035156] + xrt.assert_equal(new_ds, expected_ds) + + ## When appending to a virtual ref with encoding, it succeeds + @pytest.mark.asyncio + async def test_append_with_multiple_root_arrays( + self, icechunk_storage: "StorageConfig", netcdf4_files_factory: Callable + ): + import xarray.testing as xrt + from icechunk import IcechunkStore + + filepath1, filepath2 = netcdf4_files_factory( + encoding={"air": {"dtype": "float64", "chunksizes": (1460, 25, 53)}} + ) + lon_manifest = gen_virtual_variable( filepath1, shape=(53,), @@ -519,7 +578,7 @@ async def test_append_virtual_ref_with_encoding( shape=(1460,), chunk_shape=(1460,), dtype=np.dtype("float32"), - offset=15495515, + offset=15498221, length=5840, dims=["time"], attrs=time_attrs, @@ -542,8 +601,7 @@ async def test_append_virtual_ref_with_encoding( dims=["time", "lat", "lon"], dtype=np.dtype("float64"), variable_name="air", - encoding={"scale_factor": scale_factor}, - offset=15419, + offset=18043, length=15476000, coords=coords1, ), @@ -554,8 +612,7 @@ async def test_append_virtual_ref_with_encoding( dims=["time", "lat", "lon"], dtype=np.dtype("float64"), variable_name="air", - encoding={"scale_factor": scale_factor}, - offset=15419, + offset=18043, length=15476000, coords=coords2, ), @@ -583,9 +640,6 @@ async def test_append_virtual_ref_with_encoding( expected_ds1, expected_ds2 = open_dataset(filepath1), open_dataset(filepath2) expected_ds = concat([expected_ds1, expected_ds2], dim="time") - # Because we encode attributes, attributes may differ, for example - # actual_range for expected_ds.air is array([185.16, 322.1 ], dtype=float32) - # but encoded it is [185.16000366210935, 322.1000061035156] xrt.assert_equal(new_ds, expected_ds) # When appending to a virtual ref with compression, it succeeds