Only metadata is written, no chunks
I did not come up with a minimal reproducible example yet, but I was hoping that someone might already have a solution to the issue that I'm encountering. I'm not sure if this is actually an issue of this package or not though.
In any case, this is my code and traceback:
In [4]: import xarray as xr
...: from tzis import tzis
...: ACCOUNT="mh0010"
...: USERNAME="m300408"
...: CONTAINER_NAME="EUREC4A_LES"
...: ZARR_DSET_NAME="experiment_1/ICON_DOM01_radiation.zarr/"
...:
...: token=tzis.get_token(ACCOUNT, USERNAME=USERNAME)
...: container = tzis.Tzis(token["OS_STORAGE_URL"],token["OS_AUTH_TOKEN"],CONTAINER_NAME,use_fsspec=True)
...: prefix_for_object_storage=ZARR_DSET_NAME
...:
...: container.open_store(prefix_for_object_storage)
...:
...: zf=xr.open_zarr("ICON_DOM01_radiation.zarr")
...: zf.to_zarr(container.store,mode='w')
...:
---------------------------------------------------------------------------
PathNotFoundError Traceback (most recent call last)
<ipython-input-4-4b9c5e661c66> in <module>
13
14 zf=xr.open_zarr("ICON_DOM01_radiation.zarr")
---> 15 zf.to_zarr(container.store,mode='w')
~/.local/lib/python3.8/site-packages/xarray/core/dataset.py in to_zarr(self, store, chunk_store, mode, synchronizer, group, encoding, compute, consolidated, append_dim, region, safe_chunks)
2029 encoding = {}
2030
-> 2031 return to_zarr(
2032 self,
2033 store=store,
~/.local/lib/python3.8/site-packages/xarray/backends/api.py in to_zarr(dataset, store, chunk_store, mode, synchronizer, group, encoding, compute, consolidated, append_dim, region, safe_chunks)
1412 writer = ArrayWriter()
1413 # TODO: figure out how to properly handle unlimited_dims
-> 1414 dump_to_store(dataset, zstore, writer, encoding=encoding)
1415 writes = writer.sync(compute=compute)
1416
~/.local/lib/python3.8/site-packages/xarray/backends/api.py in dump_to_store(dataset, store, writer, encoder, encoding, unlimited_dims)
1122 variables, attrs = encoder(variables, attrs)
1123
-> 1124 store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims)
1125
1126
~/.local/lib/python3.8/site-packages/xarray/backends/zarr.py in store(self, variables, attributes, check_encoding_set, writer, unlimited_dims)
557 )
558 if self._consolidate_on_close:
--> 559 zarr.consolidate_metadata(self.zarr_group.store)
560
561 def sync(self):
~/.local/lib/python3.8/site-packages/zarr/convenience.py in consolidate_metadata(store, metadata_key)
1132 }
1133 store[metadata_key] = json_dumps(out)
-> 1134 return open_consolidated(store, metadata_key=metadata_key)
1135
1136
~/.local/lib/python3.8/site-packages/zarr/convenience.py in open_consolidated(store, metadata_key, mode, **kwargs)
1190 # pass through
1191 chunk_store = kwargs.pop('chunk_store', None) or store
-> 1192 return open(store=meta_store, chunk_store=chunk_store, mode=mode, **kwargs)
~/.local/lib/python3.8/site-packages/zarr/convenience.py in open(store, mode, **kwargs)
104 return open_group(_store, mode=mode, **kwargs)
105 else:
--> 106 raise PathNotFoundError(path)
107
108
PathNotFoundError: nothing found at path ''
The error occurs independent of the value of use_fsspec
. When I investigate the objects on the swiftbrowser, all metafiles (.zattrs, .zarray, .zmetadata) are written, but no single chunk.
Cheers, Hauke