diff --git a/tzis/tzis.py b/tzis/tzis.py index c46f71321a5445f4936782c55449bc4b88ae60bc..09f640634e4d80b3c6ab5fff8dee30ed6f462973 100644 --- a/tzis/tzis.py +++ b/tzis/tzis.py @@ -25,6 +25,7 @@ import json from .provenance import Provenance from .catalog import * from .openmf import * +from numcodecs import Blosc OPEN_ZARR_KWARGS = dict(consolidated=True, decode_cf=True, use_cftime=True) @@ -146,17 +147,24 @@ class write_zarr(fsspec.FSMap): def _init_dataset(self, ds, store, chunkdim, overwrite): storestring, storage_options = get_storestring_and_options(store) - if overwrite: - try: - with self.fs.transaction: - self.fs.rm(self.fsspec_map.root, recursive=True) - # with self.fs.transaction: - # self.fs.mkdirs('/'.join([self.os_url,self.container,self.prefix]),exist_ok=True) - except Exception as e: - print(e) - if self.verbose: - print("Could not remove target.") - pass +# if overwrite: +# try: +# with self.fs.transaction: +# self.fs.rm(self.fsspec_map.root, recursive=True) +# while self.fs.du(self.fsspec_map.root) != 0: +# time.sleep(0.02) +# continue +# if self.fs.protocol != "swift": +# with self.fs.transaction: +# self.fs.mkdirs(os.path.dirname(self.fsspec_map.root),exist_ok=True) +# else: +# if self.verbose: +# print("Make sure the container exists already!") +# except Exception as e: +# print(e) +# if self.verbose: +# print("Could not remove target.") +# pass try: # try fsspec store: ds.to_zarr( @@ -853,6 +861,22 @@ class write_zarr(fsspec.FSMap): print(f"Compression ratio was estimated by disk size to {self._cratio}") if "chunks" in dset_to_write[self.varname].encoding.keys(): del dset_to_write[self.varname].encoding["chunks"] + + for var in dset_to_write.variables: + l_set_compressor = False + if "compressor" in dset_to_write[var].encoding.keys(): + try: + from gribscan import RawGribCodec + if isinstance(dset_to_write[var].encoding["compressor"],RawGribCodec): + l_set_compressor=True + except Exception as e: + print(e) + pass + else: + l_set_compressor = True + + if l_set_compressor: + dset_to_write[var].encoding["compressor"]=Blosc(cname="zstd", clevel=5, shuffle=2) pargs = locals() if not validity_check: