Missed one att

82965973 · wachsylon · 8eb66102 · 82965973
Commit 82965973 authored 2 years ago by wachsylon
--- a/tests/test_validations.ipynb
+++ b/tests/test_validations.ipynb
@@ -38,7 +38,7 @@
    "    SWIFT_TOKEN,\n",
    "    fsmap_name\n",
    ")\n",
-    "alltests=fsmap.fs.listdir(fsmap.fsspec_map.root)\n",
+    "alltests=fsmap.fs.listdir(fsmap.root)\n",
    "for zarr_test_output in alltests:\n",
    "    if zarr_test_output[\"type\"] == \"directory\":\n",
    "        print(zarr_test_output)\n",

 %% Cell type:code id: tags:

 ``` python
 from tzis import catalog, daskhandling, openmf, provenance, rechunker, swifthandling, tzis
 import os
 import pytest
 import xarray as xr
 import subprocess
 import fsspec
 ```

 %% Cell type:code id: tags:

 ``` python
 OS_STORAGE_URL="https://swift.dkrz.de/v1/dkrz_0b2a0dcc-1430-4a8a-9f25-a6cb8924d92b"
 SWIFT_TOKEN=os.environ.get('SWIFT_TOKEN')
 fsmap_name="tzis-tests_output-created"
 ```

 %% Cell type:code id: tags:

 ``` python
 config=[]
 testfiledir="files"
 fsmap=swifthandling.get_swift_mapper(
    OS_STORAGE_URL,
    SWIFT_TOKEN,
    fsmap_name
 )
-alltests=fsmap.fs.listdir(fsmap.fsspec_map.root)
+alltests=fsmap.fs.listdir(fsmap.root)
 for zarr_test_output in alltests:
    if zarr_test_output["type"] == "directory":
        print(zarr_test_output)
        config.append(
            (fsmap,
             zarr_test_output["name"].split('/')[-1]
            )
        )
 ```

 %% Cell type:code id: tags:

 ``` python
 def set_coords(zarr_dset):
    precoords = set(
        ["lat_bnds", "lev_bnds", "ap", "b", "ap_bnds", "b_bnds", "lon_bnds"]
    )
    coords = [x for x in zarr_dset.data_vars.variables if x in precoords]
    return zarr_dset.set_coords(coords)

 def get_encoding(zarr_dset):
    encoding = {
        var:{
            key : zarr_dset[var].encoding[key]
            for key in zarr_dset[var].encoding.keys()
            if key not in ['chunks', 'preferred_chunks', 'compressor', 'filters', 'coordinates']
        }
        for var in list(zarr_dset.variables.keys())
    }
    for var in zarr_dset.data_vars:
        encoding[var].update(dict(zlib=True, complevel=1))
    for var in zarr_dset.coords:
        encoding[var].update(dict(_FillValue=False))

    return encoding
 ```

 %% Cell type:code id: tags:

 ``` python
 @pytest.mark.parametrize("fsmap,zarr_test_output", config)
 class TestTzisValidate:
    def test_validate_tzis_open(self,
                                fsmap,
                                zarr_test_output
                               ):
        zarr_fsmap=fsspec.get_mapper(
            '/'.join(
                [
                    fsmap.root,
                    zarr_test_output
                ]
            )
        )
        zarr_dset=xr.open_zarr(zarr_fsmap,
                               consolidated=True,
                               #decode_times=True,
                               use_cftime=True)
        assert True

    def test_validate_tzis_tonetcdf(self,
                                    fsmap,
                                    zarr_test_output
                                   ):
        zarr_fsmap=fsspec.get_mapper(
            '/'.join(
                [
                    fsmap.root,
                    zarr_test_output
                ]
            )
        )

        zarr_dset=xr.open_zarr(zarr_fsmap,
                               consolidated=True,
                               #decode_times=True,
                               use_cftime=True)
        zarr_dset=set_coords(zarr_dset)
        encoding=get_encoding(zarr_dset)
        netcdf_rewritten=zarr_test_output+".nc"
        zarr_dset.isel(time=slice(0,10)).to_netcdf(netcdf_rewritten,
                          format="NETCDF4_CLASSIC",
                          unlimited_dims="time",
                          encoding=encoding)
        assert(True)
    def test_validate_tzis_compare(self,
                                   fsmap,
                                   zarr_test_output
                                  ):
        netcdf_rewritten=zarr_test_output+".nc"
        netcdf_orig=os.path.join(testfiledir, '.'.join(zarr_test_output.split('.')[:-1]))
        with open("cdo_diff.log", "a") as f:
            f.write(netcdf_rewritten)
            subprocess.run(["cdo", "diff", netcdf_rewritten, "-seltimestep,1/10", netcdf_orig],
                           stdout=f,
                           stderr=subprocess.STDOUT)
        subprocess.run(["rm", netcdf_rewritten])
        assert(True)
 ```

 %% Cell type:code id: tags:

 ``` python
 pytest.main(["-x", "./test_validations.py"])
 ```

 %% Cell type:code id: tags:

 ``` python
 ```