merged conflicts

environment.yml

@@ -3,6 +3,7 @@ name: tzisdemo
 channels:
   - conda-forge
 dependencies:
+  - python==3.10
   - pip
   - intake
   - intake-esm

tests/run_tests.sh

 #!/bin/sh
-conda install python=3.10 cdo python-cdo -c conda-forge -y
+micromamba install python=3.10 cdo python-cdo -c conda-forge -y
 cd files
 for file in $(ls -1 ./ ); do
+  echo $file
   sha=$(cat $file | grep sha256 | cut -d ':' -f 2);
   wget -q https://swift.dkrz.de/v1/dkrz_0b2a0dcc-1430-4a8a-9f25-a6cb8924d92b/tzis-tests_input-files/${sha} ;
   mv $sha $file ;

tzis/openmf.py

@@ -3,7 +3,7 @@ import xarray
 from .provenance import Provenance
 from .rechunker import calc_chunk_length, calc_other_dim_chunks
 import glob
-from copy import deepcopy
+import copy
 
 OPEN_MFDATASET_KWARGS = dict(
     decode_cf=True,
@@ -69,7 +69,7 @@ class open_mfdataset_optimize:
         self.mf_dset = None
         self.original_size = None
         self.provenance = None
-        default_kwargs=deepcopy(OPEN_MFDATASET_KWARGS)
+        default_kwargs=copy.deepcopy(OPEN_MFDATASET_KWARGS)
 
         if verbose:
             print("Resetting disk size and compression ratio")
@@ -77,6 +77,8 @@ class open_mfdataset_optimize:
         imf = mf
         if type(mf) == str:
             imf = glob.glob(mf)
+        if not imf:
+            imf = copy.deepcopy(mf)
 
         if verbose:
             print("Opening test dataset")

tzis/provenance.py

@@ -34,7 +34,7 @@ class Provenance(object):
     def __init__(self, url):
         from .tzis import __version__ as tzis_version
 
-        self._identifier = None
+        self._identifier = uuid.uuid4()
         self.tzis_version = tzis_version
 
         self.url = url
@@ -44,10 +44,18 @@ class Provenance(object):
             self.url = os.path.dirname(os.path.dirname(url))
             self.container = os.path.basename(os.path.dirname(url))
             self.prefix = os.path.basename(url)
+            if len(self.url.split('/')) < 6:
+                if len(self.url.split('/')) < 5:
+                    raise ValueError(f"The provided {url} does not include a container")
+                self.url = os.path.dirname(url)
+                self.container = os.path.basename(url)
+                self.prefix = self._identifier
+
+                
+
         # Create an empyty prov document
         self.doc = prov.ProvDocument()
 
-        self._identifier = uuid.uuid4()
         self.tracking_id = str(uuid.uuid4())
         # Declaring namespaces for various prefixes
         self.doc.add_namespace(TZIS)

tzis/swifthandling.py

@@ -77,6 +77,7 @@ def mapper_mkdirs(url, container, prefix=None):
         conn = Connection(
             preauthurl=getenv("OS_STORAGE_URL"), preauthtoken=getenv("OS_AUTH_TOKEN")
         )
+        print(f"Check if {container} is in {url}. Otherwise make it.")
         listings = fsmap.fs.listdir(url)
         if not container in [a["name"].split("/")[-1] for a in listings]:
             conn.put_container(container)
@@ -259,6 +260,7 @@ def toggle_public(container):
     acl = conn.head_container(container).get("x-container-read", "")
     if ".r:*" in acl:
         acl = acl.replace(".r:*", "")
+        acl = acl.replace(".rlistings", "")
     else:
         acl = ",".join([acl, ".r:*,.rlistings"])
     acl = ",".join(sorted(filter(None, set(acl.split(",")))))