reconsolidating zmetadata after pulling metadata for fields via yac

apps/hiopy/_zarr_utils.py

@@ -23,6 +23,7 @@ def get_time_axis(v):
     else:
         return None
 
+
 def get_var_parent_group(v):
     var_group = get_var_group(v)
     parent_var_path = var_group.attrs["hiopy::parent"]

apps/hiopy/configure/create_dataset.py

@@ -36,18 +36,14 @@ def add_coordinates(
 
     lat_list, lon_list = zip(*coordinates)
 
-    lon = dataset.create_array(
-        name=coord_names[0], dtype=np.float32, shape=(len(coordinates),)
-    )
+    lon = dataset.create_array(name=coord_names[0], dtype=np.float32, shape=(len(coordinates),))
     lon[:] = np.array(lon_list)
     lon.attrs["_ARRAY_DIMENSIONS"] = [coord_names[0]]
     lon.attrs["long_name"] = "longitude"
     lon.attrs["units"] = "degree"
     lon.attrs["standard_name"] = "grid_longitude"
 
-    lat = dataset.create_array(
-        name=coord_names[1], dtype=np.float32, shape=(len(coordinates),)
-    )
+    lat = dataset.create_array(name=coord_names[1], dtype=np.float32, shape=(len(coordinates),))
     lat[:] = np.array(lat_list)
     lat.attrs["_ARRAY_DIMENSIONS"] = [coord_names[1]]
     lat.attrs["long_name"] = "latitude"

apps/hiopy/tests/CMakeLists.txt

@@ -29,7 +29,7 @@ set_tests_properties(hiopy.create_simple_dataset_4threads PROPERTIES
 
 add_test(
   NAME hiopy.check_hierarchy
-  COMMAND ${Python_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/check_hierarchy.py" "simple_dataset.zarr"
+  COMMAND "/usr/bin/env" "python3" "${CMAKE_CURRENT_SOURCE_DIR}/check_hierarchy.py" "simple_dataset.zarr"
 )
 
 set_tests_properties(hiopy.check_hierarchy PROPERTIES

apps/hiopy/tests/check_hierarchy.py

@@ -21,7 +21,9 @@ def check_interpolation(source_var, target_var):
 
 
 def check_metadata(var):
-    assert "hiopy::copy_metadata" not in var.attrs
+    assert (
+        "hiopy::copy_metadata" not in var.attrs
+    ), f"Attributes of {var.name} {var.attrs.asdict()} is not cleaned"
 
 
 def _collect_groups(dataset, parent=None):

apps/hiopy/worker.py

@@ -134,7 +134,8 @@ def main():
                 - np.datetime64(var_group["time"].attrs["units"][len("seconds since ") :])
             ) / np.timedelta64(1, "s") + dt
             t0_idx = np.searchsorted(var_group["time"], t0)
-            assert var_group["time"][t0_idx] == t0, f"start_datetime {t0} not found in time axis at index {t0_idx} which has value {var_group['time'][t0_idx]}"
+            assert var_group["time"][t0_idx] == t0, f"start_datetime {t0} not found in time axis \
+                                    at index {t0_idx} which has value {var_group['time'][t0_idx]}"
 
             # see YAC_REDUCTION_TIME_NONE etc. (TODO: pass constants through coyote)
             time_methods2yac = {"point": 0, "sum": 1, "mean": 2, "min": 3, "max": 4}
@@ -203,13 +204,16 @@ def main():
             if "hiopy::copy_metadata" in v.attrs:
                 comp, grid, field = get_source_triple(v)
                 md_str = get_field_metadata(comp, grid, field)
-                print(md_str)
                 metadata = json.loads(md_str)
-                print(metadata)
+                logging.debug(f"Found metadata for {field}: {metadata}")
                 for key, value in metadata["cf"].items():
                     v.attrs[key] = value
                 del v.attrs["hiopy::copy_metadata"]  # copy only once
 
+        # re-consolidate the newly updated metadata
+        for ds in args.datasets:
+            zarr.consolidate_metadata(ds.store)
+
     run()
 
     for dh in data_handlers: