cdo_post example

.gitignore

 .DS_Store
 .*.un~
+sample_files
+MPI-ESM-*_output.csv
+!MPI-ESM-1-2-LR_output.csv
 
 # Created by https://www.gitignore.io/api/python
 

pymistral/cdo_post.py

@@ -90,19 +90,22 @@ def read_all_outdatatype_files_to_ds(
             ds = cdo.copy(input=path, options=options, returnXDataset=True)
             if 'time' not in ds.dims:
                 ds = ds.expand_dims('time')
-            ds.to_netcdf(f'{outpath}/sample_files/{model}_{outdatatype}.nc')
+            sample_files_path = f'{outpath}/sample_files/{expid}'
+            if not os.path.exists(sample_files_path):
+                os.makedirs(sample_files_path)
+            ds.to_netcdf(f'{sample_files_path}/{model}_{outdatatype}.nc')
             # add outdatatype
             for v in ds.data_vars:
                 ds[v].attrs['outdatatype'] = outdatatype
                 ds[v].attrs['model'] = model
                 ds[v].attrs['dims'] = list(ds[v].dims)
             ds_list.append(ds.isel(time=0).mean())
-        except:
-            print(f'{outdatatype_id} failed')
+        except Exception as e:
+            print(f'{outdatatype_id} failed: {e}')
     return xr.merge(ds_list, compat='override')
 
 
-def create_dataframe_of_output_info(ds, outpath=outpath):
+def create_dataframe_of_output_info(ds, outpath=outpath, expid=expid):
     """Create pd.Dataframe about output from `ds` and save to `outpath`."""
     df = pd.DataFrame(
         index=ds.data_vars,
@@ -121,7 +124,7 @@ def create_dataframe_of_output_info(ds, outpath=outpath):
         for c in list(df.columns):
             df[c].loc[v] = ds[v].attrs[c] if c in ds[v].attrs else ''
     df['stream'] = df['model'] + '_' + df['outdatatype']
-    df.to_csv(outpath + 'MPI-ESM-1-2-LR_output.csv')
+    df.to_csv(f'{outpath}/MPI-ESM-{expid}_output.csv')
     return df
 
 
@@ -130,6 +133,7 @@ def generate_output_df(
 ):
     """Combine all functions above to generate output from `expid` or just load
      if not `recalc`."""
+    # standard path
     path = outpath + 'MPI-ESM-1-2-LR_output.csv'
     if not recalc and os.path.exists(path):
         print(f'Read df from path: {path}')
@@ -145,7 +149,7 @@ def generate_output_df(
         ds = read_all_outdatatype_files_to_ds(
             outdatatypes, expid=expid, exppath=exppath, year=year, outpath=outpath, table_path=table_path
         )
-        output_df = create_dataframe_of_output_info(ds, outpath=outpath)
+        output_df = create_dataframe_of_output_info(ds, outpath=outpath, expid=expid)
     return output_df