From cd55ed4703dbda40516aa48337b2da2990c89da5 Mon Sep 17 00:00:00 2001
From: Fabian Wachsmann <k204210@l40030.lvt.dkrz.de>
Date: Mon, 9 Jan 2023 17:40:19 +0100
Subject: [PATCH] Updated intake master catalog to recent main version of
intake-esm
---
.../cloud-access/dkrz_catalog.yaml | 42 ++++++++---------
esm-collections/disk-access/dkrz_catalog.yaml | 47 +++++++++----------
2 files changed, 44 insertions(+), 45 deletions(-)
diff --git a/esm-collections/cloud-access/dkrz_catalog.yaml b/esm-collections/cloud-access/dkrz_catalog.yaml
index 15e0242..b1953ef 100755
--- a/esm-collections/cloud-access/dkrz_catalog.yaml
+++ b/esm-collections/cloud-access/dkrz_catalog.yaml
@@ -42,7 +42,7 @@ plugins:
sources:
dkrz_cmip5_archive:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- product_id
@@ -59,14 +59,14 @@ sources:
- uri
- format
- jblob_file
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_cmip5_archive.json
+ obj: "{{CATALOG_DIR}}/dkrz_cmip5_archive.json"
description: This is an ESM collection for CMIP5 data accessible on the DKRZ's
cera archive
driver:
- intake.open_esm_datastore
dkrz_cmip5_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- product_id
@@ -82,14 +82,14 @@ sources:
- temporal_subset
- uri
- format
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_cmip5_disk.json
+ obj: "{{CATALOG_DIR}}/dkrz_cmip5_disk.json"
description: This is an ESM collection for CMIP5 data accessible on the DKRZ's
Lustre disk storage system in /work/kd0956/CMIP5/data/cmip5/
driver:
- intake.open_esm_datastore
dkrz_cmip6_cloud:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- activity_id
@@ -103,14 +103,14 @@ sources:
- version
- uri
- format
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_cmip6_cloud.json
+ obj: "{{CATALOG_DIR}}/dkrz_cmip6_cloud.json"
description: This is an ESM collection for CMIP6 data accessible on DKRZ's swift
cloud store
driver:
- intake.open_esm_datastore
dkrz_cmip6_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- activity_id
@@ -126,13 +126,13 @@ sources:
- time_range
- uri
- format
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_cmip6_disk.json
+ obj: "{{CATALOG_DIR}}/dkrz_cmip6_disk.json"
description: This is a ESM-collection for CMIP6 data on DKRZ's disk storage system
driver:
- intake.open_esm_datastore
dkrz_cordex_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- product_id
@@ -149,14 +149,14 @@ sources:
- time_range
- uri
- format
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_cordex_disk.json
+ obj: "{{CATALOG_DIR}}/dkrz_cordex_disk.json"
description: 'This is an ESM collection for CORDEX data accessible on the DKRZ''s
disk storage system '
driver:
- intake.open_esm_datastore
dkrz_dyamond-winter_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols: &id001
- project
- institution_id
@@ -175,13 +175,13 @@ sources:
- format
- uri
- variable_id
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_dyamond-winter_disk.json
+ obj: "{{CATALOG_DIR}}/dkrz_dyamond-winter_disk.json"
description: Default catalog for ICON-ESM experiments by MPIMet
driver:
- intake.open_esm_datastore
dkrz_era5_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- era_id
@@ -195,19 +195,19 @@ sources:
- initialization_date
- uri
- format
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_era5_disk.json
+ obj: "{{CATALOG_DIR}}/dkrz_era5_disk.json"
description: This is an ESM collection for ERA5 data accessible on the DKRZ's
disk storage system in /work/bk1099/data/
driver:
- intake.open_esm_datastore
dkrz_monsoon_disk:
args:
- path: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_monsoon_disk.yaml
+ path: "{{CATALOG_DIR}}/dkrz_monsoon_disk.yaml"
description: Monsoon 2.0
driver: yaml_file_cat
dkrz_mpige_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- product_id
- institute
@@ -222,7 +222,7 @@ sources:
- version
- uri
- format
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_mpige_disk.json
+ obj: "{{CATALOG_DIR}}/dkrz_mpige_disk.json"
description: This is an ESM collection for the Max Planck Institute Grand Ensemble
(Maher et al. 2019 https://doi.org/10/gf3kgt) cmorized by CMIP5-standards accessible
on the DKRZ's Levante disk storage system in /work/mh1007/CMOR/MPI-GE
@@ -230,15 +230,15 @@ sources:
- intake.open_esm_datastore
dkrz_nextgems_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols: *id001
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_nextgems_disk.json
+ obj: "{{CATALOG_DIR}}/dkrz_nextgems_disk.json"
description: Default catalog for ICON-ESM experiments by MPIMet
driver:
- intake.open_esm_datastore
dkrz_palmod2_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- source_id
@@ -252,7 +252,7 @@ sources:
- format
- time_range
- uri
- esmcol_obj: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_palmod2_disk.json
+ obj: "{{CATALOG_DIR}}/dkrz_palmod2_disk.json"
description: This is a ESM-collection for Palmod2 data on DKRZ's disk storage
system
driver:
diff --git a/esm-collections/disk-access/dkrz_catalog.yaml b/esm-collections/disk-access/dkrz_catalog.yaml
index a7d5444..0f24a8b 100755
--- a/esm-collections/disk-access/dkrz_catalog.yaml
+++ b/esm-collections/disk-access/dkrz_catalog.yaml
@@ -42,7 +42,7 @@ plugins:
sources:
dkrz_cmip5_archive:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- product_id
@@ -59,14 +59,14 @@ sources:
- uri
- format
- jblob_file
- esmcol_obj: /pool/data/Catalogs/dkrz_cmip5_archive.json
+ obj: '{{CATALOG_DIR}}/dkrz_cmip5_archive.json'
description: This is an ESM collection for CMIP5 data accessible on the DKRZ's
cera archive
driver:
- intake.open_esm_datastore
dkrz_cmip5_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- product_id
@@ -82,14 +82,14 @@ sources:
- temporal_subset
- uri
- format
- esmcol_obj: /pool/data/Catalogs/dkrz_cmip5_disk.json
+ obj: '{{CATALOG_DIR}}/dkrz_cmip5_disk.json'
description: This is an ESM collection for CMIP5 data accessible on the DKRZ's
Lustre disk storage system in /work/kd0956/CMIP5/data/cmip5/
driver:
- intake.open_esm_datastore
dkrz_cmip6_cloud:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- activity_id
@@ -103,14 +103,13 @@ sources:
- version
- uri
- format
- esmcol_obj: /pool/data/Catalogs/dkrz_cmip6_cloud.json
+ obj: '{{CATALOG_DIR}}/dkrz_cmip6_cloud.json'
description: This is an ESM collection for CMIP6 data accessible on DKRZ's swift
cloud store
- driver:
- - intake.open_esm_datastore
+ driver: intake_esm.esm_datastore
dkrz_cmip6_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- activity_id
@@ -126,13 +125,13 @@ sources:
- time_range
- uri
- format
- esmcol_obj: /pool/data/Catalogs/dkrz_cmip6_disk.json
+ obj: '{{CATALOG_DIR}}/dkrz_cmip6_disk.json'
description: This is a ESM-collection for CMIP6 data on DKRZ's disk storage system
driver:
- intake.open_esm_datastore
dkrz_cordex_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- product_id
@@ -149,14 +148,14 @@ sources:
- time_range
- uri
- format
- esmcol_obj: /pool/data/Catalogs/dkrz_cordex_disk.json
- description: 'This is an ESM collection for CORDEX data accessible on the DKRZ''s
+ obj: '{{CATALOG_DIR}}/dkrz_cordex_disk.json'
+ description: This is an ESM collection for CORDEX data accessible on the DKRZ's
disk storage system '
driver:
- intake.open_esm_datastore
dkrz_dyamond-winter_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols: &id001
- project
- institution_id
@@ -175,13 +174,13 @@ sources:
- format
- uri
- variable_id
- esmcol_obj: /pool/data/Catalogs/dkrz_dyamond-winter_disk.json
+ obj: '{{CATALOG_DIR}}/dkrz_dyamond-winter_disk.json'
description: Default catalog for ICON-ESM experiments by MPIMet
driver:
- intake.open_esm_datastore
dkrz_era5_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- era_id
@@ -195,19 +194,19 @@ sources:
- initialization_date
- uri
- format
- esmcol_obj: /pool/data/Catalogs/dkrz_era5_disk.json
+ obj: '{{CATALOG_DIR}}/dkrz_era5_disk.json'
description: This is an ESM collection for ERA5 data accessible on the DKRZ's
disk storage system in /work/bk1099/data/
driver:
- intake.open_esm_datastore
dkrz_monsoon_disk:
args:
- path: /pool/data/Catalogs/dkrz_monsoon_disk.yaml
+ path: '{{CATALOG_DIR}}/dkrz_monsoon_disk.yaml'
description: Monsoon 2.0
driver: yaml_file_cat
dkrz_mpige_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- product_id
- institute
@@ -222,7 +221,7 @@ sources:
- version
- uri
- format
- esmcol_obj: /pool/data/Catalogs/dkrz_mpige_disk.json
+ obj: '{{CATALOG_DIR}}/dkrz_mpige_disk.json'
description: This is an ESM collection for the Max Planck Institute Grand Ensemble
(Maher et al. 2019 https://doi.org/10/gf3kgt) cmorized by CMIP5-standards accessible
on the DKRZ's Levante disk storage system in /work/mh1007/CMOR/MPI-GE
@@ -230,15 +229,15 @@ sources:
- intake.open_esm_datastore
dkrz_nextgems_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols: *id001
- esmcol_obj: /pool/data/Catalogs/dkrz_nextgems_disk.json
+ obj: '{{CATALOG_DIR}}/dkrz_nextgems_disk.json'
description: Default catalog for ICON-ESM experiments by MPIMet
driver:
- intake.open_esm_datastore
dkrz_palmod2_disk:
args:
- csv_kwargs:
+ read_csv_kwargs:
usecols:
- project
- source_id
@@ -252,7 +251,7 @@ sources:
- format
- time_range
- uri
- esmcol_obj: /pool/data/Catalogs/dkrz_palmod2_disk.json
+ obj: '{{CATALOG_DIR}}/dkrz_palmod2_disk.json'
description: This is a ESM-collection for Palmod2 data on DKRZ's disk storage
system
driver:
--
GitLab