From 7c7ad6d20fe61c97c642ea2862ea860e673e7071 Mon Sep 17 00:00:00 2001
From: Fabian Wachsmann <k204210@l40052.lvt.dkrz.de>
Date: Wed, 18 Oct 2023 11:42:17 +0200
Subject: [PATCH] Added cloud eerie
---
esm-collections/cloud-access/dkrz_eerie.yaml | 26 +++++++++++++
esm-collections/disk-access/dkrz_catalog.yaml | 37 +++++++++++--------
esm-collections/disk-access/dkrz_eerie.yaml | 26 +++++++++++++
3 files changed, 74 insertions(+), 15 deletions(-)
create mode 100644 esm-collections/cloud-access/dkrz_eerie.yaml
create mode 100644 esm-collections/disk-access/dkrz_eerie.yaml
diff --git a/esm-collections/cloud-access/dkrz_eerie.yaml b/esm-collections/cloud-access/dkrz_eerie.yaml
new file mode 100644
index 0000000..c005e9b
--- /dev/null
+++ b/esm-collections/cloud-access/dkrz_eerie.yaml
@@ -0,0 +1,26 @@
+sources:
+ dkrz_eerie_kerchunk:
+ args:
+ path: "{{CATALOG_DIR}}/dkrz_eerie_kerchunk.yaml"
+ description: This catalog contains datasets for all raw data in /work/bm1344 and accesses the data via kerchunks in /work/bm1344/DKRZ/kerchunks
+ driver: yaml_file_cat
+ dkrz_eerie_archive:
+ args:
+ path: "{{CATALOG_DIR}}/dkrz_eerie_archive.yaml"
+ description: This catalog contains datasets for archived data in /arch/bm1344
+ driver: yaml_file_cat
+ dkrz_eerie_xpublish:
+ args:
+ path: "{{CATALOG_DIR}}/dkrz_eerie_xpublish.yaml"
+ description: This catalog contains the same datasets as *dkrz_eerie_kerchunk* but data access is via the xpublish server *eerie.cloud.dkrz.de*
+ driver: yaml_file_cat
+ dkrz_eerie_evaluation:
+ args:
+ path: "{{CATALOG_DIR}}/dkrz_eerie_evaluation.yaml"
+ description: This catalog contains additional EERIE evaluation data from /work/bm1344/DKRZ
+ driver: yaml_file_cat
+ dkrz_catalog:
+ args:
+ path: "https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_catalog.yaml"
+ description: DKRZ master catalog for all /pool/data catalogs available
+ driver: yaml_file_cat
diff --git a/esm-collections/disk-access/dkrz_catalog.yaml b/esm-collections/disk-access/dkrz_catalog.yaml
index 1a7fafe..874e118 100755
--- a/esm-collections/disk-access/dkrz_catalog.yaml
+++ b/esm-collections/disk-access/dkrz_catalog.yaml
@@ -4,16 +4,17 @@ metadata:
additional_cmip6_disk_columns:
default:
- units
- - path
- opendap_url
+ - path
- long_name
type: list[str]
additional_era5_disk_columns:
default:
+ - step
- path
- - units
- - long_name
- short_name
+ - long_name
+ - units
type: list[str]
cataloonie_columns:
default:
@@ -41,6 +42,7 @@ plugins:
sources:
dkrz_cmip5_archive:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_cmip5_archive.json'
read_csv_kwargs:
usecols:
- project
@@ -58,13 +60,13 @@ sources:
- uri
- format
- jblob_file
- obj: '{{CATALOG_DIR}}/dkrz_cmip5_archive.json'
description: This is an ESM collection for CMIP5 data accessible on the DKRZ's
cera archive
driver:
- intake.open_esm_datastore
dkrz_cmip5_disk:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_cmip5_disk.json'
read_csv_kwargs:
usecols:
- project
@@ -81,13 +83,13 @@ sources:
- temporal_subset
- uri
- format
- obj: '{{CATALOG_DIR}}/dkrz_cmip5_disk.json'
description: This is an ESM collection for CMIP5 data accessible on the DKRZ's
Lustre disk storage system in /work/kd0956/CMIP5/data/cmip5/
driver:
- intake.open_esm_datastore
dkrz_cmip6_cloud:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_cmip6_cloud.json'
read_csv_kwargs:
usecols:
- project
@@ -102,12 +104,13 @@ sources:
- version
- uri
- format
- obj: '{{CATALOG_DIR}}/dkrz_cmip6_cloud.json'
description: This is an ESM collection for CMIP6 data accessible on DKRZ's swift
cloud store
- driver: intake_esm.esm_datastore
+ driver:
+ - intake.open_esm_datastore
dkrz_cmip6_disk:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_cmip6_disk.json'
read_csv_kwargs:
usecols:
- project
@@ -124,12 +127,12 @@ sources:
- time_range
- uri
- format
- obj: '{{CATALOG_DIR}}/dkrz_cmip6_disk.json'
description: This is a ESM-collection for CMIP6 data on DKRZ's disk storage system
driver:
- intake.open_esm_datastore
dkrz_cordex_disk:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_cordex_disk.json'
read_csv_kwargs:
usecols:
- project
@@ -147,13 +150,13 @@ sources:
- time_range
- uri
- format
- obj: '{{CATALOG_DIR}}/dkrz_cordex_disk.json'
- description: This is an ESM collection for CORDEX data accessible on the DKRZ's
+ description: 'This is an ESM collection for CORDEX data accessible on the DKRZ''s
disk storage system '
driver:
- intake.open_esm_datastore
dkrz_dyamond-winter_disk:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_dyamond-winter_disk.json'
read_csv_kwargs:
usecols: &id001
- project
@@ -173,12 +176,17 @@ sources:
- format
- uri
- variable_id
- obj: '{{CATALOG_DIR}}/dkrz_dyamond-winter_disk.json'
description: Default catalog for ICON-ESM experiments by MPIMet
driver:
- intake.open_esm_datastore
+ dkrz_eerie:
+ args:
+ path: '{{CATALOG_DIR}}/dkrz_eerie.yaml'
+ description: EERIE Data at DKRZ
+ driver: yaml_file_cat
dkrz_era5_disk:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_era5_disk.json'
read_csv_kwargs:
usecols:
- project
@@ -193,7 +201,6 @@ sources:
- initialization_date
- uri
- format
- obj: '{{CATALOG_DIR}}/dkrz_era5_disk.json'
description: This is an ESM collection for ERA5 data accessible on the DKRZ's
disk storage system in /work/bk1099/data/
driver:
@@ -205,6 +212,7 @@ sources:
driver: yaml_file_cat
dkrz_mpige_disk:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_mpige_disk.json'
read_csv_kwargs:
usecols:
- product_id
@@ -220,7 +228,6 @@ sources:
- version
- uri
- format
- obj: '{{CATALOG_DIR}}/dkrz_mpige_disk.json'
description: This is an ESM collection for the Max Planck Institute Grand Ensemble
(Maher et al. 2019 https://doi.org/10/gf3kgt) cmorized by CMIP5-standards accessible
on the DKRZ's Levante disk storage system in /work/mh1007/CMOR/MPI-GE
@@ -228,14 +235,15 @@ sources:
- intake.open_esm_datastore
dkrz_nextgems_disk:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_nextgems_disk.json'
read_csv_kwargs:
usecols: *id001
- obj: '{{CATALOG_DIR}}/dkrz_nextgems_disk.json'
description: Default catalog for ICON-ESM experiments by MPIMet
driver:
- intake.open_esm_datastore
dkrz_palmod2_disk:
args:
+ obj: '{{CATALOG_DIR}}/dkrz_palmod2_disk.json'
read_csv_kwargs:
usecols:
- project
@@ -250,7 +258,6 @@ sources:
- format
- time_range
- uri
- obj: '{{CATALOG_DIR}}/dkrz_palmod2_disk.json'
description: This is a ESM-collection for Palmod2 data on DKRZ's disk storage
system
driver:
diff --git a/esm-collections/disk-access/dkrz_eerie.yaml b/esm-collections/disk-access/dkrz_eerie.yaml
new file mode 100644
index 0000000..c005e9b
--- /dev/null
+++ b/esm-collections/disk-access/dkrz_eerie.yaml
@@ -0,0 +1,26 @@
+sources:
+ dkrz_eerie_kerchunk:
+ args:
+ path: "{{CATALOG_DIR}}/dkrz_eerie_kerchunk.yaml"
+ description: This catalog contains datasets for all raw data in /work/bm1344 and accesses the data via kerchunks in /work/bm1344/DKRZ/kerchunks
+ driver: yaml_file_cat
+ dkrz_eerie_archive:
+ args:
+ path: "{{CATALOG_DIR}}/dkrz_eerie_archive.yaml"
+ description: This catalog contains datasets for archived data in /arch/bm1344
+ driver: yaml_file_cat
+ dkrz_eerie_xpublish:
+ args:
+ path: "{{CATALOG_DIR}}/dkrz_eerie_xpublish.yaml"
+ description: This catalog contains the same datasets as *dkrz_eerie_kerchunk* but data access is via the xpublish server *eerie.cloud.dkrz.de*
+ driver: yaml_file_cat
+ dkrz_eerie_evaluation:
+ args:
+ path: "{{CATALOG_DIR}}/dkrz_eerie_evaluation.yaml"
+ description: This catalog contains additional EERIE evaluation data from /work/bm1344/DKRZ
+ driver: yaml_file_cat
+ dkrz_catalog:
+ args:
+ path: "https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_catalog.yaml"
+ description: DKRZ master catalog for all /pool/data catalogs available
+ driver: yaml_file_cat
--
GitLab