diff --git a/builder/data-pool_collect-create-main.ipynb b/builder/data-pool_collect-create-main.ipynb index a00e96a4122ab1cf57e016ff40b57437fd86f107..4af9eef3c34ce3b1f90b3bb708b847e43daca74f 100644 --- a/builder/data-pool_collect-create-main.ipynb +++ b/builder/data-pool_collect-create-main.ipynb @@ -33,7 +33,7 @@ "source": [ "catalogs=[cat \n", " for cat in os.listdir(TRUNK)\n", - " if cat.endswith('.json')]" + " if cat.endswith('.json') or ((cat.endswith('.yaml')) & ('dkrz_catalog.yaml' not in cat))]" ] }, { @@ -84,6 +84,8 @@ "allcols={}\n", "usecols={}\n", "for cat in catalogs:\n", + " if \"yaml\" in cat:\n", + " continue\n", " with open(TRUNK+\"/\"+cat, \"r\") as f:\n", " cat_json=json.load(f)\n", " catname='_'.join(cat.split('.')[0].split('_')[1:])\n", @@ -156,6 +158,16 @@ "for k in catalogs:\n", " kentry=k.split('.')[0]\n", " catname='_'.join(k.split('.')[0].split('_')[1:])\n", + " if \"yaml\" in k:\n", + " if \"monsoon\" in k:\n", + " sources[kentry]=dict(\n", + " args=dict(\n", + " path=\"/pool/data/Catalogs/dkrz_monsoon_disk.yaml\"\n", + " ),\n", + " description=\"Monsoon 2.0\",\n", + " driver=\"yaml_file_cat\"\n", + " )\n", + " continue\n", " kpath=TRUNK+'/'+k\n", " with open(kpath,'r') as f:\n", " content=json.load(f)\n", @@ -183,6 +195,16 @@ "sourcesdict={\"sources\":sources}" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "23fc6855-f1c0-4545-8361-e074450636a3", + "metadata": {}, + "outputs": [], + "source": [ + "sources" + ] + }, { "cell_type": "code", "execution_count": null, @@ -190,7 +212,7 @@ "metadata": {}, "outputs": [], "source": [ - "with open(\"/home/k/k204210/volume/data-infrastructure-services/intake-esm/esm-collections/disk-access/dkrz_catalog.yaml\",\"w\") as f:\n", + "with open(\"/home/k/k204210/intake-esm/esm-collections/disk-access/dkrz_catalog.yaml\",\"w\") as f:\n", " f.write(yaml.dump(header))\n", " f.write(yaml.dump(sourcesdict))" ] diff --git a/esm-collections/cloud-access/dkrz_catalog.yaml b/esm-collections/cloud-access/dkrz_catalog.yaml index b14103ef00506267392949f19a86bdfe0f21f832..15e0242f3d4d6602437dc0811929b89e358044bd 100755 --- a/esm-collections/cloud-access/dkrz_catalog.yaml +++ b/esm-collections/cloud-access/dkrz_catalog.yaml @@ -3,18 +3,18 @@ metadata: parameters: additional_cmip6_disk_columns: default: + - units - path - - long_name - opendap_url - - units + - long_name type: list[str] additional_era5_disk_columns: default: + - path - units + - long_name - step - short_name - - path - - long_name type: list[str] cataloonie_columns: default: @@ -200,6 +200,11 @@ sources: disk storage system in /work/bk1099/data/ driver: - intake.open_esm_datastore + dkrz_monsoon_disk: + args: + path: https://gitlab.dkrz.de/data-infrastructure-services/intake-esm/-/raw/master/esm-collections/cloud-access/dkrz_monsoon_disk.yaml + description: Monsoon 2.0 + driver: yaml_file_cat dkrz_mpige_disk: args: csv_kwargs: diff --git a/esm-collections/cloud-access/dkrz_monsoon_disk.yaml b/esm-collections/cloud-access/dkrz_monsoon_disk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85b6a0d6116c36f5d72ce1cf7642424a00d7a076 --- /dev/null +++ b/esm-collections/cloud-access/dkrz_monsoon_disk.yaml @@ -0,0 +1,449 @@ +sources: + grids: + args: + path: https://gitlab.gwdg.de/ican/catalogs/-/raw/main/grids/icon_grids_by_uuid.yaml + descriptions: Catalog of ICON grids + driver: yaml_file_cat + luk1000: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1000_catalog.yaml + description: Monsoon 2.0 run luk1000. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1000 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.0.tmpl + zvz0i: 1.25 + luk1001: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1001_catalog.yaml + description: Monsoon 2.0 run luk1001. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1001 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.1.tmpl + zvz0i: 1.25 + luk1002: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1002_catalog.yaml + description: Monsoon 2.0 run luk1002. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk1002 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.2.tmpl + zvz0i: 1.25 + luk1003: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1003_catalog.yaml + description: Monsoon 2.0 run luk1003. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1003 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.3.tmpl + zvz0i: 1.25 + luk1010: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1010_catalog.yaml + description: Monsoon 2.0 run luk1010. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1010 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: ssp245 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.0.tmpl + zvz0i: 1.25 + luk1011: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1011_catalog.yaml + description: Monsoon 2.0 run luk1011. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1011 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: ssp245 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.1.tmpl + zvz0i: 1.25 + luk1020: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1020_catalog.yaml + description: Monsoon 2.0 run luk1020. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1020 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.0.tmpl + zvz0i: 1.25 + luk1021: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1021_catalog.yaml + description: Monsoon 2.0 run luk1021. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1021 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.1.tmpl + zvz0i: 1.25 + luk1022: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1022_catalog.yaml + description: Monsoon 2.0 run luk1022. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk1022 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.2.tmpl + zvz0i: 1.25 + luk1023: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1023_catalog.yaml + description: Monsoon 2.0 run luk1023. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1023 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.3.tmpl + zvz0i: 1.25 + luk1030: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1030_catalog.yaml + description: Monsoon 2.0 run luk1030. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1030 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl-p1K + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.0.tmpl + zvz0i: 1.25 + luk1031: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1031_catalog.yaml + description: Monsoon 2.0 run luk1031. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1031 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl-p1K + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.1.tmpl + zvz0i: 1.25 + luk1040: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1040_catalog.yaml + description: Monsoon 2.0 run luk1040. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + checkpoint_interval: P1M + end_date: '2018-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1040 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + restart_interval: P3M + sst_sic_file: sst-sic_oper_201803-201810_R02B09.nc + start_date: '2018-04-01T00:00:00Z' + template: casestudy.tmpl + luk1041: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1041_catalog.yaml + description: Monsoon 2.0 run luk1041. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + checkpoint_interval: P1M + end_date: '2019-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1041 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + restart_interval: P3M + sst_sic_file: sst-sic_oper_201903-201910_R02B09.nc + start_date: '2019-04-01T00:00:00Z' + template: casestudy.tmpl + luk1042: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1042_catalog.yaml + description: Monsoon 2.0 run luk1042. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + checkpoint_interval: P1M + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk1042 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + restart_interval: P3M + sst_sic_file: sst-sic_oper_202003-202010_R02B09.nc + start_date: '2020-04-01T00:00:00Z' + template: casestudy.tmpl + luk1043: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1043_catalog.yaml + description: Monsoon 2.0 run luk1043. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + checkpoint_interval: P1M + end_date: '2021-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1043 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + restart_interval: P3M + sst_sic_file: sst-sic_oper_202103-202110_R02B09.nc + start_date: '2021-04-01T00:00:00Z' + template: casestudy.tmpl + luk1103: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1103_catalog.yaml + description: Monsoon 2.0 run luk1103. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1103 + icesedi_exp: 0.4 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.3.tmpl + zvz0i: 3.29 + luk1123: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1123_catalog.yaml + description: Monsoon 2.0 run luk1123. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1123 + icesedi_exp: 0.4 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.3.tmpl + zvz0i: 3.29 + luk2000: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk2000_catalog.yaml + description: Monsoon 2.0 run luk2000. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: 0039 + atm_grid_refinement: R02B10 + end_date: '2021-08-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk2000 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT20S + nodes: 310 + rcp_scenario: ssp245 + start_date: '2021-07-12T00:00:00Z' + template: casestudy.tmpl + luk2002: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk2002_catalog.yaml + description: Monsoon 2.0 run luk2002. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + end_date: '2021-08-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk2002 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + rcp_scenario: ssp245 + start_date: '2021-07-12T00:00:00Z' + template: casestudy.tmpl + luk2003: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk2003_catalog.yaml + description: Monsoon 2.0 run luk2003. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + end_date: '2021-08-07T00:00:00Z' + ensemble_id: 3 + experiment_id: luk2003 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + rcp_scenario: ssp245 + start_date: '2021-07-18T00:00:00Z' + template: casestudy.tmpl + luk3001: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk3001_catalog.yaml + description: Monsoon 2.0 run luk3001. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + end_date: '2021-08-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk3001 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + start_date: '2021-06-22T00:00:00Z' + template: idealized.tmpl + luk3002: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk3002_catalog.yaml + description: Monsoon 2.0 run luk3002. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + end_date: '2021-08-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk3002 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + start_date: '2021-06-22T00:00:00Z' + template: idealized.tmpl diff --git a/esm-collections/create_cloud_parent.ipynb b/esm-collections/create_cloud_parent.ipynb index 4b9072b8b8233602b851826955cdfd95668ae8b1..5947b79a176c195d80f5898774e7fbdb14072c0a 100644 --- a/esm-collections/create_cloud_parent.ipynb +++ b/esm-collections/create_cloud_parent.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "116bb321-0bbb-4e85-804a-8a75932d9e46", "metadata": {}, "outputs": [], @@ -14,17 +14,31 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "60ccbb3d-0f0c-467e-bb43-4c025fe3ea05", "metadata": {}, "outputs": [], "source": [ - "cats=glob.glob(\"disk-access/dkrz*.json\")" + "cats=glob.glob(\"disk-access/dkrz*.json\")\n", + "cats+=[f\n", + " for f in glob.glob(\"disk-access/dkrz*.yaml\")\n", + " if \"dkrz_catalog.yaml\" not in f\n", + " ]" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, + "id": "7dc15e3e-b391-4ac3-8364-40bdb970afec", + "metadata": {}, + "outputs": [], + "source": [ + "cats" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "b98dd45c-aa97-4b36-9fce-28158467e107", "metadata": {}, "outputs": [], @@ -32,6 +46,8 @@ "for cat in cats:\n", " targetfile=cat.split('/')[1]\n", " target=\"cloud-access/\"+targetfile\n", + " if \"dyamond\" in cat or \"nextgems\" in cat or \"monsoon\" in cat :\n", + " continue\n", " with open(cat, \"r\") as f:\n", " catjson=json.load(f)\n", " catjson[\"description\"]+=\" which will be loaded from a source file which is in the cloud (see catalog_file)\"\n", @@ -42,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "2d1dfe72-e476-45a8-a1d6-2a2b8f3cbbb3", "metadata": {}, "outputs": [], @@ -53,7 +69,59 @@ { "cell_type": "code", "execution_count": null, - "id": "2d985cb4-ebc0-4906-9f21-fed42ffcb4a3", + "id": "c93e028c-35e4-4d0f-9f7a-4d39da7c32c0", + "metadata": {}, + "outputs": [], + "source": [ + "import intake\n", + "cat=intake.open_catalog([\"/home/k/k204210/intake-esm/esm-collections/disk-access/dkrz_catalog.yaml\"])\n", + "list(cat)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f12eb1e6-ea16-429b-ac69-a5116b480260", + "metadata": {}, + "outputs": [], + "source": [ + "cat.dkrz_monsoon_disk.luk1000" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8b061f6-089d-4edd-a9eb-6a2f206acb21", + "metadata": {}, + "outputs": [], + "source": [ + "cat=intake.open_catalog([\"https://dkrz.de/s/intake\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d1c7b3c-414d-46ad-965d-b5701786d244", + "metadata": {}, + "outputs": [], + "source": [ + "list(cat)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2999fd8-5505-4d62-965a-fdc69a71b571", + "metadata": {}, + "outputs": [], + "source": [ + "cat.dkrz_mpige_disk" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf1f02ac-5c49-4148-b73e-940991dab446", "metadata": {}, "outputs": [], "source": [] @@ -61,9 +129,9 @@ ], "metadata": { "kernelspec": { - "display_name": "taucenv", + "display_name": "Python 3 (based on the module python3/2022.01)", "language": "python", - "name": "taucenv" + "name": "python3_2022_01" }, "language_info": { "codemirror_mode": { @@ -75,7 +143,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.9" } }, "nbformat": 4, diff --git a/esm-collections/disk-access/dkrz_catalog.yaml b/esm-collections/disk-access/dkrz_catalog.yaml index 7ab082dd45590b7f8b468a24bfff9a699f09545e..a7d544464d9d923fc1ddc1a51f0762dd5cb680cb 100755 --- a/esm-collections/disk-access/dkrz_catalog.yaml +++ b/esm-collections/disk-access/dkrz_catalog.yaml @@ -3,18 +3,18 @@ metadata: parameters: additional_cmip6_disk_columns: default: + - units - path - - long_name - opendap_url - - units + - long_name type: list[str] additional_era5_disk_columns: default: + - path - units + - long_name - step - short_name - - path - - long_name type: list[str] cataloonie_columns: default: @@ -200,6 +200,11 @@ sources: disk storage system in /work/bk1099/data/ driver: - intake.open_esm_datastore + dkrz_monsoon_disk: + args: + path: /pool/data/Catalogs/dkrz_monsoon_disk.yaml + description: Monsoon 2.0 + driver: yaml_file_cat dkrz_mpige_disk: args: csv_kwargs: diff --git a/esm-collections/disk-access/dkrz_monsoon_disk.yaml b/esm-collections/disk-access/dkrz_monsoon_disk.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85b6a0d6116c36f5d72ce1cf7642424a00d7a076 --- /dev/null +++ b/esm-collections/disk-access/dkrz_monsoon_disk.yaml @@ -0,0 +1,449 @@ +sources: + grids: + args: + path: https://gitlab.gwdg.de/ican/catalogs/-/raw/main/grids/icon_grids_by_uuid.yaml + descriptions: Catalog of ICON grids + driver: yaml_file_cat + luk1000: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1000_catalog.yaml + description: Monsoon 2.0 run luk1000. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1000 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.0.tmpl + zvz0i: 1.25 + luk1001: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1001_catalog.yaml + description: Monsoon 2.0 run luk1001. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1001 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.1.tmpl + zvz0i: 1.25 + luk1002: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1002_catalog.yaml + description: Monsoon 2.0 run luk1002. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk1002 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.2.tmpl + zvz0i: 1.25 + luk1003: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1003_catalog.yaml + description: Monsoon 2.0 run luk1003. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1003 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.3.tmpl + zvz0i: 1.25 + luk1010: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1010_catalog.yaml + description: Monsoon 2.0 run luk1010. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1010 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: ssp245 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.0.tmpl + zvz0i: 1.25 + luk1011: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1011_catalog.yaml + description: Monsoon 2.0 run luk1011. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1011 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: ssp245 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.1.tmpl + zvz0i: 1.25 + luk1020: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1020_catalog.yaml + description: Monsoon 2.0 run luk1020. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1020 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.0.tmpl + zvz0i: 1.25 + luk1021: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1021_catalog.yaml + description: Monsoon 2.0 run luk1021. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1021 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.1.tmpl + zvz0i: 1.25 + luk1022: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1022_catalog.yaml + description: Monsoon 2.0 run luk1022. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk1022 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.2.tmpl + zvz0i: 1.25 + luk1023: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1023_catalog.yaml + description: Monsoon 2.0 run luk1023. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1023 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.3.tmpl + zvz0i: 1.25 + luk1030: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1030_catalog.yaml + description: Monsoon 2.0 run luk1030. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1030 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl-p1K + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.0.tmpl + zvz0i: 1.25 + luk1031: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1031_catalog.yaml + description: Monsoon 2.0 run luk1031. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1031 + icesedi_exp: 0.33 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl-p1K + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.1.tmpl + zvz0i: 1.25 + luk1040: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1040_catalog.yaml + description: Monsoon 2.0 run luk1040. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + checkpoint_interval: P1M + end_date: '2018-10-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk1040 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + restart_interval: P3M + sst_sic_file: sst-sic_oper_201803-201810_R02B09.nc + start_date: '2018-04-01T00:00:00Z' + template: casestudy.tmpl + luk1041: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1041_catalog.yaml + description: Monsoon 2.0 run luk1041. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + checkpoint_interval: P1M + end_date: '2019-10-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk1041 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + restart_interval: P3M + sst_sic_file: sst-sic_oper_201903-201910_R02B09.nc + start_date: '2019-04-01T00:00:00Z' + template: casestudy.tmpl + luk1042: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1042_catalog.yaml + description: Monsoon 2.0 run luk1042. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + checkpoint_interval: P1M + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk1042 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + restart_interval: P3M + sst_sic_file: sst-sic_oper_202003-202010_R02B09.nc + start_date: '2020-04-01T00:00:00Z' + template: casestudy.tmpl + luk1043: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1043_catalog.yaml + description: Monsoon 2.0 run luk1043. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + checkpoint_interval: P1M + end_date: '2021-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1043 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + restart_interval: P3M + sst_sic_file: sst-sic_oper_202103-202110_R02B09.nc + start_date: '2021-04-01T00:00:00Z' + template: casestudy.tmpl + luk1103: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1103_catalog.yaml + description: Monsoon 2.0 run luk1103. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2020-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1103 + icesedi_exp: 0.4 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp245 + sst_scenario: piControl + start_date: '2020-04-01T00:00:00Z' + template: ensemble-1.3.tmpl + zvz0i: 3.29 + luk1123: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk1123_catalog.yaml + description: Monsoon 2.0 run luk1123. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: P7D + atm_3d_output_operation: mean + atm_grid_refinement: R02B09 + end_date: '2070-10-01T00:00:00Z' + ensemble_id: 3 + experiment_id: luk1123 + icesedi_exp: 0.4 + icon_basedir: /p/project/highresmonsoon/ICON/icon-cscs-qubicc3-monsoon2.0 + rcp_scenario: ssp585 + sst_scenario: ssp585 + start_date: '2070-04-01T00:00:00Z' + template: ensemble-1.3.tmpl + zvz0i: 3.29 + luk2000: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk2000_catalog.yaml + description: Monsoon 2.0 run luk2000. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: 0039 + atm_grid_refinement: R02B10 + end_date: '2021-08-01T00:00:00Z' + ensemble_id: 0 + experiment_id: luk2000 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT20S + nodes: 310 + rcp_scenario: ssp245 + start_date: '2021-07-12T00:00:00Z' + template: casestudy.tmpl + luk2002: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk2002_catalog.yaml + description: Monsoon 2.0 run luk2002. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + end_date: '2021-08-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk2002 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + rcp_scenario: ssp245 + start_date: '2021-07-12T00:00:00Z' + template: casestudy.tmpl + luk2003: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk2003_catalog.yaml + description: Monsoon 2.0 run luk2003. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + end_date: '2021-08-07T00:00:00Z' + ensemble_id: 3 + experiment_id: luk2003 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + rcp_scenario: ssp245 + start_date: '2021-07-18T00:00:00Z' + template: casestudy.tmpl + luk3001: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk3001_catalog.yaml + description: Monsoon 2.0 run luk3001. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + end_date: '2021-08-01T00:00:00Z' + ensemble_id: 1 + experiment_id: luk3001 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + start_date: '2021-06-22T00:00:00Z' + template: idealized.tmpl + luk3002: + args: + path: /work/ik1017/Catalogs/highresmonsoon/experiments_zarr/luk3002_catalog.yaml + description: Monsoon 2.0 run luk3002. Check `metadata` for more information. + driver: yaml_file_cat + metadata: + atm_3d_output_interval: PT3H + atm_3d_output_operation: none + atm_grid_id: '0015' + atm_grid_refinement: R02B09 + end_date: '2021-08-01T00:00:00Z' + ensemble_id: 2 + experiment_id: luk3002 + icon_basedir: /p/project/highresmonsoon/kluft1/icon-cscs/build-2.6.5-sst + model_timestep: PT40S + nodes: 76 + start_date: '2021-06-22T00:00:00Z' + template: idealized.tmpl