diff --git a/builder/data-pool_collect-create-main.ipynb b/builder/data-pool_collect-create-main.ipynb
index a00e96a4122ab1cf57e016ff40b57437fd86f107..4af9eef3c34ce3b1f90b3bb708b847e43daca74f 100644
--- a/builder/data-pool_collect-create-main.ipynb
+++ b/builder/data-pool_collect-create-main.ipynb
@@ -33,7 +33,7 @@
"source": [
"catalogs=[cat \n",
" for cat in os.listdir(TRUNK)\n",
- " if cat.endswith('.json')]"
+ " if cat.endswith('.json') or ((cat.endswith('.yaml')) & ('dkrz_catalog.yaml' not in cat))]"
]
},
{
@@ -84,6 +84,8 @@
"allcols={}\n",
"usecols={}\n",
"for cat in catalogs:\n",
+ " if \"yaml\" in cat:\n",
+ " continue\n",
" with open(TRUNK+\"/\"+cat, \"r\") as f:\n",
" cat_json=json.load(f)\n",
" catname='_'.join(cat.split('.')[0].split('_')[1:])\n",
@@ -156,6 +158,16 @@
"for k in catalogs:\n",
" kentry=k.split('.')[0]\n",
" catname='_'.join(k.split('.')[0].split('_')[1:])\n",
+ " if \"yaml\" in k:\n",
+ " if \"monsoon\" in k:\n",
+ " sources[kentry]=dict(\n",
+ " args=dict(\n",
+ " path=\"/pool/data/Catalogs/dkrz_monsoon_disk.yaml\"\n",
+ " ),\n",
+ " description=\"Monsoon 2.0\",\n",
+ " driver=\"yaml_file_cat\"\n",
+ " )\n",
+ " continue\n",
" kpath=TRUNK+'/'+k\n",
" with open(kpath,'r') as f:\n",
" content=json.load(f)\n",
@@ -183,6 +195,16 @@
"sourcesdict={\"sources\":sources}"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "23fc6855-f1c0-4545-8361-e074450636a3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sources"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -190,7 +212,7 @@
"metadata": {},
"outputs": [],
"source": [
- "with open(\"/home/k/k204210/volume/data-infrastructure-services/intake-esm/esm-collections/disk-access/dkrz_catalog.yaml\",\"w\") as f:\n",
+ "with open(\"/home/k/k204210/intake-esm/esm-collections/disk-access/dkrz_catalog.yaml\",\"w\") as f:\n",
" f.write(yaml.dump(header))\n",
" f.write(yaml.dump(sourcesdict))"
]
diff --git a/esm-collections/create_cloud_parent.ipynb b/esm-collections/create_cloud_parent.ipynb
index 4b9072b8b8233602b851826955cdfd95668ae8b1..5947b79a176c195d80f5898774e7fbdb14072c0a 100644
--- a/esm-collections/create_cloud_parent.ipynb
+++ b/esm-collections/create_cloud_parent.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"id": "116bb321-0bbb-4e85-804a-8a75932d9e46",
"metadata": {},
"outputs": [],
@@ -14,17 +14,31 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"id": "60ccbb3d-0f0c-467e-bb43-4c025fe3ea05",
"metadata": {},
"outputs": [],
"source": [
- "cats=glob.glob(\"disk-access/dkrz*.json\")"
+ "cats=glob.glob(\"disk-access/dkrz*.json\")\n",
+ "cats+=[f\n",
+ " for f in glob.glob(\"disk-access/dkrz*.yaml\")\n",
+ " if \"dkrz_catalog.yaml\" not in f\n",
+ " ]"
]
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
+ "id": "7dc15e3e-b391-4ac3-8364-40bdb970afec",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cats"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
"id": "b98dd45c-aa97-4b36-9fce-28158467e107",
"metadata": {},
"outputs": [],
@@ -32,6 +46,8 @@
"for cat in cats:\n",
" targetfile=cat.split('/')[1]\n",
" target=\"cloud-access/\"+targetfile\n",
+ " if \"dyamond\" in cat or \"nextgems\" in cat or \"monsoon\" in cat :\n",
+ " continue\n",
" with open(cat, \"r\") as f:\n",
" catjson=json.load(f)\n",
" catjson[\"description\"]+=\" which will be loaded from a source file which is in the cloud (see catalog_file)\"\n",
@@ -42,7 +58,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"id": "2d1dfe72-e476-45a8-a1d6-2a2b8f3cbbb3",
"metadata": {},
"outputs": [],
@@ -53,7 +69,59 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "2d985cb4-ebc0-4906-9f21-fed42ffcb4a3",
+ "id": "c93e028c-35e4-4d0f-9f7a-4d39da7c32c0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import intake\n",
+ "cat=intake.open_catalog([\"/home/k/k204210/intake-esm/esm-collections/disk-access/dkrz_catalog.yaml\"])\n",
+ "list(cat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f12eb1e6-ea16-429b-ac69-a5116b480260",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cat.dkrz_monsoon_disk.luk1000"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8b061f6-089d-4edd-a9eb-6a2f206acb21",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cat=intake.open_catalog([\"https://dkrz.de/s/intake\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7d1c7b3c-414d-46ad-965d-b5701786d244",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "list(cat)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f2999fd8-5505-4d62-965a-fdc69a71b571",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cat.dkrz_mpige_disk"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cf1f02ac-5c49-4148-b73e-940991dab446",
"metadata": {},
"outputs": [],
"source": []
@@ -61,9 +129,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "taucenv",
+ "display_name": "Python 3 (based on the module python3/2022.01)",
"language": "python",
- "name": "taucenv"
+ "name": "python3_2022_01"
},
"language_info": {
"codemirror_mode": {
@@ -75,7 +143,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.4"
+ "version": "3.9.9"
}
},
"nbformat": 4,