Commit 8231f240 authored by Fabian Wachsmann's avatar Fabian Wachsmann
Browse files

updated era5 tut

parent 495c4775
Pipeline #19352 passed with stage
in 29 seconds
......@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
......@@ -11,11 +11,11 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"nbs = !find ./ -mindepth 2 -name \"*.ipynb\" ! -name \"*checkpoint.ipynb\" -type f "
"nbs = !find ./ -mindepth 2 -name \"*.ipynb\" ! -name \"*checkpoint.ipynb\" ! -name \"*era5*\" -type f "
]
},
{
......@@ -84,9 +84,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": "Python 3 (based on the module python3/2022.01)",
"language": "python",
"name": "python3"
"name": "python3_2022_01"
},
"language_info": {
"codemirror_mode": {
......@@ -98,7 +98,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.0"
"version": "3.9.9"
}
},
"nbformat": 4,
......
%% Cell type:code id: tags:
``` python
import json
```
%% Cell type:code id: tags:
``` python
nbs = !find ./ -mindepth 2 -name "*.ipynb" ! -name "*checkpoint.ipynb" -type f
nbs = !find ./ -mindepth 2 -name "*.ipynb" ! -name "*checkpoint.ipynb" ! -name "*era5*" -type f
```
%% Cell type:code id: tags:
``` python
for nb in nbs:
#!jupyter nbconvert --clear-output --inplace {nb}
!jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace {nb}
with open(nb, 'r') as jsonFile:
nbdata=json.load(jsonFile)
nbdata["metadata"]["kernelspec"]["name"]="python3"
nbdata["metadata"]["kernelspec"]["display_name"]="python3"
with open(nb, 'w') as jsonFile:
json.dump(nbdata, jsonFile, indent=1, ensure_ascii=False)
```
%%%% Output: stream
[NbConvertApp] Converting notebook ./hands-on_excercises/1_hands-on_find_data_intake.ipynb to notebook
[NbConvertApp] Writing 4392 bytes to ./hands-on_excercises/1_hands-on_find_data_intake.ipynb
[NbConvertApp] Converting notebook ./hands-on_excercises/2_hands-on_tropical_nights_intake_xarray_cmip6.ipynb to notebook
[NbConvertApp] Writing 4843 bytes to ./hands-on_excercises/2_hands-on_tropical_nights_intake_xarray_cmip6.ipynb
[NbConvertApp] Converting notebook ./hands-on_solutions/2_hands-on_tropical_nights_intake_xarray_cmip6_solution.ipynb to notebook
[NbConvertApp] Writing 7892 bytes to ./hands-on_solutions/2_hands-on_tropical_nights_intake_xarray_cmip6_solution.ipynb
[NbConvertApp] Converting notebook ./hands-on_solutions/1_hands-on_find_data_intake_solution.ipynb to notebook
[NbConvertApp] Writing 4999 bytes to ./hands-on_solutions/1_hands-on_find_data_intake_solution.ipynb
[NbConvertApp] Converting notebook ./demo/use-case_simple-vis_xarray-matplotlib_cmip6.ipynb to notebook
[NbConvertApp] Writing 8696 bytes to ./demo/use-case_simple-vis_xarray-matplotlib_cmip6.ipynb
[NbConvertApp] Converting notebook ./demo/use-case_calculate-frost-days_intake-xarray_cmip6.ipynb to notebook
[NbConvertApp] Writing 12892 bytes to ./demo/use-case_calculate-frost-days_intake-xarray_cmip6.ipynb
[NbConvertApp] Converting notebook ./demo/dkrz-data-pool-cmip6.ipynb to notebook
[NbConvertApp] Writing 16043 bytes to ./demo/dkrz-data-pool-cmip6.ipynb
[NbConvertApp] Converting notebook ./demo/dkrz-intake-catalog.ipynb to notebook
[NbConvertApp] Writing 12605 bytes to ./demo/dkrz-intake-catalog.ipynb
[NbConvertApp] Converting notebook ./demo/use-case_convert-nc-to-tiff_rioxarray-xesmf_cmip.ipynb to notebook
[NbConvertApp] Writing 13772 bytes to ./demo/use-case_convert-nc-to-tiff_rioxarray-xesmf_cmip.ipynb
[NbConvertApp] Converting notebook ./demo/use-case_advanced_summer_days_intake_xarray_cmip6.ipynb to notebook
[NbConvertApp] Writing 15325 bytes to ./demo/use-case_advanced_summer_days_intake_xarray_cmip6.ipynb
[NbConvertApp] Converting notebook ./demo/tutorial_esmvaltool.ipynb to notebook
[NbConvertApp] Writing 5332 bytes to ./demo/tutorial_esmvaltool.ipynb
[NbConvertApp] Converting notebook ./demo/dkrz-jupyterhub-notebook.ipynb to notebook
[NbConvertApp] Writing 5453 bytes to ./demo/dkrz-jupyterhub-notebook.ipynb
[NbConvertApp] Converting notebook ./demo/tutorial_intake.ipynb to notebook
[NbConvertApp] Writing 18803 bytes to ./demo/tutorial_intake.ipynb
[NbConvertApp] Converting notebook ./demo/use-case_multimodel-comparison_xarray-cdo_cmip6.ipynb to notebook
[NbConvertApp] Writing 17702 bytes to ./demo/use-case_multimodel-comparison_xarray-cdo_cmip6.ipynb
[NbConvertApp] Converting notebook ./demo/use-case_plot-unstructured_psyplot_cmip6.ipynb to notebook
[NbConvertApp] Writing 9342 bytes to ./demo/use-case_plot-unstructured_psyplot_cmip6.ipynb
[NbConvertApp] Converting notebook ./demo/use-case_global-yearly-mean-anomaly_xarray-hvplot_cmip6.ipynb to notebook
[NbConvertApp] Writing 14045 bytes to ./demo/use-case_global-yearly-mean-anomaly_xarray-hvplot_cmip6.ipynb
%% Cell type:code id: tags:
``` python
```
......
......@@ -119,7 +119,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
......@@ -137,7 +137,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
......@@ -156,9 +156,28 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"['dkrz_cmip5_archive',\n",
" 'dkrz_cmip5_disk',\n",
" 'dkrz_cmip6_cloud',\n",
" 'dkrz_cmip6_disk',\n",
" 'dkrz_cordex_disk',\n",
" 'dkrz_dyamond-winter_disk',\n",
" 'dkrz_era5_disk',\n",
" 'dkrz_nextgems_disk',\n",
" 'dkrz_palmod2_disk']"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(dkrz_catalog)"
]
......@@ -172,9 +191,18 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/sw/spack-levante/mambaforge-4.11.0-0-Linux-x86_64-sobz6z/lib/python3.9/site-packages/intake_esm/utils.py:96: DtypeWarning: Columns (13,14) have mixed types. Specify dtype option on import or set low_memory=False.\n",
" return pd.read_csv(catalog_path, **csv_kwargs), catalog_path\n"
]
}
],
"source": [
"col=dkrz_catalog.dkrz_era5_disk"
]
......@@ -188,9 +216,20 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"\"This is an ESM collection for ERA5 data accessible on the DKRZ's disk storage system in /work/bk1099/data/\""
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"col.description"
]
......@@ -204,9 +243,94 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/html": [
"<p><strong>dkrz_era5_disk catalog with 16 dataset(s) from 571576 asset(s)</strong>:</p> <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>unique</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>project</th>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>era_id</th>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>dataType</th>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>level_type</th>\n",
" <td>3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>frequency</th>\n",
" <td>3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>stepType</th>\n",
" <td>6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>table_id</th>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>code</th>\n",
" <td>111</td>\n",
" </tr>\n",
" <tr>\n",
" <th>validation_date</th>\n",
" <td>16326</td>\n",
" </tr>\n",
" <tr>\n",
" <th>initialization_date</th>\n",
" <td>561</td>\n",
" </tr>\n",
" <tr>\n",
" <th>uri</th>\n",
" <td>571576</td>\n",
" </tr>\n",
" <tr>\n",
" <th>format</th>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"col"
]
......@@ -261,9 +385,21 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'default': ['step', 'long_name', 'short_name', 'path', 'units'],\n",
" 'type': 'list[str]'}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dkrz_catalog.metadata[\"parameters\"][\"additional_era5_columns\"]"
]
......@@ -277,9 +413,18 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/sw/spack-levante/mambaforge-4.11.0-0-Linux-x86_64-sobz6z/lib/python3.9/site-packages/intake_esm/utils.py:96: DtypeWarning: Columns (13,14) have mixed types. Specify dtype option on import or set low_memory=False.\n",
" return pd.read_csv(catalog_path, **csv_kwargs), catalog_path\n"
]
}
],
"source": [
"cols=dkrz_catalog._entries[\"dkrz_era5_disk\"]._open_args[\"csv_kwargs\"][\"usecols\"]+dkrz_catalog.metadata[\"parameters\"][\"additional_era5_columns\"][\"default\"]\n",
"col=dkrz_catalog.dkrz_era5_disk(csv_kwargs=dict(usecols=cols))"
......@@ -303,9 +448,20 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'dataType': {'count': 2, 'values': ['fc', 'an']}}"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"col.unique(\"dataType\")"
]
......@@ -321,11 +477,22 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"metadata": {
"tags": []
},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'frequency': {'count': 3, 'values': ['hourly', 'monthly', 'daily']}}"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"col.unique(\"frequency\")"
]
......@@ -339,9 +506,21 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 11,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'level_type': {'count': 3,\n",
" 'values': ['pressure_level', 'surface', 'model_level']}}"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"col.unique(\"level_type\")"
]
......@@ -356,9 +535,21 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 12,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'stepType': {'count': 6,\n",
" 'values': ['accum', 'avgid', 'max', 'instant', 'avgad', 'avgua']}}"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"col.unique(\"stepType\")"
]
......@@ -385,9 +576,28 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 13,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"[('an', 'model_level', 'hourly'),\n",
" ('an', 'pressure_level', 'daily'),\n",
" ('an', 'pressure_level', 'hourly'),\n",
" ('an', 'surface', 'daily'),\n",
" ('an', 'surface', 'hourly'),\n",
" ('an', 'surface', 'monthly'),\n",
" ('fc', 'surface', 'daily'),\n",
" ('fc', 'surface', 'hourly'),\n",
" ('fc', 'surface', 'monthly')]"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(col.df.groupby([\"dataType\", \"level_type\", \"frequency\"]).groups.keys())"
]
......@@ -403,7 +613,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
......@@ -421,9 +631,36 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 15,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'long_name': {'count': 16,\n",
" 'values': ['Vorticity (relative)',\n",
" 'V component of wind',\n",
" 'Ozone mass mixing ratio',\n",
" 'U component of wind',\n",
" 'Relative humidity',\n",
" 'Specific humidity',\n",
" 'Fraction of cloud cover',\n",
" 'Specific rain water content',\n",
" 'Specific cloud ice water content',\n",
" 'Potential vorticity',\n",
" 'Specific cloud liquid water content',\n",
" 'Temperature',\n",
" 'Divergence',\n",
" 'Vertical velocity',\n",
" 'Geopotential',\n",
" 'Specific snow water content']}}"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat.unique(\"long_name\")"
]
......@@ -437,7 +674,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
......@@ -454,9 +691,20 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 17,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'short_name': {'count': 1, 'values': ['t']}}"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"temp_hourly_pl.unique(\"short_name\")"
]
......@@ -483,11 +731,94 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 18,
"metadata": {
"tags": []
},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"--> The keys in the returned dictionary of datasets are constructed as follows:\n",
"\t'table_id.stepType.level_type.frequency'\n"
]
},
{
"data": {
"text/html": [
"\n",
"<style>\n",
" /* Turns off some styling */\n",
" progress {\n",
" /* gets rid of default border in Firefox and Opera. */\n",
" border: none;\n",
" /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
" background-size: auto;\n",
" }\n",
" .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
" background: #F44336;\n",
" }\n",
"</style>\n"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
" <div>\n",
" <progress value='1' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" 100.00% [1/1 00:00<00:00]\n",
" </div>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"{'128.0.instant.pressure_level.hourly': <xarray.Dataset>\n",
" Dimensions: (time: 24, isobaricInhPa: 37, values: 542080)\n",
" Coordinates:\n",
" number int64 ...\n",
" * time (time) datetime64[ns] 1980-01-31 ... 1980-01-31T23:00:00\n",
" step timedelta64[ns] ...\n",
" * isobaricInhPa (isobaricInhPa) int64 1000 975 950 925 900 875 ... 7 5 3 2 1\n",
" latitude (values) float64 dask.array<chunksize=(542080,), meta=np.ndarray>\n",
" longitude (values) float64 dask.array<chunksize=(542080,), meta=np.ndarray>\n",
" valid_time (time) datetime64[ns] dask.array<chunksize=(1,), meta=np.ndarray>\n",
" Dimensions without coordinates: values\n",
" Data variables:\n",
" t (time, isobaricInhPa, values) float32 dask.array<chunksize=(1, 37, 542080), meta=np.ndarray>\n",
" Attributes:\n",
" GRIB_edition: 1\n",
" GRIB_centre: ecmf\n",
" GRIB_centreDescription: European Centre for Medium-Range Weather Forecasts\n",
" GRIB_subCentre: 0\n",
" Conventions: CF-1.7\n",
" institution: European Centre for Medium-Range Weather Forecasts\n",
" history: 2022-06-21T18:04:52 GRIB to CDM+CF via cfgrib-0....\n",
" intake_esm_varname: 130\n",
" intake_esm_dataset_key: 128.0.instant.pressure_level.hourly}"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"temp_hourly_pl_xr_dict=temp_hourly_pl.to_dataset_dict(cdf_kwargs={\"engine\":\"cfgrib\",\n",
" \"chunks\":{\"time\":1}\n",
......@@ -505,9 +836,22 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 19,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"{'aggregations': [{'attribute_name': 'code', 'type': 'union'}],\n",
" 'variable_column_name': 'code',\n",
" 'groupby_attrs': ['table_id', 'stepType', 'level_type', 'frequency']}"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"temp_hourly_pl.esmcol_data[\"aggregation_control\"]"
]
......@@ -521,7 +865,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
......@@ -546,7 +890,7 @@
},