diff --git a/notebooks/tzis-introduction-dkrz.ipynb b/notebooks/tzis-introduction-dkrz.ipynb
index 27284f64f713ac7dd917e053ce5b3712173d3c01..02c5c7a7e2512c4c236e0013df48d2656ec22578 100644
--- a/notebooks/tzis-introduction-dkrz.ipynb
+++ b/notebooks/tzis-introduction-dkrz.ipynb
@@ -2,7 +2,11 @@
"cells": [
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
"source": [
"# Introduction to **Tzis** (To Zarr In Swift) at DKRZ\n",
"\n",
@@ -18,7 +22,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
"source": [
"## [Zarr](https://zarr.readthedocs.io/en/stable/)\n",
"\n",
@@ -41,15 +49,23 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"> \"What is a chunk?\" 🤔"
]
},
{
"cell_type": "code",
- "execution_count": 3,
- "metadata": {},
+ "execution_count": 10,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"data": {
@@ -405,303 +421,21 @@
" stroke: currentColor;\n",
" fill: currentColor;\n",
"}\n",
- "</style><pre class='xr-text-repr-fallback'><xarray.Dataset>\n",
- "Dimensions: (bnds: 2, lat: 192, lon: 384, time: 1032)\n",
+ "</style><pre class='xr-text-repr-fallback'><xarray.DataArray 'tas' (time: 1032, lat: 192, lon: 384)>\n",
+ "dask.array<concatenate, shape=(1032, 192, 384), dtype=float32, chunksize=(60, 192, 384), chunktype=numpy.ndarray>\n",
"Coordinates:\n",
- " * time (time) datetime64[ns] 2015-01-16T12:00:00 ... 2100-12-16T12:00:00\n",
- " * lat (lat) float64 -89.28 -88.36 -87.42 -86.49 ... 87.42 88.36 89.28\n",
- " * lon (lon) float64 0.0 0.9375 1.875 2.812 ... 356.2 357.2 358.1 359.1\n",
- " height float64 2.0\n",
- "Dimensions without coordinates: bnds\n",
- "Data variables:\n",
- " time_bnds (time, bnds) datetime64[ns] dask.array<chunksize=(60, 2), meta=np.ndarray>\n",
- " lat_bnds (time, lat, bnds) float64 dask.array<chunksize=(60, 192, 2), meta=np.ndarray>\n",
- " lon_bnds (time, lon, bnds) float64 dask.array<chunksize=(60, 384, 2), meta=np.ndarray>\n",
- " tas (time, lat, lon) float32 dask.array<chunksize=(60, 192, 384), meta=np.ndarray>\n",
- "Attributes: (12/47)\n",
- " Conventions: CF-1.7 CMIP-6.2\n",
- " activity_id: ScenarioMIP AerChemMIP\n",
- " branch_method: standard\n",
- " branch_time_in_child: 60265.0\n",
- " branch_time_in_parent: 60265.0\n",
- " contact: cmip6-mpi-esm@dkrz.de\n",
- " ... ...\n",
- " title: MPI-ESM1-2-HR output prepared for CMIP6\n",
- " variable_id: tas\n",
- " variant_label: r1i1p1f1\n",
- " license: CMIP6 model data produced by DKRZ is licensed und...\n",
- " cmor_version: 3.4.0\n",
- " tracking_id: hdl:21.14100/d6f0f6b2-b4c2-460b-bc5e-f1c72f932399</pre><div class='xr-wrap' hidden><div class='xr-header'><div class='xr-obj-type'>xarray.Dataset</div></div><ul class='xr-sections'><li class='xr-section-item'><input id='section-85cf516b-64e1-44d0-b6e4-79df9541cd19' class='xr-section-summary-in' type='checkbox' disabled ><label for='section-85cf516b-64e1-44d0-b6e4-79df9541cd19' class='xr-section-summary' title='Expand/collapse section'>Dimensions:</label><div class='xr-section-inline-details'><ul class='xr-dim-list'><li><span>bnds</span>: 2</li><li><span class='xr-has-index'>lat</span>: 192</li><li><span class='xr-has-index'>lon</span>: 384</li><li><span class='xr-has-index'>time</span>: 1032</li></ul></div><div class='xr-section-details'></div></li><li class='xr-section-item'><input id='section-916b975e-8b43-4933-ad44-c059eaa1b262' class='xr-section-summary-in' type='checkbox' checked><label for='section-916b975e-8b43-4933-ad44-c059eaa1b262' class='xr-section-summary' >Coordinates: <span>(4)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2015-01-16T12:00:00 ... 2100-12-...</div><input id='attrs-15e0ca84-7e97-4f35-9f35-69219f0a75a8' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-15e0ca84-7e97-4f35-9f35-69219f0a75a8' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-4c9848c9-d462-4a27-9069-9bbd9ddcbf2f' class='xr-var-data-in' type='checkbox'><label for='data-4c9848c9-d462-4a27-9069-9bbd9ddcbf2f' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>bounds :</span></dt><dd>time_bnds</dd><dt><span>axis :</span></dt><dd>T</dd><dt><span>long_name :</span></dt><dd>time</dd><dt><span>standard_name :</span></dt><dd>time</dd></dl></div><div class='xr-var-data'><pre>array(['2015-01-16T12:00:00.000000000', '2015-02-15T00:00:00.000000000',\n",
- " '2015-03-16T12:00:00.000000000', ..., '2100-10-16T12:00:00.000000000',\n",
- " '2100-11-16T00:00:00.000000000', '2100-12-16T12:00:00.000000000'],\n",
- " dtype='datetime64[ns]')</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>lat</span></div><div class='xr-var-dims'>(lat)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>-89.28 -88.36 ... 88.36 89.28</div><input id='attrs-f611fff9-ac39-4411-8a9c-902d94882387' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-f611fff9-ac39-4411-8a9c-902d94882387' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-5e68e07e-bc54-43fc-af9c-7816a6d9ec6d' class='xr-var-data-in' type='checkbox'><label for='data-5e68e07e-bc54-43fc-af9c-7816a6d9ec6d' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>bounds :</span></dt><dd>lat_bnds</dd><dt><span>units :</span></dt><dd>degrees_north</dd><dt><span>axis :</span></dt><dd>Y</dd><dt><span>long_name :</span></dt><dd>Latitude</dd><dt><span>standard_name :</span></dt><dd>latitude</dd></dl></div><div class='xr-var-data'><pre>array([-89.284228, -88.357004, -87.424304, -86.490367, -85.55596 , -84.621327,\n",
- " -83.686567, -82.751728, -81.816839, -80.881913, -79.946962, -79.011992,\n",
- " -78.077007, -77.142011, -76.207005, -75.271992, -74.336973, -73.401949,\n",
- " -72.466921, -71.531889, -70.596854, -69.661816, -68.726776, -67.791734,\n",
- " -66.856691, -65.921645, -64.986599, -64.051551, -63.116502, -62.181452,\n",
- " -61.246401, -60.311349, -59.376297, -58.441244, -57.50619 , -56.571136,\n",
- " -55.636081, -54.701026, -53.76597 , -52.830914, -51.895857, -50.960801,\n",
- " -50.025743, -49.090686, -48.155628, -47.22057 , -46.285512, -45.350454,\n",
- " -44.415395, -43.480336, -42.545277, -41.610218, -40.675159, -39.740099,\n",
- " -38.805039, -37.86998 , -36.93492 , -35.99986 , -35.064799, -34.129739,\n",
- " -33.194679, -32.259618, -31.324558, -30.389497, -29.454436, -28.519375,\n",
- " -27.584315, -26.649254, -25.714193, -24.779132, -23.84407 , -22.909009,\n",
- " -21.973948, -21.038887, -20.103825, -19.168764, -18.233703, -17.298641,\n",
- " -16.36358 , -15.428518, -14.493457, -13.558395, -12.623333, -11.688272,\n",
- " -10.75321 , -9.818148, -8.883087, -7.948025, -7.012963, -6.077901,\n",
- " -5.14284 , -4.207778, -3.272716, -2.337654, -1.402593, -0.467531,\n",
- " 0.467531, 1.402593, 2.337654, 3.272716, 4.207778, 5.14284 ,\n",
- " 6.077901, 7.012963, 7.948025, 8.883087, 9.818148, 10.75321 ,\n",
- " 11.688272, 12.623333, 13.558395, 14.493457, 15.428518, 16.36358 ,\n",
- " 17.298641, 18.233703, 19.168764, 20.103825, 21.038887, 21.973948,\n",
- " 22.909009, 23.84407 , 24.779132, 25.714193, 26.649254, 27.584315,\n",
- " 28.519375, 29.454436, 30.389497, 31.324558, 32.259618, 33.194679,\n",
- " 34.129739, 35.064799, 35.99986 , 36.93492 , 37.86998 , 38.805039,\n",
- " 39.740099, 40.675159, 41.610218, 42.545277, 43.480336, 44.415395,\n",
- " 45.350454, 46.285512, 47.22057 , 48.155628, 49.090686, 50.025743,\n",
- " 50.960801, 51.895857, 52.830914, 53.76597 , 54.701026, 55.636081,\n",
- " 56.571136, 57.50619 , 58.441244, 59.376297, 60.311349, 61.246401,\n",
- " 62.181452, 63.116502, 64.051551, 64.986599, 65.921645, 66.856691,\n",
- " 67.791734, 68.726776, 69.661816, 70.596854, 71.531889, 72.466921,\n",
- " 73.401949, 74.336973, 75.271992, 76.207005, 77.142011, 78.077007,\n",
- " 79.011992, 79.946962, 80.881913, 81.816839, 82.751728, 83.686567,\n",
- " 84.621327, 85.55596 , 86.490367, 87.424304, 88.357004, 89.284228])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>lon</span></div><div class='xr-var-dims'>(lon)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>0.0 0.9375 1.875 ... 358.1 359.1</div><input id='attrs-38da3e89-77fa-4b5e-93c0-706b052928e7' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-38da3e89-77fa-4b5e-93c0-706b052928e7' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-4a2fbdcb-ec9a-4da9-b646-c0b38e3fc071' class='xr-var-data-in' type='checkbox'><label for='data-4a2fbdcb-ec9a-4da9-b646-c0b38e3fc071' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>bounds :</span></dt><dd>lon_bnds</dd><dt><span>units :</span></dt><dd>degrees_east</dd><dt><span>axis :</span></dt><dd>X</dd><dt><span>long_name :</span></dt><dd>Longitude</dd><dt><span>standard_name :</span></dt><dd>longitude</dd></dl></div><div class='xr-var-data'><pre>array([ 0. , 0.9375, 1.875 , ..., 357.1875, 358.125 , 359.0625])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>height</span></div><div class='xr-var-dims'>()</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>2.0</div><input id='attrs-32724d88-7f27-4267-a8bc-0286c3e25a54' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-32724d88-7f27-4267-a8bc-0286c3e25a54' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-cea93ae2-2e15-4666-a615-8f71eda19916' class='xr-var-data-in' type='checkbox'><label for='data-cea93ae2-2e15-4666-a615-8f71eda19916' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>units :</span></dt><dd>m</dd><dt><span>axis :</span></dt><dd>Z</dd><dt><span>positive :</span></dt><dd>up</dd><dt><span>long_name :</span></dt><dd>height</dd><dt><span>standard_name :</span></dt><dd>height</dd></dl></div><div class='xr-var-data'><pre>array(2.)</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-57750866-940b-4aaf-b2ba-0b43aa23956c' class='xr-section-summary-in' type='checkbox' checked><label for='section-57750866-940b-4aaf-b2ba-0b43aa23956c' class='xr-section-summary' >Data variables: <span>(4)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span>time_bnds</span></div><div class='xr-var-dims'>(time, bnds)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>dask.array<chunksize=(60, 2), meta=np.ndarray></div><input id='attrs-d58798d5-c3e8-438b-b9c8-1f1244b7f3c7' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-d58798d5-c3e8-438b-b9c8-1f1244b7f3c7' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-6ca97bd7-93c3-4527-87eb-602b79e820b3' class='xr-var-data-in' type='checkbox'><label for='data-6ca97bd7-93c3-4527-87eb-602b79e820b3' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><table>\n",
- "<tr>\n",
- "<td>\n",
- "<table>\n",
- " <thead>\n",
- " <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
- " </thead>\n",
- " <tbody>\n",
- " <tr><th> Bytes </th><td> 16.51 kB </td> <td> 960 B </td></tr>\n",
- " <tr><th> Shape </th><td> (1032, 2) </td> <td> (60, 2) </td></tr>\n",
- " <tr><th> Count </th><td> 54 Tasks </td><td> 18 Chunks </td></tr>\n",
- " <tr><th> Type </th><td> datetime64[ns] </td><td> numpy.ndarray </td></tr>\n",
- " </tbody>\n",
- "</table>\n",
- "</td>\n",
- "<td>\n",
- "<svg width=\"75\" height=\"170\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
- "\n",
- " <!-- Horizontal lines -->\n",
- " <line x1=\"0\" y1=\"0\" x2=\"25\" y2=\"0\" style=\"stroke-width:2\" />\n",
- " <line x1=\"0\" y1=\"6\" x2=\"25\" y2=\"6\" />\n",
- " <line x1=\"0\" y1=\"13\" x2=\"25\" y2=\"13\" />\n",
- " <line x1=\"0\" y1=\"20\" x2=\"25\" y2=\"20\" />\n",
- " <line x1=\"0\" y1=\"27\" x2=\"25\" y2=\"27\" />\n",
- " <line x1=\"0\" y1=\"34\" x2=\"25\" y2=\"34\" />\n",
- " <line x1=\"0\" y1=\"41\" x2=\"25\" y2=\"41\" />\n",
- " <line x1=\"0\" y1=\"48\" x2=\"25\" y2=\"48\" />\n",
- " <line x1=\"0\" y1=\"55\" x2=\"25\" y2=\"55\" />\n",
- " <line x1=\"0\" y1=\"62\" x2=\"25\" y2=\"62\" />\n",
- " <line x1=\"0\" y1=\"69\" x2=\"25\" y2=\"69\" />\n",
- " <line x1=\"0\" y1=\"76\" x2=\"25\" y2=\"76\" />\n",
- " <line x1=\"0\" y1=\"83\" x2=\"25\" y2=\"83\" />\n",
- " <line x1=\"0\" y1=\"90\" x2=\"25\" y2=\"90\" />\n",
- " <line x1=\"0\" y1=\"97\" x2=\"25\" y2=\"97\" />\n",
- " <line x1=\"0\" y1=\"104\" x2=\"25\" y2=\"104\" />\n",
- " <line x1=\"0\" y1=\"111\" x2=\"25\" y2=\"111\" />\n",
- " <line x1=\"0\" y1=\"118\" x2=\"25\" y2=\"118\" />\n",
- " <line x1=\"0\" y1=\"120\" x2=\"25\" y2=\"120\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Vertical lines -->\n",
- " <line x1=\"0\" y1=\"0\" x2=\"0\" y2=\"120\" style=\"stroke-width:2\" />\n",
- " <line x1=\"25\" y1=\"0\" x2=\"25\" y2=\"120\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Colored Rectangle -->\n",
- " <polygon points=\"0.000000,0.000000 25.412617,0.000000 25.412617,120.000000 0.000000,120.000000\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
- "\n",
- " <!-- Text -->\n",
- " <text x=\"12.706308\" y=\"140.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >2</text>\n",
- " <text x=\"45.412617\" y=\"60.000000\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,45.412617,60.000000)\">1032</text>\n",
- "</svg>\n",
- "</td>\n",
- "</tr>\n",
- "</table></div></li><li class='xr-var-item'><div class='xr-var-name'><span>lat_bnds</span></div><div class='xr-var-dims'>(time, lat, bnds)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>dask.array<chunksize=(60, 192, 2), meta=np.ndarray></div><input id='attrs-acc494d5-1d89-42aa-ad78-16fbbb7132fc' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-acc494d5-1d89-42aa-ad78-16fbbb7132fc' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-41298e6c-dcf4-49aa-bc7e-95ac521800fd' class='xr-var-data-in' type='checkbox'><label for='data-41298e6c-dcf4-49aa-bc7e-95ac521800fd' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><table>\n",
- "<tr>\n",
- "<td>\n",
- "<table>\n",
- " <thead>\n",
- " <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
- " </thead>\n",
- " <tbody>\n",
- " <tr><th> Bytes </th><td> 3.17 MB </td> <td> 184.32 kB </td></tr>\n",
- " <tr><th> Shape </th><td> (1032, 192, 2) </td> <td> (60, 192, 2) </td></tr>\n",
- " <tr><th> Count </th><td> 72 Tasks </td><td> 18 Chunks </td></tr>\n",
- " <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
- " </tbody>\n",
- "</table>\n",
- "</td>\n",
- "<td>\n",
- "<svg width=\"156\" height=\"162\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
- "\n",
- " <!-- Horizontal lines -->\n",
- " <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
- " <line x1=\"10\" y1=\"41\" x2=\"80\" y2=\"112\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Vertical lines -->\n",
- " <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"41\" style=\"stroke-width:2\" />\n",
- " <line x1=\"14\" y1=\"4\" x2=\"14\" y2=\"45\" />\n",
- " <line x1=\"18\" y1=\"8\" x2=\"18\" y2=\"49\" />\n",
- " <line x1=\"22\" y1=\"12\" x2=\"22\" y2=\"54\" />\n",
- " <line x1=\"26\" y1=\"16\" x2=\"26\" y2=\"58\" />\n",
- " <line x1=\"30\" y1=\"20\" x2=\"30\" y2=\"62\" />\n",
- " <line x1=\"34\" y1=\"24\" x2=\"34\" y2=\"66\" />\n",
- " <line x1=\"38\" y1=\"28\" x2=\"38\" y2=\"70\" />\n",
- " <line x1=\"42\" y1=\"32\" x2=\"42\" y2=\"74\" />\n",
- " <line x1=\"46\" y1=\"36\" x2=\"46\" y2=\"78\" />\n",
- " <line x1=\"51\" y1=\"41\" x2=\"51\" y2=\"82\" />\n",
- " <line x1=\"55\" y1=\"45\" x2=\"55\" y2=\"86\" />\n",
- " <line x1=\"59\" y1=\"49\" x2=\"59\" y2=\"90\" />\n",
- " <line x1=\"63\" y1=\"53\" x2=\"63\" y2=\"95\" />\n",
- " <line x1=\"67\" y1=\"57\" x2=\"67\" y2=\"99\" />\n",
- " <line x1=\"71\" y1=\"61\" x2=\"71\" y2=\"103\" />\n",
- " <line x1=\"75\" y1=\"65\" x2=\"75\" y2=\"107\" />\n",
- " <line x1=\"79\" y1=\"69\" x2=\"79\" y2=\"111\" />\n",
- " <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"112\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Colored Rectangle -->\n",
- " <polygon points=\"10.000000,0.000000 80.588235,70.588235 80.588235,112.286857 10.000000,41.698621\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
- "\n",
- " <!-- Horizontal lines -->\n",
- " <line x1=\"10\" y1=\"0\" x2=\"35\" y2=\"0\" style=\"stroke-width:2\" />\n",
- " <line x1=\"14\" y1=\"4\" x2=\"39\" y2=\"4\" />\n",
- " <line x1=\"18\" y1=\"8\" x2=\"43\" y2=\"8\" />\n",
- " <line x1=\"22\" y1=\"12\" x2=\"47\" y2=\"12\" />\n",
- " <line x1=\"26\" y1=\"16\" x2=\"51\" y2=\"16\" />\n",
- " <line x1=\"30\" y1=\"20\" x2=\"55\" y2=\"20\" />\n",
- " <line x1=\"34\" y1=\"24\" x2=\"60\" y2=\"24\" />\n",
- " <line x1=\"38\" y1=\"28\" x2=\"64\" y2=\"28\" />\n",
- " <line x1=\"42\" y1=\"32\" x2=\"68\" y2=\"32\" />\n",
- " <line x1=\"46\" y1=\"36\" x2=\"72\" y2=\"36\" />\n",
- " <line x1=\"51\" y1=\"41\" x2=\"76\" y2=\"41\" />\n",
- " <line x1=\"55\" y1=\"45\" x2=\"80\" y2=\"45\" />\n",
- " <line x1=\"59\" y1=\"49\" x2=\"84\" y2=\"49\" />\n",
- " <line x1=\"63\" y1=\"53\" x2=\"88\" y2=\"53\" />\n",
- " <line x1=\"67\" y1=\"57\" x2=\"92\" y2=\"57\" />\n",
- " <line x1=\"71\" y1=\"61\" x2=\"96\" y2=\"61\" />\n",
- " <line x1=\"75\" y1=\"65\" x2=\"101\" y2=\"65\" />\n",
- " <line x1=\"79\" y1=\"69\" x2=\"105\" y2=\"69\" />\n",
- " <line x1=\"80\" y1=\"70\" x2=\"106\" y2=\"70\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Vertical lines -->\n",
- " <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
- " <line x1=\"35\" y1=\"0\" x2=\"106\" y2=\"70\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Colored Rectangle -->\n",
- " <polygon points=\"10.000000,0.000000 35.412617,0.000000 106.000852,70.588235 80.588235,70.588235\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
- "\n",
- " <!-- Horizontal lines -->\n",
- " <line x1=\"80\" y1=\"70\" x2=\"106\" y2=\"70\" style=\"stroke-width:2\" />\n",
- " <line x1=\"80\" y1=\"112\" x2=\"106\" y2=\"112\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Vertical lines -->\n",
- " <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"112\" style=\"stroke-width:2\" />\n",
- " <line x1=\"106\" y1=\"70\" x2=\"106\" y2=\"112\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Colored Rectangle -->\n",
- " <polygon points=\"80.588235,70.588235 106.000852,70.588235 106.000852,112.286857 80.588235,112.286857\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
- "\n",
- " <!-- Text -->\n",
- " <text x=\"93.294544\" y=\"132.286857\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >2</text>\n",
- " <text x=\"126.000852\" y=\"91.437546\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,126.000852,91.437546)\">192</text>\n",
- " <text x=\"35.294118\" y=\"96.992739\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,35.294118,96.992739)\">1032</text>\n",
- "</svg>\n",
- "</td>\n",
- "</tr>\n",
- "</table></div></li><li class='xr-var-item'><div class='xr-var-name'><span>lon_bnds</span></div><div class='xr-var-dims'>(time, lon, bnds)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>dask.array<chunksize=(60, 384, 2), meta=np.ndarray></div><input id='attrs-b830e4da-f978-47dc-8179-f6aac5a88aa3' class='xr-var-attrs-in' type='checkbox' disabled><label for='attrs-b830e4da-f978-47dc-8179-f6aac5a88aa3' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-fed4c6f5-66cf-46ab-8232-91b4b7b932dd' class='xr-var-data-in' type='checkbox'><label for='data-fed4c6f5-66cf-46ab-8232-91b4b7b932dd' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'></dl></div><div class='xr-var-data'><table>\n",
- "<tr>\n",
- "<td>\n",
- "<table>\n",
- " <thead>\n",
- " <tr><td> </td><th> Array </th><th> Chunk </th></tr>\n",
- " </thead>\n",
- " <tbody>\n",
- " <tr><th> Bytes </th><td> 6.34 MB </td> <td> 368.64 kB </td></tr>\n",
- " <tr><th> Shape </th><td> (1032, 384, 2) </td> <td> (60, 384, 2) </td></tr>\n",
- " <tr><th> Count </th><td> 72 Tasks </td><td> 18 Chunks </td></tr>\n",
- " <tr><th> Type </th><td> float64 </td><td> numpy.ndarray </td></tr>\n",
- " </tbody>\n",
- "</table>\n",
- "</td>\n",
- "<td>\n",
- "<svg width=\"156\" height=\"165\" style=\"stroke:rgb(0,0,0);stroke-width:1\" >\n",
- "\n",
- " <!-- Horizontal lines -->\n",
- " <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
- " <line x1=\"10\" y1=\"44\" x2=\"80\" y2=\"115\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Vertical lines -->\n",
- " <line x1=\"10\" y1=\"0\" x2=\"10\" y2=\"44\" style=\"stroke-width:2\" />\n",
- " <line x1=\"14\" y1=\"4\" x2=\"14\" y2=\"48\" />\n",
- " <line x1=\"18\" y1=\"8\" x2=\"18\" y2=\"52\" />\n",
- " <line x1=\"22\" y1=\"12\" x2=\"22\" y2=\"56\" />\n",
- " <line x1=\"26\" y1=\"16\" x2=\"26\" y2=\"61\" />\n",
- " <line x1=\"30\" y1=\"20\" x2=\"30\" y2=\"65\" />\n",
- " <line x1=\"34\" y1=\"24\" x2=\"34\" y2=\"69\" />\n",
- " <line x1=\"38\" y1=\"28\" x2=\"38\" y2=\"73\" />\n",
- " <line x1=\"42\" y1=\"32\" x2=\"42\" y2=\"77\" />\n",
- " <line x1=\"46\" y1=\"36\" x2=\"46\" y2=\"81\" />\n",
- " <line x1=\"51\" y1=\"41\" x2=\"51\" y2=\"85\" />\n",
- " <line x1=\"55\" y1=\"45\" x2=\"55\" y2=\"89\" />\n",
- " <line x1=\"59\" y1=\"49\" x2=\"59\" y2=\"93\" />\n",
- " <line x1=\"63\" y1=\"53\" x2=\"63\" y2=\"98\" />\n",
- " <line x1=\"67\" y1=\"57\" x2=\"67\" y2=\"102\" />\n",
- " <line x1=\"71\" y1=\"61\" x2=\"71\" y2=\"106\" />\n",
- " <line x1=\"75\" y1=\"65\" x2=\"75\" y2=\"110\" />\n",
- " <line x1=\"79\" y1=\"69\" x2=\"79\" y2=\"114\" />\n",
- " <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"115\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Colored Rectangle -->\n",
- " <polygon points=\"10.000000,0.000000 80.588235,70.588235 80.588235,115.239398 10.000000,44.651163\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
- "\n",
- " <!-- Horizontal lines -->\n",
- " <line x1=\"10\" y1=\"0\" x2=\"35\" y2=\"0\" style=\"stroke-width:2\" />\n",
- " <line x1=\"14\" y1=\"4\" x2=\"39\" y2=\"4\" />\n",
- " <line x1=\"18\" y1=\"8\" x2=\"43\" y2=\"8\" />\n",
- " <line x1=\"22\" y1=\"12\" x2=\"47\" y2=\"12\" />\n",
- " <line x1=\"26\" y1=\"16\" x2=\"51\" y2=\"16\" />\n",
- " <line x1=\"30\" y1=\"20\" x2=\"55\" y2=\"20\" />\n",
- " <line x1=\"34\" y1=\"24\" x2=\"60\" y2=\"24\" />\n",
- " <line x1=\"38\" y1=\"28\" x2=\"64\" y2=\"28\" />\n",
- " <line x1=\"42\" y1=\"32\" x2=\"68\" y2=\"32\" />\n",
- " <line x1=\"46\" y1=\"36\" x2=\"72\" y2=\"36\" />\n",
- " <line x1=\"51\" y1=\"41\" x2=\"76\" y2=\"41\" />\n",
- " <line x1=\"55\" y1=\"45\" x2=\"80\" y2=\"45\" />\n",
- " <line x1=\"59\" y1=\"49\" x2=\"84\" y2=\"49\" />\n",
- " <line x1=\"63\" y1=\"53\" x2=\"88\" y2=\"53\" />\n",
- " <line x1=\"67\" y1=\"57\" x2=\"92\" y2=\"57\" />\n",
- " <line x1=\"71\" y1=\"61\" x2=\"96\" y2=\"61\" />\n",
- " <line x1=\"75\" y1=\"65\" x2=\"101\" y2=\"65\" />\n",
- " <line x1=\"79\" y1=\"69\" x2=\"105\" y2=\"69\" />\n",
- " <line x1=\"80\" y1=\"70\" x2=\"106\" y2=\"70\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Vertical lines -->\n",
- " <line x1=\"10\" y1=\"0\" x2=\"80\" y2=\"70\" style=\"stroke-width:2\" />\n",
- " <line x1=\"35\" y1=\"0\" x2=\"106\" y2=\"70\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Colored Rectangle -->\n",
- " <polygon points=\"10.000000,0.000000 35.412617,0.000000 106.000852,70.588235 80.588235,70.588235\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
- "\n",
- " <!-- Horizontal lines -->\n",
- " <line x1=\"80\" y1=\"70\" x2=\"106\" y2=\"70\" style=\"stroke-width:2\" />\n",
- " <line x1=\"80\" y1=\"115\" x2=\"106\" y2=\"115\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Vertical lines -->\n",
- " <line x1=\"80\" y1=\"70\" x2=\"80\" y2=\"115\" style=\"stroke-width:2\" />\n",
- " <line x1=\"106\" y1=\"70\" x2=\"106\" y2=\"115\" style=\"stroke-width:2\" />\n",
- "\n",
- " <!-- Colored Rectangle -->\n",
- " <polygon points=\"80.588235,70.588235 106.000852,70.588235 106.000852,115.239398 80.588235,115.239398\" style=\"fill:#ECB172A0;stroke-width:0\"/>\n",
- "\n",
- " <!-- Text -->\n",
- " <text x=\"93.294544\" y=\"135.239398\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" >2</text>\n",
- " <text x=\"126.000852\" y=\"92.913817\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(-90,126.000852,92.913817)\">384</text>\n",
- " <text x=\"35.294118\" y=\"99.945280\" font-size=\"1.0rem\" font-weight=\"100\" text-anchor=\"middle\" transform=\"rotate(45,35.294118,99.945280)\">1032</text>\n",
- "</svg>\n",
- "</td>\n",
- "</tr>\n",
- "</table></div></li><li class='xr-var-item'><div class='xr-var-name'><span>tas</span></div><div class='xr-var-dims'>(time, lat, lon)</div><div class='xr-var-dtype'>float32</div><div class='xr-var-preview xr-preview'>dask.array<chunksize=(60, 192, 384), meta=np.ndarray></div><input id='attrs-e673fbcd-bff8-4c02-bb00-23ff8583a42d' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-e673fbcd-bff8-4c02-bb00-23ff8583a42d' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-090a39e5-80cb-42a5-9101-0c033a86fb20' class='xr-var-data-in' type='checkbox'><label for='data-090a39e5-80cb-42a5-9101-0c033a86fb20' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>air_temperature</dd><dt><span>long_name :</span></dt><dd>Near-Surface Air Temperature</dd><dt><span>comment :</span></dt><dd>near-surface (usually, 2 meter) air temperature</dd><dt><span>units :</span></dt><dd>K</dd><dt><span>cell_methods :</span></dt><dd>area: time: mean</dd><dt><span>cell_measures :</span></dt><dd>area: areacella</dd><dt><span>history :</span></dt><dd>2019-07-20T13:41:51Z altered by CMOR: Treated scalar dimension: 'height'. 2019-07-20T13:41:51Z altered by CMOR: replaced missing value flag (-9e+33) with standard missing value (1e+20). 2019-07-20T13:41:51Z altered by CMOR: Converted type from 'd' to 'f'. 2019-07-20T13:41:52Z altered by CMOR: Inverted axis: lat.</dd></dl></div><div class='xr-var-data'><table>\n",
+ " * time (time) datetime64[ns] 2015-01-16T12:00:00 ... 2100-12-16T12:00:00\n",
+ " * lat (lat) float64 -89.28 -88.36 -87.42 -86.49 ... 87.42 88.36 89.28\n",
+ " * lon (lon) float64 0.0 0.9375 1.875 2.812 ... 356.2 357.2 358.1 359.1\n",
+ " height float64 2.0\n",
+ "Attributes:\n",
+ " standard_name: air_temperature\n",
+ " long_name: Near-Surface Air Temperature\n",
+ " comment: near-surface (usually, 2 meter) air temperature\n",
+ " units: K\n",
+ " cell_methods: area: time: mean\n",
+ " cell_measures: area: areacella\n",
+ " history: 2019-07-20T13:41:51Z altered by CMOR: Treated scalar dime...</pre><div class='xr-wrap' hidden><div class='xr-header'><div class='xr-obj-type'>xarray.DataArray</div><div class='xr-array-name'>'tas'</div><ul class='xr-dim-list'><li><span class='xr-has-index'>time</span>: 1032</li><li><span class='xr-has-index'>lat</span>: 192</li><li><span class='xr-has-index'>lon</span>: 384</li></ul></div><ul class='xr-sections'><li class='xr-section-item'><div class='xr-array-wrap'><input id='section-5881eb1c-0a74-4e0f-87f5-088c2a72fb9b' class='xr-array-in' type='checkbox' checked><label for='section-5881eb1c-0a74-4e0f-87f5-088c2a72fb9b' title='Show/hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-array-preview xr-preview'><span>dask.array<chunksize=(60, 192, 384), meta=np.ndarray></span></div><div class='xr-array-data'><table>\n",
"<tr>\n",
"<td>\n",
"<table>\n",
@@ -793,60 +527,78 @@
"</svg>\n",
"</td>\n",
"</tr>\n",
- "</table></div></li></ul></div></li><li class='xr-section-item'><input id='section-686af0de-ec7d-49ac-a6a8-6bb2212ac945' class='xr-section-summary-in' type='checkbox' ><label for='section-686af0de-ec7d-49ac-a6a8-6bb2212ac945' class='xr-section-summary' >Attributes: <span>(47)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>Conventions :</span></dt><dd>CF-1.7 CMIP-6.2</dd><dt><span>activity_id :</span></dt><dd>ScenarioMIP AerChemMIP</dd><dt><span>branch_method :</span></dt><dd>standard</dd><dt><span>branch_time_in_child :</span></dt><dd>60265.0</dd><dt><span>branch_time_in_parent :</span></dt><dd>60265.0</dd><dt><span>contact :</span></dt><dd>cmip6-mpi-esm@dkrz.de</dd><dt><span>creation_date :</span></dt><dd>2019-07-20T13:41:51Z</dd><dt><span>data_specs_version :</span></dt><dd>01.00.30</dd><dt><span>experiment :</span></dt><dd>gap-filling scenario reaching 7.0 based on SSP3</dd><dt><span>experiment_id :</span></dt><dd>ssp370</dd><dt><span>external_variables :</span></dt><dd>areacella</dd><dt><span>forcing_index :</span></dt><dd>1</dd><dt><span>frequency :</span></dt><dd>mon</dd><dt><span>further_info_url :</span></dt><dd>https://furtherinfo.es-doc.org/CMIP6.DKRZ.MPI-ESM1-2-HR.ssp370.none.r1i1p1f1</dd><dt><span>grid :</span></dt><dd>gn</dd><dt><span>grid_label :</span></dt><dd>gn</dd><dt><span>history :</span></dt><dd>2019-07-20T13:41:51Z ; CMOR rewrote data to be consistent with CMIP6, CF-1.7 CMIP-6.2 and CF standards.</dd><dt><span>initialization_index :</span></dt><dd>1</dd><dt><span>institution :</span></dt><dd>Deutsches Klimarechenzentrum, Hamburg 20146, Germany</dd><dt><span>institution_id :</span></dt><dd>DKRZ</dd><dt><span>mip_era :</span></dt><dd>CMIP6</dd><dt><span>nominal_resolution :</span></dt><dd>100 km</dd><dt><span>parent_activity_id :</span></dt><dd>CMIP</dd><dt><span>parent_experiment_id :</span></dt><dd>historical</dd><dt><span>parent_mip_era :</span></dt><dd>CMIP6</dd><dt><span>parent_source_id :</span></dt><dd>MPI-ESM1-2-HR</dd><dt><span>parent_time_units :</span></dt><dd>days since 1850-1-1 00:00:00</dd><dt><span>parent_variant_label :</span></dt><dd>r1i1p1f1</dd><dt><span>physics_index :</span></dt><dd>1</dd><dt><span>product :</span></dt><dd>model-output</dd><dt><span>project_id :</span></dt><dd>CMIP6</dd><dt><span>realization_index :</span></dt><dd>1</dd><dt><span>realm :</span></dt><dd>atmos</dd><dt><span>references :</span></dt><dd>MPI-ESM: Mauritsen, T. et al. (2019), Developments in the MPIâ€M Earth System Model version 1.2 (MPIâ€ESM1.2) and Its Response to Increasing CO2, J. Adv. Model. Earth Syst.,11, 998-1038, doi:10.1029/2018MS001400,\n",
- "Mueller, W.A. et al. (2018): A highâ€resolution version of the Max Planck Institute Earth System Model MPIâ€ESM1.2â€HR. J. Adv. Model. EarthSyst.,10,1383–1413, doi:10.1029/2017MS001217</dd><dt><span>source :</span></dt><dd>MPI-ESM1.2-HR (2017): \n",
- "aerosol: none, prescribed MACv2-SP\n",
- "atmos: ECHAM6.3 (spectral T127; 384 x 192 longitude/latitude; 95 levels; top level 0.01 hPa)\n",
- "atmosChem: none\n",
- "land: JSBACH3.20\n",
- "landIce: none/prescribed\n",
- "ocean: MPIOM1.63 (tripolar TP04, approximately 0.4deg; 802 x 404 longitude/latitude; 40 levels; top grid cell 0-12 m)\n",
- "ocnBgchem: HAMOCC6\n",
- "seaIce: unnamed (thermodynamic (Semtner zero-layer) dynamic (Hibler 79) sea ice model)</dd><dt><span>source_id :</span></dt><dd>MPI-ESM1-2-HR</dd><dt><span>source_type :</span></dt><dd>AOGCM</dd><dt><span>sub_experiment :</span></dt><dd>none</dd><dt><span>sub_experiment_id :</span></dt><dd>none</dd><dt><span>table_id :</span></dt><dd>Amon</dd><dt><span>table_info :</span></dt><dd>Creation Date:(09 May 2019) MD5:e6ef8ececc8f338646ebfb3aeed36bfc</dd><dt><span>title :</span></dt><dd>MPI-ESM1-2-HR output prepared for CMIP6</dd><dt><span>variable_id :</span></dt><dd>tas</dd><dt><span>variant_label :</span></dt><dd>r1i1p1f1</dd><dt><span>license :</span></dt><dd>CMIP6 model data produced by DKRZ is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.</dd><dt><span>cmor_version :</span></dt><dd>3.4.0</dd><dt><span>tracking_id :</span></dt><dd>hdl:21.14100/d6f0f6b2-b4c2-460b-bc5e-f1c72f932399</dd></dl></div></li></ul></div></div>"
+ "</table></div></div></li><li class='xr-section-item'><input id='section-588d3696-888c-4842-890a-b95b83e4fce5' class='xr-section-summary-in' type='checkbox' checked><label for='section-588d3696-888c-4842-890a-b95b83e4fce5' class='xr-section-summary' >Coordinates: <span>(4)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><ul class='xr-var-list'><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>time</span></div><div class='xr-var-dims'>(time)</div><div class='xr-var-dtype'>datetime64[ns]</div><div class='xr-var-preview xr-preview'>2015-01-16T12:00:00 ... 2100-12-...</div><input id='attrs-8716d40b-e260-4061-8463-ec65eb8149a5' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-8716d40b-e260-4061-8463-ec65eb8149a5' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-20ababca-157f-416d-b82d-83060324d6ed' class='xr-var-data-in' type='checkbox'><label for='data-20ababca-157f-416d-b82d-83060324d6ed' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>bounds :</span></dt><dd>time_bnds</dd><dt><span>axis :</span></dt><dd>T</dd><dt><span>long_name :</span></dt><dd>time</dd><dt><span>standard_name :</span></dt><dd>time</dd></dl></div><div class='xr-var-data'><pre>array(['2015-01-16T12:00:00.000000000', '2015-02-15T00:00:00.000000000',\n",
+ " '2015-03-16T12:00:00.000000000', ..., '2100-10-16T12:00:00.000000000',\n",
+ " '2100-11-16T00:00:00.000000000', '2100-12-16T12:00:00.000000000'],\n",
+ " dtype='datetime64[ns]')</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>lat</span></div><div class='xr-var-dims'>(lat)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>-89.28 -88.36 ... 88.36 89.28</div><input id='attrs-4aa09ffa-b6f9-4acc-a6c0-8847b1fffb35' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-4aa09ffa-b6f9-4acc-a6c0-8847b1fffb35' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-8349cb38-2f75-4beb-9869-3d3b80f9fb80' class='xr-var-data-in' type='checkbox'><label for='data-8349cb38-2f75-4beb-9869-3d3b80f9fb80' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>bounds :</span></dt><dd>lat_bnds</dd><dt><span>units :</span></dt><dd>degrees_north</dd><dt><span>axis :</span></dt><dd>Y</dd><dt><span>long_name :</span></dt><dd>Latitude</dd><dt><span>standard_name :</span></dt><dd>latitude</dd></dl></div><div class='xr-var-data'><pre>array([-89.284228, -88.357004, -87.424304, -86.490367, -85.55596 , -84.621327,\n",
+ " -83.686567, -82.751728, -81.816839, -80.881913, -79.946962, -79.011992,\n",
+ " -78.077007, -77.142011, -76.207005, -75.271992, -74.336973, -73.401949,\n",
+ " -72.466921, -71.531889, -70.596854, -69.661816, -68.726776, -67.791734,\n",
+ " -66.856691, -65.921645, -64.986599, -64.051551, -63.116502, -62.181452,\n",
+ " -61.246401, -60.311349, -59.376297, -58.441244, -57.50619 , -56.571136,\n",
+ " -55.636081, -54.701026, -53.76597 , -52.830914, -51.895857, -50.960801,\n",
+ " -50.025743, -49.090686, -48.155628, -47.22057 , -46.285512, -45.350454,\n",
+ " -44.415395, -43.480336, -42.545277, -41.610218, -40.675159, -39.740099,\n",
+ " -38.805039, -37.86998 , -36.93492 , -35.99986 , -35.064799, -34.129739,\n",
+ " -33.194679, -32.259618, -31.324558, -30.389497, -29.454436, -28.519375,\n",
+ " -27.584315, -26.649254, -25.714193, -24.779132, -23.84407 , -22.909009,\n",
+ " -21.973948, -21.038887, -20.103825, -19.168764, -18.233703, -17.298641,\n",
+ " -16.36358 , -15.428518, -14.493457, -13.558395, -12.623333, -11.688272,\n",
+ " -10.75321 , -9.818148, -8.883087, -7.948025, -7.012963, -6.077901,\n",
+ " -5.14284 , -4.207778, -3.272716, -2.337654, -1.402593, -0.467531,\n",
+ " 0.467531, 1.402593, 2.337654, 3.272716, 4.207778, 5.14284 ,\n",
+ " 6.077901, 7.012963, 7.948025, 8.883087, 9.818148, 10.75321 ,\n",
+ " 11.688272, 12.623333, 13.558395, 14.493457, 15.428518, 16.36358 ,\n",
+ " 17.298641, 18.233703, 19.168764, 20.103825, 21.038887, 21.973948,\n",
+ " 22.909009, 23.84407 , 24.779132, 25.714193, 26.649254, 27.584315,\n",
+ " 28.519375, 29.454436, 30.389497, 31.324558, 32.259618, 33.194679,\n",
+ " 34.129739, 35.064799, 35.99986 , 36.93492 , 37.86998 , 38.805039,\n",
+ " 39.740099, 40.675159, 41.610218, 42.545277, 43.480336, 44.415395,\n",
+ " 45.350454, 46.285512, 47.22057 , 48.155628, 49.090686, 50.025743,\n",
+ " 50.960801, 51.895857, 52.830914, 53.76597 , 54.701026, 55.636081,\n",
+ " 56.571136, 57.50619 , 58.441244, 59.376297, 60.311349, 61.246401,\n",
+ " 62.181452, 63.116502, 64.051551, 64.986599, 65.921645, 66.856691,\n",
+ " 67.791734, 68.726776, 69.661816, 70.596854, 71.531889, 72.466921,\n",
+ " 73.401949, 74.336973, 75.271992, 76.207005, 77.142011, 78.077007,\n",
+ " 79.011992, 79.946962, 80.881913, 81.816839, 82.751728, 83.686567,\n",
+ " 84.621327, 85.55596 , 86.490367, 87.424304, 88.357004, 89.284228])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span class='xr-has-index'>lon</span></div><div class='xr-var-dims'>(lon)</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>0.0 0.9375 1.875 ... 358.1 359.1</div><input id='attrs-bd196b60-c19a-47a4-8504-b19322268875' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-bd196b60-c19a-47a4-8504-b19322268875' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-e76048d1-61be-43aa-bbe2-6985b8d42fbd' class='xr-var-data-in' type='checkbox'><label for='data-e76048d1-61be-43aa-bbe2-6985b8d42fbd' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>bounds :</span></dt><dd>lon_bnds</dd><dt><span>units :</span></dt><dd>degrees_east</dd><dt><span>axis :</span></dt><dd>X</dd><dt><span>long_name :</span></dt><dd>Longitude</dd><dt><span>standard_name :</span></dt><dd>longitude</dd></dl></div><div class='xr-var-data'><pre>array([ 0. , 0.9375, 1.875 , ..., 357.1875, 358.125 , 359.0625])</pre></div></li><li class='xr-var-item'><div class='xr-var-name'><span>height</span></div><div class='xr-var-dims'>()</div><div class='xr-var-dtype'>float64</div><div class='xr-var-preview xr-preview'>2.0</div><input id='attrs-0497fb77-e513-4443-b55b-8f8f95720e26' class='xr-var-attrs-in' type='checkbox' ><label for='attrs-0497fb77-e513-4443-b55b-8f8f95720e26' title='Show/Hide attributes'><svg class='icon xr-icon-file-text2'><use xlink:href='#icon-file-text2'></use></svg></label><input id='data-2e10db33-4eb8-4a9c-9c81-c2ea354eb97c' class='xr-var-data-in' type='checkbox'><label for='data-2e10db33-4eb8-4a9c-9c81-c2ea354eb97c' title='Show/Hide data repr'><svg class='icon xr-icon-database'><use xlink:href='#icon-database'></use></svg></label><div class='xr-var-attrs'><dl class='xr-attrs'><dt><span>units :</span></dt><dd>m</dd><dt><span>axis :</span></dt><dd>Z</dd><dt><span>positive :</span></dt><dd>up</dd><dt><span>long_name :</span></dt><dd>height</dd><dt><span>standard_name :</span></dt><dd>height</dd></dl></div><div class='xr-var-data'><pre>array(2.)</pre></div></li></ul></div></li><li class='xr-section-item'><input id='section-f47cc70c-5d45-44bb-91cf-67a5309e5ae8' class='xr-section-summary-in' type='checkbox' checked><label for='section-f47cc70c-5d45-44bb-91cf-67a5309e5ae8' class='xr-section-summary' >Attributes: <span>(7)</span></label><div class='xr-section-inline-details'></div><div class='xr-section-details'><dl class='xr-attrs'><dt><span>standard_name :</span></dt><dd>air_temperature</dd><dt><span>long_name :</span></dt><dd>Near-Surface Air Temperature</dd><dt><span>comment :</span></dt><dd>near-surface (usually, 2 meter) air temperature</dd><dt><span>units :</span></dt><dd>K</dd><dt><span>cell_methods :</span></dt><dd>area: time: mean</dd><dt><span>cell_measures :</span></dt><dd>area: areacella</dd><dt><span>history :</span></dt><dd>2019-07-20T13:41:51Z altered by CMOR: Treated scalar dimension: 'height'. 2019-07-20T13:41:51Z altered by CMOR: replaced missing value flag (-9e+33) with standard missing value (1e+20). 2019-07-20T13:41:51Z altered by CMOR: Converted type from 'd' to 'f'. 2019-07-20T13:41:52Z altered by CMOR: Inverted axis: lat.</dd></dl></div></li></ul></div></div>"
],
"text/plain": [
- "<xarray.Dataset>\n",
- "Dimensions: (bnds: 2, lat: 192, lon: 384, time: 1032)\n",
+ "<xarray.DataArray 'tas' (time: 1032, lat: 192, lon: 384)>\n",
+ "dask.array<concatenate, shape=(1032, 192, 384), dtype=float32, chunksize=(60, 192, 384), chunktype=numpy.ndarray>\n",
"Coordinates:\n",
- " * time (time) datetime64[ns] 2015-01-16T12:00:00 ... 2100-12-16T12:00:00\n",
- " * lat (lat) float64 -89.28 -88.36 -87.42 -86.49 ... 87.42 88.36 89.28\n",
- " * lon (lon) float64 0.0 0.9375 1.875 2.812 ... 356.2 357.2 358.1 359.1\n",
- " height float64 2.0\n",
- "Dimensions without coordinates: bnds\n",
- "Data variables:\n",
- " time_bnds (time, bnds) datetime64[ns] dask.array<chunksize=(60, 2), meta=np.ndarray>\n",
- " lat_bnds (time, lat, bnds) float64 dask.array<chunksize=(60, 192, 2), meta=np.ndarray>\n",
- " lon_bnds (time, lon, bnds) float64 dask.array<chunksize=(60, 384, 2), meta=np.ndarray>\n",
- " tas (time, lat, lon) float32 dask.array<chunksize=(60, 192, 384), meta=np.ndarray>\n",
- "Attributes: (12/47)\n",
- " Conventions: CF-1.7 CMIP-6.2\n",
- " activity_id: ScenarioMIP AerChemMIP\n",
- " branch_method: standard\n",
- " branch_time_in_child: 60265.0\n",
- " branch_time_in_parent: 60265.0\n",
- " contact: cmip6-mpi-esm@dkrz.de\n",
- " ... ...\n",
- " title: MPI-ESM1-2-HR output prepared for CMIP6\n",
- " variable_id: tas\n",
- " variant_label: r1i1p1f1\n",
- " license: CMIP6 model data produced by DKRZ is licensed und...\n",
- " cmor_version: 3.4.0\n",
- " tracking_id: hdl:21.14100/d6f0f6b2-b4c2-460b-bc5e-f1c72f932399"
+ " * time (time) datetime64[ns] 2015-01-16T12:00:00 ... 2100-12-16T12:00:00\n",
+ " * lat (lat) float64 -89.28 -88.36 -87.42 -86.49 ... 87.42 88.36 89.28\n",
+ " * lon (lon) float64 0.0 0.9375 1.875 2.812 ... 356.2 357.2 358.1 359.1\n",
+ " height float64 2.0\n",
+ "Attributes:\n",
+ " standard_name: air_temperature\n",
+ " long_name: Near-Surface Air Temperature\n",
+ " comment: near-surface (usually, 2 meter) air temperature\n",
+ " units: K\n",
+ " cell_methods: area: time: mean\n",
+ " cell_measures: area: areacella\n",
+ " history: 2019-07-20T13:41:51Z altered by CMOR: Treated scalar dime..."
]
},
- "execution_count": 3,
+ "execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import xarray\n",
- "xarray.open_mfdataset(\"/mnt/lustre02/work/ik1017/CMIP6/data/CMIP6/ScenarioMIP/DKRZ/MPI-ESM1-2-HR/ssp370/r1i1p1f1/Amon/tas/gn/v20190710/tas_Amon_MPI-ESM1-2-HR_ssp370_r1i1p1f1_gn_*\")"
+ "dset=xarray.open_mfdataset(\"/mnt/lustre02/work/ik1017/CMIP6/data/CMIP6/ScenarioMIP/DKRZ/MPI-ESM1-2-HR/ssp370/r1i1p1f1/Amon/tas/gn/v20190710/tas_Amon_MPI-ESM1-2-HR_ssp370_r1i1p1f1_gn_*\")\n",
+ "dset.tas"
]
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"> \"That sounds like [NetCDF](https://github.com/Unidata/netcdf-c)\"\n",
"\n",
@@ -864,8 +616,60 @@
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": 13,
"metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "height\tlat lat_bnds lon lon_bnds tas time time_bnds\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "<xarray.backends.zarr.ZarrStore at 0x7f2c4142d130>"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "directorystore=\"/home/dkrz/k204210/example\"\n",
+ "dset.to_zarr(directorystore, mode='w')\n",
+ "!ls {directorystore}\n",
+ "\n",
+ "import zarr\n",
+ "store = zarr.ZipStore('/home/dkrz/k204210/example.zip', mode='w')\n",
+ "dset.to_zarr(store)\n",
+ "\n",
+ "#also possible: different databases, sql, ...."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "source": [
+ "Zarr has a **semantic**:\n",
+ "- It uses hidden `json` formatted \"descriptors\" which contain information about how the data is shaped\n",
+ "- They have standard names like `.zmetadata` or `.zattrs`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"> \"Well... that is... technical performance stuff. What about advantages for data managers?\" \n",
"\n",
@@ -876,7 +680,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
"source": [
"## (Cloud) Object storages\n",
"\n",
@@ -891,7 +699,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"> \"That sounds like a downgrade in comparison to filesystems\"\n",
"\n",
@@ -906,7 +718,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"> \"What is the interest of DKRZ?\"\n",
"\n",
@@ -923,7 +739,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"### 📢 DKRZ's Openstack Swift Object Storage 📢\n",
"\n",
@@ -936,7 +756,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"> \"OK and how do users benefit from climate data in object storages?\"\n",
"\n",
@@ -949,7 +773,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"Georg Siemund, a bachelor student from University Hamburg has made some tests for throughput of zarr data: <br>\n",
"<img src=\"https://swift.dkrz.de/v1/dkrz_485fa9f6d16841fc847fb61048428062/zarr-imgs/Bildschirmfoto%20vom%202021-09-09%2014-58-47.png\" width=\"500\">"
@@ -957,18 +785,26 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"But even stronger arguments arise when using the **synergy** of zarr in object storages..."
]
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
"source": [
"## Zarr data in Object Storages\n",
"\n",
- "One challenge is that data transfer is expensive: It is not effective to download a dataset (file level) to a local computer if only a part of that (chunk) is actually needed for the analysis.\n",
+ "One challenge of data infrastrucutre is that data transfer is expensive: It is not effective to download a dataset (file level) to a local computer if only a part of that (chunk) is actually needed for the analysis.\n",
"\n",
"- `Zarr` allows programs to identify _chunks_ corresponding to the desired subset of the data before the download so that the **volume of data transfer is reduced**.\n",
"- The `Zarr` formatted data can be accessed via `http` so that no software on the cloud repository site is required. \n",
@@ -978,7 +814,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"- Meta data is saved next to the binary data. That allows programs to quickly create a virtual representation of large and complex datasets. The data is therefore **Analysis ready**.\n",
"\n",
@@ -989,7 +829,11 @@
{
"cell_type": "code",
"execution_count": 27,
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"name": "stdout",
@@ -1023,7 +867,11 @@
{
"cell_type": "code",
"execution_count": 26,
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"data": {
@@ -1673,7 +1521,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"- At this point, we have not downloaded any data yet\n",
"- The chunk sizes are optimized for swift cloud storage\n",
@@ -1682,7 +1534,11 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"The ultimate goal is therefore to provide users with a workflow that looks like:\n",
"<img src=\"https://swift.dkrz.de/v1/dkrz_485fa9f6d16841fc847fb61048428062/zarr-imgs/Bildschirmfoto%20vom%202021-09-09%2016-24-29.png\"></img>\n",
@@ -1692,16 +1548,24 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
"source": [
"## Workflow to bring NetCDF **To Zarr in Swift**"
]
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
- "Follow the path of a long and lonesome history...\n",
+ "Follow the path of a long and cumbersome history...\n",
"\n",
"1. While for S3 and GCS store backends for zarr already existed, a **Swift-store for Zarr** needed to be developed. That was done by Pavan: https://github.com/siligam/zarr-swiftstore 🥳 🥳 🥳 (biggest milestone).\n",
"2. The chunk size needed to be optimized for Swift Store\n",
@@ -1710,12 +1574,16 @@
" - Larger chunk size leads to problems with the Zarr compressors. 5GB is the limit by Swift\n",
"3. Validation of chunk writes had to be included.\n",
"\n",
- "Smaller issues like `dask` array memory problems needed to be solved. The `fsspec` lib was included into `xarray`."
+ "Smaller issues like `dask` array memory problems needed to be solved. The `fsspec` lib was included into `xarray` and `zarr`."
]
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
"In the end, we now have a first prototype of a **program**! Which is **tzis**!"
]
@@ -1723,7 +1591,11 @@
{
"cell_type": "code",
"execution_count": 7,
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"name": "stdout",
@@ -1799,20 +1671,31 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"source": [
- "### Necessary input\n",
+ "### Input and Conditions\n",
"\n",
"1. We need the storage URL and the auth token for Swift\n",
"2. We have to decide how the container should be named\n",
"3. Only one variable is written per dataset and need to be selected.\n",
- "4. A file or a list of files needs to be passed."
+ "4. A file or a list of files needs to be passed.\n",
+ "\n",
+ "- Only `time` can be used as chunk dimension. Datasets without `time` will be written directly (\"unmodified\") to storage.\n",
+ "- Consolidated stores are used: Metadata of many files are saved into one (no `atts` are lost)."
]
},
{
"cell_type": "code",
- "execution_count": 1,
- "metadata": {},
+ "execution_count": 4,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [],
"source": [
"import tzis\n",
@@ -1843,92 +1726,38 @@
},
{
"cell_type": "code",
- "execution_count": 5,
- "metadata": {},
+ "execution_count": 2,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Help on SwiftStore in module zarrswift.storage object:\n",
+ "Help on class Tzis in module tzis.tzis:\n",
"\n",
- "class SwiftStore(collections.abc.MutableMapping)\n",
- " | SwiftStore(container, prefix='', storage_options=None)\n",
- " | \n",
- " | Storage class using Openstack Swift Object Store.\n",
- " | \n",
- " | Parameters\n",
- " | ----------\n",
- " | container: string\n",
- " | swift container to use. It is created if it does not already exists\n",
- " | prefix: string\n",
- " | sub-directory path with in the container to store data\n",
- " | storage_options: dict\n",
- " | authentication information to connect to the swift store.\n",
- " | \n",
- " | Examples\n",
- " | --------\n",
- " | \n",
- " | >>> import os\n",
- " | >>> from zarrswift import SwiftStore\n",
- " | >>> getenv = os.environ.get\n",
- " | >>> options = {'preauthurl': getenv('OS_STORAGE_URL'),\n",
- " | ... 'preauthtoken': getenv('OS_AUTH_TOKEN')}\n",
- " | >>> store = SwiftStore(container=\"demo\", prefix=\"zarr_demo\", storage_options=options)\n",
- " | >>> root = zarr.group(store=store, overwrite=True)\n",
- " | >>> z = root.zeros('foo/bar', shape=(10, 10), chunks=(5, 5), dtype='i4')\n",
- " | >>> z[:] = 42\n",
- " | \n",
- " | Method resolution order:\n",
- " | SwiftStore\n",
- " | collections.abc.MutableMapping\n",
- " | collections.abc.Mapping\n",
- " | collections.abc.Collection\n",
- " | collections.abc.Sized\n",
- " | collections.abc.Iterable\n",
- " | collections.abc.Container\n",
- " | builtins.object\n",
+ "class Tzis(builtins.object)\n",
+ " | Tzis(os_url, os_token, os_container, os_name=None, mf_dset=None, varname=None, verbose=False, xarray_kwargs=None)\n",
" | \n",
" | Methods defined here:\n",
" | \n",
- " | __contains__(self, name)\n",
- " | \n",
- " | __delitem__(self, name)\n",
- " | \n",
- " | __eq__(self, other)\n",
- " | Return self==value.\n",
- " | \n",
- " | __getitem__(self, name)\n",
- " | \n",
- " | __getstate__(self)\n",
- " | \n",
- " | __init__(self, container, prefix='', storage_options=None)\n",
+ " | __init__(self, os_url, os_token, os_container, os_name=None, mf_dset=None, varname=None, verbose=False, xarray_kwargs=None)\n",
" | Initialize self. See help(type(self)) for accurate signature.\n",
" | \n",
- " | __iter__(self)\n",
- " | \n",
- " | __len__(self)\n",
- " | \n",
- " | __setitem__(self, name, value)\n",
- " | \n",
- " | __setstate__(self, state)\n",
- " | \n",
- " | clear(self)\n",
- " | D.clear() -> None. Remove all items from D.\n",
+ " | open_mf_dataset(self, mf, xarray_kwargs=None)\n",
" | \n",
- " | getsize(self, path=None)\n",
+ " | open_store(self, os_name)\n",
" | \n",
- " | keys(self)\n",
- " | D.keys() -> a set-like object providing a view on D's keys\n",
+ " | write_by_region(self, chunked_ds, store, startchunk, validity_check, chunkdim, varname)\n",
" | \n",
- " | listdir(self, path=None)\n",
+ " | write_directly(self, dset=None, store=None)\n",
" | \n",
- " | rmdir(self, path=None)\n",
+ " | write_to_swift(self, chunkdim='time', target_mb=1000, startchunk=0, validity_check=False, maxretries=3)\n",
" | \n",
- " | ----------------------------------------------------------------------\n",
- " | Readonly properties defined here:\n",
- " | \n",
- " | url\n",
+ " | write_with_validation_and_retries(self, ds, varname, store, chunkdim, target_mb, startchunk, validity_check, maxretries)\n",
" | \n",
" | ----------------------------------------------------------------------\n",
" | Data descriptors defined here:\n",
@@ -1938,101 +1767,48 @@
" | \n",
" | __weakref__\n",
" | list of weak references to the object (if defined)\n",
- " | \n",
- " | ----------------------------------------------------------------------\n",
- " | Data and other attributes defined here:\n",
- " | \n",
- " | __abstractmethods__ = frozenset()\n",
- " | \n",
- " | __hash__ = None\n",
- " | \n",
- " | ----------------------------------------------------------------------\n",
- " | Methods inherited from collections.abc.MutableMapping:\n",
- " | \n",
- " | pop(self, key, default=<object object at 0x7fb79b5af150>)\n",
- " | D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n",
- " | If key is not found, d is returned if given, otherwise KeyError is raised.\n",
- " | \n",
- " | popitem(self)\n",
- " | D.popitem() -> (k, v), remove and return some (key, value) pair\n",
- " | as a 2-tuple; but raise KeyError if D is empty.\n",
- " | \n",
- " | setdefault(self, key, default=None)\n",
- " | D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D\n",
- " | \n",
- " | update(self, other=(), /, **kwds)\n",
- " | D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\n",
- " | If E present and has a .keys() method, does: for k in E: D[k] = E[k]\n",
- " | If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v\n",
- " | In either case, this is followed by: for k, v in F.items(): D[k] = v\n",
- " | \n",
- " | ----------------------------------------------------------------------\n",
- " | Methods inherited from collections.abc.Mapping:\n",
- " | \n",
- " | get(self, key, default=None)\n",
- " | D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.\n",
- " | \n",
- " | items(self)\n",
- " | D.items() -> a set-like object providing a view on D's items\n",
- " | \n",
- " | values(self)\n",
- " | D.values() -> an object providing a view on D's values\n",
- " | \n",
- " | ----------------------------------------------------------------------\n",
- " | Data and other attributes inherited from collections.abc.Mapping:\n",
- " | \n",
- " | __reversed__ = None\n",
- " | \n",
- " | ----------------------------------------------------------------------\n",
- " | Class methods inherited from collections.abc.Collection:\n",
- " | \n",
- " | __subclasshook__(C) from abc.ABCMeta\n",
- " | Abstract classes can override this to customize issubclass().\n",
- " | \n",
- " | This is invoked early on by abc.ABCMeta.__subclasscheck__().\n",
- " | It should return True, False or NotImplemented. If it returns\n",
- " | NotImplemented, the normal algorithm is used. Otherwise, it\n",
- " | overrides the normal algorithm (and the outcome is cached).\n",
- " | \n",
- " | ----------------------------------------------------------------------\n",
- " | Class methods inherited from collections.abc.Iterable:\n",
- " | \n",
- " | __class_getitem__ = GenericAlias(...) from abc.ABCMeta\n",
- " | Represent a PEP 585 generic type\n",
- " | \n",
- " | E.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,).\n",
"\n"
]
}
],
"source": [
- "help(container.store)"
+ "import tzis\n",
+ "help(tzis.Tzis)"
]
},
{
"cell_type": "code",
- "execution_count": 6,
- "metadata": {},
+ "execution_count": 5,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"data": {
"text/plain": [
- "<zarrswift.storage.SwiftStore at 0x7fb790632790>"
+ "<zarrswift.storage.SwiftStore at 0x7fb3ed581370>"
]
},
- "execution_count": 6,
+ "execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
+ "#Still have to open a _zarr store_ inside the container\n",
"container.open_store(\"tastest\")"
]
},
{
"cell_type": "code",
"execution_count": 9,
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"data": {
@@ -2719,7 +2495,11 @@
{
"cell_type": "code",
"execution_count": 7,
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"name": "stdout",
@@ -2789,7 +2569,11 @@
{
"cell_type": "code",
"execution_count": 4,
- "metadata": {},
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
"outputs": [
{
"data": {
@@ -3417,22 +3201,42 @@
"source": [
"import xarray\n",
"import swiftenvbk1088\n",
+ "#must be publically available, otherwise use a swift-zarrstore\n",
"xarray.open_zarr(swiftenvbk1088.OS_STORAGE_URL+\"/\"+\"tastest\"+\"/\"+\"tastest\", consolidated=True)"
]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "source": [
+ "### Ongoing activities around DKRZ\n",
+ "\n",
+ "- Which is the most suitable **chunk dimension**?\n",
+ "- **Geomar**: high-resolution ocean-model outputs to zarr (Katharina Höflich)\n",
+ "- **AWI**: *Pyfesom*: https://github.com/FESOM/pyfesom2/blob/master/pyfesom2/datasets.py (Suvi Cheedela)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "source": [
+ "A **Show case** is provided by a prototype webapplication service which allows to upload data from lustre to swift with `tzis` on a VM."
+ ]
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python [conda env:root] *",
+ "display_name": "Python [conda env:testTzis2]",
"language": "python",
- "name": "conda-root-py"
+ "name": "conda-env-testTzis2-py"
},
"language_info": {
"codemirror_mode": {
@@ -3444,7 +3248,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.8"
+ "version": "3.9.4"
}
},
"nbformat": 4,