Commit 95106086 authored by Georg Siemund's avatar Georg Siemund
Browse files

Replace zarr_objectstorage_performancetest.ipynb

parent e950e6a6
......@@ -19,13 +19,6 @@
"import fsspec"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
......@@ -51,42 +44,19 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dset_dict = cat.to_dataset_dict(zarr_kwargs={'consolidated': True})\n",
"list(dset_dict.keys())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ds = dset_dict['ScenarioMIP.DKRZ.MPI-ESM1-2-HR.ssp370.day.gn']\n",
"ds"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"data = ds.tasmax.data\n",
"cat = col.search(variable_id=\"tasmax\", table_id=\"day\")\n",
"zarrpath = cat.unique(\"zarr_path\")[\"zarr_path\"][\"values\"][0]\n",
"fsmap = fsspec.get_mapper(zarrpath)\n",
"openzarr = xarray.open_zarr(fsmap, consolidated=True)\n",
"data = openzarr.tasmax.data\n",
"data"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#ds.tasmax[0, 0].plot(figsize=(16, 8), center=False, robust=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
......@@ -119,7 +89,7 @@
"metadata": {},
"outputs": [],
"source": [
"%time dsa.store(data[0, :660], null_store, lock=False)"
"%time dsa.store(data, null_store, lock=False)"
]
},
{
......@@ -164,17 +134,8 @@
" def dataframe(self):\n",
" return pd.DataFrame(self.diagnostics)\n",
"\n",
"diag_timer = DiagnosticTimer()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"chunksize = np.prod(data.chunksize) * data.dtype.itemsize\n",
"chunksize"
"diag_timer = DiagnosticTimer()\n",
"chunksize = np.prod(data.chunksize) * data.dtype.itemsize"
]
},
{
......@@ -223,7 +184,19 @@
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
"source": [
"client.close()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df = diag_timer.dataframe()\n",
"df"
]
},
{
"cell_type": "code",
......@@ -235,9 +208,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "zarr_test",
"display_name": "Python 3 bleeding edge (using the module anaconda3/bleeding_edge)",
"language": "python",
"name": "zarr_test"
"name": "anaconda3_bleeding"
},
"language_info": {
"codemirror_mode": {
......@@ -249,7 +222,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.9"
"version": "3.6.10"
}
},
"nbformat": 4,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment