Commit 95106086 authored by Georg Siemund's avatar Georg Siemund
Browse files

Replace zarr_objectstorage_performancetest.ipynb

parent e950e6a6
......@@ -15,16 +15,10 @@
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
#use intake to browse catalog
col_url = "https://swift.dkrz.de/v1/dkrz_a44962e3ba914c309a7421573a6949a6/intake-esm/swift-cmip6.json"
col = intake.open_esm_datastore(col_url)
```
......@@ -37,37 +31,21 @@
```
%% Cell type:code id: tags:
``` python
dset_dict = cat.to_dataset_dict(zarr_kwargs={'consolidated': True})
list(dset_dict.keys())
```
%% Cell type:code id: tags:
``` python
ds = dset_dict['ScenarioMIP.DKRZ.MPI-ESM1-2-HR.ssp370.day.gn']
ds
```
%% Cell type:code id: tags:
``` python
data = ds.tasmax.data
cat = col.search(variable_id="tasmax", table_id="day")
zarrpath = cat.unique("zarr_path")["zarr_path"]["values"][0]
fsmap = fsspec.get_mapper(zarrpath)
openzarr = xarray.open_zarr(fsmap, consolidated=True)
data = openzarr.tasmax.data
data
```
%% Cell type:code id: tags:
``` python
#ds.tasmax[0, 0].plot(figsize=(16, 8), center=False, robust=True)
```
%% Cell type:code id: tags:
``` python
class DevNullStore:
def __init__(self):
pass
......@@ -84,11 +62,11 @@
```
%% Cell type:code id: tags:
``` python
%time dsa.store(data[0, :660], null_store, lock=False)
%time dsa.store(data, null_store, lock=False)
```
%% Cell type:markdown id: tags:
## Set up Cluster
......@@ -121,17 +99,11 @@
def dataframe(self):
return pd.DataFrame(self.diagnostics)
diag_timer = DiagnosticTimer()
```
%% Cell type:code id: tags:
``` python
chunksize = np.prod(data.chunksize) * data.dtype.itemsize
chunksize
```
%% Cell type:code id: tags:
``` python
......@@ -165,11 +137,18 @@
```
%% Cell type:code id: tags:
``` python
client.close()
```
%% Cell type:code id: tags:
``` python
df = diag_timer.dataframe()
df
```
%% Cell type:code id: tags:
``` python
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment