Skip to content
Snippets Groups Projects
Commit f37c45bf authored by Nils Brüggemann's avatar Nils Brüggemann
Browse files

Merge with master.

parents 6db0b41f ff3bf26c
No related branches found
No related tags found
No related merge requests found
Pipeline #63480 passed
pages:
script:
- apt-get update
- apt-get -y upgrade
- apt-get -y install make
- conda env create -f ci/requirements_py311.yml
- source activate pyicon_py311
- pip install -e .
- cd doc
- apk add --repository https://dl-cdn.alpinelinux.org/alpine/edge/community pandoc
- pip install nbsphinx nbsphinx_link pydata_sphinx_theme
- make html
- mv _build/html/* ../public/
tags:
- sphinx
- conda
only:
- master
......@@ -15,4 +19,4 @@ pages:
artifacts:
paths:
- public
expire_in: 5min
expire_in: 10min
......@@ -56,3 +56,6 @@ Once, `cartopy` is installed in your environment:
```bash
pip install git+https://gitlab.dkrz.de/m300602/pyicon.git
```
## Developing
When adding new functions, make sure to document them with a docstring. This should detail what the function does, the arguments and what type of objects it returns. Examples are encouraged. We use so-called "numpy" style docstrings which are then automatically rendered into the sphinx documentation. A guide to numpy style docstrings is available [here](https://numpydoc.readthedocs.io/en/latest/format.html) and they even produce some nice [examples](https://numpydoc.readthedocs.io/en/latest/example.html#example).
\ No newline at end of file
name: pyicon_py311
channels:
- conda-forge
dependencies:
- python=3.11
- numpy
- dask
- matplotlib
- cartopy
- netcdf4
- xarray
- zarr
- pandas
- cmocean
- scipy
- ipdb
- ipywidgets
- jupytext
- pytest
- healpy
- pandoc
- sphinx # Documentation
- nbsphinx # Documentation
- nbsphinx-link # Documentation
- sphinx-copybutton # Documentation
- pydata-sphinx-theme # Documentation
......@@ -8,7 +8,32 @@ API for pyicon
Some semi-automatically generate documentation about the pyicon API.
.. automodule:: pyicon
:members:
:undoc-members:
:show-inheritance:
pyicon.pyicon_accessor
---------------------
.. automodule:: pyicon.pyicon_accessor
:members:
:undoc-members:
:private-members:
pyicon.pyicon_calc_xr
---------------------
.. automodule:: pyicon.pyicon_calc_xr
:members:
:undoc-members:
:private-members:
pyicon.pyicon_calc
---------------------
.. automodule:: pyicon.pyicon_calc
:members:
:undoc-members:
:private-members:
pyicon.pyicon_IconData
----------------------
......@@ -16,8 +41,36 @@ pyicon.pyicon_IconData
.. automodule:: pyicon.pyicon_IconData
:members:
pyicon.pyicon_params
---------------------
.. automodule:: pyicon.pyicon_params
:members:
:undoc-members:
:private-members:
pyicon.pyicon_plotting
----------------------
.. automodule:: pyicon.pyicon_plotting
:members:
pyicon.pyicon_simulation
---------------------
.. automodule:: pyicon.pyicon_simulation
:members:
:undoc-members:
:private-members:
pyicon.pyicon_tb
---------------------
.. automodule:: pyicon.pyicon_tb
:members:
:undoc-members:
:private-members:
pyicon.pyicon_thermo
---------------------
.. automodule:: pyicon.pyicon_thermo
:members:
:undoc-members:
:private-members:
\ No newline at end of file
......@@ -18,8 +18,7 @@
import os
import sys
# sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('../..')) # Source code dir relative to this file
sys.path.insert(0, os.path.abspath('..')) # Source code dir relative to this file
# -- General configuration ------------------------------------------------
......@@ -35,12 +34,11 @@ extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.todo',
'sphinx.ext.imgmath',
# 'sphinx.ext.autosummary',
'nbsphinx',
'nbsphinx_link',
'sphinx_copybutton',
# "IPython.sphinxext.ipython_directive",
# "IPython.sphinxext.ipython_console_highlighting",
'sphinx.ext.autosummary',
'sphinx.ext.napoleon',
]
# Add any paths that contain templates here, relative to this directory.
......@@ -54,6 +52,8 @@ source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
autosummary_generate = True
napoleon_numpy_docstring = True
# !by_nils
html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html'] }
......@@ -165,6 +165,3 @@ texinfo_documents = [
author, 'pyicon', 'One line description of project.',
'Miscellaneous'),
]
......@@ -27,10 +27,6 @@ Pyicon is hosted here: `<https://gitlab.dkrz.de/m300602/pyicon/>`_
pyicon-view
example_notebooks
api
..
example_notebooks
Indices and tables
==================
......
This diff is collapsed.
import sys, glob, os
import json
from pathlib import Path
# --- calculations
import numpy as np
from scipy import interpolate
......@@ -1901,6 +1902,115 @@ def plot(data,
# --- original grid
lonlat_for_mask=False,
):
"""plot map of data
Parameters
----------
data : xr.DataArray
data to be
Plot : pyicon.Plot, optional
plotting canvas to draw on, by default None
ax : matplotlib.axes, optional
axis to plot on to , by default None
cax : matplotlib.axes, optional
axis to plot colorbar on, by default None
asp : float, optional
aspect ratio, by default None
fig_size_fac : float, optional
_description_, by default 2.0
mask_data : bool, optional
mask data where data = 0 (NaN value in ICON), by default True
logplot : bool, optional
logarithmic colormap, by default False
lon_reg : _type_, optional
longitude range to plot, by default None
lat_reg : _type_, optional
latitudes range to plot, by default None
central_longitude : str or float, optional
central longitude of plot, by default 'auto'
clim : str or (float, float), optional
colorbar limits, by default 'auto'
cmap : str or colormap, optional
colormap to use, by default 'auto'
conts : _type_, optional
_description_, by default None
contfs : _type_, optional
_description_, by default None
clevs : _type_, optional
_description_, by default None
use_pcol_or_contf : bool, optional
_description_, by default True
contcolor : str, optional
colour of contours, by default 'k'
cincr : float, optional
_description_, by default -1.0
clabel : bool, optional
whether to label contours, by default False
cbticks : str or list, optional
ticks for colorbar, by default 'auto'
xlabel : str, optional
label for x-axis, by default ''
ylabel : str, optional
label fo y-axis, by default ''
xticks : str or list, optional
tick labels for x-axis, by default 'auto'
yticks : str or list, optional
tick labels for y-axis, by default 'auto'
template : str, optional
_description_, by default 'none'
cbar_str : str, optional
label for colorbar, by default 'auto'
cbar_pos : str, optional
position of colorbar, by default 'bottom'
title_right : str, optional
title for rhs of plot, by default 'auto'
title_left : str, optional
title for lhs of plot, by default 'auto'
title_center : str, optional
title for centre of plot, by default 'auto'
projection : str, optional
projection for map, by default 'pc'
coastlines_color : str, optional
colour of coastlines, by default 'k'
land_facecolor : str, optional
colour of land, by default '0.7'
axes_facecolor : str, optional
colour of background axes, by default '0.7'
noland : bool, optional
whether to not plot land, by default False
do_plot_settings : bool, optional
whether to apply defauly plot settings, by default True
do_xyticks : bool, optional
_description_, by default True
do_gridlines : bool, optional
whether to plot gridlindes, by default False
gname : str, optional
name of the grid the data is on, by default 'auto'. Typically the name
of a subdirectory of pyicon.params["path_grid"].
fpath_tgrid : str, optional
path to the triangulation grid, by default 'auto'.
plot_method : "nn" or "tgrid", optional
whether to use perform nearest neighbour (nn) interpolation or plot on
the native triangular grid (tgrid), by default 'nn'
res : float, optional
resolution for healpix grids, by default 0.3
fpath_ckdtree : str, optional
path to the ckdtree, by default 'auto'
coordinates : str, optional
the coordinates of the variable to plotted, by default 'clat clon'.
Typically set to "xlon xlat" where x is either "c", "e" or "v" for cell,
edge or vertex points.
lonlat_for_mask : bool, optional
_description_, by default False
Returns
-------
ax : matplotlib.axes
axes that have been plotted to
hm : _type_
_description_
"""
# --- derive plot boundaries
......@@ -1946,37 +2056,41 @@ def plot(data,
# --- identify grid file names and paths
path_grid = params['path_grid']
if gname=='auto':
try:
Dgrid = identify_grid(data, path_grid)
except:
# This doesn't always work, lets try another approach
try:
Dgrid = identify_grid(data, path_grid)
gname = Dgrid['name']
Dgrid = identify_grid(
data, path_grid, uuidOfHGrid=data.attrs['uuidOfHGrid']
)
except:
try:
Dgrid = identify_grid(data, path_grid, uuidOfHGrid=data.attrs['uuidOfHGrid'])
gname = Dgrid['name']
except:
gname = 'none'
if fpath_tgrid=='auto':
Dgrid = dict()
if gname == "auto":
try:
Dgrid = identify_grid(data, path_grid)
fpath_tgrid = Dgrid['fpath_grid']
except:
fpath_tgrid = 'from_file'
if fpath_ckdtree=='auto':
fpath_ckdtree = f'{path_grid}/{gname}/ckdtree/rectgrids/{gname}_res{res:3.2f}_180W-180E_90S-90N.nc'
if grid_type=='auto':
if Dgrid["name"].startswith("healpix"):
gname = Dgrid["name"]
except KeyError:
gname = "none"
if fpath_tgrid == "auto":
try:
fpath_tgrid = Dgrid["fpath_grid"]
except KeyError:
fpath_tgrid = "from_file"
if grid_type == 'auto':
if gname.startswith("healpix"):
grid_type = 'healpix'
else:
grid_type = 'native'
if fpath_ckdtree == 'auto':
fpath_ckdtree = f'{path_grid}/{gname}/ckdtree/rectgrids/{gname}_res{res:3.2f}_180W-180E_90S-90N.nc'
# --- rename dimensions
if 'cells' in data.dims:
data = data.rename(cells='ncells')
#for dim in data.dims:
# if dim.startswith('layers'):
# data = data.rename({dim: 'depth'})
# --- infer depth name
depth_name = identify_depth_name(data)
......@@ -1988,22 +2102,46 @@ def plot(data,
# ---
if data.ndim!=1:
raise ValueError(f'::: Error: Wrong dimension of data: {data.dims}.')
#if 'grid_mapping' in list(data.attrs):
# plot_method = 'healpix'
# --- interpolate and cut to region
if grid_type=='native' and plot_method=='nn':
datai = interp_to_rectgrid_xr(data.compute(), fpath_ckdtree, lon_reg=lon_reg, lat_reg=lat_reg, coordinates=coordinates,
antialias=antialias, mask_to_zero=mask_to_zero,
)
if grid_type == 'native' and plot_method == 'nn':
# We need fpath_ckdtree so check it is there.
if not Path(fpath_ckdtree).exists():
if gname == "none":
raise FileNotFoundError(
f"Unable to find file `fpath_ckdtree={fpath_ckdtree}`. \
This may be because `gname='none'`. Try either setting \
`gname` explicitly or `fpath_ckdtree` explicitly. `gname` \
typically takes the name of a subdirectory of the folder \
{params['path_grid']}"
)
else:
raise FileNotFoundError(
f"Unable to find file `fpath_ckdtree={fpath_ckdtree}`. \
Try setting `fpath_ckdtree` explicitly."
)
datai = interp_to_rectgrid_xr(
data.compute(), fpath_ckdtree,
lon_reg=lon_reg, lat_reg=lat_reg, coordinates=coordinates,
antialias=antialias, mask_to_zero=mask_to_zero,
)
lon = datai.lon
lat = datai.lat
elif grid_type=='native' and plot_method=='tgrid':
print('Deriving triangulation object, this can take a while...')
if fpath_tgrid != 'from_file':
ds_tgrid = xr.open_dataset(fpath_tgrid)
else:
raise NotImplementedError(
"Function not yet ready for calling with \
`fpath_tgrid='from_file'`, `grid_type='native'` and \
`plot_method='tgrid`"
)
# In the below code it isn't clear what ds should be?
ds_tgrid = xr.Dataset()
ntr = ds.clon.size
vlon = ds.clon_bnds.data.reshape(ntr*3)
......@@ -2032,6 +2170,11 @@ def plot(data,
datai = hp_to_rectgrid(data, lon_reg=lon_reg, lat_reg=lat_reg, res=res)
lon = datai.lon
lat = datai.lat
else:
raise ValueError(
"Invalid combination of `grid_type` and `plot_method` \
arguments."
)
# --- title, colorbar, and x/y label strings
if cbar_str=='auto':
......@@ -2055,7 +2198,6 @@ def plot(data,
cbar_str = f'{data.name}'
if (title_right=='auto') and ('time' in data.coords):
tstr = str(data.time.data)
#tstr = tstr.split('T')[0].replace('-', '')+'T'+tstr.split('T')[1].split('.')[0].replace(':','')+'Z'
tstr = tstr.split('.')[0]
title_right = tstr
elif (title_right=='full_time') and ('time' in data.dims):
......
import pytest
from pathlib import Path
import pytest
import xarray as xr
import pyicon as pyic
@pytest.fixture()
def raw_grid():
cur_dir = Path(__file__).parent.resolve()
grid_path = cur_dir / "test_data/icon_grid_0014_R02B04_O.nc"
def lazy_raw_grid():
path_data = Path(pyic.params["path_example_data"])
path_data.mkdir(parents=True, exist_ok=True)
grid_path = path_data / "icon_grid_0014_R02B04_O.nc"
if not grid_path.exists():
import requests
grid_download_link = "http://icon-downloads.mpimet.mpg.de/grids/public/mpim/0014/icon_grid_0014_R02B04_O.nc"
try:
r = requests.get(grid_download_link, allow_redirects=True)
with open(grid_path, "wb") as grid_file:
grid_file.write(r.content)
except:
raise FileNotFoundError("{grid_path} does not exist and unable to \
download it")
raise FileNotFoundError(
"{grid_path} does not exist and unable to \
download it"
)
ds_grid = xr.open_dataset(grid_path)
ds_grid = xr.open_dataset(grid_path, chunks="auto")
return ds_grid
@pytest.fixture()
def processed_tgrid(raw_grid):
return pyic.convert_tgrid_data(raw_grid)
def eager_raw_grid(lazy_raw_grid):
ds_raw_grid = lazy_raw_grid.compute()
return ds_raw_grid
@pytest.fixture()
def examp_icon_dataset():
#cur_dir = Path(__file__).parent.resolve()
#fpath_data = cur_dir / "test_data/icon_example_data_r2b6.nc"
path_data = Path(pyic.params['path_example_data'])
def lazy_processed_tgrid(lazy_raw_grid):
return pyic.convert_tgrid_data(lazy_raw_grid)
@pytest.fixture()
def eager_processed_tgrid(eager_raw_grid):
return pyic.convert_tgrid_data(eager_raw_grid)
@pytest.fixture()
def lazy_examp_icon_dataset():
path_data = Path(pyic.params["path_example_data"])
path_data.mkdir(parents=True, exist_ok=True)
fpath_data = path_data / "icon_example_data_r2b4.nc"
if not fpath_data.exists():
import requests
download_link = "https://swift.dkrz.de/v1/dkrz_07387162e5cd4c81b1376bd7c648bb60/pyicon_example_data/icon_example_data_r2b4.nc"
download_link = "https://swift.dkrz.de/v1/dkrz_83018ad4-3c8d-4c7d-b684-7ba0742caa1a/pyicon_test_data/icon_example_data_r2b4.nc?temp_url_sig=03fb5d20a44832c7cf736ab83c1be3936364dbf6&temp_url_expires=2026-11-24T11:53:16Z"
try:
r = requests.get(download_link, allow_redirects=True)
r = requests.get(download_link, allow_redirects=True, stream=True)
with open(fpath_data, "wb") as fobj:
fobj.write(r.content)
except:
raise FileNotFoundError("{fpath_data} does not exist and unable to \
download it")
raise FileNotFoundError(
"{fpath_data} does not exist and unable to \
download it"
)
ds = xr.open_dataset(fpath_data)
ds = xr.open_dataset(fpath_data, chunks="auto")
return ds
@pytest.fixture()
def examp_icon_dataarray(examp_icon_dataset):
return examp_icon_dataset["to"]
def eager_examp_icon_dataset(lazy_examp_icon_dataset):
return lazy_examp_icon_dataset.compute()
@pytest.fixture()
def lazy_examp_icon_dataarray(lazy_examp_icon_dataset):
return lazy_examp_icon_dataset["to"]
@pytest.fixture()
def eager_examp_icon_dataarray(eager_examp_icon_dataset):
return eager_examp_icon_dataset["to"]
import subprocess
import pytest
import pyicon as pyic
from pathlib import Path
def test_docs_build(tmpdir_factory):
base_dir = Path(pyic.__file__).parent / ".."
doc_dir = base_dir.resolve() / "doc"
build_dir = tmpdir_factory.mktemp("_build")
# Run the sphinx-build command
try:
subprocess.check_call(["sphinx-build", "-b", "html", doc_dir, build_dir])
except subprocess.CalledProcessError as e:
pytest.fail(f"Documentation build failed with exit code {e.returncode}")
......@@ -2,10 +2,17 @@ from itertools import product
import pytest
import numpy as np
import pyicon as pyic
from .conftest import raw_grid, processed_tgrid
from .conftest import (
eager_raw_grid,
eager_processed_tgrid,
lazy_raw_grid,
lazy_processed_tgrid,
)
def test_convert_tgrid_data(raw_grid):
@pytest.mark.parametrize("raw_grid", ["lazy_raw_grid", "eager_raw_grid"])
def test_convert_tgrid_data(raw_grid, request):
raw_grid = request.getfixturevalue(raw_grid)
converted_tgrid = pyic.convert_tgrid_data(raw_grid)
# Check conversion to pythonic indexing of neighbour info has worked
......@@ -24,14 +31,17 @@ def test_convert_tgrid_data(raw_grid):
assert converted_tgrid[info].min().values == 0 or -1
if info.startswith("v") or info.startswith("edge_vertices"):
assert converted_tgrid[info].max().values == \
converted_tgrid.dims["vertex"] - 1
assert (
converted_tgrid[info].max().values == converted_tgrid.dims["vertex"] - 1
)
elif info.startswith("e"):
assert converted_tgrid[info].max().values == \
converted_tgrid.dims["edge"] - 1
assert (
converted_tgrid[info].max().values == converted_tgrid.dims["edge"] - 1
)
elif info.startswith("c") or info.startswith("a"):
assert converted_tgrid[info].max().values == \
converted_tgrid.dims["cell"] - 1
assert (
converted_tgrid[info].max().values == converted_tgrid.dims["cell"] - 1
)
# Conversion of ecv lat and lon to degrees
for point, dim in product("ecv", ("lat", "lon")):
......@@ -50,9 +60,16 @@ def test_convert_tgrid_data(raw_grid):
assert "cell" in converted_tgrid.dims
@pytest.mark.parametrize("tgrid", ["raw_grid", "processed_tgrid"])
@pytest.mark.parametrize(
"tgrid",
[
"lazy_raw_grid",
"lazy_processed_tgrid",
"eager_raw_grid",
"eager_processed_tgrid",
],
)
def test_xr_crop_tgrid(tgrid, request):
# Set ireg_c and crop the grid
tgrid = request.getfixturevalue(tgrid)
for point, dim in product("cev", ["lon", "lat"]):
......@@ -60,10 +77,20 @@ def test_xr_crop_tgrid(tgrid, request):
if tgrid[coord].units == "radian":
tgrid[coord] = np.degrees(tgrid[coord])
ireg_c = tgrid["cell"].where(
(tgrid["clon"] > -5) & (tgrid["clon"] < 5)
& (tgrid["clat"] > -5) & (tgrid["clat"] < 5),
drop=True).astype("int32")
tgrid["clon"] = tgrid["clon"].compute()
tgrid["clat"] = tgrid["clat"].compute()
ireg_c = (
tgrid["cell"]
.where(
(tgrid["clon"] > -5)
& (tgrid["clon"] < 5)
& (tgrid["clat"] > -5)
& (tgrid["clat"] < 5),
drop=True,
)
.astype("int32")
)
# This checks ireg_c is as expected
assert ireg_c.sum() == 301614
......@@ -89,8 +116,26 @@ def test_xr_crop_tgrid(tgrid, request):
assert cropped_tgrid["ireg_v"].sum() == 135385
assert cropped_tgrid["ireg_v"].prod() == -1427286351937536000
# Try running the example code from the docstring
clon = tgrid.clon.compute().data * 180.0 / np.pi
clat = tgrid.clat.compute().data * 180.0 / np.pi
lon_reg = [6, 10]
lat_reg = [-32, -30]
ireg_c = np.where(
(clon > lon_reg[0])
& (clon <= lon_reg[1])
& (clat > lat_reg[0])
& (clat <= lat_reg[1])
)[0]
pyic.xr_crop_tgrid(tgrid, ireg_c)
@pytest.mark.parametrize(
"processed_tgrid", ["lazy_processed_tgrid", "eager_processed_tgrid"]
)
def test_nabla_funcs(processed_tgrid, request):
processed_tgrid = request.getfixturevalue(processed_tgrid)
def test_nabla_funcs(processed_tgrid):
# Want to check curl of a gradient
gradient = pyic.xr_calc_grad(processed_tgrid, processed_tgrid["clon"])
curl_of_grad = pyic.xr_calc_curl(processed_tgrid, gradient)
......
......@@ -2,19 +2,23 @@ import pytest
import pyicon as pyic
import xarray as xr
import matplotlib.pyplot as plt
from .conftest import examp_icon_dataarray
from .conftest import lazy_examp_icon_dataarray, eager_examp_icon_dataarray
def test_plot(examp_icon_dataarray):
pyic.plot(examp_icon_dataarray.isel(time=0, depth=0))
return
def test_plot_sec(examp_icon_dataarray):
pyic.plot_sec(examp_icon_dataarray.isel(time=0), section='170W')
@pytest.mark.parametrize(
"examp_icon_dataarray", ["lazy_examp_icon_dataarray", "eager_examp_icon_dataarray"]
)
@pytest.mark.parametrize("grid_type,", ["auto", "native"])
def test_plot(examp_icon_dataarray, grid_type, request):
examp_icon_dataarray = request.getfixturevalue(examp_icon_dataarray)
pyic.plot(examp_icon_dataarray.isel(time=0, depth=0), grid_type=grid_type)
return
#def test_shade():
# dai = pyic.interp_to_rectgrid_xr(da)
# dai = dai.compute()
# pyic.shade(dai.lon, dai.lat, dai)
# return
@pytest.mark.parametrize(
"examp_icon_dataarray", ["lazy_examp_icon_dataarray", "eager_examp_icon_dataarray"]
)
def test_plot_sec(examp_icon_dataarray, request):
examp_icon_dataarray = request.getfixturevalue(examp_icon_dataarray)
pyic.plot_sec(examp_icon_dataarray.isel(time=0), section="170W")
return
import numpy as np
from pyicon.pyicon_thermo import (
_calculate_mpiom_density,
_calculate_insitu_temperature
)
from pyicon.pyicon_thermo import _calculate_mpiom_density, _calculate_insitu_temperature
def test_mpiom_density():
"""the test values come from A3.2 of Gill (1982), Atmospheric & Ocean
......@@ -11,9 +9,12 @@ def test_mpiom_density():
assert np.allclose(_calculate_mpiom_density(0, 5, 0), 999.96675, atol=1e-5)
assert np.allclose(_calculate_mpiom_density(35, 5, 0), 1027.67547, atol=1e-5)
assert np.allclose(_calculate_mpiom_density(35, 25, 1000), 1062.53817, atol=1e-5)
def test_insitu_temperature():
"""the test values come from A3.5 of Gill (1982), Atmospheric & Ocean
Dynamics
"""
assert np.allclose(_calculate_insitu_temperature(25, 8.4678516, 1000), 10, atol=1e-7)
\ No newline at end of file
assert np.allclose(
_calculate_insitu_temperature(25, 8.4678516, 1000), 10, atol=1e-7
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment