diff --git a/README.md b/README.md index 9f78e598c4b1ea9396c0646f1dba4b174a5889d8..3f26818b75a1681a9ca705c832ad49e98122dbdc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,76 @@ # Catalog for Pamtra insitu scripts with YAC # Setup on levante -TODO +To setup the pamtra-insitu workflow on levante follow the following steps: + +Create and activate a python environment (do **not** use the python executable that comes with `module load python3`) +```bash +/sw/spack-levante/python-3.9.9-fwvsvi/bin/python -m venv venv --prompt pamtra-insitu +. venv/bin/activate +pip install --upgrade pip +pip install wheel numpy matplotlib scipy numexpr netcdf4 cython +``` + +Build Pamtra and install it in your venv +```bash +git clone --recursive https://github.com/igmk/pamtra.git +cd pamtra +spack load /bcn7mbu # gcc + +# Now, let's modify the Makefile for openblas +sed 's/-llapack//g' Makefile > Makefile.levante +sed -i 's%-lblas% -L/sw/spack-levante/openblas-0.3.18-tpmfvw/lib/ -L/sw/spack-levante/fftw-3.3.10-fnfhvr/lib/ -lopenblas%g' Makefile.levante + +make -f Makefile.levante -j1 + +# add rpath +cd python/pyPamtra +RPATH="$(patchelf --print-rpath pyPamtraLib*.so):/sw/spack-levante/fftw-3.3.10-fnfhvr/lib/:/sw/spack-levante/openblas-0.3.18-tpmfvw/lib/" +patchelf --set-rpath ${RPATH} pyPamtraLib*.so +cd - + +cp -r python/pyPamtra ../venv/lib64/python3.9/site-packages/ + +cd .. +``` + +Build ICON +```bash +curl -s -L https://gitlab.dkrz.de/icon/icon-model/-/archive/release-2024.07-public/icon-model-release-2024.07-public.tar.gz | tar xzf - +mv icon-model-release-2024.07-public/ icon +cd icon +mkdir build +cd build +../config/dkrz/levante.gcc --enable-opnemp --enable-python-bindings --with-pic +make -j32 +pip install externals/yac/python +``` +With this pamtra, yac and icon should be in place and ready to use. + +# Prepare runscripts +To attach a pamtra component to ICON you need to modify the runscript as follows: + +To enable the "output_coupling" add `coupled_to_ocean = .TRUE.` to the `coupling_mode_nml`: +``` +&coupling_mode_nml + coupled_to_ocean = .TRUE. +/ +``` + +The we need to add an additional python process to MPI. We use the `--multi-prog` feature of `srun`. +Add this just before the `srun` (`${START}`) call +```bash +cat > mpmd.conf << EOF +/path/to/venv/bin/python path/to/pamtra/component.py +* ${MODEL} +EOF +``` +(adjust the pathes) and replace the line +``` +${START} ${MODEL} +``` +with +``` +${START} --multi-prog mpmd.conf +``` +(This does only work with slurm.)