Skip to content
Snippets Groups Projects
Commit 32dc1085 authored by Siddhant Tibrewal's avatar Siddhant Tibrewal
Browse files

wip: boilerplate for setting up local dev env for yaco

parent c7244209
No related branches found
No related tags found
1 merge request!46Build process for multiple compilers across different machines
Hide whitespace changes
Inline Side-by-side
Showing
with 466 additions and 0 deletions
scripts/setup/build-dependencies.sh 0 → 100644
+ 361
− 0
View file @ 32dc1085
#!/bin/bash
# More safety by turning some bugs into errors.
# Without `errexit` you don’t need ! and can replace
# ${PIPESTATUS[0]} with a simple $?, but I prefer safety.
set -o errexit -o pipefail -o noclobber -o nounset
REPO_TAG_AEC="v1.1.2"
REPO_TAG_ECBUILD="3.8.0"
REPO_TAG_ECCODES="2.38.3"
REPO_TAG_ECKIT="1.25.0"
REPO_TAG_METKIT="1.11.0"
REPO_TAG_FDB="5.11.94"
REPO_TAG_FYAML="v0.9"
REPO_TAG_CFITSIO="4.1.0"
REPO_TAG_HEALPIX="trunk"
REPO_TAG_YAC="release-3.4.0_p2"
REPO_TAG_YAXT="release-0.11.3"
# we trust that the yac version in the checked out external folder of yaco is the right one
# otherwise, this is a source of version conflict if used from within a repository and not as a standalone script
# TODO: if this is acceptable, may be extend it to other submodules of this repo
ROOT_DIR=$(pwd)
THREADS=8
function print_versions {
echo "ECBUILD $REPO_TAG_ECBUILD"
echo "ECCODES $REPO_TAG_ECCODES"
echo "ECKIT $REPO_TAG_ECKIT"
echo "METKIT $REPO_TAG_METKIT"
echo "FDB $REPO_TAG_FDB"
echo "FYAML $REPO_TAG_FYAML"
echo "CFITSIO $REPO_TAG_CFITSIO"
echo "HEALPIX $REPO_TAG_HEALPIX"
echo "YAC $REPO_TAG_YAC"
echo "YAXT $REPO_TAG_YAXT"
}
function usage {
echo "This script is intendet to build YACO with its dependencies on Levante/LUMI"
echo "Each checked out branch/version of the repositories will get their own repo/install/build folder"
echo ""
echo "--- Workflow --------------------------------------------------------------"
echo " build-dependencies -j <thread_count>"
echo " starts the building process with given number of threads"
echo ""
echo "--- Options ---------------------------------------------------------------"
echo " -j Threadcount"
echo " --versions Prints the versions that will be installed"
echo " --work_dir <name> Sets the name of the work dir folder which"
echo " will be created in the process."
echo " --dependency <name>/all Set the library name to be built eg: healpix"
echo " valid is one of [ecbuild, eckit, eccodes, metkit, fdb, fyaml, cfitsio, healpix, yac, yaxt]"
echo " defaults to building all depedencies"
}
function build_dependency {
library_name=$1
if [[ "${library_name,,}" == "all" ]]; then
build_all
else
install_${library_name}
fi
}
VALID_ARGS=$(getopt -o j:h --long versions,help,work_dir,dependency: -- "$@") || exit 1
eval set -- "$VALID_ARGS"
while true; do
case "$1" in
-j)
THREADS=$2
shift 2
;;
--dependency)
build_dependency "$2"
shift 2
exit
;;
--versions)
print_versions
shift
exit
;;
--work_dir)
WORK_DIR_NAME=$2
shift 2
;;
? | -h | --help)
usage
exit
;;
--) shift
break
;;
esac
done
shift "$(( OPTIND - 1 ))"
YACO_WORK_DIR=${ROOT_DIR}/$WORK_DIR_NAME
GIT_REPOS=$YACO_WORK_DIR/external
INSTALL_FOLDER=$YACO_WORK_DIR/install
function debug {
echo "$1"
}
function clone_to {
if [ -d "$2" ]
then
cd "$2"
echo "repo already loaded: skipping git clone"
else
if [ -z "$3" ]; then
echo "Using default branch"
git clone --depth=1 --recursive "$1" "$2"
else
git clone --depth=1 --recursive --branch "$3" "$1" "$2"
fi
fi
}
function install_via_cmake {
REPO_NAME=$1
shift
REPO_TAG=$1
shift
GIT_URL=$1
shift
local REPO_TAG=${REPO_TAG/\//-}
local VERSIONED_FOLDER=$REPO_NAME-$REPO_TAG
local REPO_PATH=$GIT_REPOS/$VERSIONED_FOLDER
local BUILD_FOLDER=$REPO_PATH/build
clone_to "$GIT_URL" "$REPO_PATH" "$REPO_TAG"
mkdir -p "$BUILD_FOLDER"
cd "$BUILD_FOLDER"
echo "$REPO_PATH"
pwd
cmake "$REPO_PATH" -DCMAKE_INSTALL_PREFIX="$INSTALL_FOLDER" -DCMAKE_PREFIX_PATH="$INSTALL_FOLDER" "$@"
make -j "$THREADS"
make install
}
function install_aec {
FOLDER_NAME="aec"
GIT_URL=git@gitlab.dkrz.de:k202009/libaec.git
install_via_cmake "$FOLDER_NAME" "$REPO_TAG_AEC" "$GIT_URL"
}
function install_ecbuild {
FOLDER_NAME="ecbuild"
GIT_URL=https://github.com/ecmwf/ecbuild
install_via_cmake "$FOLDER_NAME" "$REPO_TAG_ECBUILD" "$GIT_URL"
}
function install_eccodes {
REPO_NAME="eccodes"
GIT_URL=https://github.com/ecmwf/eccodes
install_via_cmake "$REPO_NAME" "$REPO_TAG_ECCODES" "$GIT_URL" -DENABLE_AEC=ON
}
function install_eckit {
REPO_NAME="eckit"
GIT_URL="https://github.com/ecmwf/eckit"
install_via_cmake "$REPO_NAME" "$REPO_TAG_ECKIT" "$GIT_URL"
}
function install_metkit {
REPO_NAME="metkit"
GIT_URL="https://github.com/ecmwf/metkit"
install_via_cmake "$REPO_NAME" "$REPO_TAG_METKIT" "$GIT_URL"
}
function install_fdb {
REPO_NAME="fdb"
GIT_URL="https://github.com/ecmwf/fdb"
install_via_cmake "$REPO_NAME" "$REPO_TAG_FDB" "$GIT_URL"
}
function install_fyaml {
GIT_URL="https://github.com/pantoniou/libfyaml.git"
REPO_NAME="fyaml"
install_via_cmake "$REPO_NAME" "$REPO_TAG_FYAML" "$GIT_URL"
}
function install_CFITSIO {
REPO_NAME="cfitsio"
GIT_URL="https://github.com/healpy/cfitsio.git"
install_via_cmake "$REPO_NAME" "$REPO_TAG_CFITSIO" "$GIT_URL"
}
function install_healpix {
REPO_NAME="healpix"
VERSIONED_FOLDER=$REPO_NAME-$REPO_TAG_HEALPIX
REPO_PATH=$GIT_REPOS/$VERSIONED_FOLDER
#-------------------------------------------------------------------------------------
clone_to https://github.com/healpy/healpixmirror.git "$REPO_PATH" "$REPO_TAG_HEALPIX"
#-------------------------------------------------------------------------------------
# install libsharp
LIBSHARP_ROOT=$REPO_PATH/src/common_libraries/libsharp
LIBSHARP_BUILD_FOLDER=$LIBSHARP_ROOT/build
cd "$LIBSHARP_ROOT"
autoreconf -ifv
mkdir -p "$LIBSHARP_BUILD_FOLDER"
cd "$LIBSHARP_BUILD_FOLDER"
"$LIBSHARP_ROOT"/configure --prefix="$INSTALL_FOLDER"
make -j "$THREADS"
make install
#-------------------------------------------------------------------------------------
HEALPIX_ROOT=$REPO_PATH/src/cxx
HEALPIX_BUILD_DIR=$HEALPIX_ROOT/build
cd "$HEALPIX_ROOT"
autoreconf -ifv
mkdir -p "$HEALPIX_BUILD_DIR"
cd "$HEALPIX_BUILD_DIR"
$HEALPIX_ROOT/configure --prefix=$INSTALL_FOLDER
make -j "$THREADS"
make install
}
function install_yaxt {
REPO_NAME="yaxt"
VERSIONED_FOLDER=$REPO_NAME-$REPO_TAG_YAXT
REPO_PATH=$GIT_REPOS/$VERSIONED_FOLDER
#-------------------------------------------------------------------------------------
clone_to https://gitlab.dkrz.de/dkrz-sw/yaxt.git "$REPO_PATH" "$REPO_TAG_YAXT"
#-------------------------------------------------------------------------------------
# install yac
cd "$REPO_PATH"
autoreconf -ifv
YAXT_BUILD_FOLDER=$REPO_PATH/build
mkdir -p "$YAXT_BUILD_FOLDER"
cd "$YAXT_BUILD_FOLDER"
echo $PKG_CONFIG_PATH
$REPO_PATH/configure --prefix=$INSTALL_FOLDER
make -j "$THREADS"
make install
}
function install_yac {
REPO_NAME="yac"
VERSIONED_FOLDER=$REPO_NAME-$REPO_TAG_YAC
REPO_PATH=$GIT_REPOS/$VERSIONED_FOLDER
#-------------------------------------------------------------------------------------
clone_to https://gitlab.dkrz.de/dkrz-sw/yac.git "$REPO_PATH" "$REPO_TAG_YAC"
#-------------------------------------------------------------------------------------
# install yac
cd "$REPO_PATH"
autoreconf -ifv
YAC_BUILD_FOLDER=$REPO_PATH/build
mkdir -p "$YAC_BUILD_FOLDER"
cd "$YAC_BUILD_FOLDER"
echo $PKG_CONFIG_PATH
CC=mpicc FC=mpif90 $REPO_PATH/configure --disable-mpi-checks --disable-netcdf --prefix=$INSTALL_FOLDER \
--with-yaxt-include=$INSTALL_FOLDER/include \
--with-yaxt-lib=$INSTALL_FOLDER/lib \
--with-fyaml-include=$INSTALL_FOLDER/include \
--with-fyaml-lib=$INSTALL_FOLDER/lib
make -j "$THREADS"
make install
}
function make_yaco() {
# it is expected that the main branch is a production branch
REPO_NAME="yaco"
BRANCH_NAME="cleanup_build_process"
VERSIONED_FOLDER=$REPO_NAME-${BRANCH_NAME/\//-}
REPO_PATH=$YACO_WORK_DIR/$VERSIONED_FOLDER
GIT_URL=https://gitlab.dkrz.de/mpim-sw/yaco.git
clone_to "$GIT_URL" "$REPO_PATH" "$BRANCH_NAME"
BUILD_FOLDER=$REPO_PATH/build
mkdir -p "$BUILD_FOLDER" && cd "$BUILD_FOLDER"
cmake "$REPO_PATH" -DCMAKE_INSTALL_PREFIX="$INSTALL_FOLDER" -DCMAKE_PREFIX_PATH=$INSTALL_FOLDER -DENABLE_NETCDF=OFF -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_BUILD_TYPE=RelWithDebInfo;
make -j "$THREADS"
make install
}
function build_all {
debug "========================"
debug "---- handling aec"
install_aec
debug "------------------------"
debug "---- handling ecbuild"
install_ecbuild
debug "------------------------"
debug "---- handling eccodes"
install_eccodes
debug "------------------------"
debug "---- handling eckit"
install_eckit
debug "------------------------"
debug "---- handling metkit"
install_metkit
debug "------------------------"
debug "---- handling fdb"
install_fdb
debug "========================"
debug "== handling yaml"
install_fyaml
debug "========================"
debug "== handling CFITSIO"
install_CFITSIO
debug "========================"
debug "== handling SHARP and HEALPIX"
install_healpix
debug "========================"
debug "== handling YAXT & YAC"
install_yaxt
install_yac
debug "========================"
}
function main {
echo "ROOT_DIR: $ROOT_DIR"
echo "GIT_REPOS: $GIT_REPOS"
echo "INSTALL_FOLDER: $INSTALL_FOLDER"
echo "YACO_WORK_DIR: $YACO_WORK_DIR"
mkdir -p "$GIT_REPOS"
mkdir -p "$INSTALL_FOLDER"
mkdir -p "$YACO_WORK_DIR"
build_all
}
main
scripts/setup/build-yaco.sh 0 → 100755
+ 75
− 0
View file @ 32dc1085
#!/bin/bash
# More safety by turning some bugs into errors.
# Without `errexit` you don’t need ! and can replace
# ${PIPESTATUS[0]} with a simple $?, but I prefer safety.
set -o errexit -o pipefail -o noclobber -o nounset
function usage {
echo "This script is intendet to build YACO with its dependencies on Levante/LUMI"
echo "Each checked out branch/version of the repositories will get their own repo/install/build folder"
echo ""
echo "--- Workflow --------------------------------------------------------------"
echo " (1) yaco-build --modules <lumi|levante>"
echo " prints the commands to load echo all required"
echo " software from given machine."
echo " (2) yaco-build -j <thread_count>"
echo " starts the building process with given number of threads"
echo " "
echo "--- Info ------------------------------------------------------------------"
echo " --modules <levante, lumi> prints the necessary calls to load all"
echo " all required software from given machine"
echo "--- Options ---------------------------------------------------------------"
echo " --machine <levante, lumi> Build for target machine"
echo " -j Threadcount"
echo " --versions Prints the versions that will be installed"
echo " --work_dir <name> Sets the name of the work dir folder which"
echo " will be created in the process."
}
VALID_ARGS=$(getopt -o j:h --long help,prefix,work_dir,root_dir: -- "$@") || exit 1
THREADS=8
ROOT_DIR=$(pwd)/..
BUILD_PATH=$ROOT_DIR/build_test
INSTALL_FOLDER=$BUILD_PATH/install
eval set -- "$VALID_ARGS"
while true; do
case "$1" in
-j)
THREADS=$2
shift 2
;;
--prefix)
INSTALL_FOLDER "$2"
shift 2
;;
--work_dir)
BUILD_PATH=$2
shift 2
;;
--root_dir)
ROOT_DIR=$3
shift 2
;;
? | -h | --help)
usage
exit
;;
--) shift
break
;;
esac
done
shift "$(( OPTIND - 1 ))"
# build yaco
mkdir -p "$BUILD_PATH"
cd "$BUILD_PATH"
cmake "$ROOT_DIR" -DCMAKE_PREFIX_PATH=$INSTALL_FOLDER -DENABLE_NETCDF=OFF -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_BUILD_TYPE=RelWithDebInfo;
cmake --build . -j $THREADS
cmake --install . --prefix=$INSTALL_FOLDER
\ No newline at end of file
scripts/setup/generic.gcc-13.0 0 → 100644
+ 2
− 0
View file @ 32dc1085
WORK_DIR_NAME="deploy_yaco_20241119_132122CET"
scripts/setup/levante.intel-2021.5.0 0 → 100644
+ 8
− 0
View file @ 32dc1085
function echo_software_stack_levante {
echo module load openmpi/4.1.2-intel-2021.5.0
echo module load hdf5/1.12.1-intel-2021.5.0
echo module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0
echo module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0
echo spack load openblas@0.3.18%intel@2021.5.0
echo spack load /evf4hh6 #libaec 1.0.6 @ intel 2012.5.0
}
\ No newline at end of file
scripts/setup/lumi.cce-16.0.1 0 → 100644
+ 10
− 0
View file @ 32dc1085
function echo_software_stack_lumi {
modules1='craype-x86-milan craype-network-ofi craype cray-mpich PrgEnv-cray libfabric perftools-base craype-accel-amd-gfx90a rocm cray-hdf5 cray-netcdf'
modules2='cce/16.0.1.1'
modules_c17='PrgEnv-cray cce/17.0.1 craype-x86-milan cray-mpich/8.1.29 cray-hdf5 cray-netcdf cray-libsci libfabric'
echo "module purge"
echo "module load ${modules1}"
echo "unset INCLUDE_PATH_X86_64"
echo "module load ${modules2}"
}
scripts/setup/lumi.cce.17.0.1 0 → 100644
+ 10
− 0
View file @ 32dc1085
function echo_software_stack_lumi {
modules1='craype-x86-milan craype-network-ofi craype cray-mpich PrgEnv-cray libfabric perftools-base craype-accel-amd-gfx90a rocm cray-hdf5 cray-netcdf'
modules2='cce/16.0.1.1'
modules_c17='PrgEnv-cray cce/17.0.1 craype-x86-milan cray-mpich/8.1.29 cray-hdf5 cray-netcdf cray-libsci libfabric'
echo "module purge"
echo "module load ${modules1}"
echo "unset INCLUDE_PATH_X86_64"
echo "module load ${modules2}"
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Imprint | Privacy Policy