added build script for yaco

e33a64b8 · Oliver Heidmann · a1cf52ce · e33a64b8
Commit e33a64b8 authored 1 year ago by Oliver Heidmann
--- a/scripts/yaco-build.sh
+++ b/scripts/yaco-build.sh
+#!/bin/bash
+
+WORK_DIR_NAME="yaco_test"
+
+REPO_TAG_AEC="v1.1.2"
+REPO_TAG_ECBUILD="3.8.0"
+REPO_TAG_ECCODES="2.32.3"
+REPO_TAG_ECKIT="1.25.0"
+REPO_TAG_METKIT="1.11.0"
+REPO_TAG_FDB="5.11.94"
+REPO_TAG_FYAML="master"
+REPO_TAG_YAML_CPP="yaml-cpp-0.7.0"
+REPO_TAG_CFITSIO="4.1.0"
+REPO_TAG_HEALPIX="trunk"
+REPO_TAG_YACO="300910/develop"
+YACO_YAC_VERSION="release-3.0.3"
+
+function print_versions {
+
+echo "ECBUILD   $REPO_TAG_ECBUILD"
+echo "ECCODES   $REPO_TAG_ECCODES"
+echo "ECKIT     $REPO_TAG_ECKIT"
+echo "METKIT    $REPO_TAG_METKIT"
+echo "FDB       $REPO_TAG_FDB"
+echo "FYAML     $REPO_TAG_FYAML"
+echo "YAML_CPP  $REPO_TAG_YAML_CPP"
+echo "CFITSIO   $REPO_TAG_CFITSIO"
+echo "HEALPIX   $REPO_TAG_HEALPIX"
+echo "YACO      $REPO_TAG_YACO"
+echo "YACO_YAC  $YACO_YAC_VERSION"
+
+}
+#------------------------------------------------------------------
+# More safety, by turning some bugs into errors.
+# Without `errexit` you don’t need ! and can replace
+# ${PIPESTATUS[0]} with a simple $?, but I prefer safety.
+set -o errexit -o pipefail -o noclobber -o nounset
+
+ROOT_DIR=$(pwd)
+THREADS=4
+LIB_LOCATIONS="" #needs to be set by the functions
+
+function usage {
+  echo "This script is intendet to build YACO with its dependencies on Levante/LUMI"
+  echo "Each checked out branch/version of the repositories will get their own repo/install/build folder"
+  echo ""
+  echo "--- Workflow --------------------------------------------------------------"
+  echo "   (1) yaco-build --modules <lumi|levante>"
+  echo "          prints the commands to load echo all required"
+  echo "          software from given machine."
+  echo "   (2) yaco-build -j <thread_count>"
+  echo "          starts the building process with given number of threads"
+  echo "   "
+  echo "--- Info ------------------------------------------------------------------"
+  echo "   --modules <levante, lumi>       prints the necessary calls to load all"
+  echo "                                   all required software from given machine"
+  echo "--- Options ---------------------------------------------------------------"
+  echo "   --machine <levante, lumi>       Build for target machine"
+  echo "   -j                              Threadcount"
+  echo "   --versions                      Prints the versions that will be installed"
+  echo "   --work_dir <name>               Sets the name of the work dir folder which"
+  echo "                                   will be created in the process."
+
+}
+
+
+function echo_software_stack_lumi {
+modules1='craype-x86-milan craype-network-ofi craype cray-mpich PrgEnv-cray libfabric perftools-base craype-accel-amd-gfx90a rocm cray-hdf5 cray-netcdf'
+modules2='cce/16.0.1.1'
+
+echo "module purge"
+echo "module load ${modules1}"
+echo "unset INCLUDE_PATH_X86_64" 
+echo "module load ${modules2}"
+}
+
+function echo_software_stack_levante {
+  echo module load openmpi/4.1.2-intel-2021.5.0
+  echo module load hdf5/1.12.1-intel-2021.5.0
+  echo module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0
+  echo module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0
+  echo spack load openblas@0.3.18%intel@2021.5.0
+  echo spack load libfyaml@0.7.12
+  echo spack load git
+  echo spack load /evf4hh6 #libaec 1.0.6 @ intel 2012.5.0
+}
+function handle_module_option {
+  machine_name=$1
+  if [[ "${machine_name,,}" == "lumi" ]]; then
+    echo_software_stack_lumi
+  elif [[ "${machine_name,,}" == "levante" ]]; then
+    echo_software_stack_levante
+  else
+    echo "Error, machine name <$1> not found. <lumi,levante>"
+  fi
+}
+
+function handle_machine_option {
+  machine_name=$1
+  if [[ "${machine_name,,}" == "lumi" ]]; then
+    MACHINE=$machine_name
+  elif [[ "${machine_name,,}" == "levante" ]]; then
+    MACHINE=$machine_name
+  else
+    echo "Error, machine name <$1> not found. <lumi,levante>"
+  fi
+}
+
+VALID_ARGS=$(getopt -o j:h --long versions,work_dir,modules:,machine:,install-dir,lib-install-dir: -- "$@")
+if [[ $? -ne 0 ]]; then
+  exit 1;
+fi
+
+eval set -- "$VALID_ARGS"
+while [ : ]; do
+  case "$1" in
+    -j)
+      THREADS=$2
+      shift 2
+      ;;
+    --modules)
+      handle_module_option $2
+      shift 2
+      exit
+      ;;
+    --versions)
+      print_versions
+      shift
+      exit
+      ;;
+    --work_dir)
+      WORK_DIR_NAME=$2
+      shift 2
+      ;;
+    --machine)
+      handle_machine_option "$2"
+      shift 2
+      ;;
+    ? | -h | --help)
+      usage
+      exit
+      ;;
+    --) shift
+      break
+      ;;
+  esac
+done
+shift "$(($OPTIND -1))"
+
+YACO_WORK_DIR=${ROOT_DIR}/$WORK_DIR_NAME
+GIT_REPOS=$YACO_WORK_DIR/external
+BUILD_FOLDERS=$YACO_WORK_DIR/build
+INSTALL_FOLDER=$YACO_WORK_DIR/install
+
+
+function debug {
+  echo $1
+}
+
+function create_folder {
+  if [ -d $1 ] ; then return ; fi
+  mkdir -p $1
+}
+
+function clone_to {
+  if [ -d $2 ] ; then cd $2; echo "repo already loaded: skipping git clone";
+  else
+    git clone --recursive $1 $2 ;
+    if [ -z "$3" ]; then
+      echo "Using default branch"
+    else
+      cd $2; git checkout $3
+    fi
+  fi
+}
+
+function install_via_cmake {
+  REPO_NAME=$1
+  REPO_TAG=$2
+  GIT_URL=$3
+  if [ -z ${4+x} ] ; then
+    EXTRA=""
+  else
+    EXTRA=$4
+  fi
+
+  local REPO_TAG=${REPO_TAG/\//-}
+  local VERSIONED_FOLDER=$REPO_NAME-$REPO_TAG
+  local REPO_PATH=$GIT_REPOS/$VERSIONED_FOLDER
+  local BUILD_FOLDER=$BUILD_FOLDERS/$VERSIONED_FOLDER
+  local INSTALL_PATH=$INSTALL_FOLDER/$VERSIONED_FOLDER
+
+  clone_to $GIT_URL $REPO_PATH $REPO_TAG
+  create_folder $BUILD_FOLDER
+  cd $BUILD_FOLDER
+  echo $REPO_PATH
+  echo $(pwd)
+  cmake .. $REPO_PATH -DCMAKE_INSTALL_PREFIX=$INSTALL_PATH -DCMAKE_PREFIX_PATH=$INSTALL_FOLDER $EXTRA
+  make -j $THREADS
+  make install
+}
+function install_aec {
+  FOLDER_NAME="aec"
+  GIT_URL=git@gitlab.dkrz.de:k202009/libaec.git
+
+  install_via_cmake $FOLDER_NAME $REPO_TAG_AEC $GIT_URL
+
+}
+
+function install_ecbuild {
+  FOLDER_NAME="ecbuild"
+  GIT_URL=https://github.com/ecmwf/ecbuild
+
+  install_via_cmake $FOLDER_NAME $REPO_TAG_ECBUILD $GIT_URL
+
+}
+
+function install_eccodes {
+  REPO_NAME="eccodes"
+  GIT_URL=https://github.com/ecmwf/eccodes
+
+  install_via_cmake $REPO_NAME $REPO_TAG_ECCODES $GIT_URL "-DENABLE_AEC=ON -DCMAKE_PREFIX_PATH=$INSTALL_FOLDER/ecbuild-$REPO_TAG_ECBUILD;$INSTALL_FOLDER/aec-$REPO_TAG_AEC"
+}
+
+function install_eckit {
+  REPO_NAME="eckit"
+  GIT_URL="https://github.com/ecmwf/eckit"
+
+  install_via_cmake $REPO_NAME $REPO_TAG_ECKIT $GIT_URL -DCMAKE_PREFIX_PATH=$INSTALL_FOLDER/ecbuild-$REPO_TAG_ECBUILD
+}
+
+function install_metkit {
+  REPO_NAME="metkit"
+  GIT_URL="https://github.com/ecmwf/metkit"
+
+  install_via_cmake $REPO_NAME $REPO_TAG_METKIT $GIT_URL -DCMAKE_PREFIX_PATH=$INSTALL_FOLDER
+}
+
+function install_fdb {
+  REPO_NAME="fdb"
+  GIT_URL="https://github.com/ecmwf/fdb"
+
+  install_via_cmake $REPO_NAME $REPO_TAG_FDB $GIT_URL -DCMAKE_PREFIX_PATH=$INSTALL_FOLDER
+}
+
+function install_fyaml {
+  GIT_URL="https://github.com/pantoniou/libfyaml.git"
+  REPO_NAME="fyaml"
+  install_via_cmake $REPO_NAME $REPO_TAG_FYAML $GIT_URL
+}
+function install_CFITSIO {
+  REPO_NAME="cfitsio"
+  GIT_URL="https://github.com/healpy/cfitsio.git"
+  install_via_cmake $REPO_NAME $REPO_TAG_CFITSIO $GIT_URL
+}
+
+function install_healpix {
+  REPO_NAME="healpix"
+
+  VERSIONED_FOLDER=$REPO_NAME-$REPO_TAG_HEALPIX
+  REPO_PATH=$GIT_REPOS/$VERSIONED_FOLDER
+  #-------------------------------------------------------------------------------------
+  clone_to https://github.com/healpy/healpixmirror.git $REPO_PATH $REPO_TAG_HEALPIX
+  #-------------------------------------------------------------------------------------
+  # install libsharp
+  REPO_NAME="libsharp"
+  SHARP_BUILD_FOLDER=$BUILD_FOLDERS/$VERSIONED_FOLDER
+  SHARP_CONFIGURE_LOCATION=$REPO_PATH/src/common_libraries/libsharp
+  LIB_SHARP_LOCATION=$INSTALL_FOLDER/$VERSIONED_FOLDER
+  SHARP_BUILD_FOLDER=$BUILD_FOLDERS/$VERSIONED_FOLDER
+
+  cd $SHARP_CONFIGURE_LOCATION
+  autoreconf -i
+
+  create_folder $SHARP_BUILD_FOLDER
+  cd $SHARP_BUILD_FOLDER
+  $SHARP_CONFIGURE_LOCATION/configure --prefix=$LIB_SHARP_LOCATION
+
+  make -j $THREADS
+  make install
+
+  #-------------------------------------------------------------------------------------
+  REPO_NAME="cxx-healpix"
+  VERSIONED_FOLDER=$REPO_NAME-$REPO_TAG_HEALPIX
+  CXX_CONFIGURE_LOCATION=$REPO_PATH/src/cxx
+  CXX_BUILD_DIR=$BUILD_FOLDERS/$VERSIONED_FOLDER
+
+  cd $CXX_CONFIGURE_LOCATION
+  autoreconf -i
+
+  create_folder $CXX_BUILD_DIR
+  cd $CXX_BUILD_DIR
+  CFITSIO_LOC="$INSTALL_FOLDER/cfitsio-$REPO_TAG_CFITSIO"
+
+  INCLUDE_FLAGS="-I$LIB_SHARP_LOCATION/include -I$CFITSIO_LOC/include"
+  TEST_LDFLAGS="-L$CFITSIO_LOC/lib -L$LIB_SHARP_LOCATION/lib -L$CFITSIO_LOC/lib64 -Wl,-rpath,$CFITSIO_LOC/lib64"
+
+
+  $CXX_CONFIGURE_LOCATION/configure LDFLAGS="$TEST_LDFLAGS" CFLAGS="$INCLUDE_FLAGS" CXXFLAGS="$INCLUDE_FLAGS" --prefix=$INSTALL_FOLDER/$VERSIONED_FOLDER
+
+  make -j $THREADS
+  make install
+
+}
+
+# YAC-IO
+function make_yaco() {
+
+  REPO_NAME="yaco"
+  VERSIONED_FOLDER=$REPO_NAME-${REPO_TAG_YACO/\//-}
+  REPO_PATH=$GIT_REPOS/$VERSIONED_FOLDER
+  GIT_URL=https://gitlab.dkrz.de/across/yaco.git
+
+  clone_to $GIT_URL $REPO_PATH $REPO_TAG_YACO
+
+  echo $INSTALL_FOLDER
+  LIBS="-DCMAKE_PREFIX_PATH=\
+$INSTALL_FOLDER/eccodes-$REPO_TAG_ECCODES;\
+$INSTALL_FOLDER/eckit-$REPO_TAG_ECKIT;\
+$INSTALL_FOLDER/fdb-$REPO_TAG_FDB;\
+$INSTALL_FOLDER/cxx-healpix-$REPO_TAG_HEALPIX;\
+$INSTALL_FOLDER/cfitsio-$REPO_TAG_CFITSIO";
+
+  if [ ${MACHINE,,} == "lumi" ];  then
+    LIBS+="$INSTALL_FOLDER/fyaml-$REPO_TAG_FYAML;"
+  fi
+
+
+  echo $LIBS
+  cd $REPO_PATH/external/yac
+  git checkout $YACO_YAC_VERSION
+  BUILD_FOLDER=$BUILD_FOLDERS/$VERSIONED_FOLDER
+  create_folder $BUILD_FOLDER
+  cd $BUILD_FOLDER
+  cmake .. $REPO_PATH "$LIBS" -DENABLE_NETCDF=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_BUILD_TYPE=RelWithDebInfo;
+  make -j $THREADS
+
+}
+
+
+function default {
+
+  echo ROOT_DIR: $ROOT_DIR
+  echo YACO_WORK_DIR: $YACO_WORK_DIR
+  echo GIT_REPOS: $GIT_REPOS
+  echo BUILD_FOLDERS: $BUILD_FOLDERS
+  echo INSTALL_FOLDER: $INSTALL_FOLDER
+  create_folder $GIT_REPOS
+  create_folder $BUILD_FOLDERS
+  create_folder $INSTALL_FOLDER
+  if cd $YACO_WORK_DIR; then echo "directory found" ; else create_folder $YACO_WORK_DIR; fi
+
+  #debug "== handling FDB"
+  #debug "---- handling ecbuild"
+  #install_ecbuild
+  #debug "------------------------"
+  #debug "---- handling eccodes"
+  #install_eccodes
+  #debug "------------------------"
+  #debug "---- handling eckit"
+  #install_eckit
+  #debug "------------------------"
+  #debug "---- handling metkit"
+  #install_metkit
+  #debug "------------------------"
+  #debug "---- handling fdb"
+  #install_fdb
+  #debug "========================"
+  #debug "== handling aec"
+  #install_aec
+  #debug "========================"
+  #debug "== handlinghandling yaml"
+  #install_fyaml
+  #debug "========================"
+  #debug "== handlingCFITSIO"
+  #install_CFITSIO
+  #debug "========================"
+  #debug "== handlingSHARP"
+  #install_healpix
+  #debug "========================"
+  debug "== handlingYACO"
+  make_yaco
+  debug "========================"
+}
+
+
+default