From 4e4f255ddf5a44cf31351178ef97a7c1fb5bdb2c Mon Sep 17 00:00:00 2001
From: Sergey Kosukhin <sergey.kosukhin@mpimet.mpg.de>
Date: Tue, 12 Jul 2022 13:21:43 +0200
Subject: [PATCH] Make CI script for ICON parallel integration to Levante more
robust.
---
.../levante-dkrz/test.icon-pio.intel-classic | 148 +++++++++++++++---
1 file changed, 130 insertions(+), 18 deletions(-)
diff --git a/.ci/bb/levante-dkrz/test.icon-pio.intel-classic b/.ci/bb/levante-dkrz/test.icon-pio.intel-classic
index 3bfa59906..b64f626d5 100755
--- a/.ci/bb/levante-dkrz/test.icon-pio.intel-classic
+++ b/.ci/bb/levante-dkrz/test.icon-pio.intel-classic
@@ -9,27 +9,69 @@ top_srcdir=$(cd "${script_dir}/../../.."; pwd)
. "${script_dir}/utils.sh"
init_intelclassic202150
-test -f "${top_srcdir}/configure" || "${top_srcdir}/autogen.sh"
+# Make sure that the libtool script has a chance to erroneously link to the
+# shared versions of YAXT and PPM:
+for file in \
+ "${PPM_ROOT}/lib/libscalesppmcore.so" \
+ "${YAXT_ROOT}/lib/libyaxt.so" \
+ "${YAXT_ROOT}/lib/libyaxt_c.so"; do
+ test -r "${file}" || {
+ echo "ERROR: file '${file}' not found" >&2
+ exit 1
+ }
+done
# When configured as part of ICON, C headers, Fortran modules and library files
-# of YAXT and PPM become available only at the build time. The following
-# imitates that.
-
-WRONG_YAXT_ROOT=$(init_gcc1120 >/dev/null && echo "${YAXT_ROOT}")
-export PKG_CONFIG_PATH="${WRONG_YAXT_ROOT}/lib/pkgconfig"
-export LD_LIBRARY_PATH="${WRONG_YAXT_ROOT}/lib:${LD_LIBRARY_PATH-}"
-
+# of YAXT and PPM become available only at the build time. We mock that by
+# creating two empty directories for YAXT and PPM before running the configure
+# script of CDI and copy the required files into them before running make:
LOCAL_YAXT_ROOT="$(pwd)/cdi-test-yaxt"
LOCAL_PPM_ROOT="$(pwd)/cdi-test-ppm"
-
rm -rf "${LOCAL_YAXT_ROOT}" "${LOCAL_PPM_ROOT}"
+mkdir "${LOCAL_YAXT_ROOT}" "${LOCAL_PPM_ROOT}"
+
+# Make sure that CDI gets linked to the right instances of YAXT even if certain
+# environment variables and linker flags point to the wrong existing instance:
+WRONG_YAXT_LIBDIR="$(init_gcc1120 >/dev/null && echo "${YAXT_ROOT}")/lib"
-# There is not way for the configure script of ICON to provide the configure
+for file in \
+ "${WRONG_YAXT_LIBDIR}/libyaxt.la" \
+ "${WRONG_YAXT_LIBDIR}/libyaxt_c.la" \
+ "${WRONG_YAXT_LIBDIR}/pkgconfig/yaxt.pc" \
+ "${WRONG_YAXT_LIBDIR}/pkgconfig/yaxt_c.pc"; do
+ test -f "${file}" || {
+ echo "ERROR: file '${file}' not found" >&2
+ exit 1
+ }
+done
+
+export PKG_CONFIG_PATH="${WRONG_YAXT_LIBDIR}/pkgconfig"
+export LD_LIBRARY_PATH="${WRONG_YAXT_LIBDIR}:${LD_LIBRARY_PATH-}"
+LDFLAGS="-L${WRONG_YAXT_LIBDIR}"
+
+# There is no way for the configure script of ICON to provide the configure
# script of CDI with the path to the installation of NetCDF in use. Here we
# imitate a situation when 'nc-config' from another (wrong) installation of
-# NetCDF is in the PATH.
+# NetCDF is in the ${PATH}.
WRONG_NETCDF_ROOT=$(init_gcc1120 >/dev/null && echo "${NETCDF_ROOT}")
-export PATH="${WRONG_NETCDF_ROOT}/bin:${PATH-}"
+WRONG_NETCDF_BINDIR="${WRONG_NETCDF_ROOT}/bin"
+
+# It looks like we cannot properly check whether the wrong 'nc-config' in the
+# ${PATH} causes any inconsistency. All we can do is run a not very reliable
+# indirect check of whether CDI gets built without the unwanted dependency on
+# cURL:
+nc_has_dap=`"${WRONG_NETCDF_BINDIR}/nc-config" --has-dap` || {
+ echo "ERROR: failed to check whether NetCDF in '${WRONG_NETCDF_ROOT}' is built with DAP support" >&2
+ exit 1
+}
+test "x${nc_has_dap}" = 'xyes' || {
+ echo "ERROR: NetCDF in '${WRONG_NETCDF_ROOT}' is built without DAP support" >&2
+ exit 1
+}
+
+export PATH="${WRONG_NETCDF_BINDIR}:${PATH-}"
+
+test -f "${top_srcdir}/configure" || "${top_srcdir}/autogen.sh"
"${top_srcdir}/configure" \
--disable-cdi-app \
@@ -67,7 +109,7 @@ CXX=no \
F77=no \
FC="${MPIFC}" \
FCFLAGS="-gdwarf-4 -g -pc64 -fp-model source" \
-LDFLAGS="-L${NETCDF_ROOT}/lib -L${ECCODES_ROOT}/lib64" \
+LDFLAGS="${LDFLAGS} -L${NETCDF_ROOT}/lib -L${ECCODES_ROOT}/lib64" \
LIBS='-Wl,--as-needed -lnetcdf -leccodes' \
MPIROOT= \
MPI_C_INCLUDE= \
@@ -93,15 +135,85 @@ acx_cv_have_nc4hdf5=no \
acx_cv_have_netcdf2=yes \
acx_cv_have_netcdf4=yes \
acx_cv_have_pnetcdf=no \
-acx_cv_option_search_PPM_initialize_c="${LOCAL_PPM_ROOT}/lib/libscalesppmcore.a" \
-acx_cv_option_search_xt_initialized_c="${LOCAL_YAXT_ROOT}/lib/libyaxt_c.a" \
-acx_cv_option_search_xt_initialized_fc="${LOCAL_YAXT_ROOT}/lib/libyaxt.a ${LOCAL_YAXT_ROOT}/lib/libyaxt_c.a"
+acx_cv_option_search_PPM_initialize_c="${LOCAL_PPM_ROOT}/lib/libscalesppmcore.la" \
+acx_cv_option_search_xt_initialized_c="${LOCAL_YAXT_ROOT}/lib/libyaxt_c.la" \
+acx_cv_option_search_xt_initialized_fc="${LOCAL_YAXT_ROOT}/lib/libyaxt.la ${LOCAL_YAXT_ROOT}/lib/libyaxt_c.la"
-rsync -uavz "${YAXT_ROOT}/" "${LOCAL_YAXT_ROOT}" --exclude='*.la' --exclude='*.pc' --exclude='*.so' --exclude='*.so.*'
-rsync -uavz "${PPM_ROOT}/" "${LOCAL_PPM_ROOT}" --exclude='*.la' --exclude='*.pc' --exclude='*.so' --exclude='*.so.*'
+
+# Create local copies of YAXT and PPM and make them similar to what they look like
+# when the libraries are built inside the build system of ICON:
+rsync -uavz "${YAXT_ROOT}/" "${LOCAL_YAXT_ROOT}/" --exclude='*.pc' --exclude='*.so' --exclude='*.so.*'
+rsync -uavz "${PPM_ROOT}/" "${LOCAL_PPM_ROOT}/" --exclude='*.pc' --exclude='*.so' --exclude='*.so.*'
+sed -i -E -e "{
+# The build system of ICON disables the shared versions of the libraries
+# (i.e. configures them with '--disable-shared'):
+s/^dlname=.*/dlname=''/
+s/^library_names=.*/library_names=''/
+# The linking to the libraries is done without installing them:
+s/^installed=.*/installed=no/
+}" `find "${LOCAL_YAXT_ROOT}" "${LOCAL_PPM_ROOT}" -name '*.la'`
make -j8
+# The current general approach in the build system of ICON is to avoid building
+# the shared versions of the bundled libraries and link to the static ones
+# without -L and -l flags. Instead, to avoid possible ambiguity, the linking is
+# done by passing paths to static library as positional arguments for the
+# linker. That, however, does not work well with libtool scripts as they inject
+# all files referred by the positional arguments into the generated static
+# libraries (archives). That is usually fine with the GNU linker but is a
+# problem for the MacOS one, which might ignore the whole library file:
+# ld: warning: ignoring file libcdipio.a, building for macOS-x86_64 but attempting to link with file built for macOS-x86_64
+# To circumvent the problem, we tell the configure of CDI to link to YAXT and
+# PPM using libtool .la files and check that libcdipio.a does not contain any
+# other archive files, which the linker might potentially not be able to handle:
+tested_file='src/.libs/libcdipio.a'
+invalid_entries=`ar t "${tested_file}" | sed -n '/\.a$/p'` || {
+ echo "ERROR: failed to check $1 for invalid entries" >&2
+ exit 1
+}
+if test -n "${invalid_entries}"; then
+ {
+ echo "ERROR: '${tested_file}' has invalid entries:" >&2
+ echo "${invalid_entries}" >&2
+ } >&2
+ exit 1
+fi
+
+make -j8 check TESTS= XFAIL_TESTS=
+
+# Check that a test program is linked to YAXT and PPM statically as it should
+# when built inside the build system of ICON:
+tested_file='tests/pio_write.parallel'
+invalid_deps=`ldd "${tested_file}" | sed -E -n '/libscalesppmcore|libyaxt/p'` || {
+ echo "ERROR: failed to check '${tested_file}' for invalid dependencies" >&2
+ exit 1
+}
+if test -n "${invalid_deps}"; then
+ {
+ echo "ERROR: '${tested_file}' has invalid dependencies:" >&2
+ echo "${invalid_deps}" >&2
+ } >&2
+ exit 1
+fi
+
+# Check that we do not link to the instance of YAXT from ${WRONG_YAXT_LIBDIR}
+# (we might fail earlier if ${WRONG_YAXT_LIBDIR} contains the shared version of
+# the library):
+make -C examples/pio collectData2003.parallel
+
+# Check whether CDI is built without the unwanted dependency on cURL:
+tested_file='./app/cdi'
+cdi_built_with=`"${tested_file}" -d 2>&1 | sed -E -n '/^[ ]*with:/p'` && test -n "${cdi_built_with}" || {
+ echo "ERROR: failed to get the expected debug output from '${tested_file}'" >&2
+ exit 1
+}
+case " ${cdi_built_with} " in
+ *' OPeNDAP '*)
+ echo "ERROR: CDI is built with the unwanted dependency on cURL (i.e. OPeNDAP support is enabled)" 2>&1
+ exit 1 ;;
+esac
+
make -j8 check || { cat tests/test-suite.log; exit 1; }
check_all_tests_passed tests/test-suite.log
--
GitLab