CMakeLists.txt

@@ -30,6 +30,10 @@ elseif("${MU_ARCH}" STREQUAL "a100")
     set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE)
     set(Kokkos_ENABLE_CUDA ON CACHE BOOL "" FORCE)
     set(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "" FORCE)
+elseif("${MU_ARCH}" STREQUAL "mi250x")
+    set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE)
+    set(Kokkos_ENABLE_HIP ON CACHE BOOL "" FORCE)
+    set(Kokkos_ARCH_AMD_GFX90A ON CACHE BOOL "" FORCE)
 else()
     message(FATAL_ERROR "${MU_ARCH} is not a valid/tested configuration, select architecture, x86_64, a100, h100, mi250x, mi300a(-unified), h100(-unified)")
 endif()
@@ -53,7 +57,11 @@ endif()
 add_executable(fdemo fdriver.f90 main.cpp)
 target_link_libraries(fdemo PUBLIC Kokkos::kokkos)
 if (WITH_OPENACC_FORTRAN)
-  find_package(OpenACC REQUIRED OpenACC_Fortran)
+  #find_package(OpenACC REQUIRED OpenACC_Fortran)
+  find_package(OpenACC)
+  if (NOT ${OpenACC_Fortran_FOUND})
+    message(FATAL_ERROR "OpenACC_Fortran_FOUND=${OpenACC_Fortran_FOUND}")
+  endif()
   target_link_libraries(fdemo PUBLIC OpenACC::OpenACC_Fortran )
 endif()
 set_property(TARGET fdemo PROPERTY LINKER_LANGUAGE Fortran)
@@ -65,3 +73,7 @@ if ("${MU_ARCH}" STREQUAL "a100")
     target_link_options(demo PUBLIC "-gpu=pinned")
 endif()
 
+if ("${MU_ARCH}" STREQUAL "mi250x")
+  add_compile_definitions(DEMO_DEVICE)
+endif()
+

conf.sh

+# source this
+
+case $(hostname) in
+    levante*.dkrz.de|vader*.dkrz.de)
+        export DEVICE=a100
+        ;;
+    nid*) # lumi-g
+        export DEVICE=mi250x
+        ;;
+    *)
+        echo "unknown system: $(hostname)"
+        exit 1
+        ;;
+esac
+
+echo "$0: DEVICE=$DEVICE"

debug_script.sh

@@ -8,12 +8,15 @@ ulimit -s unlimited
 
 set -e
 
-export VERBOSE_LEVEL=2
+# set DEVICE:
+source conf.sh
 
+export VERBOSE_LEVEL=2
+        
 if [ "$1" == 'gpu' ]
 then
     #rm -rf build_gpu
-    cmake -B build_gpu -S . -DMU_ARCH=a100 -DCMAKE_CXX_FLAGS="-O0 -g"
+    cmake -B build_gpu -S . -DMU_ARCH=$DEVICE -DCMAKE_CXX_FLAGS="-O0 -g"
     cmake --build build_gpu --parallel 
 
     ncells=(8)

lumi-cpu-job.sh

+#!/bin/bash
+#SBATCH --job-name=demo_cpu
+#SBATCH --output=demo_cpu.o%j
+#SBATCH --error=demo_cpu.o%j
+#SBATCH --partition=dev-g
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=8
+#SBATCH --time=00:10:00
+#SBATCH --account=project_465001141
+
+export VERBOSE=1
+#./debug_script.sh
+./script.sh

lumi-gpu-job.sh

+#!/bin/bash
+#SBATCH --job-name=demo_gpu   # Job name
+#SBATCH --output=demo_gpu.o%j # Name of stdout output file
+#SBATCH --error=demo_gpu.o%j  # Name of stderr error file
+#SBATCH --partition=dev-g  # partition name : standard-g
+#SBATCH --gpus=1
+#SBATCH --time=00:10:00       # Run time (d-hh:mm:ss)
+#SBATCH --account=project_465001141  # Project for billing
+
+
+pwd
+
+#./debug_script.sh gpu
+./script.sh gpu

lumi_cpu_setup.sh

+# source this
+module load LUMI/24.03
+module load buildtools/24.03
+module load googletest/1.14.0-cpeCray-24.03
+
+
+
+module load PrgEnv-cray
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:$LD_LIBRARY_PATH
+echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+
+# load GPU support
+#module load craype-accel-amd-gfx90a 
+module load rocm
+module load cce/17.0.1 craype-x86-milan
+
+# new: python with jinja2
+#source ~/venv/bin/activate
+
+#export CRAYPE_LINK_TYPE=dynamic
+export CXX=CC
+export FC=ftn

lumi_gpu_setup.sh

+# source this
+module load LUMI/24.03
+module load buildtools/24.03
+module load googletest/1.14.0-cpeCray-24.03
+
+
+module load PrgEnv-cray
+export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:$LD_LIBRARY_PATH
+echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+
+# load GPU support
+module load craype-accel-amd-gfx90a 
+module load rocm
+
+# new: python with jinja2
+source ~/venv/bin/activate
+
+export CRAYPE_LINK_TYPE=dynamic
+export CXX=CC
+export FC=ftn

main.cpp

@@ -749,7 +749,7 @@ bool is_device_ptr(const void* ptr) {
 #ifdef KOKKOS_ENABLE_HIP
   constexpr bool exec_on_hip = std::is_same<ExecSpace,Kokkos::HIP>::value;
   if constexpr (exec_on_hip) {
-    hipPointerAttributes att;
+    hipPointerAttribute_t att;
     auto err = hipPointerGetAttributes(&att, ptr);
     assert(err == hipSuccess);
     if (att.devicePointer && ptr == att.devicePointer ) {

script.sh

@@ -3,16 +3,21 @@
 #gcc
 #nvhpc/24.7-gcc-11.2.0 
 #export LD_LIBRARY_PATH
-export VERBOSE_LEVEL=2
+
 
 ulimit -s unlimited
 
 set -e
 
+# set DEVICE:
+source conf.sh
+
+export VERBOSE_LEVEL=2
+
 if [ "$1" == 'gpu' ]
 then
     #rm -rf build_gpu
-    cmake -B build_gpu -S . -DMU_ARCH=a100 -DCMAKE_CXX_FLAGS="-O3"
+    cmake -B build_gpu -S . -DMU_ARCH=$DEVICE -DCMAKE_CXX_FLAGS="-O3"
     cmake --build build_gpu --parallel 
 
     ncells=(5000064)
@@ -21,7 +26,7 @@ then
     #nproma=$((449*29))
 else 
     #rm -rf build
-    cmake -B build -S . -DMU_ARCH=x86_64 -DCMAKE_CXX_FLAGS="-O3"
+    cmake -B build -S . -DMU_ARCH=x86_64 -DCMAKE_CXX_FLAGS="-O3" -DCMAKE_Fortran_FLAGS="-O3 -fopenmp"
     cmake --build build --parallel
 
     ncells=(5000064)
@@ -31,6 +36,7 @@ else
     export OMP_PROC_BIND=close
     export OMP_PLACES=cores
     export OMP_NUM_THREADS=8
+    echo "OMP_NUM_THREADS=${OMP_NUM_THREADS}"
 fi
 
 for jb in ${ncells[*]}; do