Skip to content
GitLab
Commit cdb75fac authored by Deike Kleberg's avatar Deike Kleberg
Browse files

pioInit(): less arguments, new testprgr pio_write.F90, config, Makefiles, runscripts for tornado

parent 16ba2827
Hide whitespace changes
Inline Side-by-side
.gitattributes
View file @ cdb75fac
......@@ -22,6 +22,7 @@ config/install-sh -text
config/interface.rb -text
config/missing -text
config/mkinstalldirs -text
/config_tornado.sh -text
/configure -text
/configure.ac -text
doc/cdi_cman.pdf -text
......@@ -115,12 +116,18 @@ interfaces/testdata/gridTest.nc -text
interfaces/testdata/mulval.grb -text
interfaces/testdata/mulval.nc -text
m4/acx_sl_mod_suffix.m4 -text
pioExamples/Makefile_tornado_parallel -text
pioExamples/Makefile_tornado_serial -text
pioExamples/README -text
pioExamples/cdi_write_f.F90 -text
pioExamples/cdi_write_more_nodes.F90 -text
pioExamples/cdi_write_more_nodes.job -text
pioExamples/cdi_write_parallel.job -text
pioExamples/cdi_write_serial.job -text
pioExamples/pio_write.F90 -text
pioExamples/write_tornado_parallel.job -text
pioExamples/write_tornado_ps.job -text
pioExamples/write_tornado_serial.job -text
src/Makefile.am -text
src/Makefile.in -text
src/basetime.c -text
......
config_tornado.sh 0 → 100755
View file @ cdb75fac
#! /bin/bash
#-----------------------------------------------------------------------------
. /sw/share/Modules/init/bash
module load GCC/4.3.3
module load NAG/5.2.721
module load OpenMPI
export AIO_LIB="-laio -lrt"
export IB_LIB="-lrdmacm -libverbs -lnuma -ldl -Wl,--export-dynamic -lnsl -lutil -lm -ldl"
export MPIROOT=/sw/sles10-x64/ofed/openmpi-1.4.2-nag52
export MPI_INCLUDE=-I${MPIROOT}/include
export MPI_LIB="-L${MPIROOT}/lib -WL,-pthread -lmpi_f90 -lmpi_f77 -lmpi -lopen-rte -lopen-pal"
export CC=gcc
export FC=nagfor
export F77=nagfor
export CPPFLAGS="-DNAGf90Fortran -DHAVE_LIBPTHREAD"
export CFLAGS="${MPI_INCLUDE} -g -pedantic -Wall -Wextra"
#export LIBS="${AIO_LIB}"
export LIBS="${AIO_LIB} -lpthread"
export LDFLAGS=-g
export VALGRINDROOT=/sw/sles10-x64/valgrind-3.5.0
# don't use --enable-mpi
cwd=$(pwd -P)
make distclean
#./configure
./configure --prefix=${MPIROOT} --enable-debug \
--enable-memchecker --with-valgrind=${VALGRINDROOT}
cd src
gcc -o make_fint make_fint.c
./make_fint cdi.h
cd $cwd
makefiles=$(find . -name Makefile)
for file in $makefiles
do
sed -i "s:LIBS =:LIBS = ${MPI_LIB} ${IB_LIB}:" $file
done
make -j 4
if [ ! -e lib ]
then
mkdir lib
fi
cp src/libcdi.a lib/libcdi_parallel.a
make distclean
export CPPFLAGS="-DNAGf90Fortran -DHAVE_LIBPTHREAD -DNOMPI"
export CFLAGS=
./configure
cd src
gcc -o make_fint make_fint.c
./make_fint cdi.h
cd $cwd
make -j 4
cp src/libcdi.a lib/libcdi_serial.a
pioExamples/Makefile_tornado_parallel 0 → 100755
View file @ cdb75fac
MPIROOT = /sw/sles10-x64/ofed/openmpi-1.4.0-nag52
MPI_LIB = -L$(MPIROOT)/lib -Wl,-pthread -lmpi_f90 -lmpi_f77 -lmpi -lopen-rte -lopen-pal -lrdmacm -libverbs -lnuma -ldl -Wl,--export-dynamic -lnsl -lutil -lm -ldl
MPI_INCLUDE = -I$(MPIROOT)/include
NETCDFROOT = /sw/sles10-x64/netcdf-4.0.1
NETCDF_LIB = -L$(NETCDFROOT)/lib -lnetcdf
NETCDF_INCLUDE = -I$(NETCDFROOT)/include
HDF5ROOT = /sw/sles10-x64/hdf5-1.8.2
HDF5_LIB = -L$(HDF5ROOT)/lib -lhdf5_hl -lhdf5
HDF5_INCLUDE = -I$(HDF5ROOT)/include
SZIPROOT = /sw/sles10-x64/szip-2.1
SZIP_LIB = -L$(SZIPROOT)/lib -lsz
SZIP_INCLUDE = -I$(SZIPROOT)/include
ZLIBROOT = /sw/sles10-x64/zlib-1.2.3
ZLIB_LIB = -L$(ZLIBROOT)/lib -lz
ZLIB_INCLUDE = -I$(ZLIBROOT)/include
CDILIBROOT = ..
CDILIB_LIB = -L$(CDILIBROOT)/lib -lcdi_parallel
CDILIB_INCLUDE = -I$(CDILIBROOT)/include
INCLUDES = $(CDILIB_INCLUDE) $(MPI_INCLUDE) $(NETCDF_INCLUDE) $(HDF5_INCLUDE) $(SZIP_INCLUDE) $(ZLIB_INCLUDE) -g
#LIBS = $(CDILIB_LIB) $(MPI_LIB) $(NETCDF_LIB) $(HDF5_LIB) $(SZIP_LIB) $(ZLIB_LIB) -laio -lrt
LIBS = $(CDILIB_LIB) $(MPI_LIB) $(NETCDF_LIB) $(HDF5_LIB) $(SZIP_LIB) $(ZLIB_LIB) -laio -lrt -lpthread
SRCS = pio_write.F90
OBJS = pio_write.o
FC = nagfor
FFLAGS = $(INCLUDES) -mismatch_all -w=uep
LDFLAGS = $(INCLUDES)
%.o: %.F90
$(FC) $(FFLAGS) -c $<
all: pio_write_parallel
pio_write_parallel: $(OBJS)
$(FC) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
pio_write.o: pio_write.F90
clean:
-rm -f pio_write_parallel *.o *.mod
\ No newline at end of file
pioExamples/Makefile_tornado_serial 0 → 100755
View file @ cdb75fac
NETCDFROOT = /sw/sles10-x64/netcdf-4.0.1
NETCDF_LIB = -L$(NETCDFROOT)/lib -lnetcdf
NETCDF_INCLUDE = -I$(NETCDFROOT)/include
HDF5ROOT = /sw/sles10-x64/hdf5-1.8.2
HDF5_LIB = -L$(HDF5ROOT)/lib -lhdf5_hl -lhdf5
HDF5_INCLUDE = -I$(HDF5ROOT)/include
SZIPROOT = /sw/sles10-x64/szip-2.1
SZIP_LIB = -L$(SZIPROOT)/lib -lsz
SZIP_INCLUDE = -I$(SZIPROOT)/include
ZLIBROOT = /sw/sles10-x64/zlib-1.2.3
ZLIB_LIB = -L$(ZLIBROOT)/lib -lz
ZLIB_INCLUDE = -I$(ZLIBROOT)/include
CDILIBROOT = ..
CDILIB_LIB = -L$(CDILIBROOT)/lib -lcdi_serial
CDILIB_INCLUDE = -I$(CDILIBROOT)/include
INCLUDES = $(CDILIB_INCLUDE) $(NETCDF_INCLUDE) $(HDF5_INCLUDE) $(SZIP_INCLUDE) $(ZLIB_INCLUDE) -g
LIBS = $(CDILIB_LIB) $(NETCDF_LIB) $(HDF5_LIB) $(SZIP_LIB) $(ZLIB_LIB) -laio -lrt
SRCS = pio_write.F90
OBJS = pio_write.o
FC = nagfor
FFLAGS = -fpp -DNOMPI $(INCLUDES) -mismatch_all -w=uep
LDFLAGS = $(INCLUDES)
%.o: %.F90
$(FC) $(FFLAGS) -c $<
all: pio_write_serial
pio_write_serial: $(OBJS)
$(FC) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
pio_write.o: pio_write.F90
clean:
-rm -f pio_write_serial *.o *.mod
\ No newline at end of file
pioExamples/cdi_write_more_nodes.F90
View file @ cdb75fac
......@@ -568,7 +568,7 @@ CONTAINS
REAL ( dp ) :: dist
DO i = 1, NLON
lons ( i ) = 360.0 / NLON * ( i - 1 )
lons ( i ) = ( 360.0 / NLON ) * ( i - 1 )
END DO
dist = 180.0 / NLAT
......
pioExamples/pio_write.F90 0 → 100755
View file @ cdb75fac
! mpxlf95_r -o cdi_write_parallel -I../src cdi_write_f.F90 -L../src -lcdi
!-qextname -L/sw/aix53/szip-2.1/lib -lsz -L/sw/aix61/netcdf-4.0.1-ibm/lib
! -lnetcdf -L/sw/aix61/hdf5-1.8.4-patch1/lib -lhdf5_hl -lhdf5 -lz
! TODO
! varId (:,:) RESTRICTED TO STREAMSIZES(1) module loadbalancing
! catch cases with nonsensical relation nnodes/nstreams/sizesstreams
!--------------------------------------------------------------------------
!--------------------------------------------------------------------------
MODULE loadbalancing
CONTAINS
SUBROUTINE mapProblems ( problemSizes, problemMapping, nProblems, nCalculator )
!... data dictionary
IMPLICIT NONE
INTEGER, INTENT ( in ) :: nProblems
INTEGER, INTENT ( in ) :: problemSizes ( nProblems )
INTEGER, INTENT ( out ) :: problemMapping ( nProblems )
INTEGER, INTENT ( in ) :: nCalculator
INTEGER, PARAMETER :: INITVALUE1 = 99
INTEGER, PARAMETER :: INITVALUE2 = 99
INTEGER, PARAMETER :: TOLERANCE = 0
INTEGER, PARAMETER :: NOINDEX = -1
INTEGER, PARAMETER :: MIN = -99
INTEGER :: sortedProblems(nProblems,2)
INTEGER :: load(nCalculator)
REAL :: mean
REAL :: capacityLeft
INTEGER :: currCapacity,nextCapacity
INTEGER :: currIndex,nextIndex
INTEGER :: i,j
!... init
problemMapping = INITVALUE1
load = INITVALUE2
DO i = 1,nProblems
sortedProblems(i,1) = problemSizes(i)
sortedProblems(i,2) = i
ENDDO
CALL iQsort(sortedProblems)
mean = SUM(problemSizes)/float(nCalculator)
!... loop over problems, biggest problem first
prbl: DO i = 1,nProblems
!... 1. loop over PEs, find a capacity that fits
fit: DO j = 1,nCalculator
capacityLeft = ABS(mean-load(j)-sortedProblems(i,1))
IF(capacityLeft<=TOLERANCE)THEN
problemMapping(sortedProblems(i,2)) = j
load(j) = load(j)+sortedProblems(i,1)
WRITE(*,*)'MAIN: 1. loop used!'
EXIT fit
ENDIF
ENDDO fit
IF(problemMapping(sortedProblems(i,2))/=INITVALUE1) CYCLE prbl
currCapacity = MIN
currIndex = NOINDEX
!... potential 2. loop over PEs, take PE with the least load
leastload: DO j = 1,nCalculator
nextIndex = j
nextCapacity = mean-load(j)
IF(nextCapacity>currCapacity)THEN
currIndex = nextIndex
currCapacity = nextCapacity
ENDIF
ENDDO leastload
problemMapping(sortedProblems(i,2))=currIndex
load(currIndex) = load(currIndex)+sortedProblems(i,1)
ENDDO prbl
CONTAINS
RECURSIVE SUBROUTINE iQsort(A)
IMPLICIT NONE
INTEGER, INTENT(inout) :: A(:,:)
INTEGER:: pivot(1,2),temp(1,2)
INTEGER :: n,pivotIndex,currIndex
IF(SIZE(A,2)/=2) STOP 'MAIN: error: subroutine iQsort(A), dimension(A,2) has to be 2.'
n = SIZE(A,1)
IF(n<=1)RETURN
pivot(1,:) = A(n,:)
pivotIndex = n
currIndex = 1
DO WHILE(currIndex < pivotIndex)
IF(A(currIndex,1)<pivot(1,1))THEN
temp = pivot
A(pivotIndex,:) = A(currIndex,:)
A(currIndex,:) = A(pivotIndex-1,:)
A(pivotIndex-1,:) = temp(1,:)
pivotIndex = pivotIndex-1
ELSE
currIndex = currIndex+1
ENDIF
ENDDO
IF(pivotIndex /= 1) CALL iQsort(A(1:pivotIndex-1,:))
IF(pivotIndex /= n) CALL iQsort(A(pivotIndex+1:n,:))
END SUBROUTINE iQsort
END SUBROUTINE mapProblems
END MODULE loadbalancing
!---------------------------------------------------------
!---------------------------------------------------------
PROGRAM CDIWRITE
USE loadbalancing
IMPLICIT NONE
INCLUDE 'cdi.inc'
#ifndef NOMPI
INCLUDE 'mpif.h'
#endif
INTEGER, PARAMETER :: dp = SELECTED_REAL_KIND(12,307)
INTEGER, PARAMETER :: MAXTYPES = 4
INTEGER, PARAMETER :: MAXNODES = 249
INTEGER, PARAMETER :: ddebug = 0
INTEGER, PARAMETER :: NLON = 384 ! Number of longitudes 384
INTEGER, PARAMETER :: NLAT = 192 ! Number of latitudes 192
INTEGER, PARAMETER :: NLEV = 96 ! Number of levels 96
INTEGER, PARAMETER :: NTIME = 10 ! Number of time steps 124
INTEGER, PARAMETER :: NSTREAMS = 10 ! Number of files 7
INTEGER, PARAMETER :: STREAMSIZES ( NSTREAMS ) = (/ 35, 35, 35, 35, 35, &
35, 35, 35, 35, 35 /)
CHARACTER*(*), PARAMETER :: STREAMNAMES ( NSTREAMS ) = (/ 'dat0.grb', &
'dat1.grb', 'dat2.grb', 'dat3.grb', 'dat4.grb', 'dat5.grb', 'dat6.grb', &
'dat7.grb', 'dat8.grb', 'dat9.grb' /)
CHARACTER*(*), PARAMETER :: PTYPENAMES ( 7 ) = (/ &
'PIO_NONE ', &
'PIO_MPI_NONB ', &
'PIO_POSIX_NONB ', &
'PIO_POSIX_ASYNCH ', &
'PIO_POSIX_FPGUARD_SENDRECV ', &
'PIO_POSIX_FPGUARD_THREAD_REFUSE', &
'PIO_POSIX_FPGUARD_THREAD '/)
REAL(dp):: LONS(NLON), LATS(NLAT), LEVS(NLEV)
REAL(dp):: var(NLON*NLAT*NLEV)
INTEGER :: gridID, zaxisID, taxisID
INTEGER, ALLOCATABLE :: vlistID ( : ), streamID ( : )
INTEGER, ALLOCATABLE :: varIDs ( :, : )
INTEGER :: sID, tsID, i, j, nmiss, status
INTEGER :: pioComm, pioCollectorComm_NODE, rank, error
INTEGER :: PTYPE
INTEGER :: NNODES, nstreams_NODE, myNODE
INTEGER, ALLOCATABLE :: streamsizes_NODE ( : )
CHARACTER(len=8), ALLOCATABLE :: streamnames_NODE ( : )
INTEGER, ALLOCATABLE :: streamfirsts_PE ( : ), streamlasts_PE ( : )
INTEGER :: collectingData
INTEGER :: ncollectors_NODE
REAL(dp) :: timerSTART, timerSTOP
REAL(dp) :: sumOPEN, sumCLOSE, sumWRITE
#ifndef NOMPI
CALL MPI_INIT ( error )
CALL MPI_COMM_DUP ( MPI_COMM_WORLD, pioComm, error )
CALL MPI_ERRHANDLER_SET ( pioComm, MPI_ERRORS_RETURN, error )
CALL MPI_COMM_RANK ( pioComm, rank, error )
CALL readArgs( PTYPE )
IF ( ddebug == 1 ) THEN
WRITE ( *, * ) 'MAIN after readArgs: pe', rank, ', PTYPE=', PTYPE
END IF
#endif
collectingData = pioInit( PTYPE, pioComm, &
myNODE, NNODES, pioCollectorComm_NODE )
IF ( ddebug == 1 ) THEN
#ifndef NOMPI
WRITE ( *, * ) 'MAIN after pioInit: pe', rank, ', NNODES=', NNODES, &
', myNODE=', myNODE
#else
WRITE ( *, * ) 'MAIN after pioInit: NNODES=', NNODES, &
', myNODE=', myNODE
#endif
END IF
IF ( collectingData /= 1 ) THEN
GO TO 1
END IF
CALL mapFilesOnNodes
IF ( ddebug == 1 ) THEN
#ifndef NOMPI
WRITE ( *, * ) 'MAIN after mapFilesOnNodes: pe', rank, &
', nstreams_NODE=', nstreams_NODE, ', streamnames_NODE=', &
streamnames_NODE, ', streamsizes_NODE=', streamsizes_NODE
#else
WRITE ( *, * ) 'MAIN after mapFilesOnNodes: nstreams_NODE=', nstreams_NODE, &
', streamnames_NODE=', streamnames_NODE, ', streamsizes_NODE=', &
streamsizes_NODE
#endif
END IF
CALL mapVarsOnPes
IF ( ddebug == 1 ) THEN
#ifndef NOMPI
WRITE ( *, * ) 'MAIN after mapVarsOnPes: pe', rank, ', streamfirsts_PE=', &
streamfirsts_PE, 'streamlasts_PE=', streamlasts_PE
#else
WRITE ( *, * ) 'MAIN after mapVarsOnPes: streamfirsts_PE=', &
streamfirsts_PE, 'streamlasts_PE=', streamlasts_PE
#endif
END IF
CALL initCoords ()
ALLOCATE ( vlistID ( nstreams_NODE ), stat = status )
ALLOCATE ( streamID ( nstreams_NODE ), stat = status )
ALLOCATE ( varIDs ( streamsizes_NODE ( 1 ), nstreams_NODE ), stat = status )
! Create a regular lon/lat grid
gridID = gridCreate ( GRID_LONLAT, NLON*NLAT )
CALL gridDefXsize ( gridID, NLON )
CALL gridDefYsize ( gridID, NLAT )
CALL gridDefXvals ( gridID, lons )
CALL gridDefYvals ( gridID, lats )
! Create a pressure level Z-axis
zaxisID = zaxisCreate ( ZAXIS_PRESSURE, NLEV )
CALL zaxisDefLevels ( zaxisID, levs )
! Create a variable list
DO i = 1, nstreams_NODE
vlistID ( i ) = vlistCreate()
END DO
! Define the variables and the variable names
DO i = 1, nstreams_NODE
DO j = 1, streamsizes_NODE ( 1 )
varIDs ( j, i ) = vlistDefVar ( vlistID ( i ), gridID, zaxisID, TIME_VARIABLE )
CALL vlistDefVarName ( vlistID ( i ), varIDs ( j, i ), "varname")
END DO
END DO
! Create a Time axis
taxisID = taxisCreate(TAXIS_ABSOLUTE)
! Assign the Time axis to the variable list
DO i = 1, nstreams_NODE
CALL vlistDefTaxis ( vlistID ( i ), taxisID)
END DO
sumOPEN = 0.0
sumCLOSE = 0.0
sumWRITE = 0.0
#ifndef NOMPI
timerSTART = MPI_WTIME()
#else
CALL CPU_TIME ( timerSTART )
#endif
! Open files and create a dataset in GRB fromat
DO i = 1, nstreams_NODE
streamID ( i ) = streamOpenWrite ( streamnames_NODE ( i ), FILETYPE_GRB )
IF ( streamID ( i ) < 0 ) THEN
WRITE(0,*) cdiStringError ( streamID ( i ))
#ifndef NOMPI
CALL MPI_ABORT ( pioComm, 1, error )
#else
STOP
#endif
END IF
END DO
#ifndef NOMPI
timerSTOP = MPI_WTIME()
#else
CALL CPU_TIME ( timerSTOP )
#endif
sumOPEN = timerSTOP - timerSTART
CALL initVar ( var, 1, 1 )
! Assign the variable list to the dataset
DO i = 1, nstreams_NODE
CALL streamDefVlist ( streamID ( i ), vlistID ( i ))
END DO
nmiss = 0
! Loop over the number of time steps
DO tsID = 0, NTIME-1
! Set the verification date to 1985-01-01 + tsID
CALL taxisDefVdate ( taxisID, 19850101+tsID )
! Set the verification time to 12:00:00
CALL taxisDefVtime ( taxisID, 120000 )
DO i = 1, nstreams_NODE
! Define the time step
status = streamDefTimestep ( streamID ( i ), tsID )
DO sID = streamfirsts_PE ( i ), streamlasts_PE ( i )
#ifndef NOMPI
timerSTART = MPI_WTIME()
#else
CALL CPU_TIME ( timerSTART )
#endif
! Write var
CALL streamWriteVar ( streamID ( i ), varIDs ( sID, i ), var, nmiss)
#ifndef NOMPI
timerSTOP = MPI_WTIME()
#else
CALL CPU_TIME ( timerSTOP )
#endif
sumWRITE = sumWRITE + timerSTOP - timerSTART
END DO
END DO
END DO
#ifndef NOMPI
timerSTART = MPI_WTIME()
#else
CALL CPU_TIME ( timerSTART )
#endif
! Close the output stream
DO i = 1, nstreams_NODE
CALL streamClose ( streamID ( i ))
END DO
#ifndef NOMPI
timerSTOP = MPI_WTIME()
#else
CALL CPU_TIME ( timerSTOP )
#endif
sumCLOSE = timerSTOP - timerSTART
! Destroy the objects
DO i = 1, nstreams_NODE
CALL vlistDestroy ( vlistID ( i ))
END DO
CALL taxisDestroy(taxisID)
CALL zaxisDestroy(zaxisID)
CALL gridDestroy(gridID)
DEALLOCATE ( streamsizes_NODE, stat = status )
DEALLOCATE ( streamnames_NODE, stat = status )
DEALLOCATE ( streamfirsts_PE, stat = status )
DEALLOCATE ( streamlasts_PE, stat = status )
DEALLOCATE ( vlistID, stat = status )
DEALLOCATE ( streamID, stat = status )
DEALLOCATE ( varIDs, stat = status )
1 CALL pioFinalize ()