Skip to content
GitLab
Commit c0179ef9 authored by Deike Kleberg's avatar Deike Kleberg
Browse files

mapping vars on iopes.

parent 7540f873
Hide whitespace changes
Inline Side-by-side
.gitattributes
View file @ c0179ef9
......@@ -143,6 +143,7 @@ pioExamples/cdi_write_more_nodes.F90 -text
pioExamples/cdi_write_more_nodes.job -text
pioExamples/cdi_write_parallel.job -text
pioExamples/cdi_write_serial.job -text
pioExamples/collectData.c -text
pioExamples/compareResourcesArray.c -text
pioExamples/myModel.c -text
pioExamples/myModel2003.F90 -text
......
pioExamples/Makefile.am
View file @ c0179ef9
noinst_PROGRAMS=myModel
if USE_MPI
noinst_PROGRAMS+=pio_write myModel2003 compareResourcesArray
noinst_PROGRAMS+=pio_write myModel2003 compareResourcesArray collectData
endif
AM_CFLAGS=-I$(top_srcdir)/src
......@@ -20,4 +20,8 @@ myModel2003_LDADD=-L../src -lcdi
compareResourcesArray_SOURCES=compareResourcesArray.c
compareResourcesArray_LDADD=-L../src -lcdi
\ No newline at end of file
compareResourcesArray_LDADD=-L../src -lcdi
collectData_SOURCES=collectData.c
collectData_LDADD=-L../src -lcdi
\ No newline at end of file
pioExamples/Makefile.in
View file @ c0179ef9
......@@ -33,7 +33,7 @@ POST_UNINSTALL = :
build_triplet = @build@
host_triplet = @host@
noinst_PROGRAMS = myModel$(EXEEXT) $(am__EXEEXT_1)
@USE_MPI_TRUE@am__append_1 = pio_write myModel2003 compareResourcesArray
@USE_MPI_TRUE@am__append_1 = pio_write myModel2003 compareResourcesArray collectData
subdir = pioExamples
DIST_COMMON = README $(srcdir)/Makefile.am $(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
......@@ -49,8 +49,12 @@ mkinstalldirs = $(SHELL) $(top_srcdir)/config/mkinstalldirs
CONFIG_HEADER = $(top_builddir)/src/config.h
CONFIG_CLEAN_FILES =
@USE_MPI_TRUE@am__EXEEXT_1 = pio_write$(EXEEXT) myModel2003$(EXEEXT) \
@USE_MPI_TRUE@ compareResourcesArray$(EXEEXT)
@USE_MPI_TRUE@ compareResourcesArray$(EXEEXT) \
@USE_MPI_TRUE@ collectData$(EXEEXT)
PROGRAMS = $(noinst_PROGRAMS)
am_collectData_OBJECTS = collectData.$(OBJEXT)
collectData_OBJECTS = $(am_collectData_OBJECTS)
collectData_DEPENDENCIES =
am_compareResourcesArray_OBJECTS = compareResourcesArray.$(OBJEXT)
compareResourcesArray_OBJECTS = $(am_compareResourcesArray_OBJECTS)
compareResourcesArray_DEPENDENCIES =
......@@ -84,10 +88,10 @@ CCLD = $(CC)
LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
--mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
$(LDFLAGS) -o $@
SOURCES = $(compareResourcesArray_SOURCES) $(myModel_SOURCES) \
$(myModel2003_SOURCES) $(pio_write_SOURCES)
DIST_SOURCES = $(compareResourcesArray_SOURCES) $(myModel_SOURCES) \
$(myModel2003_SOURCES) $(pio_write_SOURCES)
SOURCES = $(collectData_SOURCES) $(compareResourcesArray_SOURCES) \
$(myModel_SOURCES) $(myModel2003_SOURCES) $(pio_write_SOURCES)
DIST_SOURCES = $(collectData_SOURCES) $(compareResourcesArray_SOURCES) \
$(myModel_SOURCES) $(myModel2003_SOURCES) $(pio_write_SOURCES)
ETAGS = etags
CTAGS = ctags
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
......@@ -258,6 +262,8 @@ myModel2003_SOURCES = myModel2003.F90
myModel2003_LDADD = -L../src -lcdi
compareResourcesArray_SOURCES = compareResourcesArray.c
compareResourcesArray_LDADD = -L../src -lcdi
collectData_SOURCES = collectData.c
collectData_LDADD = -L../src -lcdi
all: all-am
.SUFFIXES:
......@@ -298,6 +304,9 @@ clean-noinstPROGRAMS:
echo " rm -f $$p $$f"; \
rm -f $$p $$f ; \
done
collectData$(EXEEXT): $(collectData_OBJECTS) $(collectData_DEPENDENCIES)
@rm -f collectData$(EXEEXT)
$(LINK) $(collectData_OBJECTS) $(collectData_LDADD) $(LIBS)
compareResourcesArray$(EXEEXT): $(compareResourcesArray_OBJECTS) $(compareResourcesArray_DEPENDENCIES)
@rm -f compareResourcesArray$(EXEEXT)
$(LINK) $(compareResourcesArray_OBJECTS) $(compareResourcesArray_LDADD) $(LIBS)
......@@ -317,6 +326,7 @@ mostlyclean-compile:
distclean-compile:
-rm -f *.tab.c
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/collectData.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/compareResourcesArray.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/myModel.Po@am__quote@
......
pioExamples/collectData.c 0 → 100644
View file @ c0179ef9
#if defined (HAVE_CONFIG_H)
# include "config.h"
#endif
#define nProcsIO 1
#define nNodes 1
#ifdef USE_MPI
#include <mpi.h>
#endif
#include "cdi.h"
#include "pio_c_temp.h"
#include "pio_util.h"
extern void reshArrayPrint ( char * );
static int hasLocalFile[] = { 0 };
#define nlon 12 // Number of longitudes
#define nlat 6 // Number of latitudes
#define nlev 5 // Number of levels
#define ntsteps 3 // Number of time steps
/*
int nodeSizes[nNodes] = {1,2};
int nStreams = 6;
int streamSizes[6] = {5,127,19,18,9,52};
int vlistSizes[230] = {
1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,
47,47,47,47,47,47,47,47,47,47,
1,1,1,
11,11,11,11,11,11,11,11,
11,11,11,11,11,11,11,11,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,5,
1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,
11,11,11,11,11,11,11,11,
11,11,11,11,11,11,11,11,
11,11,11,11,11,11,11,11,
11,11,11,11,11,11,11,11,
11,11,11,11,11 };
*/
int nodeSizes[nNodes] = {1};
int nStreams = 4;
int streamSizes[4] = {2,5,1,3};
int vlistSizes[11] = {5,1,7,3,2,47,11,11,5,47,1 };
void modelRun ()
{
int gridID, zaxisID, taxisID;
int vlistID, varID, streamID, tsID;
int i, nmiss = 0, rank;
double lons[nlon] = {0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330};
double lats[nlat] = {-75, -45, -15, 15, 45, 75};
double levs[nlev] = {101300, 92500, 85000, 50000, 20000};
double var[nlon*nlat*nlev];
xdebug ();
namespaceInit ( 1, hasLocalFile );
gridID = gridCreate(GRID_LONLAT, nlon*nlat);
gridDefXsize(gridID, nlon);
gridDefYsize(gridID, nlat);
gridDefXvals(gridID, lons);
gridDefYvals(gridID, lats);
zaxisID = zaxisCreate(ZAXIS_PRESSURE, nlev);
zaxisDefLevels(zaxisID, levs);
vlistID = vlistCreate();
varID = vlistDefVar(vlistID, gridID, zaxisID, TIME_VARIABLE);
vlistDefVarName(vlistID, varID, "myVar");
{
int globfac[] = { 23, 42 };
vlistDefAttInt(vlistID, varID, "seer's globule factors", DATATYPE_INT16,
2, globfac); // bug vlistAttunpack: ..DefAtt.. can't be left out
}
taxisID = taxisCreate(TAXIS_ABSOLUTE);
vlistDefTaxis(vlistID, taxisID);
xdebug ();
streamID = streamOpenWrite("example.grb", FILETYPE_GRB);
if ( streamID < 0 )
{
fprintf(stderr, "%s\n", cdiStringError(streamID));
return;
}
xdebug ();
streamDefVlist(streamID, vlistID);
#ifdef USE_MPI
xdebug ();
pioMetadata2 ( nNodes, nodeSizes );
#endif
xdebug ();
for ( tsID = 0; tsID < ntsteps; tsID++ )
{
taxisDefVdate(taxisID, 19850101+tsID);
taxisDefVtime(taxisID, 120000);
streamDefTimestep(streamID, tsID);
for ( i = 0; i < nlon*nlat*nlev; i++ ) var[i] = 2.2;
streamWriteVar(streamID, varID, var, nmiss);
}
#ifdef USE_MPI
MPI_Comm_rank ( MPI_COMM_WORLD, &rank );
if ( rank == 0 )
#endif
reshArrayPrint ( "reshArrayModel" );
xdebug ();
streamClose(streamID);
vlistDestroy(vlistID);
taxisDestroy(taxisID);
zaxisDestroy(zaxisID);
gridDestroy(gridID);
xdebug ( "out" );
}
int main (int argc, char *argv[])
{
int sizeGlob, rankGlob;
#ifdef USE_MPI
MPI_Comm commGlob, commModel;
MPI_Init(&argc, &argv);
MPI_Comm_dup ( MPI_COMM_WORLD, &commGlob );
MPI_Comm_set_errhandler ( commGlob, MPI_ERRORS_RETURN );
MPI_Comm_size ( commGlob, &sizeGlob );
MPI_Comm_rank ( commGlob, &rankGlob );
if ( ! TEST_LOCAL )
if ( nProcsIO <=0 || nProcsIO > ( float ) sizeGlob / 2.0 )
pcdiAbort ( "bad distribution of tasks on PEs", __FILE__, __func__,
__LINE__ );
#endif
xdebug ();
#ifdef USE_MPI
commModel = pioInit_c ( commGlob, nProcsIO, 0 );
#endif
modelRun ();
#ifdef USE_MPI
pioFinalize ();
MPI_Finalize ();
#endif
return 0;
}
pioExamples/compareResourcesArray.c
View file @ c0179ef9
......@@ -132,11 +132,12 @@ int defineTaxis ()
return taxisID;
}
int defineStream ()
void defineStream ( int streamID, int vlistID )
{
int streamID = CDI_UNDEFID;
return streamID;
streamDefByteorder ( streamID, 1 );
streamDefCompType ( streamID, 2 );
streamDefCompLevel ( streamID, 3 );
streamDefVlist(streamID, vlistID);
}
int defineVlist ( int gridID, int zaxisID, int taxisID )
......@@ -197,11 +198,9 @@ void modelRun ()
instID = defineInstitute ();
modelID = defineModel ( instID );
vlistID = defineVlist ( gridID, zaxisID, taxisID);
streamID = streamOpenWrite("example.grb", FILETYPE_GRB);
if ( streamID < 0 ) xabort ( "Could not open file" );
streamDefVlist(streamID, vlistID);
defineStream ( streamID, vlistID );
reshPackBufferCreate ( &sendBuffer, &bufferSize, MPI_COMM_WORLD );
xmpi ( MPI_Send ( sendBuffer, bufferSize, MPI_PACKED, 0, 0, MPI_COMM_WORLD ));
......
src/cdi.h
View file @ c0179ef9
......@@ -203,18 +203,21 @@ extern "C" {
#define PIO_POSIX_FPGUARD_THREAD_REFUSE 5
#define PIO_POSIX_FPGUARD_THREAD 6
#define TEST_LOCAL 1
#define TEST_LOCAL 0
int pioInit ( int, int, int );
void pioFinalize ( void );
void pioMetadata ( void );
void pioMetadata2 ( int, int * );
void namespaceInit ( int, int * );
void namespaceSetActive ( int );
void gridGetIndexArray ( int, int * );
void zaxisGetIndexArray ( int, int * );
void gridGetIndexArray ( int, int * );
void streamGetIndexArray ( int, int * );
void zaxisGetIndexAtrray ( int, int * );
void vlistDefVarIOrank ( int, int, int );
/* CDI control routines */
......@@ -302,6 +305,7 @@ char *streamFilesuffix(int filetype);
int streamNtsteps(int streamID);
off_t streamNvals(int streamID);
int streamInqNvars ( int streamID );
/* STREAM var I/O routines */
......
src/cgribexlib.c
View file @ c0179ef9
......@@ -1222,6 +1222,8 @@ gribExSP(int *isec0, int *isec1, int *isec2, float *fsec2sp, int *isec3,
for ( j = 0; j < inum; j++ ) fsec4dp[j] = fsec4sp[j];
printf ( "########## %s, %s, %d\n", __FILE__, __func__, __LINE__ );
gribExDP(isec0, isec1, isec2, fsec2dp, isec3,
fsec3dp, isec4, fsec4dp, klenp, kgrib,
kleng, kword, hoper, kret);
......@@ -1236,6 +1238,8 @@ gribExSP(int *isec0, int *isec1, int *isec2, float *fsec2sp, int *isec3,
for ( j = 0; j < 10; j++ ) fsec2dp[j] = 0.0;
for ( j = 0; j < 2; j++ ) fsec3dp[j] = 0.0;
printf ( "########## %s, %s, %d\n", __FILE__, __func__, __LINE__ );
gribExDP(isec0, isec1, isec2, fsec2dp, isec3,
fsec3dp, isec4, fsec4dp, klenp, kgrib,
kleng, kword, hoper, kret);
......
src/file.c
View file @ c0179ef9
......@@ -24,6 +24,7 @@ size_t getpagesize(void);
#ifdef USE_MPI
#include "pio.h"
#include "cdi.h"
#include "namespace.h"
extern pioInfo *pioinfo;
#endif
......
src/namespace.c
View file @ c0179ef9
......@@ -73,10 +73,11 @@ void namespaceInit ( int nspn, int * argHasLocalFile )
assert(nspn <= NUM_NAMESPACES && nspn >= 1 );
nNamespaces = nspn;
if ( nspn > 1 )
if ( nspn >= 1 )
{
hasLocalFile = xmalloc ( nspn * sizeof ( int ));
for ( i = 0; i < nspn; i++ ) hasLocalFile[i] = argHasLocalFile[i];
for ( i = 0; i < nspn; i++ )
hasLocalFile[i] = argHasLocalFile[i];
}
}
......
src/namespace.h
View file @ c0179ef9
......@@ -16,6 +16,7 @@ namespaceTuple_t namespaceResHDecode ( int );
int namespaceHasLocalFile ( int );
void namespaceCleanup ( void );
int namespaceAdaptKey ( int, int );
int namespaceAdaptKey2 ( int );
#endif
/*
......
src/pio.c
View file @ c0179ef9
......@@ -208,7 +208,7 @@ void setPioCommunicator ( MPI_Comm *myComm, MPI_Comm commF2C, int *color,
{
strncpy ( hostname, myHost, len );
hostname [ len ] = '\0';
fprintf ( stderr, "pe%d: myHost = %s\n", rank, hostname );
xdebug ( "myHost = %s\n", hostname );
}
allHosts = ( char ** ) xmalloc ( size * sizeof ( char * ));
......@@ -287,6 +287,8 @@ MPI_Comm bInit ( int ptype, MPI_Comm comm, int *color, int *nnodes )
pioinfo = ( pioInfo * ) xmalloc ( sizeof ( pioInfo ));
xdebug ();
pioinfo->type = ptype;
setPioCommunicator ( & ( pioinfo->comm ), comm, color, nnodes );
......@@ -295,17 +297,24 @@ MPI_Comm bInit ( int ptype, MPI_Comm comm, int *color, int *nnodes )
xabort ( "PIOTYPE, NNODES: not a valid combination" );
pioinfo->color = *color;
xdebug ();
MPI_Comm_rank ( pioinfo->comm, &( pioinfo->rank ));
MPI_Comm_size ( pioinfo->comm, &( pioinfo->size ));
if ( ddebug == MAXDEBUG && pioinfo->rank == 0 )
fprintf ( stderr,
xdebug ();
// if ( ddebug == MAXDEBUG && pioinfo->rank == 0 )
//fprintf ( stderr,
xdebug(
"pe%d in bInit(), ptype=%d, initial_buffersize=%ld: "
"after init pioinfo ...\n",
pioinfo->rank, pioinfo->type, initial_buffersize );
pioinfo->collectorComm = MPI_COMM_NULL;
xdebug ();
switch ( pioinfo->type )
{
case PIO_NONE:
......@@ -335,8 +344,7 @@ MPI_Comm bInit ( int ptype, MPI_Comm comm, int *color, int *nnodes )
}
#endif
if ( ddebug == MAXDEBUG )
fprintf ( stderr, "pe in bInit out\n" );
xdebug ("bInit out\n" );
#ifdef USE_MPI
return pioinfo->collectorComm;
......
src/pio_interface.c
View file @ c0179ef9
......@@ -19,7 +19,7 @@
#include "pio_server.h"
#include "namespace.h"
#include "resource_handle.h"
#include "stream_int.h"
#endif
#define xprintArray(ps,array,n,datatype) printArray ( ps, array, n, datatype, \
......@@ -91,6 +91,8 @@ MPI_Comm pioInit_c ( MPI_Comm comm, int nIOP, int mode )
MPI_Finalize ();
exit ( 0 );
}
xdebug ( "out" );
return commModel;
}
......@@ -329,13 +331,86 @@ void varMapGen ( int * vSizes, int * sSizes, int * varMapping,
/************************************************************************/
static
void mapVars ( int nNodes, int * nodeSizes )
{
int nStreams, nVars, * resHs, * streamSizes, * varSizes, * varMapping;
int i, j, k = 0;
#ifdef USE_MPI
nStreams = streamSize ();
xdebug ( "nStreams=%d", nStreams );
resHs = xmalloc ( nStreams * sizeof ( int ));
streamSizes = xmalloc ( nStreams * sizeof ( int ));
streamGetIndexArray ( nStreams, resHs );
for ( i = 0; i < nStreams; i++ )
{
streamSizes[i] = streamInqNvars ( * ( resHs + i ));
xdebug ( "streamSizes[%d]=%d", i, * ( streamSizes + i ));
}
nVars = xsum ( nStreams, streamSizes );
xdebug ( "nVars=%d", nVars );
varSizes = xmalloc ( nVars * sizeof ( int ));
varMapping = xmalloc ( nVars * sizeof ( int ));
for ( i = 0; i < nStreams; i++ )
for ( j = 0; j < * ( streamSizes + i ); j++ )
varSizes[k] += vlistInqVarSize ( streamInqVlist ( * resHs + i ), j );
varMapGen ( varSizes, streamSizes, varMapping,
nStreams, nodeSizes, nNodes );
//reshSetVarMapping ( varMapping );
if ( varMapping ) free ( varMapping );
if ( varSizes ) free ( varSizes );
if ( streamSizes ) free ( streamSizes );
if ( resHs ) free ( resHs );
#endif
}
/************************************************************************/
static
void modelWinCreate ( void )
{
#ifdef USE_MPI
xdebug ();
#endif
}
/************************************************************************/
void pioMetadata ( void )
{
#ifdef USE_MPI
char * buffer;
int bufferSize;
if ( rankGlob < nProcsIO )
{
reshPackBufferCreate ( &buffer, &bufferSize, commsIO[rankGlob] );
MPI_Send ( buffer, bufferSize, MPI_PACKED, nProcsCalc,
METADATA, * ( commsIO + rankGlob ));
xdebug ( "sent message metadata" );
reshPackBufferDestroy ( &buffer );
}
#endif
}
/************************************************************************/
void pioMetadata ()
void pioMetadata2 ( int nNodes, int * nodeSizes )
{
#ifdef USE_MPI
char * buffer;
int bufferSize;
mapVars ( nNodes, nodeSizes );
if ( rankGlob < nProcsIO )
{
reshPackBufferCreate ( &buffer, &bufferSize, commsIO[rankGlob] );
......@@ -347,11 +422,15 @@ void pioMetadata ()
reshPackBufferDestroy ( &buffer );
}
modelWinCreate ();
#endif
}
/****************************************************/
/**
@brief is invoked by the calculator PEs, to inform
the I/O PEs that no more data will be written.
......@@ -377,6 +456,17 @@ void pioFinalize ()
#endif
}
/************************************************************************/
void pioWriteVar ()
{
#ifdef USE_MPI
#endif
}
/************************************************************************/
/*
* Local Variables:
......
src/pio_server.c
View file @ c0179ef9
......@@ -22,6 +22,13 @@ extern void reshArrayPrint ( char * );
MPI_Comm * commPio;
/************************************************************************/
void serverWinCreate ()
{
xdebug ();
}
/************************************************************************/
......@@ -48,10 +55,13 @@ void IOServer ( MPI_Comm comm, int ptype )
nfinished = 0;
commColl = bInit ( ptype, comm, &color, &nnodes );
xdebug();
for ( ;; )
{
MPI_Probe ( MPI_ANY_SOURCE, MPI_ANY_TAG, commCalc, &status );
xdebug();
source = status.MPI_SOURCE;
tag = status.MPI_TAG;
......@@ -81,13 +91,16 @@ void IOServer ( MPI_Comm comm, int ptype )
if ( ddebug >= 2 ) xdebugMsg ( tag, source, nfinished );
serverWinCreate ();
break;
default:
MPI_Recv ( &iBuffer, 1, MPI_INTEGER, source,
tag, commCalc, &status );
case WRITEVAR:
xdebug("WRITEVAR");
break;
if ( ddebug >= 2 ) xdebugMsg ( tag, source, nfinished );
default: