centralize communicators

.gitattributes

@@ -207,6 +207,8 @@ src/namespace.h -text
 src/pio.c -text
 src/pio.h -text
 src/pio_c_temp.h -text
+src/pio_comm.c -text
+src/pio_comm.h -text
 src/pio_dbuffer.c -text
 src/pio_impl.h -text
 src/pio_interface.c -text

pioExamples/collectData.c

@@ -76,7 +76,7 @@ void modelRun ()
 
 #ifdef USE_MPI
   xmpi ( MPI_Comm_rank ( MPI_COMM_WORLD, &rank ));
-  pioMetadata ( nNodes, nodeSizes );
+  pioEndDef ( nNodes, nodeSizes );
 #endif 
 
    

pioExamples/myModel.c

@@ -182,7 +182,7 @@ void modelRun ()
   streamDefVlist(streamID, vlistID);
 
 #ifdef USE_MPI
-  pioMetadata ( nNodes, nodeSizes );
+  pioEndDef ( nNodes, nodeSizes );
 #endif 
 
   /*

src/Makefile.am

@@ -54,6 +54,8 @@ libcdi_la_SOURCES = 	 \
 	pio.c            \
 	pio.h            \
 	pio_c_temp.h     \
+	pio_comm.c       \
+	pio_comm.h       \
 	pio_dbuffer.c    \
 	pio_impl.h 	 \
 	pio_interface.c  \

src/Makefile.in

@@ -81,15 +81,16 @@ am_libcdi_la_OBJECTS = basetime.lo binary.lo calendar.lo cdf.lo \
 	cdf_int.lo cdi_error.lo cdi_util.lo cdiFortran.lo \
 	cgribexlib.lo dmemory.lo error.lo extralib.lo file.lo \
 	gaussgrid.lo gribapi.lo grid.lo ieglib.lo institution.lo \
-	model.lo namespace.lo pio.lo pio_dbuffer.lo pio_interface.lo \
-	pio_mpinonb.lo pio_posixfpguardsendrecv.lo pio_queue.lo \
-	pio_rpc.lo pio_server.lo pio_util.lo resource_handle.lo \
-	servicelib.lo stream_cdf.lo stream_cgribex.lo stream_ext.lo \
-	stream_grb.lo stream_gribapi.lo stream_history.lo \
-	stream_ieg.lo stream_int.lo stream_record.lo stream_srv.lo \
-	stream_var.lo table.lo taxis.lo timebase.lo tsteps.lo util.lo \
-	varscan.lo version.lo vlist.lo vlist_att.lo vlist_var.lo \
-	zaxis.lo stream.lo swap.lo
+	model.lo namespace.lo pio.lo pio_comm.lo pio_dbuffer.lo \
+	pio_interface.lo pio_mpinonb.lo pio_posixfpguardsendrecv.lo \
+	pio_queue.lo pio_rpc.lo pio_server.lo pio_util.lo \
+	resource_handle.lo servicelib.lo stream_cdf.lo \
+	stream_cgribex.lo stream_ext.lo stream_grb.lo \
+	stream_gribapi.lo stream_history.lo stream_ieg.lo \
+	stream_int.lo stream_record.lo stream_srv.lo stream_var.lo \
+	table.lo taxis.lo timebase.lo tsteps.lo util.lo varscan.lo \
+	version.lo vlist.lo vlist_att.lo vlist_var.lo zaxis.lo \
+	stream.lo swap.lo
 libcdi_la_OBJECTS = $(am_libcdi_la_OBJECTS)
 @ENABLE_CDI_LIB_FALSE@am_libcdi_la_rpath =
 @ENABLE_CDI_LIB_TRUE@am_libcdi_la_rpath = -rpath $(libdir)
@@ -323,6 +324,8 @@ libcdi_la_SOURCES = \
 	pio.c            \
 	pio.h            \
 	pio_c_temp.h     \
+	pio_comm.c       \
+	pio_comm.h       \
 	pio_dbuffer.c    \
 	pio_impl.h 	 \
 	pio_interface.c  \
@@ -505,6 +508,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/model.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/namespace.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pio.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pio_comm.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pio_dbuffer.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pio_interface.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pio_mpinonb.Plo@am__quote@

src/cdi.h

@@ -209,7 +209,7 @@ extern "C" {
 
 int      pioInit            ( int, int, int );
 void     pioFinalize        ( void );
-void     pioMetadata        ( int, int * );
+void     pioEndDef          ( int, int * );
 void     pioWriteTimestep   ( int, int, int );
 
 void     pioNamespaceInit      ( int, int * );

src/cdi.inc

 ! This file was automatically generated, don't edit!
 !
-! Fortran interface for CDI library version 1.5.0
 !
 ! Author:
 ! -------
-! Uwe Schulzweida, MPI-MET, Hamburg,   July 2011
+! Uwe Schulzweida, MPI-MET, Hamburg,   September 2011
 !
 
       INTEGER    CDI_MAX_NAME          
@@ -344,10 +343,16 @@
 !                     pioFinalize
       EXTERNAL        pioFinalize
 
-!                     pioMetadata
+!                     pioEndDef
 !                                    (INTEGER         ,
 !                                     INTEGER         )
-      EXTERNAL        pioMetadata
+      EXTERNAL        pioEndDef
+
+!                     pioWriteTimestep
+!                                    (INTEGER         ,
+!                                     INTEGER         ,
+!                                     INTEGER         )
+      EXTERNAL        pioWriteTimestep
 
 !                     pioNamespaceInit
 !                                    (INTEGER         ,
@@ -387,6 +392,18 @@
 !                                     INTEGER         )
       EXTERNAL        vlistInqVarIOrank
 
+      INTEGER         vlistInqVarDecoChunk
+!                                    (INTEGER         ,
+!                                     INTEGER         ,
+!                                     INTEGER         )
+      EXTERNAL        vlistInqVarDecoChunk
+
+      INTEGER         vlistInqVarDecoOff
+!                                    (INTEGER         ,
+!                                     INTEGER         ,
+!                                     INTEGER         )
+      EXTERNAL        vlistInqVarDecoOff
+
       CHARACTER*80    cdiStringError
 !                                    (INTEGER         cdiErrno)
       EXTERNAL        cdiStringError

src/cdiFortran.c

@@ -58,7 +58,8 @@
 
 FCALLSCFUN3 (INT, pioInit, PIOINIT, pioinit, INT, INT, INT)
 FCALLSCSUB0 (pioFinalize, PIOFINALIZE, piofinalize)
-FCALLSCSUB2 (pioMetadata, PIOMETADATA, piometadata, INT, PINT)
+FCALLSCSUB2 (pioEndDef, PIOENDDEF, pioenddef, INT, PINT)
+FCALLSCSUB3 (pioWriteTimestep, PIOWRITETIMESTEP, piowritetimestep, INT, INT, INT)
 FCALLSCSUB2 (pioNamespaceInit, PIONAMESPACEINIT, pionamespaceinit, INT, PINT)
 FCALLSCSUB0 (pioNamespaceCleanup, PIONAMESPACECLEANUP, pionamespacecleanup)
 FCALLSCSUB1 (pioNamespaceSetActive, PIONAMESPACESETACTIVE, pionamespacesetactive, INT)
@@ -67,6 +68,8 @@ FCALLSCSUB2 (streamGetIndexArray, STREAMGETINDEXARRAY, streamgetindexarray, INT,
 FCALLSCSUB2 (zaxisGetIndexArray, ZAXISGETINDEXARRAY, zaxisgetindexarray, INT, PINT)
 FCALLSCSUB3 (vlistDefVarIOrank, VLISTDEFVARIORANK, vlistdefvariorank, INT, INT, INT)
 FCALLSCFUN2 (INT, vlistInqVarIOrank, VLISTINQVARIORANK, vlistinqvariorank, INT, INT)
+FCALLSCFUN3 (INT, vlistInqVarDecoChunk, VLISTINQVARDECOCHUNK, vlistinqvardecochunk, INT, INT, INT)
+FCALLSCFUN3 (INT, vlistInqVarDecoOff, VLISTINQVARDECOOFF, vlistinqvardecooff, INT, INT, INT)
 FCALLSCFUN1 (STRING, cdiStringError, CDISTRINGERROR, cdistringerror, INT)
 FCALLSCSUB1 (cdiDebug, CDIDEBUG, cdidebug, INT)
 FCALLSCFUN0 (STRING, cdiLibraryVersion, CDILIBRARYVERSION, cdilibraryversion)

src/make_fint.c

@@ -4,7 +4,7 @@
 #include <ctype.h>
 #include <time.h>
 
-#include "config.h"
+//#include "config.h"
 
 typedef struct
 {
@@ -116,7 +116,7 @@ void fortran_interface(char *fname, char *fnameinc, char *fnameint)
 
   fprintf(fpinc, "! This file was automatically generated, don't edit!\n");
   fprintf(fpinc, "!\n");
-  fprintf(fpinc, "! Fortran interface for CDI library version %s\n", VERSION);
+  //  fprintf(fpinc, "! Fortran interface for CDI library version %s\n", VERSION);
   fprintf(fpinc, "!\n");
   fprintf(fpinc, "! Author:\n");
   fprintf(fpinc, "! -------\n");

src/pio_comm.c

+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include "pio_comm.h"
+#include "cdi.h"
+#include "pio_util.h"
+
+#ifdef USE_MPI
+
+pioInfo_t * pioinfo2 = NULL;
+
+void pioinfoInit ( pioInfo_t * p )
+{
+  p->IOModus         = CDI_UNDEFID;
+  p->nProcsIO        = CDI_UNDEFID;
+
+  p->commGlob        = MPI_COMM_NULL;
+  p->sizeGlob        = CDI_UNDEFID;
+  p->rankGlob        = CDI_UNDEFID;
+
+  p->commPio         = MPI_COMM_NULL;
+  p->sizePio         = CDI_UNDEFID;
+  p->rankPio         = CDI_UNDEFID;
+
+  p->commModel       = MPI_COMM_NULL;
+  p->sizeModel       = CDI_UNDEFID;
+  p->rankModel       = CDI_UNDEFID;
+
+  p->commNode        = MPI_COMM_NULL;
+  p->sizeNode        = CDI_UNDEFID;
+  p->rankNode        = CDI_UNDEFID;
+  p->hostname[0]     = 0;
+  p->hostID          = CDI_UNDEFID;
+  p->specialRankNode = CDI_UNDEFID;
+     
+  p->commColl        = MPI_COMM_NULL;
+  p->sizeColl        = CDI_UNDEFID;
+  p->rankColl        = CDI_UNDEFID;
+     
+  p->commCalc        = MPI_COMM_NULL;
+  p->sizeCalc        = CDI_UNDEFID;
+  p->rankCalc        = CDI_UNDEFID;
+     
+  p->commsIO         = NULL;
+}
+
+
+void pioInfoNew ( void )
+{
+  pioinfo2 = xmalloc ( sizeof ( pioInfo_t ));
+  pioinfoInit ( pioinfo2 );
+}
+
+
+void pioInfoDestroy ( void )
+{
+  int i;
+
+  assert ( pioinfo2 );
+
+  if ( pioinfo2->commsIO ) 
+    {
+      for ( i = 0; i < pioinfo2->nProcsIO; i++ )
+	if ( pioinfo2->commsIO[i] != MPI_COMM_NULL )
+	  xmpi ( MPI_Comm_free ( &pioinfo2->commsIO[i] ));
+      free ( &pioinfo2->commsIO ); 
+    }  
+  if ( pioinfo2->commCalc != MPI_COMM_NULL )
+    xmpi ( MPI_Comm_free ( &pioinfo2->commCalc ));
+  if ( pioinfo2->commColl != MPI_COMM_NULL )
+    xmpi ( MPI_Comm_free ( &pioinfo2->commColl ));
+  if ( pioinfo2->commNode != MPI_COMM_NULL )
+    xmpi ( MPI_Comm_free ( &pioinfo2->commNode ));
+  if ( pioinfo2->commModel != MPI_COMM_NULL )
+    xmpi ( MPI_Comm_free ( &pioinfo2->commModel ));
+  if ( pioinfo2->commPio != MPI_COMM_NULL )
+    xmpi ( MPI_Comm_free ( &pioinfo2->commPio ));    
+
+  free ( pioinfo2 );
+}
+
+
+
+
+#endif

src/pio_comm.h

+#ifndef PIO_INFO_
+#define PIO_INFO_
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#ifdef USE_MPI
+
+#include <mpi.h>
+
+typedef struct {
+  int IOModus;
+  int nProcsIO;
+
+  MPI_Comm commGlob;
+  int sizeGlob;
+  int rankGlob;
+
+  MPI_Comm commPio;
+  int sizePio;
+  int rankPio;
+
+  MPI_Comm commModel;
+  int sizeModel;
+  int rankModel;
+
+  MPI_Comm commNode;
+  int sizeNode;
+  int rankNode;
+
+  int specialRankNode;
+  char hostname [ MPI_MAX_PROCESSOR_NAME + 1 ];
+  int hostID;
+
+  MPI_Comm commColl;
+  int sizeColl;
+  int rankColl;
+
+  MPI_Comm commCalc;
+  int sizeCalc;
+  int rankCalc;
+ 
+  MPI_Comm * commsIO;  
+} pioInfo_t;
+
+extern pioInfo_t * pioinfo2;
+
+void pioInfoNew ( void );
+void pioInfoDestroy ( void );
+
+
+#endif
+
+#endif
+/*
+ * Local Variables:
+ * c-file-style: "Java"
+ * c-basic-offset: 2
+ * indent-tabs-mode: nil
+ * show-trailing-whitespace: t
+ * require-trailing-newline: t
+ * End:
+ */

src/pio_interface.c

@@ -10,6 +10,7 @@
 
 #ifdef USE_MPI
 #include "pio_interface.h"
+#include "pio_comm.h"
 #include "pio_rpc.h"
 #include "pio_server.h"
 #include "namespace.h"
@@ -78,6 +79,12 @@ MPI_Comm pioInit_c ( MPI_Comm comm, int nIOP, int IOmode )
 
   xmpi ( MPI_Comm_rank ( commGlob, &rankGlob ));
   xmpi ( MPI_Comm_set_name ( commGlob, "commGlob" ));
+
+
+  pioInfoNew ();
+  xdebug ( "pioinfo2->sizeGlob=%d", pioinfo2->sizeGlob );
+  pioInfoDestroy ();
+
   
   if ( ddebug >= 1 ) xdebugComm (  &commGlob );
   
@@ -524,7 +531,7 @@ void pioBuffer ( int streamID, int varID, const double *data, int nmiss )
 
 /************************************************************************/
 
-void  pioMetadata ( int nNodes, int * nodeSizes )
+void  pioEndDef ( int nNodes, int * nodeSizes )
 {
 #ifdef USE_MPI
   char   * buffer;
@@ -537,7 +544,7 @@ void  pioMetadata ( int nNodes, int * nodeSizes )
       reshPackBufferCreate ( &buffer, &bufferSize, commsIO[rankGlob] );
       
       xmpi ( MPI_Send ( buffer, bufferSize, MPI_PACKED, nProcsCalc, 
-                        METADATA, * ( commsIO + rankGlob )));
+                        RESOURCES, * ( commsIO + rankGlob )));
       
       xdebug ( "sent message metadata" );
       

src/pio_rpc.h

@@ -20,7 +20,7 @@ extern MPI_Comm commCalc;
 typedef enum
   {
     FINALIZE,
-    METADATA,
+    RESOURCES,
     WINCREATE,
     WRITETS
   } command;

src/pio_server.c

@@ -278,7 +278,7 @@ void IOServer ( MPI_Comm comm, int ptype )
 	  
 	  break;
 
-	case METADATA:
+	case RESOURCES:
 	  xdebugMsg (  tag, source, nfinished );
 	  xmpi ( MPI_Get_count ( &status, MPI_CHAR, &size ));
 	  buffer = xmalloc ( size * sizeof ( char ));

src/pio_util.c

@@ -9,7 +9,7 @@
 #include "cdi.h"
 
 char commands[][13] = { "FINALIZE\0",
-                        "METADATA\0",
+                        "RESOURCES\0",
                         "WINCREATE\0",
                         "WRITETS\0"};