Commit 138511fc authored by Deike Kleberg's avatar Deike Kleberg
Browse files

interim 1

parent 1015800e
......@@ -8,11 +8,12 @@
#include <string.h>
#include <unistd.h>
#include "cdi.h"
#include "pio.h"
#include "cdi.h"
#include "pio_util.h"
#ifdef USE_MPI
#include "pio_comm.h"
#include "pio_impl.h"
#endif
......@@ -268,6 +269,11 @@ void setPioCommunicator ( MPI_Comm *myComm, MPI_Comm commF2C, int *color,
xmpi ( MPI_Comm_split ( commF2C, *color, key, myComm));
commDefCommNode ( * myComm );
commDefHostName ( myHost, len );
commDefHostID ( * color );
free ( allHosts0 );
free ( allHosts );
free ( myHost );
......
......@@ -14,42 +14,40 @@
pioInfo_t * pioinfo2 = NULL;
static
void pioinfoInit ( pioInfo_t * p )
{
p->IOModus = CDI_UNDEFID;
p->nProcsIO = CDI_UNDEFID;
p->nProcsModel = CDI_UNDEFID;
p->role = CDI_UNDEFID;
p->commGlob = MPI_COMM_NULL;
p->sizeGlob = CDI_UNDEFID;
p->rankGlob = CDI_UNDEFID;
p->commPio = MPI_COMM_NULL;
p->sizePio = CDI_UNDEFID;
p->rankPio = CDI_UNDEFID;
p->commModel = MPI_COMM_NULL;
p->sizeModel = CDI_UNDEFID;
p->rankModel = CDI_UNDEFID;
p->commNode = MPI_COMM_NULL;
p->sizeNode = CDI_UNDEFID;
p->rankNode = CDI_UNDEFID;
p->hostname[0] = 0;
p->hostID = CDI_UNDEFID;
p->specialRankNode = CDI_UNDEFID;
p->IOModus = CDI_UNDEFID;
p->nProcsIO = CDI_UNDEFID;
p->nProcsModel = CDI_UNDEFID;
p->isProcIO = CDI_UNDEFID;
p->commGlob = MPI_COMM_NULL;
p->sizeGlob = CDI_UNDEFID;
p->rankGlob = CDI_UNDEFID;
p->commPio = MPI_COMM_NULL;
p->sizePio = CDI_UNDEFID;
p->rankPio = CDI_UNDEFID;
p->commNode = MPI_COMM_NULL;
p->sizeNode = CDI_UNDEFID;
p->rankNode = CDI_UNDEFID;
p->hostname[0] = 0;
p->nodeInfo.hostID = CDI_UNDEFID;
p->nodeInfo.isProcColl = CDI_UNDEFID;
p->specialRankNode = CDI_UNDEFID;
p->commColl = MPI_COMM_NULL;
p->sizeColl = CDI_UNDEFID;
p->rankColl = CDI_UNDEFID;
p->commColl = MPI_COMM_NULL;
p->sizeColl = CDI_UNDEFID;
p->rankColl = CDI_UNDEFID;
p->commCalc = MPI_COMM_NULL;
p->sizeCalc = CDI_UNDEFID;
p->rankCalc = CDI_UNDEFID;
p->commCalc = MPI_COMM_NULL;
p->sizeCalc = CDI_UNDEFID;
p->rankCalc = CDI_UNDEFID;
p->commsIO = NULL;
p->commsIO = NULL;
}
......@@ -140,11 +138,11 @@ int commInqNProcsIO ( void )
}
int commInqRole ( void )
int commInqIsProcIO ( void )
{
assert ( pioinfo2 != NULL &&
pioinfo2->role != CDI_UNDEFID );
return pioinfo2->role;
pioinfo2->isProcIO != CDI_UNDEFID );
return pioinfo2->isProcIO;
}
......@@ -184,16 +182,12 @@ void commDefCommPio ( void )
nProcsModel = pioinfo2->sizeGlob - pioinfo2->nProcsIO;
pioinfo2->role = pioinfo2->rankGlob >= nProcsModel ? ISPROCIO : ISPROCMODEL;
pioinfo2->isProcIO = pioinfo2->rankGlob >= nProcsModel ? 1 : 0;
xmpi ( MPI_Comm_split ( pioinfo2->commGlob, pioinfo2->role, 0,
xmpi ( MPI_Comm_split ( pioinfo2->commGlob, pioinfo2->isProcIO, 0,
&pioinfo2->commPio ));
xmpi ( MPI_Comm_size ( pioinfo2->commPio, &pioinfo2->sizePio ));
xmpi ( MPI_Comm_rank ( pioinfo2->commPio, &pioinfo2->rankPio ));
pioinfo2->commModel = pioinfo2->commPio;
pioinfo2->sizeModel = pioinfo2->sizePio;
pioinfo2->rankModel = pioinfo2->rankPio;
}
......@@ -208,8 +202,64 @@ MPI_Comm commInqCommPio ( void )
MPI_Comm commInqCommModel( void )
{
assert ( pioinfo2 != NULL &&
pioinfo2->commModel != MPI_COMM_NULL );
return pioinfo2->commModel;
pioinfo2->commPio != MPI_COMM_NULL );
return pioinfo2->commPio;
}
void commDefIsProcColl ( int isProcColl )
{
assert ( pioinfo2 != NULL );
pioinfo2->nodeInfo.isProcColl = isProcColl;
}
void commDefCommNode ( MPI_Comm commNode )
{
assert ( pioinfo2 != NULL );
pioinfo2->commNode = commNode;
}
void commDefHostName ( char * hostname, size_t len )
{
assert ( pioinfo2 != NULL );
strncpy ( &pioinfo2->hostname, hostname, len );
}
void commDefHostID ( int hostID )
{
assert ( pioinfo2 != NULL );
pioinfo2->nodeInfo.hostID = hostID;
}
void commSendNodeInfo ( void )
{
int rootRank = 0;
assert ( pioinfo2 != NULL &&
pioinfo2->nodeInfo.hostID != CDI_UNDEFID &&
pioinfo2->nodeInfo.isProcColl != CDI_UNDEFID );
commPrint ( stdout);
xmpi ( MPI_Send ( &pioinfo2->nodeInfo, sizeNodeInfo, MPI_INTEGER, rootRank,
NODEINFO, pioinfo2->commGlob ));
}
void commRecvNodeInfo ( int rank )
{
nodeInfo_t nodeInfo;
MPI_Status status;
xmpi ( MPI_Recv ( &nodeInfo, sizeNodeInfo, MPI_INTEGER, rank,
NODEINFO, pioinfo2->commGlob, &status ));
xdebug ("#### recvd from PE%d nodeInfo: nodeID=%d, isProcColl=%d",
rank, nodeInfo.hostID, nodeInfo.isProcColl );
}
......@@ -225,7 +275,7 @@ void commPrint ( FILE * fp )
fprintf ( fp, "# IOModus = %d\n", pioinfo2->IOModus );
fprintf ( fp, "# nProcsIO = %d\n", pioinfo2->nProcsIO );
fprintf ( fp, "# nProcsModel = %d\n", pioinfo2->nProcsModel );
fprintf ( fp, "# role = %d\n", pioinfo2->role );
fprintf ( fp, "# isProcIO = %d\n", pioinfo2->isProcIO );
fprintf ( fp, "#\n" );
fprintf ( fp, "# commGlob = %d\n", pioinfo2->commGlob );
fprintf ( fp, "# sizeGlob = %d\n", pioinfo2->sizeGlob );
......@@ -235,16 +285,13 @@ void commPrint ( FILE * fp )
fprintf ( fp, "# sizePio = %d\n", pioinfo2->sizePio );
fprintf ( fp, "# rankPio = %d\n", pioinfo2->rankPio );
fprintf ( fp, "#\n" );
fprintf ( fp, "# commModel = %d\n", pioinfo2->commModel );
fprintf ( fp, "# sizeModel = %d\n", pioinfo2->sizeModel );
fprintf ( fp, "# rankModel = %d\n", pioinfo2->rankModel );
fprintf ( fp, "#\n" );
fprintf ( fp, "# commNode = %d\n", pioinfo2->commNode );
fprintf ( fp, "# sizeNode = %d\n", pioinfo2->sizeNode );
fprintf ( fp, "# rankNode = %d\n", pioinfo2->rankNode );
fprintf ( fp, "# hostname = %s\n", pioinfo2->hostname[0] == 0 ? "nn" :
pioinfo2->hostname );
fprintf ( fp, "# hostID = %d\n", pioinfo2->hostID );
fprintf ( fp, "# hostID = %d\n", pioinfo2->nodeInfo.hostID );
fprintf ( fp, "# isProcColl = %d\n", pioinfo2->nodeInfo.isProcColl );
fprintf ( fp, "# specialRankNode = %d\n", pioinfo2->specialRankNode );
fprintf ( fp, "#\n" );
fprintf ( fp, "# commColl = %d\n", pioinfo2->commColl );
......@@ -259,7 +306,9 @@ void commPrint ( FILE * fp )
fprintf ( fp, "#\n" );
for ( i = 0; i < pioinfo2->nProcsIO; i++ )
fprintf ( fp, "# commsIO[%d] = %d\n", i, pioinfo2->commsIO[i] );
}
}
else
fprintf ( fp, "# commsIO = NULL\n" );
fprintf ( fp, "#\n" );
fprintf ( fp, "############################\n" );
fprintf ( fp, "\n" );
......
......@@ -11,11 +11,16 @@
#include <stdio.h>
#include "cdi.h"
typedef struct {
int hostID;
int isProcColl;
} nodeInfo_t;
typedef struct {
int IOModus;
int nProcsIO;
int nProcsModel;
int role;
int isProcIO;
MPI_Comm commGlob;
int sizeGlob;
......@@ -25,17 +30,12 @@ typedef struct {
int sizePio;
int rankPio;
MPI_Comm commModel;
int sizeModel;
int rankModel;
MPI_Comm commNode;
int sizeNode;
int rankNode;
int specialRankNode;
char hostname [ MPI_MAX_PROCESSOR_NAME + 1 ];
int hostID;
nodeInfo_t nodeInfo;
int specialRankNode;
MPI_Comm commColl;
int sizeColl;
......@@ -48,29 +48,37 @@ typedef struct {
MPI_Comm * commsIO;
} pioInfo_t;
extern pioInfo_t * pioinfo2;
enum { MAXNPROCSIO = 256,
MAXIOMODUS = PIO_POSIX_FPGUARD_SENDRECV};
enum { ISPROCMODEL, ISPROCIO, ISPROCCOLL, ISPROCSPEC };
enum { NODEINFO = 9999, sizeNodeInfo = 2 };
extern pioInfo_t * pioinfo2;
void commInit ( void );
void commDestroy ( void );
void commDefCommGlob ( MPI_Comm );
MPI_Comm commInqCommGlob ( void );
int commInqSizeGlob ( void );
int commInqRankGlob ( void );
void commDefNProcsIO ( int );
int commInqNProcsIO ( void );
void commInit ( void );
void commDestroy ( void );
void commDefCommGlob ( MPI_Comm );
MPI_Comm commInqCommGlob ( void );
int commInqSizeGlob ( void );
int commInqRankGlob ( void );
void commDefNProcsIO ( int );
int commInqNProcsIO ( void );
int commInqNProcsModel ( void );
void commDefIOModus ( int );
int commInqIOModus ( void );
void commDefCommPio ( void );
MPI_Comm commInqCommPio ( void );
MPI_Comm commInqCommModel( void );
int commInqRole ( void );
void commPrint ( FILE * );
void commDefIOModus ( int );
int commInqIOModus ( void );
void commDefCommPio ( void );
MPI_Comm commInqCommPio ( void );
MPI_Comm commInqCommModel ( void );
int commInqIsProcIO ( void );
void commDefIsProcColl ( int );
void commDefCommNode ( MPI_Comm );
void commDefHostName ( char *, size_t );
void commDefHostID ( int );
void commPrint ( FILE * );
void commSendNodeInfo ( void );
void commRecvNodeInfo ( int );
#endif
......
......@@ -64,9 +64,9 @@ int pioInit ( int comm, int nIOP, int mode )
#ifdef USE_MPI
MPI_Comm pioInit_c ( MPI_Comm comm, int nIOP, int IOModus )
{
MPI_Comm commModel;
int isCalcPE;
int sizeGlob, rankGlob;
int buffer, sizeGlob, rankGlob, nProcsModel, i;
MPI_Comm commGlob;
int rankBcast = 0;
commInit ();
commDefCommGlob ( comm );
......@@ -76,10 +76,12 @@ MPI_Comm pioInit_c ( MPI_Comm comm, int nIOP, int IOModus )
commsIOCreate ();
if ( commInqRankGlob () == 0 ) commPrint ( stdout );
rankGlob = commInqRankGlob ();
if ( rankGlob == rankBcast ) commPrint ( stdout );
//commPrint ( stdout );
if ( commInqRole () == ISPROCIO )
if ( commInqIsProcIO ())
{
IOServer ();
commsIOCleanup ();
......@@ -87,6 +89,17 @@ MPI_Comm pioInit_c ( MPI_Comm comm, int nIOP, int IOModus )
exit ( 0 );
}
else if ( rankGlob == rankBcast )
{
sizeGlob = commInqSizeGlob ();
nProcsModel = commInqNProcsModel ();
for ( i = nProcsModel; i < sizeGlob; i++ )
commRecvNodeInfo ( i );
}
return commInqCommModel ();
}
#endif
......
......@@ -10,6 +10,7 @@
#include "mpi.h"
#include "pio.h"
#include "pio_comm.h"
#include "pio_impl.h"
#include "pio_util.h"
......@@ -361,14 +362,18 @@ int fowMPINONB ( const char *filename )
int initMPINONB ()
{
int collectingData = 1;
MPI_Comm commGlob = commInqCommGlob ();
xmpi ( MPI_Comm_dup ( pioinfo->comm, &( pioinfo->collectorComm )));
bibAFiledataM = queueInit ( destroyAFiledataMPINONB, compareNamesMPINONB );
commDefIsProcColl ( 1 );
commSendNodeInfo ();
bibAFiledataM = queueInit ( destroyAFiledataMPINONB, compareNamesMPINONB );
if ( bibAFiledataM == NULL )
xabort ( "queueInit did not succeed" );
xabort ( "queueInit did not succeed" );
return collectingData;
}
......
......@@ -39,8 +39,6 @@ void commsIOCreate ()
commsIO = ( MPI_Comm * ) xmalloc ( nProcsIO * sizeof ( MPI_Comm ));
strncpy ( name, "commsIO_", 8 );
name[9] = '\0';
xdebug ("######### nProcsCalc=%d", nProcsCalc );
ranks = ( int * ) xmalloc (( nProcsCalc + 1 ) * sizeof ( int ));
for ( i = 0; i < nProcsCalc; i++ )
......
......@@ -14,8 +14,8 @@
#include <assert.h>
#include "limits.h"
#include "cdi.h"
#include "pio_comm.h"
#include "pio.h"
#include "pio_comm.h"
#include "pio_rpc.h"
#include "pio_util.h"
#include "stream_int.h"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment