Commit 5ff3abc5 authored by Thomas Jahns's avatar Thomas Jahns 🤸
Browse files

Add balanced PIO task distribution scheme.

* Also add corresponding switch to test code.
parent 924c3014
......@@ -4,6 +4,16 @@
This appendix provide a brief listing of the C language bindings of the
CDI library routines:
\section*{\tt \htmlref{cdiPioCSRBalanced}{cdiPioCSRBalanced}}
\begin{verbatim}
int cdiPioCSRBalanced (MPI_Comm commSuper, int IOMode, int nProcsIO);
\end{verbatim}
return role codes appropriate to use \textit{nProcsIO}
tasks distributed on evenly spaced ranks as I/O servers.
\section*{\tt \htmlref{cdiPioCSRFirstN}{cdiPioCSRFirstN}}
\begin{verbatim}
......
......@@ -4,6 +4,17 @@
This appendix provide a brief listing of the Fortran language bindings of the
CDI library routines:
\section*{\tt \htmlref{cdiPioCSRBalanced}{cdiPioCSRBalanced}}
\begin{verbatim}
INTEGER FUNCTION cdiPioCSRBalanced (INTEGER commSuper, INTEGER IOMode,
INTEGER nProcsIO)
\end{verbatim}
return role codes appropriate to use \textit{nProcsIO}
tasks distributed on evenly spaced ranks as I/O servers.
\section*{\tt \htmlref{cdiPioCSRFirstN}{cdiPioCSRFirstN}}
\begin{verbatim}
......
......@@ -63,6 +63,10 @@ int cdiPioCSRLastN(MPI_Comm commSuper, int IOMode, int nProcsIO);
\textit{nProcsIO} tasks as I/O servers */
int cdiPioCSRFirstN(MPI_Comm commSuper, int IOMode, int nProcsIO);
/* cdiPioCSRBalanced: return role codes appropriate to use \textit{nProcsIO}
* tasks distributed on evenly spaced ranks as I/O servers */
int cdiPioCSRBalanced(MPI_Comm commSuper, int IOMode, int nProcsIO);
/* cdiPioStr2IOMode: return integer code corresponding to string
* representation of mode or -1 if no match was found */
int cdiPioStr2IOMode(const char *modeStr);
......
......@@ -103,6 +103,12 @@
! INTEGER nProcsIO)
EXTERNAL cdiPioCSRFirstN
INTEGER cdiPioCSRBalanced
! (INTEGER commSuper,
! INTEGER IOMode,
! INTEGER nProcsIO)
EXTERNAL cdiPioCSRBalanced
INTEGER cdiPioStr2IOMode
! (CHARACTER*(*) modeStr)
EXTERNAL cdiPioStr2IOMode
......
......@@ -67,6 +67,13 @@ static int cdiPioCSRFirstN_fwrap(int commSuper, int IOMode, int nProcsIO)
return v;
}
FCALLSCFUN3 (INT, cdiPioCSRFirstN_fwrap, CDIPIOCSRFIRSTN, cdipiocsrfirstn, INT, INT, INT)
static int cdiPioCSRBalanced_fwrap(int commSuper, int IOMode, int nProcsIO)
{
int v;
v = cdiPioCSRBalanced(MPI_Comm_f2c(commSuper), IOMode, nProcsIO);
return v;
}
FCALLSCFUN3 (INT, cdiPioCSRBalanced_fwrap, CDIPIOCSRBALANCED, cdipiocsrbalanced, INT, INT, INT)
FCALLSCFUN1 (INT, cdiPioStr2IOMode, CDIPIOSTR2IOMODE, cdipiostr2iomode, STRING)
FCALLSCFUN0 (INT, cdiPioConfCreate, CDIPIOCONFCREATE, cdipioconfcreate)
FCALLSCSUB1 (cdiPioConfDestroy, CDIPIOCONFDESTROY, cdipioconfdestroy, INT)
......
......@@ -71,3 +71,62 @@ cdiPioCSRSelectRange(MPI_Comm commSuper, int IOMode,
}
return role;
}
int
cdiPioCSRBalanced(MPI_Comm commSuper, int IOMode, int nProcsIO)
{
int commRank, commSize, numClients, clientsPerCollectorMin,
clientsPerCollectorMax, rest;
xmpi(MPI_Comm_size(commSuper, &commSize));
xmpi(MPI_Comm_rank(commSuper, &commRank));
numClients = commSize - nProcsIO;
int role, specialRole = PIO_ROLE_FPGUARD,
collType = PIO_ROLE_WRITER_COLLECTOR;
switch (IOMode)
{
case PIO_NONE:
xassert(commRank == 0 && commSize == 1);
role = PIO_NONE;
break;
case PIO_MPI:
case PIO_MPI_FW_ORDERED:
case PIO_MPI_FW_AT_ALL:
case PIO_MPI_FW_AT_REBLOCK:
clientsPerCollectorMin = numClients / nProcsIO;
rest = numClients % nProcsIO;
clientsPerCollectorMax = clientsPerCollectorMin + (rest != 0);
if (commRank == commSize - 1)
role = PIO_ROLE_WRITER_COLLECTOR;
else if (commRank <= (clientsPerCollectorMax + 1) * (nProcsIO - rest))
role = ((commRank + 1) % (clientsPerCollectorMax + 1)) != 0
? PIO_ROLE_CLIENT : PIO_ROLE_WRITER_COLLECTOR;
else
role = ((commRank - clientsPerCollectorMax * (nProcsIO - rest) + 1)
% (clientsPerCollectorMin + 1)) != 0
? PIO_ROLE_CLIENT : PIO_ROLE_WRITER_COLLECTOR;
break;
case PIO_WRITER:
case PIO_ASYNCH:
specialRole = PIO_ROLE_WRITER;
collType = PIO_ROLE_COLLECTOR;
/* fallthrough intentional */
case PIO_FPGUARD:
clientsPerCollectorMin = numClients / (nProcsIO - 1);
rest = numClients % nProcsIO;
clientsPerCollectorMax = clientsPerCollectorMin + (rest != 0);
if (commRank == commSize - 1)
role = specialRole;
else if (commRank <= clientsPerCollectorMax * (nProcsIO - 1 - rest))
role = (commRank + 1 % (clientsPerCollectorMax + 1)) != 0
? PIO_ROLE_CLIENT : collType;
else
role = ((commRank - clientsPerCollectorMax * (nProcsIO - 1 - rest) + 1)
% (clientsPerCollectorMin + 1)) != 0
? PIO_ROLE_CLIENT : collType;
break;
default:
xabort("Invalid mode requested %d", IOMode);
}
return role;
}
......@@ -93,10 +93,14 @@ parse_unsignedarg(const char msg[])
return (unsigned)temp;
}
typedef int (*pioRoleFunc)(MPI_Comm commSuper, int IOMode, int nProcsIO);
static void
parse_long_option(int pioConfHandle, const char *str)
parse_long_option(int pioConfHandle, pioRoleFunc *pioRoleAssign,
const char *str)
{
static const char cacheRedistStr[] = "cache-redists";
static const char cacheRedistStr[] = "cache-redists",
pioRoleSchemeOptionStr[] = "pio-role-scheme";
if (!strncmp(str, cacheRedistStr, sizeof (cacheRedistStr) - 1))
{
#ifdef USE_MPI
......@@ -111,6 +115,33 @@ parse_long_option(int pioConfHandle, const char *str)
#else
invalidOptionDie("CDI-PIO option -q%s ignored in non-MPI mode\n",
cacheRedistStr);
#endif
}
else if (!strncmp(str, pioRoleSchemeOptionStr,
sizeof (pioRoleSchemeOptionStr) - 1))
{
#ifdef USE_MPI
static const char pioRoleSchemeLastN[]="last",
pioRoleSchemeFirstN[]="first", pioRoleSchemeBalanced[]="balanced";
if (str[sizeof (pioRoleSchemeOptionStr) - 1] == '=')
{
const char *optargarg = str + sizeof (pioRoleSchemeOptionStr);
if (!strcmp(optargarg, pioRoleSchemeLastN))
*pioRoleAssign = cdiPioCSRLastN;
else if (!strcmp(optargarg, pioRoleSchemeFirstN))
*pioRoleAssign = cdiPioCSRFirstN;
else if (!strcmp(optargarg, pioRoleSchemeBalanced))
*pioRoleAssign = cdiPioCSRBalanced;
else
invalidOptionDie("unknown scheme argument \"%s\" to -q%s",
optargarg, pioRoleSchemeOptionStr);
}
else
invalidOptionDie("long option %s needs argument\n",
pioRoleSchemeOptionStr);
#else
invalidOptionDie("CDI-PIO option -q%s ignored in non-MPI mode\n",
pioRoleSchemeOptionStr);
#endif
}
else
......@@ -124,6 +155,7 @@ int main(int argc, char *argv[])
MPI_Comm commModel;
int pioConfHandle = 0;
pioRoleFunc pioRoleAssign = 0;
#ifdef USE_MPI
MPI_Comm commGlob;
int sizeGlob;
......@@ -139,6 +171,7 @@ int main(int argc, char *argv[])
xmpi ( MPI_Comm_rank ( commGlob, &rankGlob ));
pioConfHandle = cdiPioConfCreate();
pioRoleAssign = cdiPioCSRLastN;
#endif
/* seed random generator */
......@@ -189,7 +222,7 @@ int main(int argc, char *argv[])
break;
#endif
case 'q':
parse_long_option(pioConfHandle, optarg);
parse_long_option(pioConfHandle, &pioRoleAssign, optarg);
break;
case 'f':
{
......@@ -275,8 +308,8 @@ int main(int argc, char *argv[])
#ifdef USE_MPI
int pioNamespace;
cdiPioConfSetIOMode(pioConfHandle, IOMode);
cdiPioConfSetCSRole(pioConfHandle, cdiPioCSRLastN(commGlob, IOMode,
nProcsIO));
cdiPioConfSetCSRole(pioConfHandle, pioRoleAssign(commGlob, IOMode,
nProcsIO));
cdiPioConfSetPartInflate(pioConfHandle, 1.0f);
int initNamespace = namespaceGetActive();
commModel = cdiPioInit(commGlob, pioConfHandle, &pioNamespace);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment