pio_comm.c 22 KB
Newer Older
Deike Kleberg's avatar
Deike Kleberg committed
1
2
3
4
5
6
7
8
9
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <stdio.h>
#include <stdlib.h>

#include "pio_comm.h"
#include "cdi.h"
10
#include "dmemory.h"
Deike Kleberg's avatar
Deike Kleberg committed
11
12
13
14
#include "pio_util.h"

#ifdef USE_MPI

15
16
#include "cdipio.h"

17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
typedef struct {
  int IOMode;
  int nProcsIO;
  int nProcsModel;
  int isProcIO;

  MPI_Comm commGlob;
  int sizeGlob;
  int rankGlob;
  int root;

  MPI_Comm commPio;
  int sizePio;
  int rankPio;

  MPI_Comm commNode;
  int sizeNode;
  int rankNode;
  char hostname [ MPI_MAX_PROCESSOR_NAME + 1 ];
  nodeInfo_t nodeInfo;
  int specialRankNode;

  MPI_Comm commColl;
  int sizeColl;
  int rankColl;

  MPI_Comm commCalc;
44
  int rankCalc, sizeCalc;
45
46
47
48
49
50
51
52
53

  MPI_Comm * commsIO;
  int nProcsColl;
  int * procsCollMap;
  int * nodeSizes;
  int * nodeMap;
} pioInfo_t;


Deike Kleberg's avatar
Deike Kleberg committed
54
static pioInfo_t * info = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
55

Deike Kleberg's avatar
Deike Kleberg committed
56

Deike Kleberg's avatar
Deike Kleberg committed
57
static
Deike Kleberg's avatar
Deike Kleberg committed
58
void pioInfoInit ( pioInfo_t * p )
Deike Kleberg's avatar
Deike Kleberg committed
59
{
Deike Kleberg's avatar
Deike Kleberg committed
60
61
62
63
  p->IOMode               = CDI_UNDEFID;
  p->nProcsIO             = CDI_UNDEFID;
  p->nProcsModel          = CDI_UNDEFID;
  p->isProcIO             = CDI_UNDEFID;
64

Deike Kleberg's avatar
Deike Kleberg committed
65
66
67
68
  p->commGlob             = MPI_COMM_NULL;
  p->sizeGlob             = CDI_UNDEFID;
  p->rankGlob             = CDI_UNDEFID;
  p->root                 = CDI_UNDEFID;
69

Deike Kleberg's avatar
Deike Kleberg committed
70
71
  p->commPio              = MPI_COMM_NULL;
  p->sizePio              = CDI_UNDEFID;
72
73
  p->rankPio              = -1;

Deike Kleberg's avatar
Deike Kleberg committed
74
75
76
77
78
  p->commNode             = MPI_COMM_NULL;
  p->sizeNode             = CDI_UNDEFID;
  p->rankNode             = CDI_UNDEFID;
  p->hostname[0]          = 0;
  p->specialRankNode      = CDI_UNDEFID;
79

Deike Kleberg's avatar
Deike Kleberg committed
80
81
82
  p->commColl             = MPI_COMM_NULL;
  p->sizeColl             = CDI_UNDEFID;
  p->rankColl             = CDI_UNDEFID;
83

Deike Kleberg's avatar
Deike Kleberg committed
84
  p->commCalc             = MPI_COMM_NULL;
85
86
87
  p->rankCalc             = -1;
  p->sizeCalc             = -1;

Deike Kleberg's avatar
Deike Kleberg committed
88
89
90
91
  p->commsIO              = NULL;
  p->nodeInfo.hostID      = CDI_UNDEFID;
  p->nodeInfo.isProcColl  = CDI_UNDEFID;
  p->nodeInfo.nNodes      = CDI_UNDEFID;
92
  p->nProcsColl           = CDI_UNDEFID;
Deike Kleberg's avatar
interim    
Deike Kleberg committed
93
  p->procsCollMap         = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
94
95
  p->nodeSizes            = NULL;
  p->nodeMap              = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
96
97
98
}


Deike Kleberg's avatar
Deike Kleberg committed
99
void commInit ( void )
Deike Kleberg's avatar
Deike Kleberg committed
100
{
Deike Kleberg's avatar
Deike Kleberg committed
101
  xassert ( info == 0 );
Uwe Schulzweida's avatar
Uwe Schulzweida committed
102
  info = (pioInfo_t*) xmalloc ( sizeof ( pioInfo_t ));
Deike Kleberg's avatar
Deike Kleberg committed
103
  pioInfoInit ( info );
Deike Kleberg's avatar
Deike Kleberg committed
104
105
106
}


Deike Kleberg's avatar
Deike Kleberg committed
107
void commDestroy ( void )
Deike Kleberg's avatar
Deike Kleberg committed
108
{
109
  int collID;
Deike Kleberg's avatar
Deike Kleberg committed
110

Deike Kleberg's avatar
Deike Kleberg committed
111
  xassert ( info != NULL );
Deike Kleberg's avatar
Deike Kleberg committed
112

Deike Kleberg's avatar
Deike Kleberg committed
113
114
  if ( info->nodeMap   != NULL ) free ( info->nodeMap );
  if ( info->nodeSizes != NULL ) free ( info->nodeSizes );
Deike Kleberg's avatar
Deike Kleberg committed
115

Thomas Jahns's avatar
Thomas Jahns committed
116
  if ( info->commsIO != NULL )
Deike Kleberg's avatar
Deike Kleberg committed
117
    {
Deike Kleberg's avatar
Deike Kleberg committed
118
119
      for ( collID = 0; collID < info->nProcsColl; collID++ )
	if ( info->commsIO[collID] != MPI_COMM_NULL )
Thomas Jahns's avatar
Thomas Jahns committed
120
	  xmpi(MPI_Comm_free(info->commsIO + collID));
Thomas Jahns's avatar
Thomas Jahns committed
121
      free ( info->commsIO );
Deike Kleberg's avatar
Deike Kleberg committed
122
      info->commsIO = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
123
    }
124

Deike Kleberg's avatar
Deike Kleberg committed
125
  if ( info->commColl != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
126
    {
Thomas Jahns's avatar
Thomas Jahns committed
127
      xmpi ( MPI_Comm_free ( &info->commColl ));
Deike Kleberg's avatar
Deike Kleberg committed
128
      info->commColl = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
129
    }
Thomas Jahns's avatar
Thomas Jahns committed
130
131
132

  free(info->procsCollMap);

Deike Kleberg's avatar
Deike Kleberg committed
133
  if ( info->commNode != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
134
    {
Thomas Jahns's avatar
Thomas Jahns committed
135
      xmpi ( MPI_Comm_free ( &info->commNode ));
Deike Kleberg's avatar
Deike Kleberg committed
136
      info->commNode = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
137
    }
Thomas Jahns's avatar
Thomas Jahns committed
138

Deike Kleberg's avatar
Deike Kleberg committed
139
  if ( info->commPio != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
140
    {
Thomas Jahns's avatar
Thomas Jahns committed
141
      xmpi ( MPI_Comm_free ( &info->commPio ));
Thomas Jahns's avatar
Thomas Jahns committed
142
      info->commPio = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
143
    }
Thomas Jahns's avatar
Thomas Jahns committed
144

Deike Kleberg's avatar
Deike Kleberg committed
145
146
  free ( info );
  info = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
147
148
149
}


Deike Kleberg's avatar
Deike Kleberg committed
150
151
void commDefCommGlob ( MPI_Comm c )
{
Deike Kleberg's avatar
Deike Kleberg committed
152
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
153
154
	   c != MPI_COMM_NULL );

Deike Kleberg's avatar
Deike Kleberg committed
155
156
157
158
  info->commGlob = c;
  xmpi ( MPI_Comm_size ( c, &info->sizeGlob ));
  xmpi ( MPI_Comm_rank ( c, &info->rankGlob ));
  info->root = 0;
Deike Kleberg's avatar
Deike Kleberg committed
159
160
161
162
163
}
 

MPI_Comm commInqCommGlob ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
164
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
165
166
           info->commGlob != MPI_COMM_NULL );
  return info->commGlob;
Deike Kleberg's avatar
Deike Kleberg committed
167
168
169
170
171
}


int commInqSizeGlob ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
172
  xassert ( info != NULL  &&
Deike Kleberg's avatar
Deike Kleberg committed
173
174
           info->sizeGlob != CDI_UNDEFID );
  return info->sizeGlob;
Deike Kleberg's avatar
Deike Kleberg committed
175
176
177
178
179
}


int commInqRankGlob ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
180
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
181
182
           info->rankGlob != CDI_UNDEFID );
  return info->rankGlob;
Deike Kleberg's avatar
Deike Kleberg committed
183
184
185
}


186
187
int commInqRootGlob ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
188
  xassert ( info != NULL &&
189
190
191
192
193
           info->root != CDI_UNDEFID );
  return info->root;
}


Deike Kleberg's avatar
Deike Kleberg committed
194
195
void commDefNProcsIO ( int n )
{
Deike Kleberg's avatar
Deike Kleberg committed
196
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
197
	   n >= 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
198
	   n < MAXNPROCSIO &&
Deike Kleberg's avatar
Deike Kleberg committed
199
200
201
           info->commGlob != MPI_COMM_NULL );
  info->nProcsIO = n;
  info->nProcsModel = info->sizeGlob - info->nProcsIO;
Deike Kleberg's avatar
Deike Kleberg committed
202
203
204
205
206
}


int commInqNProcsIO ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
207
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
208
209
           info->nProcsIO != CDI_UNDEFID );
  return info->nProcsIO;
Deike Kleberg's avatar
Deike Kleberg committed
210
211
212
}


Deike Kleberg's avatar
Deike Kleberg committed
213
int      commInqIsProcIO     ( void )
Deike Kleberg's avatar
Deike Kleberg committed
214
{
Deike Kleberg's avatar
Deike Kleberg committed
215
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
216
217
           info->isProcIO != CDI_UNDEFID );
  return info->isProcIO;
Deike Kleberg's avatar
Deike Kleberg committed
218
219
220
221
222
}


int commInqNProcsModel ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
223
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
224
225
           info->nProcsModel != CDI_UNDEFID );
  return info->nProcsModel;
Deike Kleberg's avatar
Deike Kleberg committed
226
227
228
}


229
void     commDefIOMode  ( int IOMode )
Deike Kleberg's avatar
Deike Kleberg committed
230
{
231
  xassert(info != NULL && IOMode >= PIO_MINIOMODE && IOMode <= PIO_MAXIOMODE );
Deike Kleberg's avatar
Deike Kleberg committed
232
  info->IOMode = IOMode;
Deike Kleberg's avatar
Deike Kleberg committed
233
234
235
}


Deike Kleberg's avatar
Deike Kleberg committed
236
int      commInqIOMode  ( void )
Deike Kleberg's avatar
Deike Kleberg committed
237
{
238
239
240
241
242
243
244
  if (info != NULL)
    {
      xassert ( info->IOMode != CDI_UNDEFID );
      return info->IOMode;
    }
  else
    return PIO_NONE;
Deike Kleberg's avatar
Deike Kleberg committed
245
246
}

Deike Kleberg's avatar
Deike Kleberg committed
247

Deike Kleberg's avatar
Deike Kleberg committed
248
249
void     commDefCommPio  ( void )
{
250
251
252
  int nProcsCalc;
  MPI_Group grpAll, grpCalc, grpPio;
  int calcRange[3], pioRange[3];
Deike Kleberg's avatar
Deike Kleberg committed
253

Deike Kleberg's avatar
Deike Kleberg committed
254
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
255
256
257
           info->commGlob != MPI_COMM_NULL &&
           info->IOMode  != CDI_UNDEFID &&
           info->nProcsIO != CDI_UNDEFID );
Deike Kleberg's avatar
Deike Kleberg committed
258

259
260
261
262
263
  nProcsCalc = info->sizeGlob - info->nProcsIO;

  info->isProcIO = info->rankGlob >= nProcsCalc ? 1 : 0;

  xmpi(MPI_Comm_group(info->commGlob, &grpAll));
Deike Kleberg's avatar
Deike Kleberg committed
264

265
266
267
  calcRange[0] = 0;
  calcRange[1] = info->sizeGlob - info->nProcsIO - 1;
  calcRange[2] = 1;
268
269
  xmpi(MPI_Group_range_incl(grpAll, info->nProcsIO==info->sizeGlob?0:1,
                            &calcRange, &grpCalc));
270
  xmpi(MPI_Comm_create(info->commGlob, grpCalc, &info->commCalc));
Deike Kleberg's avatar
Deike Kleberg committed
271

272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
  pioRange[0] = info->sizeGlob - info->nProcsIO;
  pioRange[1] = info->sizeGlob - 1;
  pioRange[2] = 1;
  xmpi(MPI_Group_range_incl(grpAll, 1, &pioRange, &grpPio));
  xmpi(MPI_Comm_create(info->commGlob, grpPio, &info->commPio));

  if (info->commPio != MPI_COMM_NULL)
    {
      xmpi(MPI_Comm_size(info->commPio, &info->sizePio));
      xmpi(MPI_Comm_rank(info->commPio, &info->rankPio));
      info->sizeCalc = nProcsCalc;
      info->rankCalc = -1;
    }
  else
    {
      info->sizePio = nProcsCalc;
      info->rankPio = -1;
      xmpi(MPI_Comm_size(info->commCalc, &info->sizeCalc));
      xmpi(MPI_Comm_rank(info->commCalc, &info->rankCalc));
    }
  xmpi(MPI_Group_free(&grpCalc));
  xmpi(MPI_Group_free(&grpPio));
  xmpi(MPI_Group_free(&grpAll));
Deike Kleberg's avatar
Deike Kleberg committed
295
296
297
298
299
}


MPI_Comm commInqCommPio  ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
300
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
301
302
           info->commPio != MPI_COMM_NULL &&
           info->isProcIO == 1 );
Deike Kleberg's avatar
Deike Kleberg committed
303
  return info->commPio;
Deike Kleberg's avatar
Deike Kleberg committed
304
305
306
}


Deike Kleberg's avatar
Deike Kleberg committed
307
MPI_Comm commInqCommModel ( void )
Deike Kleberg's avatar
Deike Kleberg committed
308
{
Deike Kleberg's avatar
Deike Kleberg committed
309
  xassert ( info != NULL &&
310
           info->commCalc != MPI_COMM_NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
311
           info->isProcIO == 0 );
312
  return info->commCalc;
Deike Kleberg's avatar
Deike Kleberg committed
313
314
315
}


316
317
int      commInqRankPio       ( void )
{ 
Deike Kleberg's avatar
Deike Kleberg committed
318
  xassert ( info != NULL &&
319
           info->rankPio >= 0 &&
320
321
322
323
324
           info->isProcIO == 1 );
  return info->rankPio;
}


Deike Kleberg's avatar
Deike Kleberg committed
325
326
int      commInqRankModel       ( void )
{ 
Deike Kleberg's avatar
Deike Kleberg committed
327
  xassert ( info != NULL &&
328
           info->rankCalc >= 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
329
           info->isProcIO == 0 );
330
  return info->rankCalc;
Deike Kleberg's avatar
Deike Kleberg committed
331
332
333
}


Deike Kleberg's avatar
Deike Kleberg committed
334
void     commDefCommColl  ( int isProcColl )
Deike Kleberg's avatar
Deike Kleberg committed
335
{
Deike Kleberg's avatar
Deike Kleberg committed
336
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
337
338
           info->commNode != MPI_COMM_NULL &&
           info->commColl == MPI_COMM_NULL );
Deike Kleberg's avatar
Deike Kleberg committed
339

340
  info->nodeInfo.isProcColl = isProcColl;
Deike Kleberg's avatar
Deike Kleberg committed
341
342
343
344
  xmpi ( MPI_Comm_split ( info->commNode, info->nodeInfo.isProcColl, 0, 
                          &info->commColl ));
  xmpi ( MPI_Comm_size ( info->commColl, &info->sizeColl ));
  xmpi ( MPI_Comm_rank ( info->commColl, &info->rankColl )); 
Deike Kleberg's avatar
Deike Kleberg committed
345
346
347
}


348
349
MPI_Comm commInqCommColl    ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
350
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
351
352
           info->commColl != MPI_COMM_NULL );
  return info->commColl;
353
354
355
356
357
}


int      commInqSizeColl    ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
358
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
359
360
           info->sizeColl != CDI_UNDEFID );
  return info->sizeColl; 
361
362
363
364
365
}


int      commInqRankColl    ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
366
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
367
368
           info->rankColl != CDI_UNDEFID );
  return info->rankColl; 
369
370
}

371
372
static int
cmpstringp(const void *p1, const void *p2)
Deike Kleberg's avatar
Deike Kleberg committed
373
{
374
  return strcmp(* (char * const *) p1, * (char * const *) p2);
Deike Kleberg's avatar
Deike Kleberg committed
375
376
377
378
}

void commDefCommNode ( void )
{
379
380
  int size;
  char * myHost, (*allHosts)[MPI_MAX_PROCESSOR_NAME], **sortedHosts;
Deike Kleberg's avatar
Deike Kleberg committed
381

Deike Kleberg's avatar
Deike Kleberg committed
382
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
383
           info->commPio != MPI_COMM_NULL );
Deike Kleberg's avatar
Deike Kleberg committed
384

Deike Kleberg's avatar
Deike Kleberg committed
385
  size = info->sizePio;
Deike Kleberg's avatar
Deike Kleberg committed
386

Uwe Schulzweida's avatar
Uwe Schulzweida committed
387
  myHost = (char*) xmalloc(MPI_MAX_PROCESSOR_NAME);
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
  {
    int len;
    xmpi ( MPI_Get_processor_name ( myHost, &len ));
    xassert ( myHost[0] != '\0' );
    strncpy(info->hostname, myHost, len);
    info->hostname[len] = '\0';
  }

  allHosts = xmalloc(size * MPI_MAX_PROCESSOR_NAME);
  sortedHosts = xmalloc(size * sizeof(sortedHosts[0]));

  {
    int i;
    for (i = 0; i < size; ++i)
      sortedHosts[i] = allHosts[i];
  }

  xmpi(MPI_Allgather(myHost, MPI_MAX_PROCESSOR_NAME, MPI_CHAR,
                     allHosts[0], MPI_MAX_PROCESSOR_NAME,
                     MPI_CHAR, info->commPio ));

  qsort(sortedHosts, size, sizeof (sortedHosts[0]),
        (int (*)(const void *, const void *))cmpstringp);

  {
    int i = 1, nHosts = 1;
    if (!strcmp(myHost, sortedHosts[0]))
      {
        info->nodeInfo.hostID = 1;
      }
    else
      {
        for (; i < size && strcmp(myHost, sortedHosts[i]); ++i)
          if (strcmp(sortedHosts[i - 1], sortedHosts[i]))
            {
              /* new host seen, might be ours */
              nHosts += 1;
              if (!strcmp(sortedHosts[i], myHost))
                info->nodeInfo.hostID = nHosts - 1;
            }
      }
    for (; i < size && strcmp(myHost, sortedHosts[i]); ++i)
      if (strcmp(sortedHosts[i - 1], sortedHosts[i]))
        nHosts += 1;
    info->nodeInfo.nNodes = nHosts;
  }
Deike Kleberg's avatar
Deike Kleberg committed
434

Deike Kleberg's avatar
Deike Kleberg committed
435
  xassert ( info->nodeInfo.hostID != CDI_UNDEFID );
Deike Kleberg's avatar
Deike Kleberg committed
436

Deike Kleberg's avatar
Deike Kleberg committed
437
438
439
440
  xmpi ( MPI_Comm_split ( info->commPio, info->nodeInfo.hostID, 0, 
                          &info->commNode ));
  xmpi ( MPI_Comm_size ( info->commNode, &info->sizeNode ));
  xmpi ( MPI_Comm_rank ( info->commNode, &info->rankNode ));
441
  if ( info->IOMode >= PIO_MINIOMODEWITHSPECIALPROCS )
Deike Kleberg's avatar
Deike Kleberg committed
442
    info->specialRankNode = info->sizeNode - 1;
Deike Kleberg's avatar
Deike Kleberg committed
443

444
445
446
  free(sortedHosts);
  free(allHosts);
  free(myHost);
Deike Kleberg's avatar
Deike Kleberg committed
447
448

  return;
Deike Kleberg's avatar
Deike Kleberg committed
449
450
451
}


Deike Kleberg's avatar
Deike Kleberg committed
452
MPI_Comm commInqCommNode    ( void )
Deike Kleberg's avatar
Deike Kleberg committed
453
{
Deike Kleberg's avatar
Deike Kleberg committed
454
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
455
456
           info->commNode != MPI_COMM_NULL );
  return info->commNode;
Deike Kleberg's avatar
Deike Kleberg committed
457
} 
Deike Kleberg's avatar
Deike Kleberg committed
458
459


Deike Kleberg's avatar
Deike Kleberg committed
460
461
int commInqSizeNode ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
462
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
463
464
           info->sizeNode != CDI_UNDEFID );
  return info->sizeNode;
Deike Kleberg's avatar
Deike Kleberg committed
465
466
467
}


Deike Kleberg's avatar
Deike Kleberg committed
468
int commInqRankNode ( void )
Deike Kleberg's avatar
Deike Kleberg committed
469
{
Deike Kleberg's avatar
Deike Kleberg committed
470
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
471
472
           info->rankNode != CDI_UNDEFID );
  return info->rankNode;
Deike Kleberg's avatar
Deike Kleberg committed
473
474
475
}


476
477
int commInqSpecialRankNode ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
478
  xassert ( info != NULL );
Deike Kleberg's avatar
Deike Kleberg committed
479
  return info->specialRankNode;
480
481
482
}


Deike Kleberg's avatar
Deike Kleberg committed
483
484
void commSendNodeInfo ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
485
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
486
487
488
489
490
           info->root                != CDI_UNDEFID &&
           info->nodeInfo.hostID     != CDI_UNDEFID &&
           info->nodeInfo.isProcColl != CDI_UNDEFID &&
           info->nodeInfo.nNodes     != CDI_UNDEFID &&
           info->commGlob            != MPI_COMM_NULL );
Deike Kleberg's avatar
Deike Kleberg committed
491
  
Deike Kleberg's avatar
Deike Kleberg committed
492
493
  xmpi ( MPI_Send ( &info->nodeInfo, sizeNodeInfo, MPI_INTEGER, 
                    info->root, NODEINFO, info->commGlob ));
Deike Kleberg's avatar
Deike Kleberg committed
494
495
496
}


Deike Kleberg's avatar
Deike Kleberg committed
497
498
499
void     commRecvNodeMap    ( void )
{
  MPI_Status status;
Deike Kleberg's avatar
Deike Kleberg committed
500
  int source; 
Deike Kleberg's avatar
Deike Kleberg committed
501

Deike Kleberg's avatar
Deike Kleberg committed
502
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
503
504
505
506
           info->commGlob    != MPI_COMM_NULL &&
           info->rankGlob    != CDI_UNDEFID &&
           info->nProcsModel != CDI_UNDEFID &&
           info->nodeMap     == NULL );
Deike Kleberg's avatar
Deike Kleberg committed
507

508
  source = info->root;
Deike Kleberg's avatar
Deike Kleberg committed
509

Deike Kleberg's avatar
Deike Kleberg committed
510
511
  xmpi ( MPI_Probe ( source, NODEMAP, info->commGlob, &status ));
  xmpi ( MPI_Get_count ( &status, MPI_INTEGER, &info->nProcsColl ));
Deike Kleberg's avatar
Deike Kleberg committed
512

Deike Kleberg's avatar
Deike Kleberg committed
513
514
  xdebug ( "info->nProcsColl=%d", info->nProcsColl );

Uwe Schulzweida's avatar
Uwe Schulzweida committed
515
  info->nodeMap = (int*) xmalloc ( info->nProcsColl * 
Deike Kleberg's avatar
Deike Kleberg committed
516
                                sizeof ( info->nodeMap[0] )); 
Deike Kleberg's avatar
Deike Kleberg committed
517

Deike Kleberg's avatar
Deike Kleberg committed
518
519
  xmpi ( MPI_Recv ( info->nodeMap, info->nProcsColl, MPI_INTEGER, 
                    source, NODEMAP, info->commGlob, &status ));
Deike Kleberg's avatar
Deike Kleberg committed
520
521
522
}


Deike Kleberg's avatar
Deike Kleberg committed
523
524
void     commDefNNodes      ( int nNodes )
{
Deike Kleberg's avatar
Deike Kleberg committed
525
  xassert ( info != NULL );
Deike Kleberg's avatar
Deike Kleberg committed
526
  info->nodeInfo.nNodes = nNodes;
Deike Kleberg's avatar
Deike Kleberg committed
527
528
529
}


Deike Kleberg's avatar
Deike Kleberg committed
530
531
int      commInqNNodes      ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
532
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
533
534
           info->nodeInfo.nNodes != CDI_UNDEFID );
  return info->nodeInfo.nNodes;
Deike Kleberg's avatar
Deike Kleberg committed
535
536
537
}


Deike Kleberg's avatar
Deike Kleberg committed
538
void     commEvalPhysNodes  ( void )
Deike Kleberg's avatar
Deike Kleberg committed
539
{
Deike Kleberg's avatar
Deike Kleberg committed
540
  nodeInfo_t * nodeInfo;
Deike Kleberg's avatar
Deike Kleberg committed
541
  MPI_Status status;
Deike Kleberg's avatar
interim    
Deike Kleberg committed
542
  int i, IOID, collID, size, nNodes = CDI_UNDEFID;
543
  int ** p1, ** p2,  idx;
Deike Kleberg's avatar
Deike Kleberg committed
544

Deike Kleberg's avatar
Deike Kleberg committed
545
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
546
547
548
549
550
551
           info->root        != CDI_UNDEFID &&
           info->nProcsIO    != CDI_UNDEFID &&
           info->nProcsModel != CDI_UNDEFID &&
           info->sizeGlob    != CDI_UNDEFID &&
           info->rankGlob    != CDI_UNDEFID &&
           info->commGlob    != MPI_COMM_NULL );
Deike Kleberg's avatar
Deike Kleberg committed
552

Deike Kleberg's avatar
Deike Kleberg committed
553
  size = info->nProcsIO * sizeNodeInfo;
Deike Kleberg's avatar
Deike Kleberg committed
554

Uwe Schulzweida's avatar
Uwe Schulzweida committed
555
  nodeInfo = (nodeInfo_t*) xmalloc ( size * sizeof ( int ));
Deike Kleberg's avatar
Deike Kleberg committed
556

Deike Kleberg's avatar
Deike Kleberg committed
557
  if ( info->rankGlob == info->root )
Deike Kleberg's avatar
Deike Kleberg committed
558
    {
559
      xassert ( info->rankCalc == info->root );
Deike Kleberg's avatar
Deike Kleberg committed
560
 
Deike Kleberg's avatar
Deike Kleberg committed
561
      for ( i = info->nProcsModel; i < info->sizeGlob; i++ )
Deike Kleberg's avatar
Deike Kleberg committed
562
        {
Deike Kleberg's avatar
Deike Kleberg committed
563
          IOID = i - info->nProcsModel;
Deike Kleberg's avatar
Deike Kleberg committed
564
          xmpi ( MPI_Recv ( nodeInfo + IOID, sizeNodeInfo, MPI_INTEGER, i, 
Deike Kleberg's avatar
Deike Kleberg committed
565
                            NODEINFO, info->commGlob, &status ));
Deike Kleberg's avatar
Deike Kleberg committed
566
567
568
        }
    }

Deike Kleberg's avatar
Deike Kleberg committed
569
  xmpi ( MPI_Bcast ( nodeInfo, size, MPI_INTEGER, info->root, 
570
                     info->commCalc ));
Deike Kleberg's avatar
Deike Kleberg committed
571

572
  // consistency check, count collectors
Deike Kleberg's avatar
Deike Kleberg committed
573
  for ( IOID = 0; IOID < info->nProcsIO; IOID++ )
Deike Kleberg's avatar
Deike Kleberg committed
574
    {
575
576
      if ( IOID == 0 )
        {
Deike Kleberg's avatar
Deike Kleberg committed
577
          xassert ( nodeInfo[IOID].nNodes > 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
578
                   nodeInfo[IOID].nNodes <= info->nProcsIO );
579
580
          nNodes = nodeInfo[IOID].nNodes;
        }
Deike Kleberg's avatar
Deike Kleberg committed
581
582
      else xassert ( nodeInfo[IOID].nNodes == nNodes );         
      xassert ( nodeInfo[IOID].hostID > 0 &&
583
               nodeInfo[IOID].hostID <= nNodes );
584
585
586
587
588
      if ( nodeInfo[IOID].isProcColl ) 
        {
          if ( info->nProcsColl == CDI_UNDEFID ) info->nProcsColl = 0;
          info->nProcsColl++;
        }
Deike Kleberg's avatar
Deike Kleberg committed
589
590
    }

591
592
  xdebug ( "info->nProcsColl=%d", info->nProcsColl );

Deike Kleberg's avatar
Deike Kleberg committed
593
594
  xassert ( info->nProcsColl <= info->nProcsModel );

Uwe Schulzweida's avatar
Uwe Schulzweida committed
595
  info->procsCollMap = (int*) xmalloc ( info->nProcsColl * 
Deike Kleberg's avatar
interim    
Deike Kleberg committed
596
597
                                 sizeof ( info->procsCollMap[0] )); 

Deike Kleberg's avatar
Deike Kleberg committed
598
  // define nodeSizes
Deike Kleberg's avatar
Deike Kleberg committed
599
  info->nodeInfo.nNodes = nNodes; 
Uwe Schulzweida's avatar
Uwe Schulzweida committed
600
  info->nodeSizes = (int*) xmalloc ( info->nodeInfo.nNodes * 
Deike Kleberg's avatar
Deike Kleberg committed
601
                                  sizeof ( info->nodeSizes[0] ));
Deike Kleberg's avatar
interim    
Deike Kleberg committed
602
  collID = 0;
Deike Kleberg's avatar
Deike Kleberg committed
603
  for ( IOID = 0; IOID < info->nProcsIO; IOID++ )
604
    if ( nodeInfo[IOID].isProcColl )
Deike Kleberg's avatar
interim    
Deike Kleberg committed
605
606
607
608
      {
        info->nodeSizes[nodeInfo[IOID].hostID - 1]++;
        info->procsCollMap[collID++] = IOID + info->nProcsModel;
      }
Deike Kleberg's avatar
Deike Kleberg committed
609

Deike Kleberg's avatar
Deike Kleberg committed
610
  // define nodeMap
Uwe Schulzweida's avatar
Uwe Schulzweida committed
611
  info->nodeMap   = (int*) xmalloc ( info->nProcsColl * 
Deike Kleberg's avatar
Deike Kleberg committed
612
                                  sizeof ( info->nodeMap[0] ));
Deike Kleberg's avatar
Deike Kleberg committed
613
  // helpers
Uwe Schulzweida's avatar
Uwe Schulzweida committed
614
615
  p1 = (int**) xmalloc ( info->nodeInfo.nNodes * sizeof ( p1[0] ));
  p2 = (int**) xmalloc ( info->nodeInfo.nNodes * sizeof ( p2[0] ));
Deike Kleberg's avatar
Deike Kleberg committed
616
  idx = 0;
Deike Kleberg's avatar
Deike Kleberg committed
617
  for ( i = 0; i < info->nodeInfo.nNodes; i++ )
Deike Kleberg's avatar
Deike Kleberg committed
618
    {
Deike Kleberg's avatar
Deike Kleberg committed
619
      xassert ( idx >= 0 && idx < info->nProcsColl );
Deike Kleberg's avatar
Deike Kleberg committed
620
621
622
      p1[i] = &info->nodeMap[idx];
      p2[i] = p1[i] + info->nodeSizes[i]; 
      idx += info->nodeSizes[i];
Deike Kleberg's avatar
Deike Kleberg committed
623
    }
Deike Kleberg's avatar
Deike Kleberg committed
624
  xassert ( idx == info->nProcsColl );
625
626

  // rankGlob in nodeMap
Deike Kleberg's avatar
Deike Kleberg committed
627
  for ( IOID = 0; IOID < info->nProcsIO; IOID++ )
628
629
    {
      if ( nodeInfo[IOID].isProcColl )
630
        {
Deike Kleberg's avatar
Deike Kleberg committed
631
          xassert ( p1[nodeInfo[IOID].hostID - 1] <  
632
                   p2[nodeInfo[IOID].hostID - 1] );
Deike Kleberg's avatar
Deike Kleberg committed
633
          * p1[nodeInfo[IOID].hostID - 1]++ = IOID + info->nProcsModel;
634
        }
635
636
    }

637
638
639
640
  if ( info->rankGlob == info->root )
    for ( IOID = info->nProcsModel; IOID < info->sizeGlob; IOID++ )
      xmpi ( MPI_Send ( info->nodeMap, info->nProcsColl, MPI_INTEGER, 
                        IOID, NODEMAP, info->commGlob ));
641
642
  free ( p2 );
  free ( p1 );
Deike Kleberg's avatar
Deike Kleberg committed
643
  free ( nodeInfo );
Deike Kleberg's avatar
Deike Kleberg committed
644
645
}

Deike Kleberg's avatar
Deike Kleberg committed
646

647
648
int      commCollID2RankGlob      ( int collID ) 
{
Deike Kleberg's avatar
Deike Kleberg committed
649
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
650
           info->nProcsColl != CDI_UNDEFID &&
651
           collID >= 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
652
653
654
           collID < info->nProcsColl &&
           info->nodeMap != NULL );
  return info->nodeMap[collID];
655
656
657
658
659
660
661
}


int      commRankGlob2CollID      ( int rankGlob)
{
  int out = CDI_UNDEFID, collID;

Deike Kleberg's avatar
Deike Kleberg committed
662
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
663
664
665
666
667
           info->nProcsColl != CDI_UNDEFID &&
           info->nProcsModel != CDI_UNDEFID &&
           info->sizeGlob != CDI_UNDEFID &&
           rankGlob >= info->nProcsModel &&
           rankGlob <= info->sizeGlob );
668

Deike Kleberg's avatar
Deike Kleberg committed
669
670
  for ( collID = 0; collID < info->nProcsColl; collID++ )
    if ( info->nodeMap[collID] == rankGlob ) out = collID;
671
672
673
674
675

  return out;
} 


Deike Kleberg's avatar
Deike Kleberg committed
676
677
int *    commInqNodeSizes   ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
678
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
679
680
           info->nodeSizes != NULL );
  return info->nodeSizes;
Deike Kleberg's avatar
Deike Kleberg committed
681
682
683
}


Deike Kleberg's avatar
Deike Kleberg committed
684
// collective call
685
686
void     commDefCommsIO     ( void )
{
Thomas Jahns's avatar
Thomas Jahns committed
687
  MPI_Group groupGlob;
688
  int collID, * ranks, i, currIORank;
Deike Kleberg's avatar
Deike Kleberg committed
689
690
  char name[MAXCOMMIONAME];

Deike Kleberg's avatar
Deike Kleberg committed
691
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
692
693
694
695
696
           info->nProcsColl != CDI_UNDEFID &&
           info->commsIO == NULL &&
           info->nProcsModel != CDI_UNDEFID &&
           info->commGlob != MPI_COMM_NULL );

Uwe Schulzweida's avatar
Uwe Schulzweida committed
697
  info->commsIO = (MPI_Comm*) xmalloc ( info->nProcsColl * 
Deike Kleberg's avatar
Deike Kleberg committed
698
699
700
                                sizeof ( info->commsIO[0] ));
  for ( collID = 0; collID < info->nProcsColl; collID++ )
    info->commsIO[collID] = MPI_COMM_NULL;
701

Deike Kleberg's avatar
Deike Kleberg committed
702
703
704
  strncpy ( name, "COMMSIO_", 8 ); 
  name[MAXCOMMIONAME - 1] = '\0';
  
Uwe Schulzweida's avatar
Uwe Schulzweida committed
705
  ranks = (int*) xmalloc (( info->nProcsModel + 1 ) * sizeof ( ranks[0] ));
Deike Kleberg's avatar
Deike Kleberg committed
706
  for ( i = 0; i < info->nProcsModel; i++ )
Thomas Jahns's avatar
Thomas Jahns committed
707
    ranks[i] = i;
Deike Kleberg's avatar
Deike Kleberg committed
708
  
Deike Kleberg's avatar
Deike Kleberg committed
709
  xmpi ( MPI_Comm_group ( info->commGlob, &groupGlob )); 
Deike Kleberg's avatar
Deike Kleberg committed
710

Deike Kleberg's avatar
Deike Kleberg committed
711
  for ( collID = 0; collID < info->nProcsColl; collID++ )
Deike Kleberg's avatar
Deike Kleberg committed
712
    {
Deike Kleberg's avatar
Deike Kleberg committed
713
      currIORank = info->nodeMap[collID];
Thomas Jahns's avatar
Thomas Jahns committed
714
      ranks[info->nProcsModel] = currIORank;
Thomas Jahns's avatar
Thomas Jahns committed
715
      MPI_Group currGroupIO;
Thomas Jahns's avatar
Thomas Jahns committed
716
717
718
719
720
      xmpi(MPI_Group_incl(groupGlob, info->nProcsModel + 1,
                          ranks, &currGroupIO));
      xmpi(MPI_Comm_create(info->commGlob, currGroupIO,
                           info->commsIO + collID));
      xmpi(MPI_Group_free(&currGroupIO));
Deike Kleberg's avatar
Deike Kleberg committed
721
722
      if ( info->rankGlob == currIORank )
	info->commCalc = info->commsIO[collID];
Deike Kleberg's avatar
Deike Kleberg committed
723

Deike Kleberg's avatar
Deike Kleberg committed
724
      // set names for debugging
Deike Kleberg's avatar
Deike Kleberg committed
725
726
      if ( info->rankGlob < info->nProcsModel || 
           info->rankGlob == currIORank )
Deike Kleberg's avatar
Deike Kleberg committed
727
	{
728
	  sprintf ( &name[8], "%d", collID ); 
Deike Kleberg's avatar
Deike Kleberg committed
729
	  xmpi ( MPI_Comm_set_name ( info->commsIO[collID], name ));
Deike Kleberg's avatar
Deike Kleberg committed
730
731
732
733
734
	}
    }

  if ( ddebug >= 2 )
    {
Deike Kleberg's avatar
Deike Kleberg committed
735
736
737
738
739
      if ( info->rankGlob < info->nProcsModel )
	for ( collID = 0; collID < info->nProcsColl; collID++ ) 
          xdebugComm ( info->commsIO + collID );
      else if ( info->nodeInfo.isProcColl )
	xdebugComm ( &info->commCalc );
Deike Kleberg's avatar
Deike Kleberg committed
740
741
742
743
744
745
    }

  xmpi ( MPI_Group_free ( &groupGlob ));
  free ( ranks );

  commPrint ( stdout );
746
747
748
}


749
MPI_Comm commInqCommsIO     ( int collID )
750
{
Deike Kleberg's avatar
Deike Kleberg committed
751
  xassert ( collID >= 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
752
753
754
755
756
           info != NULL &&
           info->nProcsColl != CDI_UNDEFID &&
           collID < info->nProcsColl &&
           info->commsIO != NULL &&
           info->commsIO[collID] != MPI_COMM_NULL );
757

Deike Kleberg's avatar
Deike Kleberg committed
758
  return info->commsIO[collID];
759
760
761
762
763
}


MPI_Comm commInqCommCalc    ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
764
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
765
766
           info->commCalc != MPI_COMM_NULL );
  return info->commCalc;
767
768
769
}


770
771
int      commInqNProcsColl  ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
772
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
773
774
           info->nProcsColl != CDI_UNDEFID );
  return info->nProcsColl;
775
776
777
}


Deike Kleberg's avatar
Deike Kleberg committed
778
779
780
781
void commPrint ( FILE * fp )
{
  int i;

Deike Kleberg's avatar
Deike Kleberg committed
782
783
  if ( ddebug == 0 ) return;

Deike Kleberg's avatar
Deike Kleberg committed
784
  xassert ( info != NULL );
Deike Kleberg's avatar
Deike Kleberg committed
785
786

  fprintf ( fp, "\n" );
Deike Kleberg's avatar
Deike Kleberg committed
787
  fprintf ( fp, "######## pioinfo PE%d ###########\n", info->rankGlob );
Deike Kleberg's avatar
Deike Kleberg committed
788
  fprintf ( fp, "#\n" );
Deike Kleberg's avatar
Deike Kleberg committed
789
790
791
792
  fprintf ( fp, "# IOMode      = %d\n", info->IOMode );
  fprintf ( fp, "# nProcsIO    = %d\n", info->nProcsIO );
  fprintf ( fp, "# nProcsModel = %d\n", info->nProcsModel );
  fprintf ( fp, "# isProcIO    = %d\n", info->isProcIO );
Deike Kleberg's avatar
Deike Kleberg committed
793
  fprintf ( fp, "#\n" );
794
  fprintf ( fp, "# commGlob    = %d\n", (int)MPI_Comm_c2f(info->commGlob));
Deike Kleberg's avatar
Deike Kleberg committed
795
796
797
  fprintf ( fp, "# sizeGlob    = %d\n", info->sizeGlob );
  fprintf ( fp, "# rankGlob    = %d\n", info->rankGlob );
  fprintf ( fp, "# root        = %d\n", info->root );
Deike Kleberg's avatar
Deike Kleberg committed
798
  fprintf ( fp, "#\n" );
799
  fprintf ( fp, "# commPio     = %d\n", (int)MPI_Comm_c2f(info->commPio));
Deike Kleberg's avatar
Deike Kleberg committed
800
801
  fprintf ( fp, "# sizePio     = %d\n", info->sizePio );
  fprintf ( fp, "# rankPio     = %d\n", info->rankPio );
Deike Kleberg's avatar
Deike Kleberg committed
802
  fprintf ( fp, "#\n" );
803
  fprintf ( fp, "# commNode    = %d\n", (int)MPI_Comm_c2f(info->commNode));
Deike Kleberg's avatar
Deike Kleberg committed
804
805
806
807
808
  fprintf ( fp, "# sizeNode    = %d\n", info->sizeNode );
  fprintf ( fp, "# rankNode    = %d\n", info->rankNode );
  fprintf ( fp, "# hostname    = %s\n", info->hostname[0] == 0 ? "nn" : 
	    info->hostname );
  fprintf ( fp, "# specialRankNode = %d\n", info->specialRankNode );
809
  fprintf ( fp, "#\n" );
810
  fprintf ( fp, "# commColl    = %d\n", (int)MPI_Comm_c2f(info->commColl));
Deike Kleberg's avatar
Deike Kleberg committed
811
812
  fprintf ( fp, "# sizeColl    = %d\n", info->sizeColl );
  fprintf ( fp, "# rankColl    = %d\n", info->rankColl );
813
  fprintf ( fp, "#\n" );
814
  fprintf ( fp, "# commCalc    = %d\n", (int)MPI_Comm_c2f(info->commCalc));
Deike Kleberg's avatar
Deike Kleberg committed
815
  if ( info->commsIO != NULL )
816
817
    {
      fprintf ( fp, "#\n" );
Deike Kleberg's avatar
Deike Kleberg committed
818
      for ( i = 0; i < info->nProcsColl; i++ )
819
820
	fprintf(fp, "# commsIO[%d]  = %d\n", i,
                (int)MPI_Comm_c2f(info->commsIO[i]));
821
822
823
    }
  else
    fprintf ( fp, "# commsIO     = NULL\n" );
Deike Kleberg's avatar
Deike Kleberg committed
824
825
826
827
  fprintf ( fp, "# hostID      = %d\n", info->nodeInfo.hostID );
  fprintf ( fp, "# isProcColl  = %d\n", info->nodeInfo.isProcColl );
  fprintf ( fp, "# nNodes      = %d\n", info->nodeInfo.nNodes );
  fprintf ( fp, "# nProcsColl  = %d\n", info->nProcsColl );
Deike Kleberg's avatar
interim    
Deike Kleberg committed
828
829
830
831
832
833
834
835
836
837
  if ( info->procsCollMap != NULL )
    {
      fprintf ( fp, "# procsCollMap   = " );
      xassert ( info->nProcsColl != CDI_UNDEFID );
      for ( i = 0; i < info->nProcsColl; i++ )
        fprintf ( fp, "%d ", info->procsCollMap[i] );
      fprintf ( fp, "\n" );
    }
  else     
    fprintf ( fp, "# procsCollMap   = NULL\n" );
Deike Kleberg's avatar
Deike Kleberg committed
838
  if ( info->nodeSizes != NULL )
Deike Kleberg's avatar
Deike Kleberg committed
839
840
    {
      fprintf ( fp, "# nodeSizes   = " );
Deike Kleberg's avatar
Deike Kleberg committed
841
      xassert ( info->nodeInfo.nNodes != CDI_UNDEFID );
Deike Kleberg's avatar
Deike Kleberg committed
842
843
      for ( i = 0; i < info->nodeInfo.nNodes; i++ )
        fprintf ( fp, "%d ", info->nodeSizes[i] );
Deike Kleberg's avatar
Deike Kleberg committed
844
845
846
847
      fprintf ( fp, "\n" );
    }
  else     
    fprintf ( fp, "# nodeSizes   = NULL\n" );
Deike Kleberg's avatar
Deike Kleberg committed
848
  if ( info->nodeMap != NULL )
Deike Kleberg's avatar
Deike Kleberg committed
849
850
    {
      fprintf ( fp, "# nodeMap     = " );
Deike Kleberg's avatar
Deike Kleberg committed
851
      xassert ( info->nProcsColl != CDI_UNDEFID );
Deike Kleberg's avatar
Deike Kleberg committed
852
853
      for ( i = 0; i < info->nProcsColl; i++ )
        fprintf ( fp, "%d ", info->nodeMap[i] );
Deike Kleberg's avatar
Deike Kleberg committed
854
855
856
857
      fprintf ( fp, "\n" );
    }
  else     
    fprintf ( fp, "# nodeMap     = NULL\n" );
Deike Kleberg's avatar
Deike Kleberg committed
858
859
860
861
  fprintf ( fp, "#\n" );
  fprintf ( fp, "############################\n" );
  fprintf ( fp, "\n" );
}
Deike Kleberg's avatar
Deike Kleberg committed
862
863

#endif
Deike Kleberg's avatar
Deike Kleberg committed
864
865
866
867
868
869
870
871
872
873
874
875


/************************************************************************/
/*
 * Local Variables:
 * c-file-style: "Java"
 * c-basic-offset: 2
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */