pio_comm.c 22 KB
Newer Older
Deike Kleberg's avatar
Deike Kleberg committed
1
2
3
4
5
6
7
8
9
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <stdio.h>
#include <stdlib.h>

#include "pio_comm.h"
#include "cdi.h"
10
#include "dmemory.h"
Deike Kleberg's avatar
Deike Kleberg committed
11
12
#include "pio_util.h"

13
14
#include "cdipio.h"

15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
typedef struct {
  int IOMode;
  int nProcsIO;
  int nProcsModel;
  int isProcIO;

  MPI_Comm commGlob;
  int sizeGlob;
  int rankGlob;
  int root;

  MPI_Comm commPio;
  int sizePio;
  int rankPio;

  MPI_Comm commNode;
  int sizeNode;
  int rankNode;
  char hostname [ MPI_MAX_PROCESSOR_NAME + 1 ];
  nodeInfo_t nodeInfo;
  int specialRankNode;

  MPI_Comm commColl;
  int sizeColl;
  int rankColl;

  MPI_Comm commCalc;
42
  int rankCalc, sizeCalc;
43
44
45
46
47
48
49
50
51

  MPI_Comm * commsIO;
  int nProcsColl;
  int * procsCollMap;
  int * nodeSizes;
  int * nodeMap;
} pioInfo_t;


Deike Kleberg's avatar
Deike Kleberg committed
52
static pioInfo_t * info = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
53

Deike Kleberg's avatar
Deike Kleberg committed
54

Deike Kleberg's avatar
Deike Kleberg committed
55
static
Deike Kleberg's avatar
Deike Kleberg committed
56
void pioInfoInit ( pioInfo_t * p )
Deike Kleberg's avatar
Deike Kleberg committed
57
{
Deike Kleberg's avatar
Deike Kleberg committed
58
59
60
61
  p->IOMode               = CDI_UNDEFID;
  p->nProcsIO             = CDI_UNDEFID;
  p->nProcsModel          = CDI_UNDEFID;
  p->isProcIO             = CDI_UNDEFID;
62

Deike Kleberg's avatar
Deike Kleberg committed
63
64
65
66
  p->commGlob             = MPI_COMM_NULL;
  p->sizeGlob             = CDI_UNDEFID;
  p->rankGlob             = CDI_UNDEFID;
  p->root                 = CDI_UNDEFID;
67

Deike Kleberg's avatar
Deike Kleberg committed
68
69
  p->commPio              = MPI_COMM_NULL;
  p->sizePio              = CDI_UNDEFID;
70
71
  p->rankPio              = -1;

Deike Kleberg's avatar
Deike Kleberg committed
72
73
74
75
76
  p->commNode             = MPI_COMM_NULL;
  p->sizeNode             = CDI_UNDEFID;
  p->rankNode             = CDI_UNDEFID;
  p->hostname[0]          = 0;
  p->specialRankNode      = CDI_UNDEFID;
77

Deike Kleberg's avatar
Deike Kleberg committed
78
79
80
  p->commColl             = MPI_COMM_NULL;
  p->sizeColl             = CDI_UNDEFID;
  p->rankColl             = CDI_UNDEFID;
81

Deike Kleberg's avatar
Deike Kleberg committed
82
  p->commCalc             = MPI_COMM_NULL;
83
84
85
  p->rankCalc             = -1;
  p->sizeCalc             = -1;

Deike Kleberg's avatar
Deike Kleberg committed
86
87
88
89
  p->commsIO              = NULL;
  p->nodeInfo.hostID      = CDI_UNDEFID;
  p->nodeInfo.isProcColl  = CDI_UNDEFID;
  p->nodeInfo.nNodes      = CDI_UNDEFID;
90
  p->nProcsColl           = CDI_UNDEFID;
Deike Kleberg's avatar
interim    
Deike Kleberg committed
91
  p->procsCollMap         = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
92
93
  p->nodeSizes            = NULL;
  p->nodeMap              = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
94
95
96
}


Deike Kleberg's avatar
Deike Kleberg committed
97
void commInit ( void )
Deike Kleberg's avatar
Deike Kleberg committed
98
{
Deike Kleberg's avatar
Deike Kleberg committed
99
  xassert ( info == 0 );
100
  info = xcalloc(1, sizeof (pioInfo_t));
Deike Kleberg's avatar
Deike Kleberg committed
101
  pioInfoInit ( info );
Deike Kleberg's avatar
Deike Kleberg committed
102
103
104
}


Deike Kleberg's avatar
Deike Kleberg committed
105
void commDestroy ( void )
Deike Kleberg's avatar
Deike Kleberg committed
106
{
107
  int collID;
Deike Kleberg's avatar
Deike Kleberg committed
108

Deike Kleberg's avatar
Deike Kleberg committed
109
  xassert ( info != NULL );
Deike Kleberg's avatar
Deike Kleberg committed
110

Deike Kleberg's avatar
Deike Kleberg committed
111
112
  if ( info->nodeMap   != NULL ) free ( info->nodeMap );
  if ( info->nodeSizes != NULL ) free ( info->nodeSizes );
Deike Kleberg's avatar
Deike Kleberg committed
113

Thomas Jahns's avatar
Thomas Jahns committed
114
  if ( info->commsIO != NULL )
Deike Kleberg's avatar
Deike Kleberg committed
115
    {
Deike Kleberg's avatar
Deike Kleberg committed
116
117
      for ( collID = 0; collID < info->nProcsColl; collID++ )
	if ( info->commsIO[collID] != MPI_COMM_NULL )
Thomas Jahns's avatar
Thomas Jahns committed
118
	  xmpi(MPI_Comm_free(info->commsIO + collID));
Thomas Jahns's avatar
Thomas Jahns committed
119
      free ( info->commsIO );
Deike Kleberg's avatar
Deike Kleberg committed
120
      info->commsIO = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
121
    }
122

Deike Kleberg's avatar
Deike Kleberg committed
123
  if ( info->commColl != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
124
    {
Thomas Jahns's avatar
Thomas Jahns committed
125
      xmpi ( MPI_Comm_free ( &info->commColl ));
Deike Kleberg's avatar
Deike Kleberg committed
126
      info->commColl = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
127
    }
Thomas Jahns's avatar
Thomas Jahns committed
128
129
130

  free(info->procsCollMap);

Deike Kleberg's avatar
Deike Kleberg committed
131
  if ( info->commNode != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
132
    {
Thomas Jahns's avatar
Thomas Jahns committed
133
      xmpi ( MPI_Comm_free ( &info->commNode ));
Deike Kleberg's avatar
Deike Kleberg committed
134
      info->commNode = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
135
    }
Thomas Jahns's avatar
Thomas Jahns committed
136

Deike Kleberg's avatar
Deike Kleberg committed
137
  if ( info->commPio != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
138
    {
Thomas Jahns's avatar
Thomas Jahns committed
139
      xmpi ( MPI_Comm_free ( &info->commPio ));
Thomas Jahns's avatar
Thomas Jahns committed
140
      info->commPio = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
141
    }
Thomas Jahns's avatar
Thomas Jahns committed
142

Deike Kleberg's avatar
Deike Kleberg committed
143
144
  free ( info );
  info = NULL;
Deike Kleberg's avatar
Deike Kleberg committed
145
146
147
}


Deike Kleberg's avatar
Deike Kleberg committed
148
149
void commDefCommGlob ( MPI_Comm c )
{
Deike Kleberg's avatar
Deike Kleberg committed
150
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
151
152
	   c != MPI_COMM_NULL );

Deike Kleberg's avatar
Deike Kleberg committed
153
154
155
156
  info->commGlob = c;
  xmpi ( MPI_Comm_size ( c, &info->sizeGlob ));
  xmpi ( MPI_Comm_rank ( c, &info->rankGlob ));
  info->root = 0;
Deike Kleberg's avatar
Deike Kleberg committed
157
158
159
160
161
}
 

MPI_Comm commInqCommGlob ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
162
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
163
164
           info->commGlob != MPI_COMM_NULL );
  return info->commGlob;
Deike Kleberg's avatar
Deike Kleberg committed
165
166
167
168
169
}


int commInqSizeGlob ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
170
  xassert ( info != NULL  &&
Deike Kleberg's avatar
Deike Kleberg committed
171
172
           info->sizeGlob != CDI_UNDEFID );
  return info->sizeGlob;
Deike Kleberg's avatar
Deike Kleberg committed
173
174
175
176
177
}


int commInqRankGlob ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
178
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
179
180
           info->rankGlob != CDI_UNDEFID );
  return info->rankGlob;
Deike Kleberg's avatar
Deike Kleberg committed
181
182
183
}


184
185
int commInqRootGlob ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
186
  xassert ( info != NULL &&
187
188
189
190
191
           info->root != CDI_UNDEFID );
  return info->root;
}


Deike Kleberg's avatar
Deike Kleberg committed
192
193
void commDefNProcsIO ( int n )
{
Deike Kleberg's avatar
Deike Kleberg committed
194
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
195
	   n >= 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
196
	   n < MAXNPROCSIO &&
Deike Kleberg's avatar
Deike Kleberg committed
197
198
199
           info->commGlob != MPI_COMM_NULL );
  info->nProcsIO = n;
  info->nProcsModel = info->sizeGlob - info->nProcsIO;
Deike Kleberg's avatar
Deike Kleberg committed
200
201
202
203
204
}


int commInqNProcsIO ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
205
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
206
207
           info->nProcsIO != CDI_UNDEFID );
  return info->nProcsIO;
Deike Kleberg's avatar
Deike Kleberg committed
208
209
210
}


Deike Kleberg's avatar
Deike Kleberg committed
211
int      commInqIsProcIO     ( void )
Deike Kleberg's avatar
Deike Kleberg committed
212
{
Deike Kleberg's avatar
Deike Kleberg committed
213
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
214
215
           info->isProcIO != CDI_UNDEFID );
  return info->isProcIO;
Deike Kleberg's avatar
Deike Kleberg committed
216
217
218
219
220
}


int commInqNProcsModel ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
221
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
222
223
           info->nProcsModel != CDI_UNDEFID );
  return info->nProcsModel;
Deike Kleberg's avatar
Deike Kleberg committed
224
225
226
}


227
void     commDefIOMode  ( int IOMode )
Deike Kleberg's avatar
Deike Kleberg committed
228
{
229
  xassert(info != NULL && IOMode >= PIO_MINIOMODE && IOMode <= PIO_MAXIOMODE );
Deike Kleberg's avatar
Deike Kleberg committed
230
  info->IOMode = IOMode;
Deike Kleberg's avatar
Deike Kleberg committed
231
232
233
}


Deike Kleberg's avatar
Deike Kleberg committed
234
int      commInqIOMode  ( void )
Deike Kleberg's avatar
Deike Kleberg committed
235
{
236
237
238
239
240
241
242
  if (info != NULL)
    {
      xassert ( info->IOMode != CDI_UNDEFID );
      return info->IOMode;
    }
  else
    return PIO_NONE;
Deike Kleberg's avatar
Deike Kleberg committed
243
244
}

Deike Kleberg's avatar
Deike Kleberg committed
245

Deike Kleberg's avatar
Deike Kleberg committed
246
247
void     commDefCommPio  ( void )
{
248
249
250
  int nProcsCalc;
  MPI_Group grpAll, grpCalc, grpPio;
  int calcRange[3], pioRange[3];
Deike Kleberg's avatar
Deike Kleberg committed
251

Deike Kleberg's avatar
Deike Kleberg committed
252
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
253
254
255
           info->commGlob != MPI_COMM_NULL &&
           info->IOMode  != CDI_UNDEFID &&
           info->nProcsIO != CDI_UNDEFID );
Deike Kleberg's avatar
Deike Kleberg committed
256

257
258
259
260
261
  nProcsCalc = info->sizeGlob - info->nProcsIO;

  info->isProcIO = info->rankGlob >= nProcsCalc ? 1 : 0;

  xmpi(MPI_Comm_group(info->commGlob, &grpAll));
Deike Kleberg's avatar
Deike Kleberg committed
262

263
264
265
  calcRange[0] = 0;
  calcRange[1] = info->sizeGlob - info->nProcsIO - 1;
  calcRange[2] = 1;
266
267
  xmpi(MPI_Group_range_incl(grpAll, info->nProcsIO==info->sizeGlob?0:1,
                            &calcRange, &grpCalc));
268
  xmpi(MPI_Comm_create(info->commGlob, grpCalc, &info->commCalc));
Deike Kleberg's avatar
Deike Kleberg committed
269

270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
  pioRange[0] = info->sizeGlob - info->nProcsIO;
  pioRange[1] = info->sizeGlob - 1;
  pioRange[2] = 1;
  xmpi(MPI_Group_range_incl(grpAll, 1, &pioRange, &grpPio));
  xmpi(MPI_Comm_create(info->commGlob, grpPio, &info->commPio));

  if (info->commPio != MPI_COMM_NULL)
    {
      xmpi(MPI_Comm_size(info->commPio, &info->sizePio));
      xmpi(MPI_Comm_rank(info->commPio, &info->rankPio));
      info->sizeCalc = nProcsCalc;
      info->rankCalc = -1;
    }
  else
    {
      info->sizePio = nProcsCalc;
      info->rankPio = -1;
      xmpi(MPI_Comm_size(info->commCalc, &info->sizeCalc));
      xmpi(MPI_Comm_rank(info->commCalc, &info->rankCalc));
    }
  xmpi(MPI_Group_free(&grpCalc));
  xmpi(MPI_Group_free(&grpPio));
  xmpi(MPI_Group_free(&grpAll));
Deike Kleberg's avatar
Deike Kleberg committed
293
294
295
296
297
}


MPI_Comm commInqCommPio  ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
298
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
299
300
           info->commPio != MPI_COMM_NULL &&
           info->isProcIO == 1 );
Deike Kleberg's avatar
Deike Kleberg committed
301
  return info->commPio;
Deike Kleberg's avatar
Deike Kleberg committed
302
303
304
}


Deike Kleberg's avatar
Deike Kleberg committed
305
MPI_Comm commInqCommModel ( void )
Deike Kleberg's avatar
Deike Kleberg committed
306
{
Deike Kleberg's avatar
Deike Kleberg committed
307
  xassert ( info != NULL &&
308
           info->commCalc != MPI_COMM_NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
309
           info->isProcIO == 0 );
310
  return info->commCalc;
Deike Kleberg's avatar
Deike Kleberg committed
311
312
313
}


314
315
int      commInqRankPio       ( void )
{ 
Deike Kleberg's avatar
Deike Kleberg committed
316
  xassert ( info != NULL &&
317
           info->rankPio >= 0 &&
318
319
320
321
322
           info->isProcIO == 1 );
  return info->rankPio;
}


Deike Kleberg's avatar
Deike Kleberg committed
323
324
int      commInqRankModel       ( void )
{ 
Deike Kleberg's avatar
Deike Kleberg committed
325
  xassert ( info != NULL &&
326
           info->rankCalc >= 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
327
           info->isProcIO == 0 );
328
  return info->rankCalc;
Deike Kleberg's avatar
Deike Kleberg committed
329
330
331
}


Deike Kleberg's avatar
Deike Kleberg committed
332
void     commDefCommColl  ( int isProcColl )
Deike Kleberg's avatar
Deike Kleberg committed
333
{
Deike Kleberg's avatar
Deike Kleberg committed
334
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
335
336
           info->commNode != MPI_COMM_NULL &&
           info->commColl == MPI_COMM_NULL );
Deike Kleberg's avatar
Deike Kleberg committed
337

338
  info->nodeInfo.isProcColl = isProcColl;
Deike Kleberg's avatar
Deike Kleberg committed
339
340
341
342
  xmpi ( MPI_Comm_split ( info->commNode, info->nodeInfo.isProcColl, 0, 
                          &info->commColl ));
  xmpi ( MPI_Comm_size ( info->commColl, &info->sizeColl ));
  xmpi ( MPI_Comm_rank ( info->commColl, &info->rankColl )); 
Deike Kleberg's avatar
Deike Kleberg committed
343
344
345
}


346
347
MPI_Comm commInqCommColl    ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
348
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
349
350
           info->commColl != MPI_COMM_NULL );
  return info->commColl;
351
352
353
354
355
}


int      commInqSizeColl    ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
356
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
357
358
           info->sizeColl != CDI_UNDEFID );
  return info->sizeColl; 
359
360
361
362
363
}


int      commInqRankColl    ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
364
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
365
366
           info->rankColl != CDI_UNDEFID );
  return info->rankColl; 
367
368
}

369
370
static int
cmpstringp(const void *p1, const void *p2)
Deike Kleberg's avatar
Deike Kleberg committed
371
{
372
  return strcmp(* (char * const *) p1, * (char * const *) p2);
Deike Kleberg's avatar
Deike Kleberg committed
373
374
375
376
}

void commDefCommNode ( void )
{
377
378
  int size;
  char * myHost, (*allHosts)[MPI_MAX_PROCESSOR_NAME], **sortedHosts;
Deike Kleberg's avatar
Deike Kleberg committed
379

Deike Kleberg's avatar
Deike Kleberg committed
380
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
381
           info->commPio != MPI_COMM_NULL );
Deike Kleberg's avatar
Deike Kleberg committed
382

Deike Kleberg's avatar
Deike Kleberg committed
383
  size = info->sizePio;
Deike Kleberg's avatar
Deike Kleberg committed
384

Uwe Schulzweida's avatar
Uwe Schulzweida committed
385
  myHost = (char*) xmalloc(MPI_MAX_PROCESSOR_NAME);
386
387
388
389
  {
    int len;
    xmpi ( MPI_Get_processor_name ( myHost, &len ));
    xassert ( myHost[0] != '\0' );
390
    strncpy(info->hostname, myHost, (size_t)len);
391
392
393
    info->hostname[len] = '\0';
  }

394
395
  allHosts = xmalloc((size_t)size * MPI_MAX_PROCESSOR_NAME);
  sortedHosts = xmalloc((size_t)size * sizeof (sortedHosts[0]));
396

397
398
  for (int i = 0; i < size; ++i)
    sortedHosts[i] = allHosts[i];
399
400
401
402
403

  xmpi(MPI_Allgather(myHost, MPI_MAX_PROCESSOR_NAME, MPI_CHAR,
                     allHosts[0], MPI_MAX_PROCESSOR_NAME,
                     MPI_CHAR, info->commPio ));

404
  qsort(sortedHosts, (size_t)size, sizeof (sortedHosts[0]),
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
        (int (*)(const void *, const void *))cmpstringp);

  {
    int i = 1, nHosts = 1;
    if (!strcmp(myHost, sortedHosts[0]))
      {
        info->nodeInfo.hostID = 1;
      }
    else
      {
        for (; i < size && strcmp(myHost, sortedHosts[i]); ++i)
          if (strcmp(sortedHosts[i - 1], sortedHosts[i]))
            {
              /* new host seen, might be ours */
              nHosts += 1;
              if (!strcmp(sortedHosts[i], myHost))
                info->nodeInfo.hostID = nHosts - 1;
            }
      }
    for (; i < size && strcmp(myHost, sortedHosts[i]); ++i)
      if (strcmp(sortedHosts[i - 1], sortedHosts[i]))
        nHosts += 1;
    info->nodeInfo.nNodes = nHosts;
  }
Deike Kleberg's avatar
Deike Kleberg committed
429

Deike Kleberg's avatar
Deike Kleberg committed
430
  xassert ( info->nodeInfo.hostID != CDI_UNDEFID );
Deike Kleberg's avatar
Deike Kleberg committed
431

Deike Kleberg's avatar
Deike Kleberg committed
432
433
434
435
  xmpi ( MPI_Comm_split ( info->commPio, info->nodeInfo.hostID, 0, 
                          &info->commNode ));
  xmpi ( MPI_Comm_size ( info->commNode, &info->sizeNode ));
  xmpi ( MPI_Comm_rank ( info->commNode, &info->rankNode ));
436
  if ( info->IOMode >= PIO_MINIOMODEWITHSPECIALPROCS )
Deike Kleberg's avatar
Deike Kleberg committed
437
    info->specialRankNode = info->sizeNode - 1;
Deike Kleberg's avatar
Deike Kleberg committed
438

439
440
441
  free(sortedHosts);
  free(allHosts);
  free(myHost);
Deike Kleberg's avatar
Deike Kleberg committed
442
443

  return;
Deike Kleberg's avatar
Deike Kleberg committed
444
445
446
}


Deike Kleberg's avatar
Deike Kleberg committed
447
MPI_Comm commInqCommNode    ( void )
Deike Kleberg's avatar
Deike Kleberg committed
448
{
Deike Kleberg's avatar
Deike Kleberg committed
449
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
450
451
           info->commNode != MPI_COMM_NULL );
  return info->commNode;
Deike Kleberg's avatar
Deike Kleberg committed
452
} 
Deike Kleberg's avatar
Deike Kleberg committed
453
454


Deike Kleberg's avatar
Deike Kleberg committed
455
456
int commInqSizeNode ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
457
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
458
459
           info->sizeNode != CDI_UNDEFID );
  return info->sizeNode;
Deike Kleberg's avatar
Deike Kleberg committed
460
461
462
}


Deike Kleberg's avatar
Deike Kleberg committed
463
int commInqRankNode ( void )
Deike Kleberg's avatar
Deike Kleberg committed
464
{
Deike Kleberg's avatar
Deike Kleberg committed
465
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
466
467
           info->rankNode != CDI_UNDEFID );
  return info->rankNode;
Deike Kleberg's avatar
Deike Kleberg committed
468
469
470
}


471
472
int commInqSpecialRankNode ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
473
  xassert ( info != NULL );
Deike Kleberg's avatar
Deike Kleberg committed
474
  return info->specialRankNode;
475
476
477
}


Deike Kleberg's avatar
Deike Kleberg committed
478
479
void commSendNodeInfo ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
480
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
481
482
483
484
485
           info->root                != CDI_UNDEFID &&
           info->nodeInfo.hostID     != CDI_UNDEFID &&
           info->nodeInfo.isProcColl != CDI_UNDEFID &&
           info->nodeInfo.nNodes     != CDI_UNDEFID &&
           info->commGlob            != MPI_COMM_NULL );
Deike Kleberg's avatar
Deike Kleberg committed
486
  
Deike Kleberg's avatar
Deike Kleberg committed
487
488
  xmpi ( MPI_Send ( &info->nodeInfo, sizeNodeInfo, MPI_INTEGER, 
                    info->root, NODEINFO, info->commGlob ));
Deike Kleberg's avatar
Deike Kleberg committed
489
490
491
}


Deike Kleberg's avatar
Deike Kleberg committed
492
493
494
void     commRecvNodeMap    ( void )
{
  MPI_Status status;
Deike Kleberg's avatar
Deike Kleberg committed
495
  int source; 
Deike Kleberg's avatar
Deike Kleberg committed
496

Deike Kleberg's avatar
Deike Kleberg committed
497
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
498
499
500
501
           info->commGlob    != MPI_COMM_NULL &&
           info->rankGlob    != CDI_UNDEFID &&
           info->nProcsModel != CDI_UNDEFID &&
           info->nodeMap     == NULL );
Deike Kleberg's avatar
Deike Kleberg committed
502

503
  source = info->root;
Deike Kleberg's avatar
Deike Kleberg committed
504

Deike Kleberg's avatar
Deike Kleberg committed
505
506
  xmpi ( MPI_Probe ( source, NODEMAP, info->commGlob, &status ));
  xmpi ( MPI_Get_count ( &status, MPI_INTEGER, &info->nProcsColl ));
Deike Kleberg's avatar
Deike Kleberg committed
507

Deike Kleberg's avatar
Deike Kleberg committed
508
509
  xdebug ( "info->nProcsColl=%d", info->nProcsColl );

510
511
  info->nodeMap = (int *)xmalloc((size_t)info->nProcsColl
                                 * sizeof (info->nodeMap[0]));
Deike Kleberg's avatar
Deike Kleberg committed
512

Deike Kleberg's avatar
Deike Kleberg committed
513
514
  xmpi ( MPI_Recv ( info->nodeMap, info->nProcsColl, MPI_INTEGER, 
                    source, NODEMAP, info->commGlob, &status ));
Deike Kleberg's avatar
Deike Kleberg committed
515
516
517
}


Deike Kleberg's avatar
Deike Kleberg committed
518
519
void     commDefNNodes      ( int nNodes )
{
Deike Kleberg's avatar
Deike Kleberg committed
520
  xassert ( info != NULL );
Deike Kleberg's avatar
Deike Kleberg committed
521
  info->nodeInfo.nNodes = nNodes;
Deike Kleberg's avatar
Deike Kleberg committed
522
523
524
}


Deike Kleberg's avatar
Deike Kleberg committed
525
526
int      commInqNNodes      ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
527
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
528
529
           info->nodeInfo.nNodes != CDI_UNDEFID );
  return info->nodeInfo.nNodes;
Deike Kleberg's avatar
Deike Kleberg committed
530
531
532
}


Deike Kleberg's avatar
Deike Kleberg committed
533
void     commEvalPhysNodes  ( void )
Deike Kleberg's avatar
Deike Kleberg committed
534
{
Deike Kleberg's avatar
Deike Kleberg committed
535
  nodeInfo_t * nodeInfo;
Deike Kleberg's avatar
Deike Kleberg committed
536
  MPI_Status status;
Deike Kleberg's avatar
interim    
Deike Kleberg committed
537
  int i, IOID, collID, size, nNodes = CDI_UNDEFID;
538
  int ** p1, ** p2,  idx;
Deike Kleberg's avatar
Deike Kleberg committed
539

Deike Kleberg's avatar
Deike Kleberg committed
540
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
541
542
543
544
545
546
           info->root        != CDI_UNDEFID &&
           info->nProcsIO    != CDI_UNDEFID &&
           info->nProcsModel != CDI_UNDEFID &&
           info->sizeGlob    != CDI_UNDEFID &&
           info->rankGlob    != CDI_UNDEFID &&
           info->commGlob    != MPI_COMM_NULL );
Deike Kleberg's avatar
Deike Kleberg committed
547

Deike Kleberg's avatar
Deike Kleberg committed
548
  size = info->nProcsIO * sizeNodeInfo;
Deike Kleberg's avatar
Deike Kleberg committed
549

550
  nodeInfo = (nodeInfo_t *)xmalloc((size_t)size * sizeof (int));
Deike Kleberg's avatar
Deike Kleberg committed
551

Deike Kleberg's avatar
Deike Kleberg committed
552
  if ( info->rankGlob == info->root )
Deike Kleberg's avatar
Deike Kleberg committed
553
    {
554
      xassert ( info->rankCalc == info->root );
Deike Kleberg's avatar
Deike Kleberg committed
555
 
Deike Kleberg's avatar
Deike Kleberg committed
556
      for ( i = info->nProcsModel; i < info->sizeGlob; i++ )
Deike Kleberg's avatar
Deike Kleberg committed
557
        {
Deike Kleberg's avatar
Deike Kleberg committed
558
          IOID = i - info->nProcsModel;
Deike Kleberg's avatar
Deike Kleberg committed
559
          xmpi ( MPI_Recv ( nodeInfo + IOID, sizeNodeInfo, MPI_INTEGER, i, 
Deike Kleberg's avatar
Deike Kleberg committed
560
                            NODEINFO, info->commGlob, &status ));
Deike Kleberg's avatar
Deike Kleberg committed
561
562
563
        }
    }

Deike Kleberg's avatar
Deike Kleberg committed
564
  xmpi ( MPI_Bcast ( nodeInfo, size, MPI_INTEGER, info->root, 
565
                     info->commCalc ));
Deike Kleberg's avatar
Deike Kleberg committed
566

567
  // consistency check, count collectors
Deike Kleberg's avatar
Deike Kleberg committed
568
  for ( IOID = 0; IOID < info->nProcsIO; IOID++ )
Deike Kleberg's avatar
Deike Kleberg committed
569
    {
570
571
      if ( IOID == 0 )
        {
Deike Kleberg's avatar
Deike Kleberg committed
572
          xassert ( nodeInfo[IOID].nNodes > 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
573
                   nodeInfo[IOID].nNodes <= info->nProcsIO );
574
575
          nNodes = nodeInfo[IOID].nNodes;
        }
Deike Kleberg's avatar
Deike Kleberg committed
576
577
      else xassert ( nodeInfo[IOID].nNodes == nNodes );         
      xassert ( nodeInfo[IOID].hostID > 0 &&
578
               nodeInfo[IOID].hostID <= nNodes );
579
580
581
582
583
      if ( nodeInfo[IOID].isProcColl ) 
        {
          if ( info->nProcsColl == CDI_UNDEFID ) info->nProcsColl = 0;
          info->nProcsColl++;
        }
Deike Kleberg's avatar
Deike Kleberg committed
584
585
    }

586
587
  xdebug ( "info->nProcsColl=%d", info->nProcsColl );

Deike Kleberg's avatar
Deike Kleberg committed
588
589
  xassert ( info->nProcsColl <= info->nProcsModel );

590
591
  info->procsCollMap = (int *)xmalloc((size_t)info->nProcsColl
                                      * sizeof (info->procsCollMap[0]));
Deike Kleberg's avatar
interim    
Deike Kleberg committed
592

Deike Kleberg's avatar
Deike Kleberg committed
593
  // define nodeSizes
Deike Kleberg's avatar
Deike Kleberg committed
594
  info->nodeInfo.nNodes = nNodes; 
595
596
  info->nodeSizes = xcalloc((size_t)info->nodeInfo.nNodes,
                            sizeof (info->nodeSizes[0]));
Deike Kleberg's avatar
interim    
Deike Kleberg committed
597
  collID = 0;
Deike Kleberg's avatar
Deike Kleberg committed
598
  for ( IOID = 0; IOID < info->nProcsIO; IOID++ )
599
    if ( nodeInfo[IOID].isProcColl )
Deike Kleberg's avatar
interim    
Deike Kleberg committed
600
601
602
603
      {
        info->nodeSizes[nodeInfo[IOID].hostID - 1]++;
        info->procsCollMap[collID++] = IOID + info->nProcsModel;
      }
Deike Kleberg's avatar
Deike Kleberg committed
604

Deike Kleberg's avatar
Deike Kleberg committed
605
  // define nodeMap
606
607
  info->nodeMap = (int *)xmalloc((size_t)info->nProcsColl
                                 * sizeof (info->nodeMap[0]));
Deike Kleberg's avatar
Deike Kleberg committed
608
  // helpers
609
610
  p1 = (int **)xmalloc((size_t)info->nodeInfo.nNodes * sizeof (p1[0]));
  p2 = (int **)xmalloc((size_t)info->nodeInfo.nNodes * sizeof (p2[0]));
Deike Kleberg's avatar
Deike Kleberg committed
611
  idx = 0;
Deike Kleberg's avatar
Deike Kleberg committed
612
  for ( i = 0; i < info->nodeInfo.nNodes; i++ )
Deike Kleberg's avatar
Deike Kleberg committed
613
    {
Deike Kleberg's avatar
Deike Kleberg committed
614
      xassert ( idx >= 0 && idx < info->nProcsColl );
Deike Kleberg's avatar
Deike Kleberg committed
615
616
617
      p1[i] = &info->nodeMap[idx];
      p2[i] = p1[i] + info->nodeSizes[i]; 
      idx += info->nodeSizes[i];
Deike Kleberg's avatar
Deike Kleberg committed
618
    }
Deike Kleberg's avatar
Deike Kleberg committed
619
  xassert ( idx == info->nProcsColl );
620
621

  // rankGlob in nodeMap
Deike Kleberg's avatar
Deike Kleberg committed
622
  for ( IOID = 0; IOID < info->nProcsIO; IOID++ )
623
624
    {
      if ( nodeInfo[IOID].isProcColl )
625
        {
Deike Kleberg's avatar
Deike Kleberg committed
626
          xassert ( p1[nodeInfo[IOID].hostID - 1] <  
627
                   p2[nodeInfo[IOID].hostID - 1] );
Deike Kleberg's avatar
Deike Kleberg committed
628
          * p1[nodeInfo[IOID].hostID - 1]++ = IOID + info->nProcsModel;
629
        }
630
631
    }

632
633
634
635
  if ( info->rankGlob == info->root )
    for ( IOID = info->nProcsModel; IOID < info->sizeGlob; IOID++ )
      xmpi ( MPI_Send ( info->nodeMap, info->nProcsColl, MPI_INTEGER, 
                        IOID, NODEMAP, info->commGlob ));
636
637
  free ( p2 );
  free ( p1 );
Deike Kleberg's avatar
Deike Kleberg committed
638
  free ( nodeInfo );
Deike Kleberg's avatar
Deike Kleberg committed
639
640
}

Deike Kleberg's avatar
Deike Kleberg committed
641

642
643
int      commCollID2RankGlob      ( int collID ) 
{
Deike Kleberg's avatar
Deike Kleberg committed
644
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
645
           info->nProcsColl != CDI_UNDEFID &&
646
           collID >= 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
647
648
649
           collID < info->nProcsColl &&
           info->nodeMap != NULL );
  return info->nodeMap[collID];
650
651
652
653
654
655
656
}


int      commRankGlob2CollID      ( int rankGlob)
{
  int out = CDI_UNDEFID, collID;

Deike Kleberg's avatar
Deike Kleberg committed
657
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
658
659
660
661
662
           info->nProcsColl != CDI_UNDEFID &&
           info->nProcsModel != CDI_UNDEFID &&
           info->sizeGlob != CDI_UNDEFID &&
           rankGlob >= info->nProcsModel &&
           rankGlob <= info->sizeGlob );
663

Deike Kleberg's avatar
Deike Kleberg committed
664
665
  for ( collID = 0; collID < info->nProcsColl; collID++ )
    if ( info->nodeMap[collID] == rankGlob ) out = collID;
666
667
668
669
670

  return out;
} 


Deike Kleberg's avatar
Deike Kleberg committed
671
672
int *    commInqNodeSizes   ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
673
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
674
675
           info->nodeSizes != NULL );
  return info->nodeSizes;
Deike Kleberg's avatar
Deike Kleberg committed
676
677
678
}


Deike Kleberg's avatar
Deike Kleberg committed
679
// collective call
680
681
void     commDefCommsIO     ( void )
{
Thomas Jahns's avatar
Thomas Jahns committed
682
  MPI_Group groupGlob;
683
  int collID, * ranks, i, currIORank;
Deike Kleberg's avatar
Deike Kleberg committed
684
685
  char name[MAXCOMMIONAME];

Deike Kleberg's avatar
Deike Kleberg committed
686
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
687
688
689
690
691
           info->nProcsColl != CDI_UNDEFID &&
           info->commsIO == NULL &&
           info->nProcsModel != CDI_UNDEFID &&
           info->commGlob != MPI_COMM_NULL );

692
693
  info->commsIO = (MPI_Comm *)xmalloc((size_t)info->nProcsColl
                                      * sizeof (info->commsIO[0]));
Deike Kleberg's avatar
Deike Kleberg committed
694
695
  for ( collID = 0; collID < info->nProcsColl; collID++ )
    info->commsIO[collID] = MPI_COMM_NULL;
696

697
  strncpy ( name, "COMMSIO_", 8 );
Deike Kleberg's avatar
Deike Kleberg committed
698
  name[MAXCOMMIONAME - 1] = '\0';
699
700

  ranks = (int *)xmalloc(((size_t)info->nProcsModel + 1) * sizeof (ranks[0]));
Deike Kleberg's avatar
Deike Kleberg committed
701
  for ( i = 0; i < info->nProcsModel; i++ )
Thomas Jahns's avatar
Thomas Jahns committed
702
    ranks[i] = i;
703

Deike Kleberg's avatar
Deike Kleberg committed
704
  xmpi ( MPI_Comm_group ( info->commGlob, &groupGlob )); 
Deike Kleberg's avatar
Deike Kleberg committed
705

Deike Kleberg's avatar
Deike Kleberg committed
706
  for ( collID = 0; collID < info->nProcsColl; collID++ )
Deike Kleberg's avatar
Deike Kleberg committed
707
    {
Deike Kleberg's avatar
Deike Kleberg committed
708
      currIORank = info->nodeMap[collID];
Thomas Jahns's avatar
Thomas Jahns committed
709
      ranks[info->nProcsModel] = currIORank;
Thomas Jahns's avatar
Thomas Jahns committed
710
      MPI_Group currGroupIO;
Thomas Jahns's avatar
Thomas Jahns committed
711
712
713
714
715
      xmpi(MPI_Group_incl(groupGlob, info->nProcsModel + 1,
                          ranks, &currGroupIO));
      xmpi(MPI_Comm_create(info->commGlob, currGroupIO,
                           info->commsIO + collID));
      xmpi(MPI_Group_free(&currGroupIO));
Deike Kleberg's avatar
Deike Kleberg committed
716
717
      if ( info->rankGlob == currIORank )
	info->commCalc = info->commsIO[collID];
Deike Kleberg's avatar
Deike Kleberg committed
718

Deike Kleberg's avatar
Deike Kleberg committed
719
      // set names for debugging
Deike Kleberg's avatar
Deike Kleberg committed
720
721
      if ( info->rankGlob < info->nProcsModel || 
           info->rankGlob == currIORank )
Deike Kleberg's avatar
Deike Kleberg committed
722
	{
723
	  sprintf ( &name[8], "%d", collID ); 
Deike Kleberg's avatar
Deike Kleberg committed
724
	  xmpi ( MPI_Comm_set_name ( info->commsIO[collID], name ));
Deike Kleberg's avatar
Deike Kleberg committed
725
726
727
728
729
	}
    }

  if ( ddebug >= 2 )
    {
Deike Kleberg's avatar
Deike Kleberg committed
730
731
732
733
734
      if ( info->rankGlob < info->nProcsModel )
	for ( collID = 0; collID < info->nProcsColl; collID++ ) 
          xdebugComm ( info->commsIO + collID );
      else if ( info->nodeInfo.isProcColl )
	xdebugComm ( &info->commCalc );
Deike Kleberg's avatar
Deike Kleberg committed
735
736
737
738
739
740
    }

  xmpi ( MPI_Group_free ( &groupGlob ));
  free ( ranks );

  commPrint ( stdout );
741
742
743
}


744
MPI_Comm commInqCommsIO     ( int collID )
745
{
Deike Kleberg's avatar
Deike Kleberg committed
746
  xassert ( collID >= 0 &&
Deike Kleberg's avatar
Deike Kleberg committed
747
748
749
750
751
           info != NULL &&
           info->nProcsColl != CDI_UNDEFID &&
           collID < info->nProcsColl &&
           info->commsIO != NULL &&
           info->commsIO[collID] != MPI_COMM_NULL );
752

Deike Kleberg's avatar
Deike Kleberg committed
753
  return info->commsIO[collID];
754
755
756
757
758
}


MPI_Comm commInqCommCalc    ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
759
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
760
761
           info->commCalc != MPI_COMM_NULL );
  return info->commCalc;
762
763
764
}


765
766
int      commInqNProcsColl  ( void )
{
Deike Kleberg's avatar
Deike Kleberg committed
767
  xassert ( info != NULL &&
Deike Kleberg's avatar
Deike Kleberg committed
768
769
           info->nProcsColl != CDI_UNDEFID );
  return info->nProcsColl;
770
771
772
}


Deike Kleberg's avatar
Deike Kleberg committed
773
774
775
776
void commPrint ( FILE * fp )
{
  int i;

Deike Kleberg's avatar
Deike Kleberg committed
777
778
  if ( ddebug == 0 ) return;

Deike Kleberg's avatar
Deike Kleberg committed
779
  xassert ( info != NULL );
Deike Kleberg's avatar
Deike Kleberg committed
780
781

  fprintf ( fp, "\n" );
Deike Kleberg's avatar
Deike Kleberg committed
782
  fprintf ( fp, "######## pioinfo PE%d ###########\n", info->rankGlob );
Deike Kleberg's avatar
Deike Kleberg committed
783
  fprintf ( fp, "#\n" );
Deike Kleberg's avatar
Deike Kleberg committed
784
785
786
787
  fprintf ( fp, "# IOMode      = %d\n", info->IOMode );
  fprintf ( fp, "# nProcsIO    = %d\n", info->nProcsIO );
  fprintf ( fp, "# nProcsModel = %d\n", info->nProcsModel );
  fprintf ( fp, "# isProcIO    = %d\n", info->isProcIO );
Deike Kleberg's avatar
Deike Kleberg committed
788
  fprintf ( fp, "#\n" );
789
  fprintf ( fp, "# commGlob    = %d\n", (int)MPI_Comm_c2f(info->commGlob));
Deike Kleberg's avatar
Deike Kleberg committed
790
791
792
  fprintf ( fp, "# sizeGlob    = %d\n", info->sizeGlob );
  fprintf ( fp, "# rankGlob    = %d\n", info->rankGlob );
  fprintf ( fp, "# root        = %d\n", info->root );
Deike Kleberg's avatar
Deike Kleberg committed
793
  fprintf ( fp, "#\n" );
794
  fprintf ( fp, "# commPio     = %d\n", (int)MPI_Comm_c2f(info->commPio));
Deike Kleberg's avatar
Deike Kleberg committed
795
796
  fprintf ( fp, "# sizePio     = %d\n", info->sizePio );
  fprintf ( fp, "# rankPio     = %d\n", info->rankPio );
Deike Kleberg's avatar
Deike Kleberg committed
797
  fprintf ( fp, "#\n" );
798
  fprintf ( fp, "# commNode    = %d\n", (int)MPI_Comm_c2f(info->commNode));
Deike Kleberg's avatar
Deike Kleberg committed
799
800
801
802
803
  fprintf ( fp, "# sizeNode    = %d\n", info->sizeNode );
  fprintf ( fp, "# rankNode    = %d\n", info->rankNode );
  fprintf ( fp, "# hostname    = %s\n", info->hostname[0] == 0 ? "nn" : 
	    info->hostname );
  fprintf ( fp, "# specialRankNode = %d\n", info->specialRankNode );
804
  fprintf ( fp, "#\n" );
805
  fprintf ( fp, "# commColl    = %d\n", (int)MPI_Comm_c2f(info->commColl));
Deike Kleberg's avatar
Deike Kleberg committed
806
807
  fprintf ( fp, "# sizeColl    = %d\n", info->sizeColl );
  fprintf ( fp, "# rankColl    = %d\n", info->rankColl );
808
  fprintf ( fp, "#\n" );
809
  fprintf ( fp, "# commCalc    = %d\n", (int)MPI_Comm_c2f(info->commCalc));
Deike Kleberg's avatar
Deike Kleberg committed
810
  if ( info->commsIO != NULL )
811
812
    {
      fprintf ( fp, "#\n" );
Deike Kleberg's avatar
Deike Kleberg committed
813
      for ( i = 0; i < info->nProcsColl; i++ )
814
815
	fprintf(fp, "# commsIO[%d]  = %d\n", i,
                (int)MPI_Comm_c2f(info->commsIO[i]));
816
817
818
    }
  else
    fprintf ( fp, "# commsIO     = NULL\n" );
Deike Kleberg's avatar
Deike Kleberg committed
819
820
821
822
  fprintf ( fp, "# hostID      = %d\n", info->nodeInfo.hostID );
  fprintf ( fp, "# isProcColl  = %d\n", info->nodeInfo.isProcColl );
  fprintf ( fp, "# nNodes      = %d\n", info->nodeInfo.nNodes );
  fprintf ( fp, "# nProcsColl  = %d\n", info->nProcsColl );
Deike Kleberg's avatar
interim    
Deike Kleberg committed
823
824
825
826
827
828
829
830
831
832
  if ( info->procsCollMap != NULL )
    {
      fprintf ( fp, "# procsCollMap   = " );
      xassert ( info->nProcsColl != CDI_UNDEFID );
      for ( i = 0; i < info->nProcsColl; i++ )
        fprintf ( fp, "%d ", info->procsCollMap[i] );
      fprintf ( fp, "\n" );
    }
  else     
    fprintf ( fp, "# procsCollMap   = NULL\n" );
Deike Kleberg's avatar
Deike Kleberg committed
833
  if ( info->nodeSizes != NULL )
Deike Kleberg's avatar
Deike Kleberg committed
834
835
    {
      fprintf ( fp, "# nodeSizes   = " );
Deike Kleberg's avatar
Deike Kleberg committed
836
      xassert ( info->nodeInfo.nNodes != CDI_UNDEFID );
Deike Kleberg's avatar
Deike Kleberg committed
837
838
      for ( i = 0; i < info->nodeInfo.nNodes; i++ )
        fprintf ( fp, "%d ", info->nodeSizes[i] );
Deike Kleberg's avatar
Deike Kleberg committed
839
840
841
842
      fprintf ( fp, "\n" );
    }
  else     
    fprintf ( fp, "# nodeSizes   = NULL\n" );
Deike Kleberg's avatar
Deike Kleberg committed
843
  if ( info->nodeMap != NULL )
Deike Kleberg's avatar
Deike Kleberg committed
844
845
    {
      fprintf ( fp, "# nodeMap     = " );
Deike Kleberg's avatar
Deike Kleberg committed
846
      xassert ( info->nProcsColl != CDI_UNDEFID );
Deike Kleberg's avatar
Deike Kleberg committed
847
848
      for ( i = 0; i < info->nProcsColl; i++ )
        fprintf ( fp, "%d ", info->nodeMap[i] );
Deike Kleberg's avatar
Deike Kleberg committed
849
850
851
852
      fprintf ( fp, "\n" );
    }
  else     
    fprintf ( fp, "# nodeMap     = NULL\n" );
Deike Kleberg's avatar
Deike Kleberg committed
853
854
855
856
  fprintf ( fp, "#\n" );
  fprintf ( fp, "############################\n" );
  fprintf ( fp, "\n" );
}
Deike Kleberg's avatar
Deike Kleberg committed
857

Deike Kleberg's avatar
Deike Kleberg committed
858
859
860
861
862
863
864
865
866
867
/************************************************************************/
/*
 * Local Variables:
 * c-file-style: "Java"
 * c-basic-offset: 2
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */