pio.c 8.37 KB
Newer Older
1
#ifdef HAVE_CONFIG_H
Deike Kleberg's avatar
Deike Kleberg committed
2
#include "config.h"
3
4
#endif

Deike Kleberg's avatar
Deike Kleberg committed
5
6
7
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
Deike Kleberg's avatar
Deike Kleberg committed
8
#include <string.h>
9
#include <unistd.h>
Deike Kleberg's avatar
Deike Kleberg committed
10

Thomas Jahns's avatar
Thomas Jahns committed
11
#include "cdi.h"
12
13
14
#include "pio.h"
#include "pio_util.h"

15
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
16
17
#include "cdi.h"
#include "pio_impl.h"
Deike Kleberg's avatar
Deike Kleberg committed
18
19
#endif

20
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
21

22
char * command2charP[6] = {"IO_Open_file", "IO_Close_file",
23
24
                           "IO_Get_fp","IO_Set_fp",
                           "IO_Send_buffer", "IO_Finalize"};
Deike Kleberg's avatar
Deike Kleberg committed
25

26
long initial_buffersize = 16 * 1024 * 1024;
27
/*  4 KB <= x < 256 MB */
28
/* 16 * 1024 * 1024; */
29
/* 16 * 1024; */
30
/* 4 * 1024; */
Deike Kleberg's avatar
Deike Kleberg committed
31

Deike Kleberg's avatar
Deike Kleberg committed
32
int maxPtype = 4;
33
int maxNnodes  = 249;
34
int tagKey = 100;
35
int maxErrorString = 100;
36

Deike Kleberg's avatar
Deike Kleberg committed
37
38
39
40
41
42
43
44
45
46
double startTime;
double accumProbe   = 0.0;
double accumRecv    = 0.0;
double accumSend    = 0.0;
double accumSuspend = 0.0;
double accumWait    = 0.0;
double accumWrite   = 0.0;

pioInfo *pioinfo;

47
48
char *token = "%";

49
/***************************************************************/
Deike Kleberg's avatar
Deike Kleberg committed
50

Deike Kleberg's avatar
Deike Kleberg committed
51
52
int setTag ( int ID, int sc )
{
53
  return ID * tagKey + sc;
Deike Kleberg's avatar
Deike Kleberg committed
54
55
56
57
58
59
60
61
}

/***************************************************************/

rTag * getTag ( int tag )
{
  rTag *rtag;

Deike Kleberg's avatar
Deike Kleberg committed
62
  rtag = ( rTag * ) xmalloc ( sizeof ( rTag ));
63
64
  rtag->id = tag / tagKey;
  rtag->command = tag % tagKey;
Deike Kleberg's avatar
Deike Kleberg committed
65
66
67
68
69
70

  return rtag;
}

/***************************************************************/

71
72
73
74
75
76
77
78
void ungetTag ( rTag *rtag )
{
  free ( rtag );
  return;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
79
size_t pioFileWrite ( int fileID, int tsID, const void *buffer, size_t len )
Deike Kleberg's avatar
Deike Kleberg committed
80
{
Thomas Jahns's avatar
Thomas Jahns committed
81
  size_t iret = CDI_UNDEFID;
Deike Kleberg's avatar
Deike Kleberg committed
82
83
84

  switch ( pioinfo->type )
    {
85
    case PIO_MPI_NONB:
Deike Kleberg's avatar
Deike Kleberg committed
86
      iret = fwMPINONB ( fileID, tsID, buffer, len );
Deike Kleberg's avatar
Deike Kleberg committed
87
      break;
Deike Kleberg's avatar
Deike Kleberg committed
88
#ifndef _SX
89
/*
Deike Kleberg's avatar
Deike Kleberg committed
90
    case PIO_POSIX_ASYNCH:
Deike Kleberg's avatar
Deike Kleberg committed
91
      iret = fwPOSIXASYNCH ( fileID, tsID, buffer, len );
Deike Kleberg's avatar
Deike Kleberg committed
92
      break;
93
*/
Deike Kleberg's avatar
Deike Kleberg committed
94
#endif
95
/*
96
    case PIO_POSIX_FPGUARD_SENDRECV:
Deike Kleberg's avatar
Deike Kleberg committed
97
      iret = fwPOSIXFPGUARDSENDRECV ( fileID, tsID, buffer, len );
98
      break;
99
100
*/
/*
101
    case PIO_POSIX_FPGUARD_THREAD:
Deike Kleberg's avatar
Deike Kleberg committed
102
      iret = fwPOSIXFPGUARDTHREAD ( fileID, tsID, buffer, len );
103
      break;
104
105
*/
/*
106
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
Deike Kleberg's avatar
Deike Kleberg committed
107
      iret = fwPOSIXFPGUARDTHREADREFUSE ( fileID, tsID, buffer, len );
108
      break;
109
110
*/
/*
111
    case PIO_POSIX_NONB:
Deike Kleberg's avatar
Deike Kleberg committed
112
      iret = fwPOSIXNONB ( fileID, tsID, buffer, len );
113
      break;
114
*/
Deike Kleberg's avatar
Deike Kleberg committed
115
116
117
118
119
120
121
122
123
    }

  return iret;
}

/***************************************************************/

int pioFileClose ( int id )
{
Thomas Jahns's avatar
Thomas Jahns committed
124
  int iret = CDI_UNDEFID;
Deike Kleberg's avatar
Deike Kleberg committed
125
126
  switch ( pioinfo->type )
    {
127
128
    case PIO_MPI_NONB:
      iret = fcMPINONB ( id );
Deike Kleberg's avatar
Deike Kleberg committed
129
      break;
Deike Kleberg's avatar
Deike Kleberg committed
130
#ifndef _SX
131
/*
Deike Kleberg's avatar
Deike Kleberg committed
132
133
134
    case PIO_POSIX_ASYNCH:
      iret = fcPOSIXASYNCH ( id );
      break;
135
*/
Deike Kleberg's avatar
Deike Kleberg committed
136
#endif
137
/*
138
139
140
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fcPOSIXFPGUARDSENDRECV ( id );
      break;
141
142
*/
/*
143
144
145
    case PIO_POSIX_FPGUARD_THREAD:
      iret = fcPOSIXFPGUARDTHREAD ( id );
      break;
146
147
*/
/*
148
149
150
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
      iret = fcPOSIXFPGUARDTHREADREFUSE ( id );
      break;
151
152
*/
/*
153
154
155
    case PIO_POSIX_NONB:
      iret = fcPOSIXNONB ( id );
      break;
156
*/
Deike Kleberg's avatar
Deike Kleberg committed
157
158
159
160
161
162
163
164
165
    }

  return iret;
}

/***************************************************************/

int pioFileOpenW ( const char *filename )
{
Thomas Jahns's avatar
Thomas Jahns committed
166
  int iret = CDI_UNDEFID;
167

Deike Kleberg's avatar
Deike Kleberg committed
168
169
  switch ( pioinfo->type )
    {
170
171
    case PIO_MPI_NONB:
      iret = fowMPINONB ( filename );
Deike Kleberg's avatar
Deike Kleberg committed
172
      break;
Deike Kleberg's avatar
Deike Kleberg committed
173
#ifndef _SX
174
/*
Deike Kleberg's avatar
Deike Kleberg committed
175
176
177
    case PIO_POSIX_ASYNCH:
      iret = fowPOSIXASYNCH ( filename );
      break;
178
*/
Deike Kleberg's avatar
Deike Kleberg committed
179
#endif
180
/*
181
182
183
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fowPOSIXFPGUARDSENDRECV ( filename );
      break;
184
185
*/
/*
186
187
188
    case PIO_POSIX_FPGUARD_THREAD:
      iret = fowPOSIXFPGUARDTHREAD ( filename );
      break;
189
190
*/
/*
191
192
193
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
      iret = fowPOSIXFPGUARDTHREADREFUSE ( filename );
      break;
194
195
*/
/*
196
197
198
    case PIO_POSIX_NONB:
      iret = fowPOSIXNONB ( filename );
      break;
199
*/
Deike Kleberg's avatar
Deike Kleberg committed
200
    }
201

Deike Kleberg's avatar
Deike Kleberg committed
202
203
204
205
206
  return iret;
}

/***************************************************************/

207
208
static int cmpr ( const void *a, const void *b )
{
209
210
211
212
213
  return strcmp ( *( char ** ) a, *( char ** ) b);
}

/***************************************************************/

214
215
void setPioCommunicator ( MPI_Comm *myComm, MPI_Comm commF2C, int *color,
                          int *nnodes )
Deike Kleberg's avatar
Deike Kleberg committed
216
{
217
218
  int size, rank, len, npes_node, key, test, i, j;

Deike Kleberg's avatar
Deike Kleberg committed
219
  char *myHost, **allHosts, *allHosts0, *curr;
220
221
  char hostname [ MPI_MAX_PROCESSOR_NAME + 1 ];

Deike Kleberg's avatar
Deike Kleberg committed
222
223
  xmpi ( MPI_Comm_size ( commF2C, &size ));
  xmpi ( MPI_Comm_rank ( commF2C, &rank ));
224

Deike Kleberg's avatar
Deike Kleberg committed
225
  myHost = ( char * ) xmalloc ( MPI_MAX_PROCESSOR_NAME * sizeof ( char ));
226

Deike Kleberg's avatar
Deike Kleberg committed
227
  memset ( myHost, 0, MPI_MAX_PROCESSOR_NAME * sizeof ( char ));
228

Deike Kleberg's avatar
Deike Kleberg committed
229
  xmpi ( MPI_Get_processor_name ( myHost, &len ));
Deike Kleberg's avatar
Deike Kleberg committed
230

231
  if ( myHost[0] == '\0' ) xabort ( "did not succeed to set hostname" );
Deike Kleberg's avatar
Deike Kleberg committed
232

Deike Kleberg's avatar
Deike Kleberg committed
233
  if ( ddebug == MAXDEBUG )
234
235
236
    {
      strncpy ( hostname, myHost, len );
      hostname [ len ] = '\0';
237
      xdebug ( "myHost = %s", hostname );
238
    }
239

Deike Kleberg's avatar
Deike Kleberg committed
240
  allHosts = ( char ** ) xmalloc ( size * sizeof ( char * ));
241
242
  allHosts[0] = ( char * ) xmalloc ( size * MPI_MAX_PROCESSOR_NAME *
                                     sizeof ( char ));
Deike Kleberg's avatar
Deike Kleberg committed
243
  allHosts0 = allHosts[0];
Deike Kleberg's avatar
Deike Kleberg committed
244

245
246
  for ( i = 1; i < size; i++ )
    allHosts[i] = allHosts[0] + i * MPI_MAX_PROCESSOR_NAME;
247

248
249
250
  MPI_Allgather ( myHost, MPI_MAX_PROCESSOR_NAME, MPI_CHAR,
                            & ( allHosts[0][0] ), MPI_MAX_PROCESSOR_NAME,
                            MPI_CHAR, commF2C );
251
252
253
254
255
256

  qsort ( allHosts, size, sizeof ( char * ), cmpr );

  *color = 0;
  i = 0;
  j = 0;
257

258
  while ( i < size )
259
260
    {
      curr = allHosts[i];
261
      j++;
262
      if (( test = strcmp ( myHost, curr )) == 0 ) *color = j;
263

264
      while ( ++i < size )
265
266
        if (( test = strcmp ( allHosts[i], curr )) != 0)
          break;
267
    }
Deike Kleberg's avatar
Deike Kleberg committed
268

269
  *nnodes = j;
270

271
  if ( *color == 0 ) xabort ( "Color is not set" );
272

273
  npes_node = size / ( * nnodes );
Deike Kleberg's avatar
Deike Kleberg committed
274
275
  key = rank % npes_node;

Deike Kleberg's avatar
Deike Kleberg committed
276
  xmpi ( MPI_Comm_split ( commF2C, *color, key, myComm));
277

Deike Kleberg's avatar
Deike Kleberg committed
278
  free ( allHosts0 );
279
280
  free ( allHosts );
  free ( myHost );
Deike Kleberg's avatar
Deike Kleberg committed
281

282
  xdebug ( "color=%d", *color );
283

284
285
286
287
288
289
290
  return;
}

#endif

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
291
MPI_Comm bInit ( int ptype, MPI_Comm comm, int *color, int *nnodes )
292
{
Deike Kleberg's avatar
Deike Kleberg committed
293
  int size, rank;
294
  int collectingData = 1;
295
296
  *nnodes = 1;
  *color = 1;
297

Deike Kleberg's avatar
Deike Kleberg committed
298
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
299
300
  xmpi ( MPI_Comm_size ( comm, &size ));
  xmpi ( MPI_Comm_rank ( comm, &rank ));
301
302

  if ( ptype < 0 || ptype > maxPtype )
303
    xabort ( "PIOTYPE is no valid modus" );
Deike Kleberg's avatar
Deike Kleberg committed
304
305
306

#ifdef _SX
  if ( ptype ==  PIO_POSIX_ASYNCH )
307
    xabort ( "PIO_POSIX_ASYNCH does not work on SX" );
Deike Kleberg's avatar
Deike Kleberg committed
308
#endif
309

310
  pioinfo = ( pioInfo * ) xmalloc ( sizeof ( pioInfo ));
311
312

  pioinfo->type = ptype;
Deike Kleberg's avatar
Deike Kleberg committed
313

314
  setPioCommunicator ( & ( pioinfo->comm ), comm, color, nnodes );
315

316
  if (( pioinfo->type == PIO_NONE ) && ( *nnodes != size ))
317
    xabort ( "PIOTYPE, NNODES: not a valid combination" );
318

319
  pioinfo->color = *color;
Deike Kleberg's avatar
Deike Kleberg committed
320

Deike Kleberg's avatar
Deike Kleberg committed
321
322
  xmpi ( MPI_Comm_rank ( pioinfo->comm, &( pioinfo->rank )));
  xmpi ( MPI_Comm_size ( pioinfo->comm, &( pioinfo->size )));
323

324
  xdebug( "IOPE%d in bInit(), ptype=%d, initial_buffersize=%ld: "
325
326
          "after init pioinfo ...",
          pioinfo->rank, pioinfo->type, initial_buffersize );
327

328
  pioinfo->collectorComm = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
329

Deike Kleberg's avatar
Deike Kleberg committed
330
331
  switch ( pioinfo->type )
    {
332
    case PIO_NONE:
Deike Kleberg's avatar
Deike Kleberg committed
333
      xmpi ( MPI_Comm_dup ( pioinfo->comm, &( pioinfo->collectorComm )));
334
      collectingData = 1;
335
      break;
336
    case PIO_MPI_NONB:
337
      collectingData = initMPINONB ();
Deike Kleberg's avatar
Deike Kleberg committed
338
      break;
Deike Kleberg's avatar
Deike Kleberg committed
339
#ifndef _SX
340
/*
Deike Kleberg's avatar
Deike Kleberg committed
341
    case PIO_POSIX_ASYNCH:
342
      collectingData = initPOSIXASYNCH ();
Deike Kleberg's avatar
Deike Kleberg committed
343
      break;
344
*/
Deike Kleberg's avatar
Deike Kleberg committed
345
#endif
346
/*
347
    case PIO_POSIX_FPGUARD_SENDRECV:
348
      collectingData = initPOSIXFPGUARDSENDRECV ();
349
      break;
350
351
*/
/*
352
    case PIO_POSIX_FPGUARD_THREAD:
353
      collectingData = initPOSIXFPGUARDTHREAD ();
354
      break;
355
356
*/
/*
357
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
358
      collectingData = initPOSIXFPGUARDTHREADREFUSE ();
359
      break;
360
361
*/
/*
Deike Kleberg's avatar
Deike Kleberg committed
362
    case PIO_POSIX_NONB:
363
      collectingData = initPOSIXNONB ();
364
      break;
365
*/
Deike Kleberg's avatar
Deike Kleberg committed
366
    }
367
#endif
368
369

#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
370
  return pioinfo->collectorComm;
371
372
373
#else
  return 0;
#endif
Deike Kleberg's avatar
Deike Kleberg committed
374
375
376
377
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
378
void bFinalize ()
Deike Kleberg's avatar
Deike Kleberg committed
379
{
380
#ifdef USE_MPI
381
382
  switch ( pioinfo->type )
    {
383
/*
384
385
386
    case PIO_POSIX_FPGUARD_THREAD:
      finalizePOSIXFPGUARDTHREAD ();
      break;
387
*/
388
389
    }

390
  if ( pioinfo->collectorComm != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
391
    xmpi ( MPI_Comm_free ( &( pioinfo->collectorComm )));
392
393

  if ( pioinfo->comm != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
394
    xmpi ( MPI_Comm_free ( &( pioinfo->comm )));
395

Deike Kleberg's avatar
Deike Kleberg committed
396
  free ( pioinfo );
397
#endif
Deike Kleberg's avatar
Deike Kleberg committed
398
}
399
400
401
402
403
404
405
406
407
/*
 * Local Variables:
 * c-file-style: "Java"
 * c-basic-offset: 2
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */