pio.c 8.5 KB
Newer Older
1
#ifdef HAVE_CONFIG_H
Deike Kleberg's avatar
Deike Kleberg committed
2
#include "config.h"
3
4
#endif

Deike Kleberg's avatar
Deike Kleberg committed
5
6
7
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
Deike Kleberg's avatar
Deike Kleberg committed
8
#include <string.h>
9
#include <unistd.h>
Deike Kleberg's avatar
Deike Kleberg committed
10

11
#include "pio.h"
Deike Kleberg's avatar
Deike Kleberg committed
12
#include "cdi.h"
13
14
#include "pio_util.h"

15
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
16
#include "pio_comm.h"
Deike Kleberg's avatar
Deike Kleberg committed
17
#include "pio_impl.h"
Deike Kleberg's avatar
Deike Kleberg committed
18
19
#endif

20
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
21

22
char * command2charP[6] = {"IO_Open_file", "IO_Close_file",
23
24
                           "IO_Get_fp","IO_Set_fp",
                           "IO_Send_buffer", "IO_Finalize"};
Deike Kleberg's avatar
Deike Kleberg committed
25

26
long initial_buffersize = 16 * 1024 * 1024;
27
/*  4 KB <= x < 256 MB */
28
/* 16 * 1024 * 1024; */
29
/* 16 * 1024; */
30
/* 4 * 1024; */
Deike Kleberg's avatar
Deike Kleberg committed
31

Deike Kleberg's avatar
Deike Kleberg committed
32
int maxPtype = 4;
33
int maxNnodes  = 249;
34
int tagKey = 100;
35
int maxErrorString = 100;
36

Deike Kleberg's avatar
Deike Kleberg committed
37
38
39
40
41
42
43
44
45
46
double startTime;
double accumProbe   = 0.0;
double accumRecv    = 0.0;
double accumSend    = 0.0;
double accumSuspend = 0.0;
double accumWait    = 0.0;
double accumWrite   = 0.0;

pioInfo *pioinfo;

47
48
char *token = "%";

49
/***************************************************************/
Deike Kleberg's avatar
Deike Kleberg committed
50

Deike Kleberg's avatar
Deike Kleberg committed
51
52
int setTag ( int ID, int sc )
{
53
  return ID * tagKey + sc;
Deike Kleberg's avatar
Deike Kleberg committed
54
55
56
57
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
58
tag_t * getTag ( int tag )
Deike Kleberg's avatar
Deike Kleberg committed
59
{
Deike Kleberg's avatar
Deike Kleberg committed
60
  tag_t *rtag;
Deike Kleberg's avatar
Deike Kleberg committed
61

Deike Kleberg's avatar
Deike Kleberg committed
62
  rtag = ( tag_t * ) xmalloc ( sizeof ( tag_t ));
63
64
  rtag->id = tag / tagKey;
  rtag->command = tag % tagKey;
Deike Kleberg's avatar
Deike Kleberg committed
65
66
67
68
69
70

  return rtag;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
71
void ungetTag ( tag_t *rtag )
72
73
74
75
76
77
78
{
  free ( rtag );
  return;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
79
size_t pioFileWrite ( int fileID, int tsID, const void *buffer, size_t len )
Deike Kleberg's avatar
Deike Kleberg committed
80
{
Thomas Jahns's avatar
Thomas Jahns committed
81
  size_t iret = CDI_UNDEFID;
Deike Kleberg's avatar
Deike Kleberg committed
82
83
84

  switch ( pioinfo->type )
    {
85
    case PIO_MPI_NONB:
Deike Kleberg's avatar
Deike Kleberg committed
86
      iret = fwMPINONB ( fileID, tsID, buffer, len );
Deike Kleberg's avatar
Deike Kleberg committed
87
      break;
Deike Kleberg's avatar
Deike Kleberg committed
88
#ifndef _SX
89
/*
Deike Kleberg's avatar
Deike Kleberg committed
90
    case PIO_POSIX_ASYNCH:
Deike Kleberg's avatar
Deike Kleberg committed
91
      iret = fwPOSIXASYNCH ( fileID, tsID, buffer, len );
Deike Kleberg's avatar
Deike Kleberg committed
92
      break;
93
*/
Deike Kleberg's avatar
Deike Kleberg committed
94
#endif
95
    case PIO_POSIX_FPGUARD_SENDRECV:
Deike Kleberg's avatar
Deike Kleberg committed
96
      iret = fwPOSIXFPGUARDSENDRECV ( fileID, tsID, buffer, len );
97
      break;
98
/*
99
    case PIO_POSIX_FPGUARD_THREAD:
Deike Kleberg's avatar
Deike Kleberg committed
100
      iret = fwPOSIXFPGUARDTHREAD ( fileID, tsID, buffer, len );
101
      break;
102
103
*/
/*
104
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
Deike Kleberg's avatar
Deike Kleberg committed
105
      iret = fwPOSIXFPGUARDTHREADREFUSE ( fileID, tsID, buffer, len );
106
      break;
107
108
*/
/*
109
    case PIO_POSIX_NONB:
Deike Kleberg's avatar
Deike Kleberg committed
110
      iret = fwPOSIXNONB ( fileID, tsID, buffer, len );
111
      break;
112
*/
Deike Kleberg's avatar
Deike Kleberg committed
113
114
115
116
117
118
119
120
121
    }

  return iret;
}

/***************************************************************/

int pioFileClose ( int id )
{
Thomas Jahns's avatar
Thomas Jahns committed
122
  int iret = CDI_UNDEFID;
Deike Kleberg's avatar
Deike Kleberg committed
123
124
  switch ( pioinfo->type )
    {
125
126
    case PIO_MPI_NONB:
      iret = fcMPINONB ( id );
Deike Kleberg's avatar
Deike Kleberg committed
127
      break;
Deike Kleberg's avatar
Deike Kleberg committed
128
#ifndef _SX
129
/*
Deike Kleberg's avatar
Deike Kleberg committed
130
131
132
    case PIO_POSIX_ASYNCH:
      iret = fcPOSIXASYNCH ( id );
      break;
133
*/
Deike Kleberg's avatar
Deike Kleberg committed
134
#endif
135
136
137
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fcPOSIXFPGUARDSENDRECV ( id );
      break;
138
/*
139
140
141
    case PIO_POSIX_FPGUARD_THREAD:
      iret = fcPOSIXFPGUARDTHREAD ( id );
      break;
142
143
*/
/*
144
145
146
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
      iret = fcPOSIXFPGUARDTHREADREFUSE ( id );
      break;
147
148
*/
/*
149
150
151
    case PIO_POSIX_NONB:
      iret = fcPOSIXNONB ( id );
      break;
152
*/
Deike Kleberg's avatar
Deike Kleberg committed
153
154
155
156
157
158
159
160
161
    }

  return iret;
}

/***************************************************************/

int pioFileOpenW ( const char *filename )
{
Thomas Jahns's avatar
Thomas Jahns committed
162
  int iret = CDI_UNDEFID;
163

Deike Kleberg's avatar
Deike Kleberg committed
164
165
  switch ( pioinfo->type )
    {
166
167
    case PIO_MPI_NONB:
      iret = fowMPINONB ( filename );
Deike Kleberg's avatar
Deike Kleberg committed
168
      break;
Deike Kleberg's avatar
Deike Kleberg committed
169
#ifndef _SX
170
/*
Deike Kleberg's avatar
Deike Kleberg committed
171
172
173
    case PIO_POSIX_ASYNCH:
      iret = fowPOSIXASYNCH ( filename );
      break;
174
*/
Deike Kleberg's avatar
Deike Kleberg committed
175
#endif
176
177
178
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fowPOSIXFPGUARDSENDRECV ( filename );
      break;
179
/*
180
181
182
    case PIO_POSIX_FPGUARD_THREAD:
      iret = fowPOSIXFPGUARDTHREAD ( filename );
      break;
183
184
*/
/*
185
186
187
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
      iret = fowPOSIXFPGUARDTHREADREFUSE ( filename );
      break;
188
189
*/
/*
190
191
192
    case PIO_POSIX_NONB:
      iret = fowPOSIXNONB ( filename );
      break;
193
*/
Deike Kleberg's avatar
Deike Kleberg committed
194
    }
195

Deike Kleberg's avatar
Deike Kleberg committed
196
197
198
199
200
  return iret;
}

/***************************************************************/

201
202
static int cmpr ( const void *a, const void *b )
{
203
204
205
206
207
  return strcmp ( *( char ** ) a, *( char ** ) b);
}

/***************************************************************/

208
209
void setPioCommunicator ( MPI_Comm *myComm, MPI_Comm commF2C, int *color,
                          int *nnodes )
Deike Kleberg's avatar
Deike Kleberg committed
210
{
211
212
  int size, rank, len, npes_node, key, test, i, j;

Deike Kleberg's avatar
Deike Kleberg committed
213
  char *myHost, **allHosts, *allHosts0, *curr;
214
215
  char hostname [ MPI_MAX_PROCESSOR_NAME + 1 ];

Deike Kleberg's avatar
Deike Kleberg committed
216
217
  xmpi ( MPI_Comm_size ( commF2C, &size ));
  xmpi ( MPI_Comm_rank ( commF2C, &rank ));
218

Deike Kleberg's avatar
Deike Kleberg committed
219
  myHost = ( char * ) xmalloc ( MPI_MAX_PROCESSOR_NAME * sizeof ( char ));
220

Deike Kleberg's avatar
Deike Kleberg committed
221
  memset ( myHost, 0, MPI_MAX_PROCESSOR_NAME * sizeof ( char ));
222

Deike Kleberg's avatar
Deike Kleberg committed
223
  xmpi ( MPI_Get_processor_name ( myHost, &len ));
Deike Kleberg's avatar
Deike Kleberg committed
224

225
  if ( myHost[0] == '\0' ) xabort ( "did not succeed to set hostname" );
Deike Kleberg's avatar
Deike Kleberg committed
226

Deike Kleberg's avatar
Deike Kleberg committed
227
  if ( ddebug == MAXDEBUG )
228
229
230
    {
      strncpy ( hostname, myHost, len );
      hostname [ len ] = '\0';
231
      xdebug ( "myHost = %s", hostname );
232
    }
233

Deike Kleberg's avatar
Deike Kleberg committed
234
  allHosts = ( char ** ) xmalloc ( size * sizeof ( char * ));
235
236
  allHosts[0] = ( char * ) xmalloc ( size * MPI_MAX_PROCESSOR_NAME *
                                     sizeof ( char ));
Deike Kleberg's avatar
Deike Kleberg committed
237
  allHosts0 = allHosts[0];
Deike Kleberg's avatar
Deike Kleberg committed
238

239
240
  for ( i = 1; i < size; i++ )
    allHosts[i] = allHosts[0] + i * MPI_MAX_PROCESSOR_NAME;
241

Deike Kleberg's avatar
Deike Kleberg committed
242
243
244
  xmpi ( MPI_Allgather ( myHost, MPI_MAX_PROCESSOR_NAME, MPI_CHAR,
                         & ( allHosts[0][0] ), MPI_MAX_PROCESSOR_NAME,
                         MPI_CHAR, commF2C ));
245
246
247
248
249
250

  qsort ( allHosts, size, sizeof ( char * ), cmpr );

  *color = 0;
  i = 0;
  j = 0;
251

252
  while ( i < size )
253
254
    {
      curr = allHosts[i];
255
      j++;
256
      if (( test = strcmp ( myHost, curr )) == 0 ) *color = j;
257

258
      while ( ++i < size )
259
260
        if (( test = strcmp ( allHosts[i], curr )) != 0)
          break;
261
    }
Deike Kleberg's avatar
Deike Kleberg committed
262

263
  *nnodes = j;
264

265
  if ( *color == 0 ) xabort ( "Color is not set" );
266

267
  npes_node = size / ( * nnodes );
Deike Kleberg's avatar
Deike Kleberg committed
268
269
  key = rank % npes_node;

Deike Kleberg's avatar
Deike Kleberg committed
270
  xmpi ( MPI_Comm_split ( commF2C, *color, key, myComm));
271

Deike Kleberg's avatar
Deike Kleberg committed
272
273
274
  commDefCommNode ( * myComm );
  commDefHostName ( myHost, len );
  commDefHostID   ( * color );
Deike Kleberg's avatar
Deike Kleberg committed
275
  commDefNNodes ( * nnodes );
Deike Kleberg's avatar
Deike Kleberg committed
276
277


Deike Kleberg's avatar
Deike Kleberg committed
278
  free ( allHosts0 );
279
280
  free ( allHosts );
  free ( myHost );
Deike Kleberg's avatar
Deike Kleberg committed
281

282
  xdebug ( "color=%d", *color );
283

284
285
286
287
288
289
290
  return;
}

#endif

/***************************************************************/

291
MPI_Comm backendInit ( int ptype, MPI_Comm comm, int *color, int *nnodes )
292
{
Deike Kleberg's avatar
Deike Kleberg committed
293
  int size, rank;
294
  int collectingData = 1;
295
296
  *nnodes = 1;
  *color = 1;
297

Deike Kleberg's avatar
Deike Kleberg committed
298
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
299
300
  xmpi ( MPI_Comm_size ( comm, &size ));
  xmpi ( MPI_Comm_rank ( comm, &rank ));
301
302

  if ( ptype < 0 || ptype > maxPtype )
303
    xabort ( "PIOTYPE is no valid modus" );
Deike Kleberg's avatar
Deike Kleberg committed
304
305
306

#ifdef _SX
  if ( ptype ==  PIO_POSIX_ASYNCH )
307
    xabort ( "PIO_POSIX_ASYNCH does not work on SX" );
Deike Kleberg's avatar
Deike Kleberg committed
308
#endif
309

310
  pioinfo = ( pioInfo * ) xmalloc ( sizeof ( pioInfo ));
311
312

  pioinfo->type = ptype;
Deike Kleberg's avatar
Deike Kleberg committed
313

314
  setPioCommunicator ( & ( pioinfo->comm ), comm, color, nnodes );
315

316
  if (( pioinfo->type == PIO_NONE ) && ( *nnodes != size ))
317
    xabort ( "PIOTYPE, NNODES: not a valid combination" );
318

319
  pioinfo->color = *color;
Deike Kleberg's avatar
Deike Kleberg committed
320

Deike Kleberg's avatar
Deike Kleberg committed
321
322
  xmpi ( MPI_Comm_rank ( pioinfo->comm, &( pioinfo->rank )));
  xmpi ( MPI_Comm_size ( pioinfo->comm, &( pioinfo->size )));
323

324
  xdebug( "IOPE%d in bInit(), ptype=%d, initial_buffersize=%ld: "
325
326
          "after init pioinfo ...",
          pioinfo->rank, pioinfo->type, initial_buffersize );
327

328
  pioinfo->collectorComm = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
329

Deike Kleberg's avatar
Deike Kleberg committed
330
331
  switch ( pioinfo->type )
    {
332
    case PIO_NONE:
Deike Kleberg's avatar
Deike Kleberg committed
333
      xmpi ( MPI_Comm_dup ( pioinfo->comm, &( pioinfo->collectorComm )));
334
      collectingData = 1;
335
      break;
336
    case PIO_MPI_NONB:
337
      collectingData = initMPINONB ();
Deike Kleberg's avatar
Deike Kleberg committed
338
      break;
Deike Kleberg's avatar
Deike Kleberg committed
339
#ifndef _SX
340
/*
Deike Kleberg's avatar
Deike Kleberg committed
341
    case PIO_POSIX_ASYNCH:
342
      collectingData = initPOSIXASYNCH ();
Deike Kleberg's avatar
Deike Kleberg committed
343
      break;
344
*/
Deike Kleberg's avatar
Deike Kleberg committed
345
#endif
346
    case PIO_POSIX_FPGUARD_SENDRECV:
347
      collectingData = initPOSIXFPGUARDSENDRECV ();
348
      break;
349
/*
350
    case PIO_POSIX_FPGUARD_THREAD:
351
      collectingData = initPOSIXFPGUARDTHREAD ();
352
      break;
353
354
*/
/*
355
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
356
      collectingData = initPOSIXFPGUARDTHREADREFUSE ();
357
      break;
358
359
*/
/*
Deike Kleberg's avatar
Deike Kleberg committed
360
    case PIO_POSIX_NONB:
361
      collectingData = initPOSIXNONB ();
362
      break;
363
*/
Deike Kleberg's avatar
Deike Kleberg committed
364
    }
365
#endif
366
367

#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
368
  return pioinfo->collectorComm;
369
370
371
#else
  return 0;
#endif
Deike Kleberg's avatar
Deike Kleberg committed
372
373
374
375
}

/***************************************************************/

376
void backendFinalize ()
Deike Kleberg's avatar
Deike Kleberg committed
377
{
378
#ifdef USE_MPI
379
380
  switch ( pioinfo->type )
    {
381
/*
382
383
384
    case PIO_POSIX_FPGUARD_THREAD:
      finalizePOSIXFPGUARDTHREAD ();
      break;
385
*/
386
387
    }

388
  if ( pioinfo->collectorComm != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
389
    xmpi ( MPI_Comm_free ( &( pioinfo->collectorComm )));
390
391

  if ( pioinfo->comm != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
392
    xmpi ( MPI_Comm_free ( &( pioinfo->comm )));
393

Deike Kleberg's avatar
Deike Kleberg committed
394
  free ( pioinfo );
395
#endif
Deike Kleberg's avatar
Deike Kleberg committed
396
}
397
398
399
400
401
402
403
404
405
/*
 * Local Variables:
 * c-file-style: "Java"
 * c-basic-offset: 2
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */