pio.c 8.35 KB
Newer Older
1
#ifdef HAVE_CONFIG_H
Deike Kleberg's avatar
Deike Kleberg committed
2
#include "config.h"
3
4
#endif

Deike Kleberg's avatar
Deike Kleberg committed
5
6
7
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
Deike Kleberg's avatar
Deike Kleberg committed
8
#include <string.h>
9
#include <unistd.h>
Deike Kleberg's avatar
Deike Kleberg committed
10

Thomas Jahns's avatar
Thomas Jahns committed
11
#include "cdi.h"
12
13
14
#include "pio.h"
#include "pio_util.h"

15
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
16
#include "pio_impl.h"
Deike Kleberg's avatar
Deike Kleberg committed
17
18
#endif

19
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
20

21
char * command2charP[6] = {"IO_Open_file", "IO_Close_file",
22
23
                           "IO_Get_fp","IO_Set_fp",
                           "IO_Send_buffer", "IO_Finalize"};
Deike Kleberg's avatar
Deike Kleberg committed
24

25
long initial_buffersize = 16 * 1024 * 1024;
26
/*  4 KB <= x < 256 MB */
27
/* 16 * 1024 * 1024; */
28
/* 16 * 1024; */
29
/* 4 * 1024; */
Deike Kleberg's avatar
Deike Kleberg committed
30

Deike Kleberg's avatar
Deike Kleberg committed
31
int maxPtype = 4;
32
int maxNnodes  = 249;
33
int tagKey = 100;
34
int maxErrorString = 100;
35

Deike Kleberg's avatar
Deike Kleberg committed
36
37
38
39
40
41
42
43
44
45
double startTime;
double accumProbe   = 0.0;
double accumRecv    = 0.0;
double accumSend    = 0.0;
double accumSuspend = 0.0;
double accumWait    = 0.0;
double accumWrite   = 0.0;

pioInfo *pioinfo;

46
47
char *token = "%";

48
/***************************************************************/
Deike Kleberg's avatar
Deike Kleberg committed
49

Deike Kleberg's avatar
Deike Kleberg committed
50
51
int setTag ( int ID, int sc )
{
52
  return ID * tagKey + sc;
Deike Kleberg's avatar
Deike Kleberg committed
53
54
55
56
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
57
tag_t * getTag ( int tag )
Deike Kleberg's avatar
Deike Kleberg committed
58
{
Deike Kleberg's avatar
Deike Kleberg committed
59
  tag_t *rtag;
Deike Kleberg's avatar
Deike Kleberg committed
60

Deike Kleberg's avatar
Deike Kleberg committed
61
  rtag = ( tag_t * ) xmalloc ( sizeof ( tag_t ));
62
63
  rtag->id = tag / tagKey;
  rtag->command = tag % tagKey;
Deike Kleberg's avatar
Deike Kleberg committed
64
65
66
67
68
69

  return rtag;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
70
void ungetTag ( tag_t *rtag )
71
72
73
74
75
76
77
{
  free ( rtag );
  return;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
78
size_t pioFileWrite ( int fileID, int tsID, const void *buffer, size_t len )
Deike Kleberg's avatar
Deike Kleberg committed
79
{
Thomas Jahns's avatar
Thomas Jahns committed
80
  size_t iret = CDI_UNDEFID;
Deike Kleberg's avatar
Deike Kleberg committed
81
82
83

  switch ( pioinfo->type )
    {
84
    case PIO_MPI_NONB:
Deike Kleberg's avatar
Deike Kleberg committed
85
      iret = fwMPINONB ( fileID, tsID, buffer, len );
Deike Kleberg's avatar
Deike Kleberg committed
86
      break;
Deike Kleberg's avatar
Deike Kleberg committed
87
#ifndef _SX
88
/*
Deike Kleberg's avatar
Deike Kleberg committed
89
    case PIO_POSIX_ASYNCH:
Deike Kleberg's avatar
Deike Kleberg committed
90
      iret = fwPOSIXASYNCH ( fileID, tsID, buffer, len );
Deike Kleberg's avatar
Deike Kleberg committed
91
      break;
92
*/
Deike Kleberg's avatar
Deike Kleberg committed
93
#endif
94
    case PIO_POSIX_FPGUARD_SENDRECV:
Deike Kleberg's avatar
Deike Kleberg committed
95
      iret = fwPOSIXFPGUARDSENDRECV ( fileID, tsID, buffer, len );
96
      break;
97
/*
98
    case PIO_POSIX_FPGUARD_THREAD:
Deike Kleberg's avatar
Deike Kleberg committed
99
      iret = fwPOSIXFPGUARDTHREAD ( fileID, tsID, buffer, len );
100
      break;
101
102
*/
/*
103
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
Deike Kleberg's avatar
Deike Kleberg committed
104
      iret = fwPOSIXFPGUARDTHREADREFUSE ( fileID, tsID, buffer, len );
105
      break;
106
107
*/
/*
108
    case PIO_POSIX_NONB:
Deike Kleberg's avatar
Deike Kleberg committed
109
      iret = fwPOSIXNONB ( fileID, tsID, buffer, len );
110
      break;
111
*/
Deike Kleberg's avatar
Deike Kleberg committed
112
113
114
115
116
117
118
119
120
    }

  return iret;
}

/***************************************************************/

int pioFileClose ( int id )
{
Thomas Jahns's avatar
Thomas Jahns committed
121
  int iret = CDI_UNDEFID;
Deike Kleberg's avatar
Deike Kleberg committed
122
123
  switch ( pioinfo->type )
    {
124
125
    case PIO_MPI_NONB:
      iret = fcMPINONB ( id );
Deike Kleberg's avatar
Deike Kleberg committed
126
      break;
Deike Kleberg's avatar
Deike Kleberg committed
127
#ifndef _SX
128
/*
Deike Kleberg's avatar
Deike Kleberg committed
129
130
131
    case PIO_POSIX_ASYNCH:
      iret = fcPOSIXASYNCH ( id );
      break;
132
*/
Deike Kleberg's avatar
Deike Kleberg committed
133
#endif
134
135
136
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fcPOSIXFPGUARDSENDRECV ( id );
      break;
137
/*
138
139
140
    case PIO_POSIX_FPGUARD_THREAD:
      iret = fcPOSIXFPGUARDTHREAD ( id );
      break;
141
142
*/
/*
143
144
145
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
      iret = fcPOSIXFPGUARDTHREADREFUSE ( id );
      break;
146
147
*/
/*
148
149
150
    case PIO_POSIX_NONB:
      iret = fcPOSIXNONB ( id );
      break;
151
*/
Deike Kleberg's avatar
Deike Kleberg committed
152
153
154
155
156
157
158
159
160
    }

  return iret;
}

/***************************************************************/

int pioFileOpenW ( const char *filename )
{
Thomas Jahns's avatar
Thomas Jahns committed
161
  int iret = CDI_UNDEFID;
162

Deike Kleberg's avatar
Deike Kleberg committed
163
164
  switch ( pioinfo->type )
    {
165
166
    case PIO_MPI_NONB:
      iret = fowMPINONB ( filename );
Deike Kleberg's avatar
Deike Kleberg committed
167
      break;
Deike Kleberg's avatar
Deike Kleberg committed
168
#ifndef _SX
169
/*
Deike Kleberg's avatar
Deike Kleberg committed
170
171
172
    case PIO_POSIX_ASYNCH:
      iret = fowPOSIXASYNCH ( filename );
      break;
173
*/
Deike Kleberg's avatar
Deike Kleberg committed
174
#endif
175
176
177
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fowPOSIXFPGUARDSENDRECV ( filename );
      break;
178
/*
179
180
181
    case PIO_POSIX_FPGUARD_THREAD:
      iret = fowPOSIXFPGUARDTHREAD ( filename );
      break;
182
183
*/
/*
184
185
186
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
      iret = fowPOSIXFPGUARDTHREADREFUSE ( filename );
      break;
187
188
*/
/*
189
190
191
    case PIO_POSIX_NONB:
      iret = fowPOSIXNONB ( filename );
      break;
192
*/
Deike Kleberg's avatar
Deike Kleberg committed
193
    }
194

Deike Kleberg's avatar
Deike Kleberg committed
195
196
197
198
199
  return iret;
}

/***************************************************************/

200
201
static int cmpr ( const void *a, const void *b )
{
202
203
204
205
206
  return strcmp ( *( char ** ) a, *( char ** ) b);
}

/***************************************************************/

207
208
void setPioCommunicator ( MPI_Comm *myComm, MPI_Comm commF2C, int *color,
                          int *nnodes )
Deike Kleberg's avatar
Deike Kleberg committed
209
{
210
211
  int size, rank, len, npes_node, key, test, i, j;

Deike Kleberg's avatar
Deike Kleberg committed
212
  char *myHost, **allHosts, *allHosts0, *curr;
213
214
  char hostname [ MPI_MAX_PROCESSOR_NAME + 1 ];

Deike Kleberg's avatar
Deike Kleberg committed
215
216
  xmpi ( MPI_Comm_size ( commF2C, &size ));
  xmpi ( MPI_Comm_rank ( commF2C, &rank ));
217

Deike Kleberg's avatar
Deike Kleberg committed
218
  myHost = ( char * ) xmalloc ( MPI_MAX_PROCESSOR_NAME * sizeof ( char ));
219

Deike Kleberg's avatar
Deike Kleberg committed
220
  memset ( myHost, 0, MPI_MAX_PROCESSOR_NAME * sizeof ( char ));
221

Deike Kleberg's avatar
Deike Kleberg committed
222
  xmpi ( MPI_Get_processor_name ( myHost, &len ));
Deike Kleberg's avatar
Deike Kleberg committed
223

224
  if ( myHost[0] == '\0' ) xabort ( "did not succeed to set hostname" );
Deike Kleberg's avatar
Deike Kleberg committed
225

Deike Kleberg's avatar
Deike Kleberg committed
226
  if ( ddebug == MAXDEBUG )
227
228
229
    {
      strncpy ( hostname, myHost, len );
      hostname [ len ] = '\0';
230
      xdebug ( "myHost = %s", hostname );
231
    }
232

Deike Kleberg's avatar
Deike Kleberg committed
233
  allHosts = ( char ** ) xmalloc ( size * sizeof ( char * ));
234
235
  allHosts[0] = ( char * ) xmalloc ( size * MPI_MAX_PROCESSOR_NAME *
                                     sizeof ( char ));
Deike Kleberg's avatar
Deike Kleberg committed
236
  allHosts0 = allHosts[0];
Deike Kleberg's avatar
Deike Kleberg committed
237

238
239
  for ( i = 1; i < size; i++ )
    allHosts[i] = allHosts[0] + i * MPI_MAX_PROCESSOR_NAME;
240

241
242
243
  MPI_Allgather ( myHost, MPI_MAX_PROCESSOR_NAME, MPI_CHAR,
                            & ( allHosts[0][0] ), MPI_MAX_PROCESSOR_NAME,
                            MPI_CHAR, commF2C );
244
245
246
247
248
249

  qsort ( allHosts, size, sizeof ( char * ), cmpr );

  *color = 0;
  i = 0;
  j = 0;
250

251
  while ( i < size )
252
253
    {
      curr = allHosts[i];
254
      j++;
255
      if (( test = strcmp ( myHost, curr )) == 0 ) *color = j;
256

257
      while ( ++i < size )
258
259
        if (( test = strcmp ( allHosts[i], curr )) != 0)
          break;
260
    }
Deike Kleberg's avatar
Deike Kleberg committed
261

262
  *nnodes = j;
263

264
  if ( *color == 0 ) xabort ( "Color is not set" );
265

266
  npes_node = size / ( * nnodes );
Deike Kleberg's avatar
Deike Kleberg committed
267
268
  key = rank % npes_node;

Deike Kleberg's avatar
Deike Kleberg committed
269
  xmpi ( MPI_Comm_split ( commF2C, *color, key, myComm));
270

Deike Kleberg's avatar
Deike Kleberg committed
271
  free ( allHosts0 );
272
273
  free ( allHosts );
  free ( myHost );
Deike Kleberg's avatar
Deike Kleberg committed
274

275
  xdebug ( "color=%d", *color );
276

277
278
279
280
281
282
283
  return;
}

#endif

/***************************************************************/

284
MPI_Comm backendInit ( int ptype, MPI_Comm comm, int *color, int *nnodes )
285
{
Deike Kleberg's avatar
Deike Kleberg committed
286
  int size, rank;
287
  int collectingData = 1;
288
289
  *nnodes = 1;
  *color = 1;
290

Deike Kleberg's avatar
Deike Kleberg committed
291
#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
292
293
  xmpi ( MPI_Comm_size ( comm, &size ));
  xmpi ( MPI_Comm_rank ( comm, &rank ));
294
295

  if ( ptype < 0 || ptype > maxPtype )
296
    xabort ( "PIOTYPE is no valid modus" );
Deike Kleberg's avatar
Deike Kleberg committed
297
298
299

#ifdef _SX
  if ( ptype ==  PIO_POSIX_ASYNCH )
300
    xabort ( "PIO_POSIX_ASYNCH does not work on SX" );
Deike Kleberg's avatar
Deike Kleberg committed
301
#endif
302

303
  pioinfo = ( pioInfo * ) xmalloc ( sizeof ( pioInfo ));
304
305

  pioinfo->type = ptype;
Deike Kleberg's avatar
Deike Kleberg committed
306

307
  setPioCommunicator ( & ( pioinfo->comm ), comm, color, nnodes );
308

309
  if (( pioinfo->type == PIO_NONE ) && ( *nnodes != size ))
310
    xabort ( "PIOTYPE, NNODES: not a valid combination" );
311

312
  pioinfo->color = *color;
Deike Kleberg's avatar
Deike Kleberg committed
313

Deike Kleberg's avatar
Deike Kleberg committed
314
315
  xmpi ( MPI_Comm_rank ( pioinfo->comm, &( pioinfo->rank )));
  xmpi ( MPI_Comm_size ( pioinfo->comm, &( pioinfo->size )));
316

317
  xdebug( "IOPE%d in bInit(), ptype=%d, initial_buffersize=%ld: "
318
319
          "after init pioinfo ...",
          pioinfo->rank, pioinfo->type, initial_buffersize );
320

321
  pioinfo->collectorComm = MPI_COMM_NULL;
Deike Kleberg's avatar
Deike Kleberg committed
322

Deike Kleberg's avatar
Deike Kleberg committed
323
324
  switch ( pioinfo->type )
    {
325
    case PIO_NONE:
Deike Kleberg's avatar
Deike Kleberg committed
326
      xmpi ( MPI_Comm_dup ( pioinfo->comm, &( pioinfo->collectorComm )));
327
      collectingData = 1;
328
      break;
329
    case PIO_MPI_NONB:
330
      collectingData = initMPINONB ();
Deike Kleberg's avatar
Deike Kleberg committed
331
      break;
Deike Kleberg's avatar
Deike Kleberg committed
332
#ifndef _SX
333
/*
Deike Kleberg's avatar
Deike Kleberg committed
334
    case PIO_POSIX_ASYNCH:
335
      collectingData = initPOSIXASYNCH ();
Deike Kleberg's avatar
Deike Kleberg committed
336
      break;
337
*/
Deike Kleberg's avatar
Deike Kleberg committed
338
#endif
339
    case PIO_POSIX_FPGUARD_SENDRECV:
340
      collectingData = initPOSIXFPGUARDSENDRECV ();
341
      break;
342
/*
343
    case PIO_POSIX_FPGUARD_THREAD:
344
      collectingData = initPOSIXFPGUARDTHREAD ();
345
      break;
346
347
*/
/*
348
    case PIO_POSIX_FPGUARD_THREAD_REFUSE:
349
      collectingData = initPOSIXFPGUARDTHREADREFUSE ();
350
      break;
351
352
*/
/*
Deike Kleberg's avatar
Deike Kleberg committed
353
    case PIO_POSIX_NONB:
354
      collectingData = initPOSIXNONB ();
355
      break;
356
*/
Deike Kleberg's avatar
Deike Kleberg committed
357
    }
358
#endif
359
360

#ifdef USE_MPI
Deike Kleberg's avatar
Deike Kleberg committed
361
  return pioinfo->collectorComm;
362
363
364
#else
  return 0;
#endif
Deike Kleberg's avatar
Deike Kleberg committed
365
366
367
368
}

/***************************************************************/

369
void backendFinalize ()
Deike Kleberg's avatar
Deike Kleberg committed
370
{
371
#ifdef USE_MPI
372
373
  switch ( pioinfo->type )
    {
374
/*
375
376
377
    case PIO_POSIX_FPGUARD_THREAD:
      finalizePOSIXFPGUARDTHREAD ();
      break;
378
*/
379
380
    }

381
  if ( pioinfo->collectorComm != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
382
    xmpi ( MPI_Comm_free ( &( pioinfo->collectorComm )));
383
384

  if ( pioinfo->comm != MPI_COMM_NULL )
Deike Kleberg's avatar
Deike Kleberg committed
385
    xmpi ( MPI_Comm_free ( &( pioinfo->comm )));
386

Deike Kleberg's avatar
Deike Kleberg committed
387
  free ( pioinfo );
388
#endif
Deike Kleberg's avatar
Deike Kleberg committed
389
}
390
391
392
393
394
395
396
397
398
/*
 * Local Variables:
 * c-file-style: "Java"
 * c-basic-offset: 2
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */