pio.c 7.22 KB
Newer Older
1
#ifdef HAVE_CONFIG_H
Deike Kleberg's avatar
Deike Kleberg committed
2
#include "config.h"
3
4
#endif

5
6
7

#ifdef USE_MPI

8
9
10
#include <ctype.h>
#include <yaxt.h>

11
#include "file.h"
12
13
#include "cdi_int.h"
#include "namespace.h"
14

15
#include "pio.h"
Deike Kleberg's avatar
Deike Kleberg committed
16
17
#include "cdi.h"
#include "pio_comm.h"
Deike Kleberg's avatar
Deike Kleberg committed
18
#include "pio_impl.h"
19
20
#include "pio_interface.h"
#include "pio_rpc.h"
21
#include "pio_util.h"
Deike Kleberg's avatar
Deike Kleberg committed
22

23
char * command2charP[6] = {"IO_Open_file", "IO_Close_file",
24
25
                           "IO_Get_fp","IO_Set_fp",
                           "IO_Send_buffer", "IO_Finalize"};
Deike Kleberg's avatar
Deike Kleberg committed
26

27
long initial_buffersize = 16 * 1024 * 1024;
28
/*  4 KB <= x < 256 MB */
29
/* 16 * 1024 * 1024; */
30
/* 16 * 1024; */
31
/* 4 * 1024; */
Deike Kleberg's avatar
Deike Kleberg committed
32

33
34
35
enum {
  tagKey = 100,
};
36

Deike Kleberg's avatar
Deike Kleberg committed
37
38
39
40
41
42
43
double accumProbe   = 0.0;
double accumRecv    = 0.0;
double accumSend    = 0.0;
double accumSuspend = 0.0;
double accumWait    = 0.0;
double accumWrite   = 0.0;

44
45
char *token = "%";

46
/***************************************************************/
Deike Kleberg's avatar
Deike Kleberg committed
47

48
int encodeFileOpTag(int ID, int sc)
Deike Kleberg's avatar
Deike Kleberg committed
49
{
50
  return ID * tagKey + sc;
Deike Kleberg's avatar
Deike Kleberg committed
51
52
53
54
}

/***************************************************************/

55
struct fileOpTag decodeFileOpTag(int tag)
Deike Kleberg's avatar
Deike Kleberg committed
56
{
57
  struct fileOpTag rtag;
Deike Kleberg's avatar
Deike Kleberg committed
58

59
60
  rtag.id = tag / tagKey;
  rtag.command = tag % tagKey;
Deike Kleberg's avatar
Deike Kleberg committed
61
62
63
64
65
66

  return rtag;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
67
size_t pioFileWrite ( int fileID, int tsID, const void *buffer, size_t len )
Deike Kleberg's avatar
Deike Kleberg committed
68
{
Thomas Jahns's avatar
Thomas Jahns committed
69
  size_t iret = CDI_UNDEFID;
Deike Kleberg's avatar
Deike Kleberg committed
70

Deike Kleberg's avatar
Deike Kleberg committed
71
  switch ( commInqIOMode ())
Deike Kleberg's avatar
Deike Kleberg committed
72
    {
Deike Kleberg's avatar
Deike Kleberg committed
73
    case PIO_MPI:
Deike Kleberg's avatar
Deike Kleberg committed
74
      iret = fwMPINONB ( fileID, tsID, buffer, len );
Deike Kleberg's avatar
Deike Kleberg committed
75
      break;
Deike Kleberg's avatar
Deike Kleberg committed
76
#ifndef _SX
Deike Kleberg's avatar
Deike Kleberg committed
77
    case PIO_ASYNCH:
78
#endif
79
80
81
    case PIO_WRITER:
      iret = pioSendWrite(fileID, tsID, buffer, len);
      break;
Deike Kleberg's avatar
Deike Kleberg committed
82
    case PIO_FPGUARD:
Deike Kleberg's avatar
Deike Kleberg committed
83
      iret = fwPOSIXFPGUARDSENDRECV ( fileID, tsID, buffer, len );
84
      break;
Deike Kleberg's avatar
Deike Kleberg committed
85
86
87
88
89
90
91
92
93
    }

  return iret;
}

/***************************************************************/

int pioFileClose ( int id )
{
Thomas Jahns's avatar
Thomas Jahns committed
94
  int iret = CDI_UNDEFID;
Deike Kleberg's avatar
Deike Kleberg committed
95
  switch ( commInqIOMode ())
Deike Kleberg's avatar
Deike Kleberg committed
96
    {
Deike Kleberg's avatar
Deike Kleberg committed
97
    case PIO_MPI:
98
      iret = fcMPINONB ( id );
Deike Kleberg's avatar
Deike Kleberg committed
99
      break;
Deike Kleberg's avatar
Deike Kleberg committed
100
#ifndef _SX
Deike Kleberg's avatar
Deike Kleberg committed
101
    case PIO_ASYNCH:
Deike Kleberg's avatar
Deike Kleberg committed
102
#endif
103
104
105
    case PIO_WRITER:
      iret = pioSendClose(id);
      break;
Deike Kleberg's avatar
Deike Kleberg committed
106
    case PIO_FPGUARD:
107
108
      iret = fcPOSIXFPGUARDSENDRECV ( id );
      break;
Deike Kleberg's avatar
Deike Kleberg committed
109
110
111
112
113
114
115
    }

  return iret;
}

/***************************************************************/

116
int pioFileOpen(const char *filename, const char *mode)
Deike Kleberg's avatar
Deike Kleberg committed
117
{
Thomas Jahns's avatar
Thomas Jahns committed
118
  int iret = CDI_UNDEFID;
119

120
121
122
  if ((mode[0] != 'w' && mode[0] != 'W') || mode[0] == 0 || mode[1] != 0)
    xabort("Unsupported mode \"%s\" in parallel file open.", mode);

Deike Kleberg's avatar
Deike Kleberg committed
123
  switch ( commInqIOMode ())
Deike Kleberg's avatar
Deike Kleberg committed
124
    {
Deike Kleberg's avatar
Deike Kleberg committed
125
    case PIO_MPI:
126
      iret = fowMPINONB ( filename );
Deike Kleberg's avatar
Deike Kleberg committed
127
      break;
Deike Kleberg's avatar
Deike Kleberg committed
128
#ifndef _SX
Deike Kleberg's avatar
Deike Kleberg committed
129
    case PIO_ASYNCH:
Deike Kleberg's avatar
Deike Kleberg committed
130
#endif
131
132
133
    case PIO_WRITER:
      iret = pioSendOpen(filename);
      break;
Deike Kleberg's avatar
Deike Kleberg committed
134
    case PIO_FPGUARD:
135
136
      iret = fowPOSIXFPGUARDSENDRECV ( filename );
      break;
Deike Kleberg's avatar
Deike Kleberg committed
137
    }
138

Deike Kleberg's avatar
Deike Kleberg committed
139
140
  return iret;
}
141
142
143

/***************************************************************/

144
145
int
cdiPioStreamOpen(const char *filename, const char *filemode,
146
                 int filetype, stream_t *streamptr)
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
{
  if ( tolower ( * filemode ) == 'w' )
    {
      statusCode nspStatus = namespaceInqResStatus ();
      switch ( nspStatus )
        {
        case STAGE_DEFINITION:
          break;
        case STAGE_TIMELOOP:
          pioBufferFuncCall(STREAMOPEN, 2, filename, filetype);
          break;
        case STAGE_CLEANUP:
          xabort ( "TRANSITION TO IO PROCESSES ALREADY FINISHED." );
          break;
        default:
          xabort ( "INTERNAL ERROR" );
        }
    }
  else
    Error("cdiPIO read support not implemented");
  return 1;
}



void
cdiPioStreamDefVlist_(int streamID, int vlistID)
{
  statusCode nspStatus = namespaceInqResStatus ();
  switch ( nspStatus )
    {
    case STAGE_DEFINITION:
      break;
    case STAGE_TIMELOOP:
      pioBufferFuncCall(STREAMDEFVLIST, 2, streamID, vlistID);
      break;
    case STAGE_CLEANUP:
      xabort ( "TRANSITION TO IO PROCESSES ALREADY FINISHED." );
      break;
    default:
      xabort ( "INTERNAL ERROR" );
    }
  cdiStreamDefVlist_(streamID, vlistID);
}

/***************************************************************/

void
cdiPioStreamWriteVar_(int streamID, int varID, int memtype, const void *data,
                      int nmiss)
{
  statusCode nspStatus = namespaceInqResStatus ();
  switch ( nspStatus )
    {
    case STAGE_DEFINITION:
      xabort ( "DEFINITION STAGE: PARALLEL WRITING NOT POSSIBLE." );
      break;
    case STAGE_TIMELOOP:
      pioBufferData ( streamID, varID, data, nmiss );
      return;
    case STAGE_CLEANUP:
      xabort ( "CLEANUP STAGE: PARALLEL WRITING NOT POSSIBLE." );
      break;
    default:
      xabort ( "INTERNAL ERROR" );
    }
}

/***************************************************************/

void
cdiPioStreamWriteVarChunk_(int streamID, int varID, int memtype,
                           const int rect[][2], const void *data, int nmiss)
{
  int vlistID = streamInqVlist(streamID);
  int size = vlistInqVarSize(vlistID, varID),
    varShape[3];
  unsigned ndims = (unsigned)cdiPioQueryVarDims(varShape, vlistID, varID);
  Xt_int varShapeXt[3], chunkShape[3] = { 1, 1, 1 }, origin[3] = { 0, 0, 0 };
  /* FIXME: verify xt_int ranges are good enough */
  for (unsigned i = 0; i < 3; ++i)
    varShapeXt[i] = varShape[i];
  for (unsigned i = 0; i < ndims; ++i)
    chunkShape[i] = rect[i][1] - rect[i][0] + 1;
  int varSize = varShape[0] * varShape[1] * varShape[2];
  xassert(varSize == size);
  Xt_idxlist chunkDesc
    = xt_idxsection_new(0, ndims, varShapeXt, chunkShape, origin);
  pioBufferPartData(streamID, varID, data, nmiss, chunkDesc);
  xt_idxlist_delete(chunkDesc);
}

void
cdiPioStreamClose(stream_t *streamptr)
{
  statusCode nspStatus = namespaceInqResStatus ();
  switch ( nspStatus )
    {
    case STAGE_DEFINITION:
      break;
    case STAGE_TIMELOOP:
      pioBufferFuncCall(STREAMCLOSE, 1, streamptr->self);
      break;
    case STAGE_CLEANUP:
      break;
    default:
      xabort ( "INTERNAL ERROR" );
    }
}

/***************************************************************/

259
void backendInit ( void )
260
{
Deike Kleberg's avatar
Deike Kleberg committed
261
  int IOMode = commInqIOMode ();
262

Deike Kleberg's avatar
Deike Kleberg committed
263
  commDefCommNode ();
264

Deike Kleberg's avatar
Deike Kleberg committed
265
  xassert ( IOMode != PIO_NONE  || commInqSizeNode () == 1 );
266

Deike Kleberg's avatar
Deike Kleberg committed
267
  switch ( IOMode )
Deike Kleberg's avatar
Deike Kleberg committed
268
    {
269
    case PIO_NONE:
270
      commDefCommColl ( 1 );
Deike Kleberg's avatar
Deike Kleberg committed
271
272
273
      commSendNodeInfo ();
      commRecvNodeMap ();
      commDefCommsIO ();
274
      break;
Deike Kleberg's avatar
Deike Kleberg committed
275
    case PIO_MPI:
276
      initMPINONB ();
Deike Kleberg's avatar
Deike Kleberg committed
277
      break;
Deike Kleberg's avatar
Deike Kleberg committed
278
#ifndef _SX
Deike Kleberg's avatar
Deike Kleberg committed
279
    case PIO_ASYNCH:
Deike Kleberg's avatar
Deike Kleberg committed
280
#endif
281
282
283
    case PIO_WRITER:
      pioSendInitialize();
      break;
Deike Kleberg's avatar
Deike Kleberg committed
284
    case PIO_FPGUARD:
285
      initPOSIXFPGUARDSENDRECV ();
286
      break;
Deike Kleberg's avatar
Deike Kleberg committed
287
288
289
290
291
    }
}

/***************************************************************/

292
293
294
295
296
void backendCleanup ( void )
{
  int IOMode = commInqIOMode ();
  switch ( IOMode )
    {
297
298
    case PIO_NONE:
      break;
Deike Kleberg's avatar
Deike Kleberg committed
299
    case PIO_MPI:
300
301
      finalizeMPINONB ();
      break;
302
#ifndef _SX
Deike Kleberg's avatar
Deike Kleberg committed
303
    case PIO_ASYNCH:
304
305
306
#endif
    case PIO_WRITER:
      pioSendFinalize();
307
      break;
Deike Kleberg's avatar
Deike Kleberg committed
308
    case PIO_FPGUARD:
Deike Kleberg's avatar
Deike Kleberg committed
309
310
      finalizePOSIXFPGUARDSENDRECV ();
      break;
311
    default:
312
      xdebug("%s", " BACKENDCLEANUP FUNCTION NOT IMPLEMENTED YET.");
313
314
315
316
317
    }
}

/***************************************************************/

318
void backendFinalize ( void )
Deike Kleberg's avatar
Deike Kleberg committed
319
{
320
321
  commDestroy ();
  MPI_Finalize ();
322
  exit ( EXIT_SUCCESS );
Deike Kleberg's avatar
Deike Kleberg committed
323
}
324
#endif
325
326
327
328
329
330
331
332
333
/*
 * Local Variables:
 * c-file-style: "Java"
 * c-basic-offset: 2
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */