pio.c 5.5 KB
Newer Older
Deike Kleberg's avatar
Deike Kleberg committed
1
2
3
4
5
#ifndef NOMPI

#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
Deike Kleberg's avatar
Deike Kleberg committed
6
#include <string.h>
Deike Kleberg's avatar
Deike Kleberg committed
7
8
9
10
11
12
13

#include "mpi.h"

#include "cdi.h"
#include "pio.h"
#include "pio_impl.h"

14
bool ddebug = false;
Deike Kleberg's avatar
Deike Kleberg committed
15

Deike Kleberg's avatar
Deike Kleberg committed
16
17
const char *token_NODE = "p";

18
19
20
long initial_buffersize = 16 * 1024 * 1024;
/*  4 KB <= x < 256 MB */ 
/* 16 * 1024 * 1024; */
Deike Kleberg's avatar
Deike Kleberg committed
21
/* 256 * 1024; */
22
/* 4 * 1024; */
Deike Kleberg's avatar
Deike Kleberg committed
23
24
25
26
27
28
29
30
31
32
33

double startTime;
double accumProbe   = 0.0;
double accumRecv    = 0.0;
double accumSend    = 0.0;
double accumSuspend = 0.0;
double accumWait    = 0.0;
double accumWrite   = 0.0;

pioInfo *pioinfo;

34
35
36
37
MPI_Request request = MPI_REQUEST_NULL;

char *token = "%";

Deike Kleberg's avatar
Deike Kleberg committed
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
/*****************************************************************************/

void check_mpi ( int line, int iret )
{
  char error_string[MPI_MAX_ERROR_STRING+1];
  int len;

  if  ( iret != MPI_SUCCESS ) 
    {
      MPI_Error_string ( iret, error_string, &len ); 
      error_string[len] = '\0';
      fprintf ( stderr,"\nLine %8d MPI error %4d: %s\n\n", line, iret, 
		error_string ); 
    }

  return;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
int setTag ( int ID, int sc )
{
  return ID * 1000 + sc;
}

/***************************************************************/

rTag * getTag ( int tag )
{
  rTag *rtag;

  rtag = ( rTag * ) malloc ( sizeof ( rTag ));
  rtag->id = tag / 1000;
  rtag->command = tag % 1000;

  return rtag;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
78
79
80
81
82
83
size_t pioFileWrite ( int id, int tsId, const void *buffer, size_t len )
{
  size_t iret;

  switch ( pioinfo->type )
    {
84
85
    case PIO_MPI_NONB:
      iret = fwMPINONB ( id, tsId, buffer, len );
Deike Kleberg's avatar
Deike Kleberg committed
86
      break;
Deike Kleberg's avatar
Deike Kleberg committed
87
88
89
    case PIO_POSIX_ASYNCH:
      iret = fwPOSIXASYNCH ( id, tsId, buffer, len );
      break;
90
91
92
93
94
95
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fwPOSIXFPGUARDSENDRECV ( id, tsId, buffer, len );
      break;
    case PIO_POSIX_NONB:
      iret = fwPOSIXNONB ( id, tsId, buffer, len );
      break;
Deike Kleberg's avatar
Deike Kleberg committed
96
97
98
99
100
101
102
103
104
105
106
107
108
    }

  return iret;
}

/***************************************************************/

int pioFileClose ( int id )
{
  int iret;
  
  switch ( pioinfo->type )
    {
109
110
    case PIO_MPI_NONB:
      iret = fcMPINONB ( id );
Deike Kleberg's avatar
Deike Kleberg committed
111
      break;
Deike Kleberg's avatar
Deike Kleberg committed
112
113
114
    case PIO_POSIX_ASYNCH:
      iret = fcPOSIXASYNCH ( id );
      break;
115
116
117
118
119
120
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fcPOSIXFPGUARDSENDRECV ( id );
      break;
    case PIO_POSIX_NONB:
      iret = fcPOSIXNONB ( id );
      break;
Deike Kleberg's avatar
Deike Kleberg committed
121
122
123
124
125
126
127
128
129
130
131
132
133
    }

  return iret;
}

/***************************************************************/

int pioFileOpenW ( const char *filename )
{
  int iret;
  
  switch ( pioinfo->type )
    {
134
135
    case PIO_MPI_NONB:
      iret = fowMPINONB ( filename );
Deike Kleberg's avatar
Deike Kleberg committed
136
      break;
Deike Kleberg's avatar
Deike Kleberg committed
137
138
139
    case PIO_POSIX_ASYNCH:
      iret = fowPOSIXASYNCH ( filename );
      break;
140
141
142
143
144
145
    case PIO_POSIX_FPGUARD_SENDRECV:
      iret = fowPOSIXFPGUARDSENDRECV ( filename );
      break;
    case PIO_POSIX_NONB:
      iret = fowPOSIXNONB ( filename );
      break;
Deike Kleberg's avatar
Deike Kleberg committed
146
147
148
149
150
151
152
    }
  
  return iret;
}

/***************************************************************/

Deike Kleberg's avatar
Deike Kleberg committed
153
int pioInit ( int ptype, int comm, int *color, int *colors, int nnodes, int *pioComm, int *ncollectors )
Deike Kleberg's avatar
Deike Kleberg committed
154
{
Deike Kleberg's avatar
Deike Kleberg committed
155
  int collectingData = 1;
Deike Kleberg's avatar
Deike Kleberg committed
156
157
158
159
160
  int key, rank, size, npes_node;
  int *allcolors;
  int i,j,k,curr;

  char *cpuName, *temp;
Deike Kleberg's avatar
Deike Kleberg committed
161
162


Deike Kleberg's avatar
Deike Kleberg committed
163
  pioinfo = ( pioInfo * ) malloc ( sizeof ( pioInfo ));
Deike Kleberg's avatar
Deike Kleberg committed
164
  pioinfo->type = ptype;
Deike Kleberg's avatar
Deike Kleberg committed
165
166
167
168
169
170
171
172
173
174
175
176

  cpuName = (char * ) malloc ( 80 * sizeof(char));
  gethostname(cpuName,80);
  temp = strtok ( cpuName, token_NODE );
  *color = atoi ( temp );
  MPI_Comm_size (( MPI_Comm ) comm, &size );
  MPI_Comm_rank (( MPI_Comm ) comm, &rank );
  npes_node = size/nnodes;
  key = rank % npes_node;
  MPI_Comm_split (( MPI_Comm ) comm, *color, key, ( MPI_Comm *) pioComm );

  pioinfo->comm  = ( MPI_Comm ) *pioComm;
Deike Kleberg's avatar
Deike Kleberg committed
177
178
179
180
181
182
183
184
185
186
187
188
  MPI_Comm_rank ( pioinfo->comm, &( pioinfo->rank ));
  MPI_Comm_size ( pioinfo->comm, &( pioinfo->size ));

  if ( pioinfo->type == PIO_NONE && pioinfo->size != 1 )
    {
      fprintf ( stderr, 
		"PTYPE should be set to a parallel I/O type or npe should be 1." );
      fprintf ( stderr, "PTYPE = %d, npe = %d\n", 
		pioinfo->type , pioinfo->size );
      MPI_Abort ( pioinfo->comm, 1 );
    }

Deike Kleberg's avatar
Deike Kleberg committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
  if ( ddebug )
    fprintf (stdout, 
	     "pe%d in pioDefPtype(), size=%d, nnodes=%d, cpuName=%s, color=%d,npes_node=%d, key=%d\n", 
	     rank,size, nnodes,cpuName, *color, npes_node, key);

  allcolors = (int * ) malloc ( size * sizeof(int));
  MPI_Allgather ( color, 1, MPI_INTEGER, allcolors, 1, MPI_INTEGER, ( MPI_Comm )comm );

  k = 0;
  for ( i = 0; i < size; i++ )
    {
      curr = allcolors[i] ;
      if ( curr == -1 ) 
	continue;
      else
	{
	  *( colors + k ) = allcolors[i];
	  k++;
	  if ( k == nnodes ) break;
	}
      for ( j = i + 1; j < size; j++ )
	if ( allcolors[j] == curr ) allcolors[j] = -1;
    }

Deike Kleberg's avatar
Deike Kleberg committed
213
214
  if ( ddebug && pioinfo->rank == 0 )
    fprintf ( stdout, 
215
216
	      "pe%d in pioDefPtype(), ptype=%d, initial_buffersize=%ld: init pioinfo ...\n", 
	      pioinfo->rank, pioinfo->type, initial_buffersize );
Deike Kleberg's avatar
Deike Kleberg committed
217
218
219

  switch ( pioinfo->type )
    {
220
221
    case PIO_MPI_NONB:
      collectingData = initMPINONB ( ncollectors );
Deike Kleberg's avatar
Deike Kleberg committed
222
      break;
Deike Kleberg's avatar
Deike Kleberg committed
223
224
225
    case PIO_POSIX_ASYNCH:
      collectingData = initPOSIXASYNCH ( ncollectors );
      break;
226
227
228
229
230
231
    case PIO_POSIX_FPGUARD_SENDRECV:
      collectingData = initPOSIXFPGUARDSENDRECV ( ncollectors );
      break;
    case PIO_POSIX_NONB:
      collectingData = initPOSIXNONB ( ncollectors );
      break;
Deike Kleberg's avatar
Deike Kleberg committed
232
233
    }

Deike Kleberg's avatar
Deike Kleberg committed
234
235
236
  free ( allcolors );
  free ( cpuName );

Deike Kleberg's avatar
Deike Kleberg committed
237
238
239
240
241
242
243
244
245
246
247
248
249
  return collectingData;
}

/***************************************************************/

void pioFinalize ()
{
  free ( pioinfo );

  return;
}

#endif
250