stream.c 50.3 KB
Newer Older
1
2
#ifdef  HAVE_CONFIG_H
#include "config.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
3
4
#endif

5
6
7
8
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 600
#endif

9
#include <sys/stat.h> // struct stat
Uwe Schulzweida's avatar
Uwe Schulzweida committed
10
11
#include <ctype.h>

12
#include "binary.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
13
#include "cdi.h"
14
#include "cdi_int.h"
15
#include "cdi_cksum.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
16
#include "cdf.h"
17
18
#include "dmemory.h"
#include "error.h"
19
#include "stream_cgribex.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
20
#include "stream_grb.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
21
22
23
24
25
#include "stream_cdf.h"
#include "stream_srv.h"
#include "stream_ext.h"
#include "stream_ieg.h"
#include "file.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
26
#include "cgribex.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
27
#include "gribapi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
28
#include "vlist.h"
29
#include "serialize.h"
30
#include "resource_handle.h"
31
#include "resource_unpack.h"
32
#include "namespace.h"
33
#include "async_worker.h"
34

35

36
static stream_t *stream_new_entry(int resH);
37
static void stream_delete_entry(stream_t *streamptr);
38
39
40
41
42
43
static int streamCompareP(void * streamptr1, void * streamptr2);
static void streamDestroyP(void * streamptr);
static void streamPrintP(void * streamptr, FILE * fp);
static int streamGetPackSize(void * streamptr, void *context);
static void streamPack(void * streamptr, void * buff, int size, int * position, void *context);
static int streamTxCode(void);
44

45
46
47
48
49
50
51
52
const resOps streamOps = {
  streamCompareP,
  streamDestroyP,
  streamPrintP,
  streamGetPackSize,
  streamPack,
  streamTxCode
};
Uwe Schulzweida's avatar
Uwe Schulzweida committed
53
54


55

56
57
static
int getByteorder(int byteswap)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
58
59
60
{
  int byteorder = -1;

61
  switch (HOST_ENDIANNESS)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
62
    {
63
64
65
66
67
68
69
70
71
72
    case CDI_BIGENDIAN:
      byteorder = byteswap ? CDI_LITTLEENDIAN : CDI_BIGENDIAN;
      break;
    case CDI_LITTLEENDIAN:
      byteorder = byteswap ? CDI_BIGENDIAN : CDI_LITTLEENDIAN;
      break;
    /* FIXME: does not currently adjust for PDP endianness */
    case CDI_PDPENDIAN:
    default:
      Error("unhandled endianness");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
73
    }
74
  return byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
75
76
}

77
78
// used also in CDO
int cdiGetFiletype(const char *filename, int *byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
79
80
81
{
  int filetype = CDI_EUFTYPE;
  int swap = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
82
  int version;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
83
84
  long recpos;

85
  int fileID = fileOpen(filename, "r");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
86
87
88

  if ( fileID == CDI_UNDEFID )
    {
89
      if ( strncmp(filename, "http:", 5) == 0 || strncmp(filename, "https:", 6) == 0 )
90
	return CDI_FILETYPE_NC;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
91
      else
92
	return CDI_ESYSTEM;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
93
    }
94
  else if ( fileID == -2 ) return CDI_ETMOF;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
95

96
97
98
99
100
101
102
103
104
105
106
107
  char buffer[8];
  if ( fileRead(fileID, buffer, 8) != 8 )
    {
      struct stat buf;
      if ( stat(filename, &buf) == 0 )
        {
          if ( buf.st_size == 0 ) return CDI_EISEMPTY;
          if ( buf.st_mode&S_IFDIR ) return CDI_EISDIR;
        }

      return CDI_EUFTYPE;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
108
109
110

  fileRewind(fileID);

111
  if ( memcmp(buffer, "GRIB", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
112
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
113
114
115
      version = buffer[7];
      if ( version <= 1 )
	{
116
	  filetype = CDI_FILETYPE_GRB;
117
	  if ( CDI_Debug ) Message("found GRIB file = %s, version %d", filename, version);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
118
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
119
120
      else if ( version == 2 )
	{
121
	  filetype = CDI_FILETYPE_GRB2;
122
	  if ( CDI_Debug ) Message("found GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
123
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
124
    }
125
  else if ( memcmp(buffer, "CDF\001", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
126
    {
127
      filetype = CDI_FILETYPE_NC;
128
      if ( CDI_Debug ) Message("found CDF1 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
129
    }
130
  else if ( memcmp(buffer, "CDF\002", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
131
    {
132
      filetype = CDI_FILETYPE_NC2;
133
      if ( CDI_Debug ) Message("found CDF2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
134
    }
135
136
  else if ( memcmp(buffer, "CDF\005", 4) == 0 )
    {
137
      filetype = CDI_FILETYPE_NC5;
138
139
      if ( CDI_Debug ) Message("found CDF5 file = %s", filename);
    }
140
  else if ( memcmp(buffer+1, "HDF", 3) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
141
    {
142
      filetype = CDI_FILETYPE_NC4;
143
      if ( CDI_Debug ) Message("found HDF file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
144
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
145
#ifdef  HAVE_LIBSERVICE
Uwe Schulzweida's avatar
Uwe Schulzweida committed
146
147
  else if ( srvCheckFiletype(fileID, &swap) )
    {
148
      filetype = CDI_FILETYPE_SRV;
149
      if ( CDI_Debug ) Message("found SRV file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
150
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
151
152
#endif
#ifdef  HAVE_LIBEXTRA
Uwe Schulzweida's avatar
Uwe Schulzweida committed
153
154
  else if ( extCheckFiletype(fileID, &swap) )
    {
155
      filetype = CDI_FILETYPE_EXT;
156
      if ( CDI_Debug ) Message("found EXT file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
157
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
158
159
#endif
#ifdef  HAVE_LIBIEG
Uwe Schulzweida's avatar
Uwe Schulzweida committed
160
161
  else if ( iegCheckFiletype(fileID, &swap) )
    {
162
      filetype = CDI_FILETYPE_IEG;
163
      if ( CDI_Debug ) Message("found IEG file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
164
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
165
166
#endif
#ifdef HAVE_LIBGRIB
Uwe Schulzweida's avatar
Uwe Schulzweida committed
167
  else if ( gribCheckSeek(fileID, &recpos, &version) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
168
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
169
170
      if ( version <= 1 )
	{
171
	  filetype = CDI_FILETYPE_GRB;
172
	  if ( CDI_Debug ) Message("found seeked GRIB file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
173
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
174
      else if ( version == 2 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
175
	{
176
	  filetype = CDI_FILETYPE_GRB2;
177
	  if ( CDI_Debug ) Message("found seeked GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
178
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
179
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
180
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
181
182
183
184
185

  fileClose(fileID);

  *byteorder = getByteorder(swap);

186
  return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
187
188
189
190
191
192
193
194
}

/*
@Function  streamInqFiletype
@Title     Get the filetype

@Prototype int streamInqFiletype(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
195
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
196
197

@Description
198
The function @func{streamInqFiletype} returns the filetype of a stream.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
199
200

@Result
201
@func{streamInqFiletype} returns the type of the file format,
Uwe Schulzweida's avatar
Uwe Schulzweida committed
202
one of the set of predefined CDI file format types.
203
The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC}, @func{CDI_FILETYPE_NC2},
204
@func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_NC5}, @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
205
206
207
208
209

@EndFunction
*/
int streamInqFiletype(int streamID)
{
210
  stream_t *streamptr = stream_to_pointer(streamID);
211
  return streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
212
213
214
215
216
}


int getByteswap(int byteorder)
{
217
  int byteswap = -1;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
218

219
  switch (byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
220
    {
221
222
223
224
225
    case CDI_BIGENDIAN:
    case CDI_LITTLEENDIAN:
    case CDI_PDPENDIAN:
      byteswap = (HOST_ENDIANNESS != byteorder);
      break;
226
227
    case -1:
      break;
228
229
    default:
      Error("unexpected byteorder %d query!", byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
230
231
    }

232
  return byteswap;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
233
234
}

235
236
237

void streamDefNumWorker(int streamID, int numWorker)
{
238
239
240
241
242
  if (numWorker > 0)
    {
      stream_t *streamptr = stream_to_pointer(streamID);
      streamptr->numWorker = numWorker;
    }
243
244
245
}


246
247
248
249
250
251
/*
@Function  streamDefByteorder
@Title     Define the byte order

@Prototype void streamDefByteorder(int streamID, int byteorder)
@Parameter
252
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenWrite}.
253
254
255
256
257
    @Item  byteorder The byte order of a dataset, one of the CDI constants @func{CDI_BIGENDIAN} and
                     @func{CDI_LITTLEENDIAN}.

@Description
The function @func{streamDefByteorder} defines the byte order of a binary dataset
258
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
259
260
261

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
262
263
void streamDefByteorder(int streamID, int byteorder)
{
264
  stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
265
  streamptr->byteorder = byteorder;
266
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
267
268
269

  switch (filetype)
    {
270
#ifdef  HAVE_LIBSERVICE
271
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
272
      {
273
	srvrec_t *srvp = (srvrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
274
275
276
277
278
	srvp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
279
#ifdef  HAVE_LIBEXTRA
280
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
281
      {
282
	extrec_t *extp = (extrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
283
284
285
286
287
	extp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
288
#ifdef  HAVE_LIBIEG
289
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
290
      {
291
	iegrec_t *iegp = (iegrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
292
293
294
295
296
297
	iegp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
    }
298

299
  reshSetStatus(streamID, &streamOps, RESH_DESYNC_IN_USE);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
300
301
}

302
303
304
305
306
307
/*
@Function  streamInqByteorder
@Title     Get the byte order

@Prototype int streamInqByteorder(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
308
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
309
310
311

@Description
The function @func{streamInqByteorder} returns the byte order of a binary dataset
312
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
313
314
315
316
317
318
319

@Result
@func{streamInqByteorder} returns the type of the byte order.
The valid CDI byte order types are @func{CDI_BIGENDIAN} and @func{CDI_LITTLEENDIAN}

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
320
321
int streamInqByteorder(int streamID)
{
322
  stream_t *streamptr = stream_to_pointer(streamID);
323
  return streamptr->byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
324
325
326
}


327
const char *streamFilesuffix(int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
328
{
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
  static const char *noSuffix  = "";
  static const char *ncSuffix  = ".nc";
  static const char *grbSuffix = ".grb";
  static const char *srvSuffix = ".srv";
  static const char *extSuffix = ".ext";
  static const char *iegSuffix = ".ieg";

  switch (filetype)
    {
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
    case CDI_FILETYPE_NC5:   return ncSuffix;
    case CDI_FILETYPE_GRB:
    case CDI_FILETYPE_GRB2:  return grbSuffix;
    case CDI_FILETYPE_SRV:   return srvSuffix;
    case CDI_FILETYPE_EXT:   return extSuffix;
    case CDI_FILETYPE_IEG:   return iegSuffix;
    default:                 return noSuffix;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
350
351
352
}


353
const char *streamFilename(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
354
{
355
  stream_t *streamptr = stream_to_pointer(streamID);
356
  return streamptr->filename;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
357
358
}

359
static
360
long cdiInqTimeSize(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
361
362
{
  int tsID = 0, nrecs;
363
  stream_t *streamptr = stream_to_pointer(streamID);
364
  long ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
365

366
  if ( ntsteps == (long)CDI_UNDEFID )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
367
    while ( (nrecs = streamInqTimestep(streamID, tsID++)) )
368
      ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
369

370
  return ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
371
372
}

373
static
374
int cdiInqContents(stream_t *streamptr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
375
376
{
  int status = 0;
Thomas Jahns's avatar
Thomas Jahns committed
377
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
378
379
380

  switch (filetype)
    {
381
#ifdef  HAVE_LIBGRIB
382
383
    case CDI_FILETYPE_GRB:
    case CDI_FILETYPE_GRB2:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
384
      {
385
        status = grbInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
386
387
388
	break;
      }
#endif
389
#ifdef  HAVE_LIBSERVICE
390
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
391
      {
392
        status = srvInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
393
394
395
	break;
      }
#endif
396
#ifdef  HAVE_LIBEXTRA
397
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
398
      {
399
        status = extInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
400
401
402
	break;
      }
#endif
403
#ifdef  HAVE_LIBIEG
404
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
405
      {
406
        status = iegInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
407
408
409
	break;
      }
#endif
410
#ifdef  HAVE_LIBNETCDF
411
412
413
414
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
415
    case CDI_FILETYPE_NC5:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
416
      {
417
        status = cdfInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
418
419
420
421
422
423
	break;
      }
#endif
    default:
      {
	if ( CDI_Debug )
424
	  Message("%s support not compiled in!", strfiletype(filetype));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
425
426

	status = CDI_ELIBNAVAIL;
427
        break;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
428
429
430
431
432
      }
    }

  if ( status == 0 )
    {
Thomas Jahns's avatar
Thomas Jahns committed
433
434
435
      int vlistID = streamptr->vlistID;
      int taxisID = vlistInqTaxis(vlistID);
      if ( taxisID != CDI_UNDEFID )
436
437
438
439
440
        {
          taxis_t *taxisptr1 = &streamptr->tsteps[0].taxis;
          taxis_t *taxisptr2 = taxisPtr(taxisID);
          ptaxisCopy(taxisptr2, taxisptr1);
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
441
442
    }

443
  return status;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
444
445
}

446
447

int cdiStreamOpenDefaultDelegate(const char *filename, char filemode, int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
448
449
{
  int fileID;
450
451
  char temp[2] = { filemode, 0 };

452
  switch (filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
453
    {
454
#if defined(HAVE_LIBGRIB) && (defined(HAVE_LIBCGRIBEX) || defined(HAVE_LIBGRIB_API))
455
    case CDI_FILETYPE_GRB:
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
      {
        fileID = gribOpen(filename, temp);
        if ( fileID < 0 ) return CDI_ESYSTEM;
        if (recordBufIsToBeCreated)
          {
            streamptr->record = (Record *) Malloc(sizeof(Record));
            streamptr->record->buffer = NULL;
#ifdef  HAVE_LIBCGRIBEX
            streamptr->record->cgribexp = cgribexNew();
#else
            streamptr->record->cgribexp = NULL;
#endif
          }
        break;
      }
471
#ifdef  HAVE_LIBGRIB_API
472
    case CDI_FILETYPE_GRB2:
473
      {
474
        fileID = gribOpen(filename, temp);
475
        if ( fileID < 0 ) return CDI_ESYSTEM;
476
477
        if (recordBufIsToBeCreated)
          {
478
            streamptr->record = (Record *) Malloc(sizeof(Record));
479
480
            streamptr->record->buffer = NULL;
          }
481
482
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
483
#endif
484
#endif
485
#ifdef  HAVE_LIBSERVICE
486
    case CDI_FILETYPE_SRV:
487
      {
488
        fileID = fileOpen(filename, temp);
489
        if ( fileID < 0 ) return CDI_ESYSTEM;
490
491
        if (recordBufIsToBeCreated)
          {
492
            streamptr->record = (Record *) Malloc(sizeof(Record));
493
            streamptr->record->buffer = NULL;
494
            streamptr->record->exsep = srvNew();
495
          }
496
497
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
498
#endif
499
#ifdef  HAVE_LIBEXTRA
500
    case CDI_FILETYPE_EXT:
501
      {
502
        fileID = fileOpen(filename, temp);
503
        if ( fileID < 0 ) return CDI_ESYSTEM;
504
505
        if (recordBufIsToBeCreated)
          {
506
            streamptr->record = (Record *) Malloc(sizeof(Record));
507
            streamptr->record->buffer = NULL;
508
            streamptr->record->exsep = extNew();
509
          }
510
511
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
512
#endif
513
#ifdef  HAVE_LIBIEG
514
    case CDI_FILETYPE_IEG:
515
      {
516
        fileID = fileOpen(filename, temp);
517
        if ( fileID < 0 ) return CDI_ESYSTEM;
518
519
        if (recordBufIsToBeCreated)
          {
520
            streamptr->record = (Record *) Malloc(sizeof(Record));
521
            streamptr->record->buffer = NULL;
522
            streamptr->record->exsep = iegNew();
523
          }
524
525
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
526
#endif
527
#ifdef  HAVE_LIBNETCDF
528
529
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
530
    case CDI_FILETYPE_NC5:
531
      {
532
        fileID = cdfOpen(filename, temp, filetype);
533
534
        break;
      }
535
536
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
537
      {
538
        fileID = cdf4Open(filename, temp, &filetype);
539
        break;
540
541
542
543
544
      }
#endif
    default:
      {
        if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
545
        return CDI_ELIBNAVAIL;
546
      }
547
    }
548
549
550

  streamptr->filetype = filetype;

551
552
553
554
  return fileID;
}


555
int streamOpenID(const char *filename, char filemode, int filetype, int resH)
556
557
{
  if ( CDI_Debug )
558
    Message("Open %s mode %c file %s", strfiletype(filetype), filemode, filename?filename:"(NUL)");
559

560
  if ( ! filename || filetype < 0 ) return CDI_EINVAL;
561

562
563
  stream_t *streamptr = stream_new_entry(resH);
  int streamID = CDI_ESYSTEM;
564

565
566
567
568
  int (*streamOpenDelegate)(const char *filename, char filemode,
                            int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
    = (int (*)(const char *, char, int, stream_t *, int))
    namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
569

570
  int fileID = streamOpenDelegate(filename, filemode, filetype, streamptr, 1);
571
  if ( fileID < 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
572
573
574
575
576
    {
      streamID = fileID;
    }
  else
    {
577
578
      streamID = streamptr->self;
      if ( streamID < 0 ) return CDI_ELIMIT;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
579

580
      streamptr->filemode = filemode;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
581
582
      streamptr->filename = strdupx(filename);
      streamptr->fileID   = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
583

584
      if ( filemode == 'r' )
585
586
        {
          int vlistID = vlistCreate();
587
          if ( vlistID < 0 ) return CDI_ELIMIT;
588
589
590
591

          cdiVlistMakeInternal(vlistID);
          streamptr->vlistID = vlistID;
          /* cdiReadByteorder(streamID); */
592
          int status = cdiInqContents(streamptr);
593
594
595
596
597
598
599
600
601
602
          if ( status < 0 )
            {
              streamID = status;
            }
          else
            {
              vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
              vlistptr->ntsteps = streamptr->ntsteps;
              cdiVlistMakeImmutable(vlistID);
            }
603
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
604
    }
605

606
607
608
609
610
611
612
  if ( streamID < 0 )
    {
      Free(streamptr->record);
      stream_delete_entry(streamptr);
    }

  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
613
614
}

615
616
static
int streamOpen(const char *filename, const char *filemode, int filetype)
617
{
618
619
  if ( !filemode || strlen(filemode) != 1 ) return CDI_EINVAL;
  return streamOpenID(filename, (char)tolower(filemode[0]), filetype, CDI_UNDEFID);
620
}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
621

622
623
static
int streamOpenA(const char *filename, const char *filemode, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
624
625
{
  if ( CDI_Debug )
626
    Message("Open %s file (mode=%c); filename: %s", strfiletype(filetype), (int) *filemode, filename);
627
  if ( CDI_Debug ) printf("streamOpenA: %s\n", filename); // seg fault without this line on thunder/squall with "cdo cat x y"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
628

629
  if ( ! filename || ! filemode || filetype < 0 ) return CDI_EINVAL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
630

631
632
633
  stream_t *streamptr = stream_new_entry(CDI_UNDEFID);
  int fileID = CDI_UNDEFID;

634
  {
635
    int (*streamOpenDelegate)(const char *filename, char filemode,
636
                              int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
Thomas Jahns's avatar
Thomas Jahns committed
637
      = (int (*)(const char *, char, int, stream_t *, int))
638
      namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
639

640
    fileID = streamOpenDelegate(filename, 'r', filetype, streamptr, 1);
641
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
642

643
  if ( fileID == CDI_UNDEFID || fileID == CDI_ELIBNAVAIL || fileID == CDI_ESYSTEM ) return fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
644

645
  int streamID = streamptr->self;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
646

647
648
649
650
651
  streamptr->filemode = tolower(*filemode);
  streamptr->filename = strdupx(filename);
  streamptr->fileID   = fileID;

  streamptr->vlistID = vlistCreate();
652
  cdiVlistMakeInternal(streamptr->vlistID);
653
  /* cdiReadByteorder(streamID); */
654
  int status = cdiInqContents(streamptr);
655
  if ( status < 0 ) return status;
656
  vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
657
  vlistptr->ntsteps = (int)cdiInqTimeSize(streamID);
658
659
660
661
662

  // Needed for NetCDF4
  for ( int varID = 0; varID < vlistptr->nvars; ++varID )
    streamptr->vars[varID].defmiss = true;

663
  if ( !strcmp(filemode, "r") ) cdiVlistMakeImmutable(streamptr->vlistID);
664

665
666
  {
    void (*streamCloseDelegate)(stream_t *streamptr, int recordBufIsToBeDeleted)
667
668
      = (void (*)(stream_t *, int))
      namespaceSwitchGet(NSSWITCH_STREAM_CLOSE_BACKEND).func;
669

670
671
    streamCloseDelegate(streamptr, 0);
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
672
673
674

  switch (filetype)
    {
675
#if defined(HAVE_LIBGRIB) && (defined(HAVE_LIBCGRIBEX) || defined(HAVE_LIBGRIB_API))
676
    case CDI_FILETYPE_GRB:
677
#ifdef  HAVE_LIBGRIB_API
678
    case CDI_FILETYPE_GRB2:
679
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
680
681
      {
        fileID = gribOpen(filename, filemode);
682
        if ( fileID != CDI_UNDEFID ) gribContainersNew(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
683
684
685
	break;
      }
#endif
686
#ifdef  HAVE_LIBSERVICE
687
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
688
689
690
691
692
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
693
#ifdef  HAVE_LIBEXTRA
694
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
695
696
697
698
699
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
700
#ifdef  HAVE_LIBIEG
701
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
702
703
704
705
706
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
707
#ifdef  HAVE_LIBNETCDF
708
709
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
710
    case CDI_FILETYPE_NC5:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
711
      {
712
	fileID = cdfOpen(filename, filemode, filetype);
713
	streamptr->ncmode = 2;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
714
715
	break;
      }
716
717
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
718
      {
Deike Kleberg's avatar
Deike Kleberg committed
719
	fileID = cdf4Open(filename, filemode, &filetype);
720
	streamptr->ncmode = 2;
721
722
	break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
723
724
725
#endif
    default:
      {
726
	if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
727
	return CDI_ELIBNAVAIL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
728
729
730
731
732
733
      }
    }

  if ( fileID == CDI_UNDEFID )
    streamID = CDI_UNDEFID;
  else
734
    streamptr->fileID = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
735

736
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
737
738
739
740
}

/*
@Function  streamOpenRead
Uwe Schulzweida's avatar
Uwe Schulzweida committed
741
@Title     Open a dataset for reading
Uwe Schulzweida's avatar
Uwe Schulzweida committed
742

Uwe Schulzweida's avatar
Uwe Schulzweida committed
743
@Prototype int streamOpenRead(const char *path)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
744
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
745
    @Item  path  The name of the dataset to be read.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
746
747

@Description
748
The function @func{streamOpenRead} opens an existing dataset for reading.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
749
750

@Result
751
Upon successful completion @func{streamOpenRead} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
752
753
754
755
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
756
757
758
759
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
760
761
762
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
763
Here is an example using @func{streamOpenRead} to open an existing NetCDF
764
file named @func{foo.nc} for reading:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
765
766

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
767
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
768
769
770
771
772
773
774
775
776
   ...
int streamID;
   ...
streamID = streamOpenRead("foo.nc");
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
Thomas Jahns's avatar
Thomas Jahns committed
777
int streamOpenRead(const char *filename)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
778
{
779
780
  cdiInitialize();

781
  int byteorder = 0;
782
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
783

784
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
785

Thomas Jahns's avatar
Thomas Jahns committed
786
  int streamID = streamOpen(filename, "r", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
787

Uwe Schulzweida's avatar
Uwe Schulzweida committed
788
789
  if ( streamID >= 0 )
    {
790
      stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
791
792
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
793

794
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
795
796
797
798
799
}


int streamOpenAppend(const char *filename)
{
800
801
  cdiInitialize();

802
  int byteorder = 0;
803
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
804

805
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
806

807
  int streamID = streamOpenA(filename, "a", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
808

809
810
  if ( streamID >= 0 )
    {
811
      stream_t *streamptr = stream_to_pointer(streamID);
812
813
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
814

815
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
816
817
818
819
}

/*
@Function  streamOpenWrite
Uwe Schulzweida's avatar
Uwe Schulzweida committed
820
@Title     Create a new dataset
Uwe Schulzweida's avatar
Uwe Schulzweida committed
821

Uwe Schulzweida's avatar
Uwe Schulzweida committed
822
@Prototype int streamOpenWrite(const char *path, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
823
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
824
    @Item  path      The name of the new dataset.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
825
    @Item  filetype  The type of the file format, one of the set of predefined CDI file format types.
826
                     The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC},
827
                     @func{CDI_FILETYPE_NC2}, @func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_NC5}, @func{CDI_FILETYPE_SRV},
828
                     @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
829
830

@Description
831
The function @func{streamOpenWrite} creates a new datset.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
832
@Result
833
Upon successful completion @func{streamOpenWrite} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
834
835
836
837
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
838
839
840
841
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
842
843
844
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
845
Here is an example using @func{streamOpenWrite} to create a new NetCDF file named @func{foo.nc} for writing:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
846
847

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
848
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
849
850
851
   ...
int streamID;
   ...
852
streamID = streamOpenWrite("foo.nc", CDI_FILETYPE_NC);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
853
854
855
856
857
858
859
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
int streamOpenWrite(const char *filename, int filetype)
{
860
861
  cdiInitialize();

862
  return streamOpen(filename, "w", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
863
864
}

865
866
867
868
869
870
static
void streamDefaultValue ( stream_t * streamptr )
{
  streamptr->self              = CDI_UNDEFID;
  streamptr->accesstype        = CDI_UNDEFID;
  streamptr->accessmode        = 0;
871
  streamptr->filetype          = CDI_FILETYPE_UNDEF;
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
  streamptr->byteorder         = CDI_UNDEFID;
  streamptr->fileID            = 0;
  streamptr->filemode          = 0;
  streamptr->numvals           = 0;
  streamptr->filename          = NULL;
  streamptr->record            = NULL;
  streamptr->varsAllocated     = 0;
  streamptr->nrecs             = 0;
  streamptr->nvars             = 0;
  streamptr->vars              = NULL;
  streamptr->ncmode            = 0;
  streamptr->curTsID           = CDI_UNDEFID;
  streamptr->rtsteps           = 0;
  streamptr->ntsteps           = CDI_UNDEFID;
  streamptr->tsteps            = NULL;
  streamptr->tstepsTableSize   = 0;
  streamptr->tstepsNextID      = 0;
  streamptr->historyID         = CDI_UNDEFID;
  streamptr->vlistID           = CDI_UNDEFID;
  streamptr->globalatts        = 0;
  streamptr->localatts         = 0;
  streamptr->unreduced         = cdiDataUnreduced;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
894
895
  streamptr->sortname          = cdiSortName > 0;
  streamptr->sortparam         = cdiSortParam > 0;
896
  streamptr->have_missval      = cdiHaveMissval;
897
  streamptr->comptype          = CDI_COMPRESS_NONE;
898
899
900
901
  streamptr->complevel         = 0;

  basetimeInit(&streamptr->basetime);

Uwe Schulzweida's avatar
Uwe Schulzweida committed
902
#ifdef HAVE_LIBNETCDF
903
904
  for ( int i = 0; i < MAX_ZAXES_PS; i++ ) streamptr->zaxisID[i]  = CDI_UNDEFID;
  for ( int i = 0; i < MAX_ZAXES_PS; i++ ) streamptr->nczvarID[i] = CDI_UNDEFID;
905

906
907
908
  for ( int i = 0; i < MAX_GRIDS_PS; i++ )
    {
      streamptr->ncgrid[i].gridID = CDI_UNDEFID;
909
910
      for (size_t j = 0; j < CDF_SIZE_ncIDs; ++j)
        streamptr->ncgrid[i].ncIDs[j] = CDI_UNDEFID;
911
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
912
913
914
915
916

  streamptr->vct.ilev          = 0;
  streamptr->vct.mlev          = 0;
  streamptr->vct.ilevID        = CDI_UNDEFID;
  streamptr->vct.mlevID        = CDI_UNDEFID;