stream.c 47.2 KB
Newer Older
Uwe Schulzweida's avatar
Uwe Schulzweida committed
1
2
3
4
#if defined (HAVE_CONFIG_H)
#  include "config.h"
#endif

5
6
7
8
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 600
#endif

Uwe Schulzweida's avatar
Uwe Schulzweida committed
9
10
#include <ctype.h>

11
#include "binary.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
12
#include "cdi.h"
13
#include "cdi_int.h"
14
#include "cdi_cksum.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
15
#include "cdf.h"
16
17
#include "dmemory.h"
#include "error.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
18
#include "stream_grb.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
19
20
21
22
23
#include "stream_cdf.h"
#include "stream_srv.h"
#include "stream_ext.h"
#include "stream_ieg.h"
#include "file.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
24
#include "cgribex.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
25
#include "gribapi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
26
#include "vlist.h"
27
#include "serialize.h"
28
#include "resource_handle.h"
29
#include "resource_unpack.h"
30
31
#include "namespace.h"

32

33
static stream_t *stream_new_entry(int resH);
34
static void stream_delete_entry(stream_t *streamptr);
35
36
37
38
39
40
static int streamCompareP(void * streamptr1, void * streamptr2);
static void streamDestroyP(void * streamptr);
static void streamPrintP(void * streamptr, FILE * fp);
static int streamGetPackSize(void * streamptr, void *context);
static void streamPack(void * streamptr, void * buff, int size, int * position, void *context);
static int streamTxCode(void);
41

42
43
44
45
46
47
48
49
const resOps streamOps = {
  streamCompareP,
  streamDestroyP,
  streamPrintP,
  streamGetPackSize,
  streamPack,
  streamTxCode
};
Uwe Schulzweida's avatar
Uwe Schulzweida committed
50
51


52

53
54
static
int getByteorder(int byteswap)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
55
56
57
{
  int byteorder = -1;

58
  switch (HOST_ENDIANNESS)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
59
    {
60
61
62
63
64
65
66
67
68
69
    case CDI_BIGENDIAN:
      byteorder = byteswap ? CDI_LITTLEENDIAN : CDI_BIGENDIAN;
      break;
    case CDI_LITTLEENDIAN:
      byteorder = byteswap ? CDI_BIGENDIAN : CDI_LITTLEENDIAN;
      break;
    /* FIXME: does not currently adjust for PDP endianness */
    case CDI_PDPENDIAN:
    default:
      Error("unhandled endianness");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
70
    }
71
  return byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
72
73
}

74
75
// used also in CDO
int cdiGetFiletype(const char *filename, int *byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
76
77
78
{
  int filetype = CDI_EUFTYPE;
  int swap = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
79
  int version;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
80
  long recpos;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
81
  char buffer[8];
Uwe Schulzweida's avatar
Uwe Schulzweida committed
82

83
  int fileID = fileOpen(filename, "r");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
84
85
86

  if ( fileID == CDI_UNDEFID )
    {
87
      if ( strncmp(filename, "http:", 5) == 0 || strncmp(filename, "https:", 6) == 0 )
88
	return CDI_FILETYPE_NC;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
89
      else
90
	return CDI_ESYSTEM;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
91
92
    }

93
  if ( fileRead(fileID, buffer, 8) != 8 ) return CDI_EUFTYPE;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
94
95
96

  fileRewind(fileID);

97
  if ( memcmp(buffer, "GRIB", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
98
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
99
100
101
      version = buffer[7];
      if ( version <= 1 )
	{
102
	  filetype = CDI_FILETYPE_GRB;
103
	  if ( CDI_Debug ) Message("found GRIB file = %s, version %d", filename, version);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
104
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
105
106
      else if ( version == 2 )
	{
107
	  filetype = CDI_FILETYPE_GRB2;
108
	  if ( CDI_Debug ) Message("found GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
109
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
110
    }
111
  else if ( memcmp(buffer, "CDF\001", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
112
    {
113
      filetype = CDI_FILETYPE_NC;
114
      if ( CDI_Debug ) Message("found CDF1 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
115
    }
116
  else if ( memcmp(buffer, "CDF\002", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
117
    {
118
      filetype = CDI_FILETYPE_NC2;
119
      if ( CDI_Debug ) Message("found CDF2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
120
    }
121
  else if ( memcmp(buffer+1, "HDF", 3) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
122
    {
123
      filetype = CDI_FILETYPE_NC4;
124
      if ( CDI_Debug ) Message("found HDF file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
125
126
127
128
    }
#if  defined  (HAVE_LIBSERVICE)
  else if ( srvCheckFiletype(fileID, &swap) )
    {
129
      filetype = CDI_FILETYPE_SRV;
130
      if ( CDI_Debug ) Message("found SRV file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
131
132
133
134
135
    }
#endif
#if  defined  (HAVE_LIBEXTRA)
  else if ( extCheckFiletype(fileID, &swap) )
    {
136
      filetype = CDI_FILETYPE_EXT;
137
      if ( CDI_Debug ) Message("found EXT file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
138
139
140
141
142
    }
#endif
#if  defined  (HAVE_LIBIEG)
  else if ( iegCheckFiletype(fileID, &swap) )
    {
143
      filetype = CDI_FILETYPE_IEG;
144
      if ( CDI_Debug ) Message("found IEG file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
145
146
    }
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
147
#if  defined  (HAVE_LIBCGRIBEX)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
148
  else if ( gribCheckSeek(fileID, &recpos, &version) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
149
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
150
151
      if ( version <= 1 )
	{
152
	  filetype = CDI_FILETYPE_GRB;
153
	  if ( CDI_Debug ) Message("found seeked GRIB file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
154
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
155
      else if ( version == 2 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
156
	{
157
	  filetype = CDI_FILETYPE_GRB2;
158
	  if ( CDI_Debug ) Message("found seeked GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
159
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
160
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
161
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
162
163
164
165
166

  fileClose(fileID);

  *byteorder = getByteorder(swap);

167
  return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
168
169
170
171
172
173
174
175
}

/*
@Function  streamInqFiletype
@Title     Get the filetype

@Prototype int streamInqFiletype(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
176
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
177
178

@Description
179
The function @func{streamInqFiletype} returns the filetype of a stream.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
180
181

@Result
182
@func{streamInqFiletype} returns the type of the file format,
Uwe Schulzweida's avatar
Uwe Schulzweida committed
183
one of the set of predefined CDI file format types.
184
185
The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC}, @func{CDI_FILETYPE_NC2},
@func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
186
187
188
189
190

@EndFunction
*/
int streamInqFiletype(int streamID)
{
191
  stream_t *streamptr = stream_to_pointer(streamID);
192
  return streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
193
194
195
196
197
}


int getByteswap(int byteorder)
{
198
  int byteswap = -1;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
199

200
  switch (byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
201
    {
202
203
204
205
206
    case CDI_BIGENDIAN:
    case CDI_LITTLEENDIAN:
    case CDI_PDPENDIAN:
      byteswap = (HOST_ENDIANNESS != byteorder);
      break;
207
208
    case -1:
      break;
209
210
    default:
      Error("unexpected byteorder %d query!", byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
211
212
    }

213
  return byteswap;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
214
215
}

216
217
218
219
220
221
/*
@Function  streamDefByteorder
@Title     Define the byte order

@Prototype void streamDefByteorder(int streamID, int byteorder)
@Parameter
222
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenWrite}.
223
224
225
226
227
    @Item  byteorder The byte order of a dataset, one of the CDI constants @func{CDI_BIGENDIAN} and
                     @func{CDI_LITTLEENDIAN}.

@Description
The function @func{streamDefByteorder} defines the byte order of a binary dataset
228
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
229
230
231

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
232
233
void streamDefByteorder(int streamID, int byteorder)
{
234
  stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
235
  streamptr->byteorder = byteorder;
236
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
237
238
239
240

  switch (filetype)
    {
#if  defined  (HAVE_LIBSERVICE)
241
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
242
      {
243
	srvrec_t *srvp = (srvrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
244
245
246
247
248
249
	srvp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
250
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
251
      {
252
	extrec_t *extp = (extrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
253
254
255
256
257
258
	extp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
259
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
260
      {
261
	iegrec_t *iegp = (iegrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
262
263
264
265
266
267
	iegp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
    }
268
  reshSetStatus(streamID, &streamOps, RESH_DESYNC_IN_USE);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
269
270
}

271
272
273
274
275
276
/*
@Function  streamInqByteorder
@Title     Get the byte order

@Prototype int streamInqByteorder(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
277
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
278
279
280

@Description
The function @func{streamInqByteorder} returns the byte order of a binary dataset
281
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
282
283
284
285
286
287
288

@Result
@func{streamInqByteorder} returns the type of the byte order.
The valid CDI byte order types are @func{CDI_BIGENDIAN} and @func{CDI_LITTLEENDIAN}

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
289
290
int streamInqByteorder(int streamID)
{
291
  stream_t *streamptr = stream_to_pointer(streamID);
292
  return streamptr->byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
293
294
295
}


296
const char *streamFilesuffix(int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
297
{
298
  // static char *fileSuffix[] = {"", ".grb", ".g2", ".nc", ".nc", ".nc4", ".nc4", ".srv", ".ext", ".ieg"};
299
300
301
302
303
  /* note: the 2nd dimenstion of the fileSuffix array must be equal to or
   * larger than the length of the longest suffix (dot and \0 terminator
   * included) */
  static const char fileSuffix[][5] = {"", ".grb", ".grb", ".nc", ".nc", ".nc", ".nc", ".srv", ".ext", ".ieg"};
  int size = (int)(sizeof(fileSuffix)/sizeof(fileSuffix[0]));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
304
305

  if ( filetype > 0 && filetype < size )
306
    return fileSuffix[filetype];
Uwe Schulzweida's avatar
Uwe Schulzweida committed
307
  else
308
    return fileSuffix[0];
Uwe Schulzweida's avatar
Uwe Schulzweida committed
309
310
311
}


312
const char *streamFilename(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
313
{
314
  stream_t *streamptr = stream_to_pointer(streamID);
315
  return streamptr->filename;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
316
317
}

318
static
319
long cdiInqTimeSize(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
320
321
{
  int tsID = 0, nrecs;
322
  stream_t *streamptr = stream_to_pointer(streamID);
323
  long ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
324

325
  if ( ntsteps == (long)CDI_UNDEFID )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
326
    while ( (nrecs = streamInqTimestep(streamID, tsID++)) )
327
      ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
328

329
  return ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
330
331
}

332
static
333
int cdiInqContents(stream_t *streamptr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
334
335
{
  int status = 0;
Thomas Jahns's avatar
Thomas Jahns committed
336
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
337
338
339
340

  switch (filetype)
    {
#if  defined  (HAVE_LIBGRIB)
341
342
    case CDI_FILETYPE_GRB:
    case CDI_FILETYPE_GRB2:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
343
      {
344
        status = grbInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
345
346
347
	break;
      }
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
348
#if  defined  (HAVE_LIBSERVICE)
349
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
350
      {
351
        status = srvInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
352
353
354
355
	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
356
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
357
      {
358
        status = extInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
359
360
361
362
	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
363
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
364
      {
365
        status = iegInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
366
367
368
369
	break;
      }
#endif
#if  defined  (HAVE_LIBNETCDF)
370
371
372
373
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
374
      {
375
        status = cdfInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
376
377
378
379
380
381
	break;
      }
#endif
    default:
      {
	if ( CDI_Debug )
382
	  Message("%s support not compiled in!", strfiletype(filetype));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
383
384

	status = CDI_ELIBNAVAIL;
385
        break;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
386
387
388
389
390
      }
    }

  if ( status == 0 )
    {
Thomas Jahns's avatar
Thomas Jahns committed
391
392
393
      int vlistID = streamptr->vlistID;
      int taxisID = vlistInqTaxis(vlistID);
      if ( taxisID != CDI_UNDEFID )
394
395
396
397
398
        {
          taxis_t *taxisptr1 = &streamptr->tsteps[0].taxis;
          taxis_t *taxisptr2 = taxisPtr(taxisID);
          ptaxisCopy(taxisptr2, taxisptr1);
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
399
400
    }

401
  return status;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
402
403
}

404
int cdiStreamOpenDefaultDelegate(const char *filename, char filemode,
405
406
                                 int filetype, stream_t *streamptr,
                                 int recordBufIsToBeCreated)
407
408
409
{
  int fileID;
  switch (filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
410
411
    {
#if  defined  (HAVE_LIBGRIB)
412
    case CDI_FILETYPE_GRB:
413
#if  defined  (HAVE_LIBGRIB_API)
414
    case CDI_FILETYPE_GRB2:
415
#endif
416
      {
417
#ifndef __cplusplus
418
        fileID = gribOpen(filename, (char [2]){filemode, 0});
419
420
421
422
#else
        char temp[2] = { filemode, 0 };
        fileID = gribOpen(filename, temp);
#endif
423
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
424
425
        if (recordBufIsToBeCreated)
          {
426
            streamptr->record = (Record *) Malloc(sizeof(Record));
427
428
            streamptr->record->buffer = NULL;
          }
429
430
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
431
432
#endif
#if  defined  (HAVE_LIBSERVICE)
433
    case CDI_FILETYPE_SRV:
434
      {
435
#ifndef __cplusplus
436
        fileID = fileOpen(filename, (char [2]){filemode, 0});
437
438
439
440
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif
441
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
442
443
        if (recordBufIsToBeCreated)
          {
444
            streamptr->record = (Record *) Malloc(sizeof(Record));
445
            streamptr->record->buffer = NULL;
446
            streamptr->record->exsep  = srvNew();
447
          }
448
449
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
450
451
#endif
#if  defined  (HAVE_LIBEXTRA)
452
    case CDI_FILETYPE_EXT:
453
      {
454
#ifndef __cplusplus
455
        fileID = fileOpen(filename, (char [2]){filemode, 0});
456
457
458
459
460
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif

461
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
462
463
        if (recordBufIsToBeCreated)
          {
464
            streamptr->record = (Record *) Malloc(sizeof(Record));
465
            streamptr->record->buffer = NULL;
466
            streamptr->record->exsep  = extNew();
467
          }
468
469
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
470
471
#endif
#if  defined  (HAVE_LIBIEG)
472
    case CDI_FILETYPE_IEG:
473
      {
474
#ifndef __cplusplus
475
        fileID = fileOpen(filename, (char [2]){filemode, 0});
476
477
478
479
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif
480
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
481
482
        if (recordBufIsToBeCreated)
          {
483
            streamptr->record = (Record *) Malloc(sizeof(Record));
484
            streamptr->record->buffer = NULL;
485
            streamptr->record->exsep  = iegNew();
486
          }
487
488
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
489
490
#endif
#if  defined  (HAVE_LIBNETCDF)
491
    case CDI_FILETYPE_NC:
492
      {
493
#ifndef __cplusplus
494
        fileID = cdfOpen(filename, (char [2]){filemode, 0});
495
496
497
498
#else
        char temp[2] = { filemode, 0 };
        fileID = cdfOpen(filename, temp);
#endif
499
500
        break;
      }
501
    case CDI_FILETYPE_NC2:
502
      {
503
#ifndef __cplusplus
504
        fileID = cdfOpen64(filename, (char [2]){filemode, 0});
505
506
507
508
#else
        char temp[2] = { filemode, 0 };
        fileID = cdfOpen64(filename, temp);
#endif
509
510
        break;
      }
511
512
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
513
      {
514
#ifndef __cplusplus
515
        fileID = cdf4Open(filename, (char [2]){filemode, 0}, &filetype);
516
517
518
519
#else
        char temp[2] = { filemode, 0 };
        fileID = cdf4Open(filename, temp, &filetype);
#endif
520
        break;
521
522
523
524
525
      }
#endif
    default:
      {
        if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
526
        return CDI_ELIBNAVAIL;
527
      }
528
    }
529
530
531

  streamptr->filetype = filetype;

532
533
534
535
  return fileID;
}


536
int streamOpenID(const char *filename, char filemode, int filetype, int resH)
537
538
{
  if ( CDI_Debug )
539
    Message("Open %s mode %c file %s", strfiletype(filetype), filemode,
540
            filename?filename:"(NUL)");
541

542
  if ( ! filename || filetype < 0 ) return CDI_EINVAL;
543

544
545
  stream_t *streamptr = stream_new_entry(resH);
  int streamID = CDI_ESYSTEM;
546

547
548
549
550
  int (*streamOpenDelegate)(const char *filename, char filemode,
                            int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
    = (int (*)(const char *, char, int, stream_t *, int))
    namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
551

552
  int fileID = streamOpenDelegate(filename, filemode, filetype, streamptr, 1);
553
  if ( fileID < 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
554
555
556
557
558
    {
      streamID = fileID;
    }
  else
    {
559
560
      streamID = streamptr->self;
      if ( streamID < 0 ) return CDI_ELIMIT;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
561

562
      streamptr->filemode = filemode;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
563
564
      streamptr->filename = strdupx(filename);
      streamptr->fileID   = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
565

566
      if ( filemode == 'r' )
567
568
        {
          int vlistID = vlistCreate();
569
          if ( vlistID < 0 ) return CDI_ELIMIT;
570
571
572
573

          cdiVlistMakeInternal(vlistID);
          streamptr->vlistID = vlistID;
          /* cdiReadByteorder(streamID); */
574
          int status = cdiInqContents(streamptr);
575
576
577
578
579
580
581
582
583
584
          if ( status < 0 )
            {
              streamID = status;
            }
          else
            {
              vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
              vlistptr->ntsteps = streamptr->ntsteps;
              cdiVlistMakeImmutable(vlistID);
            }
585
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
586
    }
587

588
589
590
591
592
593
594
  if ( streamID < 0 )
    {
      Free(streamptr->record);
      stream_delete_entry(streamptr);
    }

  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
595
596
}

597
598
static
int streamOpen(const char *filename, const char *filemode, int filetype)
599
{
600
601
  if ( !filemode || strlen(filemode) != 1 ) return CDI_EINVAL;
  return streamOpenID(filename, (char)tolower(filemode[0]), filetype, CDI_UNDEFID);
602
}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
603

604
605
static
int streamOpenA(const char *filename, const char *filemode, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
606
607
{
  if ( CDI_Debug )
608
    Message("Open %s file (mode=%c); filename: %s", strfiletype(filetype), (int) *filemode, filename);
609
  if ( CDI_Debug ) printf("streamOpenA: %s\n", filename); // seg fault without this line on thunder/squall with "cdo cat x y"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
610

611
  if ( ! filename || ! filemode || filetype < 0 ) return CDI_EINVAL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
612

613
614
615
  stream_t *streamptr = stream_new_entry(CDI_UNDEFID);
  int fileID = CDI_UNDEFID;

616
  {
617
    int (*streamOpenDelegate)(const char *filename, char filemode,
618
                              int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
Thomas Jahns's avatar
Thomas Jahns committed
619
      = (int (*)(const char *, char, int, stream_t *, int))
620
      namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
621

622
    fileID = streamOpenDelegate(filename, 'r', filetype, streamptr, 1);
623
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
624

625
  if ( fileID == CDI_UNDEFID || fileID == CDI_ELIBNAVAIL || fileID == CDI_ESYSTEM ) return fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
626

627
  int streamID = streamptr->self;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
628

629
630
631
632
633
  streamptr->filemode = tolower(*filemode);
  streamptr->filename = strdupx(filename);
  streamptr->fileID   = fileID;

  streamptr->vlistID = vlistCreate();
634
  cdiVlistMakeInternal(streamptr->vlistID);
635
  /* cdiReadByteorder(streamID); */
636
  int status = cdiInqContents(streamptr);
637
  if ( status < 0 ) return status;
638
  vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
639
  vlistptr->ntsteps = (int)cdiInqTimeSize(streamID);
640
641
642
643
644

  // Needed for NetCDF4
  for ( int varID = 0; varID < vlistptr->nvars; ++varID )
    streamptr->vars[varID].defmiss = true;

645
  if ( !strcmp(filemode, "r") ) cdiVlistMakeImmutable(streamptr->vlistID);
646

647
648
  {
    void (*streamCloseDelegate)(stream_t *streamptr, int recordBufIsToBeDeleted)
649
650
      = (void (*)(stream_t *, int))
      namespaceSwitchGet(NSSWITCH_STREAM_CLOSE_BACKEND).func;
651

652
653
    streamCloseDelegate(streamptr, 0);
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
654
655
656
657

  switch (filetype)
    {
#if  defined  (HAVE_LIBGRIB)
658
    case CDI_FILETYPE_GRB:
659
#if  defined  (HAVE_LIBGRIB_API)
660
    case CDI_FILETYPE_GRB2:
661
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
662
663
      {
        fileID = gribOpen(filename, filemode);
664
        if ( fileID != CDI_UNDEFID ) gribContainersNew(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
665
666
667
668
	break;
      }
#endif
#if  defined  (HAVE_LIBSERVICE)
669
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
670
671
672
673
674
675
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
676
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
677
678
679
680
681
682
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
683
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
684
685
686
687
688
689
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBNETCDF)
690
    case CDI_FILETYPE_NC:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
691
692
      {
	fileID = cdfOpen(filename, filemode);
693
	streamptr->ncmode = 2;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
694
695
	break;
      }
696
    case CDI_FILETYPE_NC2:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
697
698
      {
	fileID = cdfOpen64(filename, filemode);
699
	streamptr->ncmode = 2;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
700
701
	break;
      }
702
703
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
704
      {
Deike Kleberg's avatar
Deike Kleberg committed
705
	fileID = cdf4Open(filename, filemode, &filetype);
706
	streamptr->ncmode = 2;
707
708
	break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
709
710
711
#endif
    default:
      {
712
	if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
713
	return CDI_ELIBNAVAIL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
714
715
716
717
718
719
      }
    }

  if ( fileID == CDI_UNDEFID )
    streamID = CDI_UNDEFID;
  else
720
    streamptr->fileID = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
721

722
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
723
724
725
726
}

/*
@Function  streamOpenRead
Uwe Schulzweida's avatar
Uwe Schulzweida committed
727
@Title     Open a dataset for reading
Uwe Schulzweida's avatar
Uwe Schulzweida committed
728

Uwe Schulzweida's avatar
Uwe Schulzweida committed
729
@Prototype int streamOpenRead(const char *path)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
730
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
731
    @Item  path  The name of the dataset to be read.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
732
733

@Description
734
The function @func{streamOpenRead} opens an existing dataset for reading.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
735
736

@Result
737
Upon successful completion @func{streamOpenRead} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
738
739
740
741
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
742
743
744
745
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
746
747
748
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
749
Here is an example using @func{streamOpenRead} to open an existing NetCDF
750
file named @func{foo.nc} for reading:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
751
752

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
753
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
754
755
756
757
758
759
760
761
762
   ...
int streamID;
   ...
streamID = streamOpenRead("foo.nc");
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
Thomas Jahns's avatar
Thomas Jahns committed
763
int streamOpenRead(const char *filename)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
764
{
765
766
  cdiInitialize();

767
  int byteorder = 0;
768
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
769

770
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
771

Thomas Jahns's avatar
Thomas Jahns committed
772
  int streamID = streamOpen(filename, "r", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
773

Uwe Schulzweida's avatar
Uwe Schulzweida committed
774
775
  if ( streamID >= 0 )
    {
776
      stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
777
778
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
779

780
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
781
782
783
784
785
}


int streamOpenAppend(const char *filename)
{
786
787
  cdiInitialize();

788
  int byteorder = 0;
789
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
790

791
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
792

793
  int streamID = streamOpenA(filename, "a", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
794

795
796
  if ( streamID >= 0 )
    {
797
      stream_t *streamptr = stream_to_pointer(streamID);
798
799
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
800

801
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
802
803
804
805
}

/*
@Function  streamOpenWrite
Uwe Schulzweida's avatar
Uwe Schulzweida committed
806
@Title     Create a new dataset
Uwe Schulzweida's avatar
Uwe Schulzweida committed
807

Uwe Schulzweida's avatar
Uwe Schulzweida committed
808
@Prototype int streamOpenWrite(const char *path, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
809
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
810
    @Item  path      The name of the new dataset.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
811
    @Item  filetype  The type of the file format, one of the set of predefined CDI file format types.
812
813
814
                     The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC},
                     @func{CDI_FILETYPE_NC2}, @func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_SRV},
                     @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
815
816

@Description
817
The function @func{streamOpenWrite} creates a new datset.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
818
@Result
819
Upon successful completion @func{streamOpenWrite} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
820
821
822
823
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
824
825
826
827
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
828
829
830
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
831
Here is an example using @func{streamOpenWrite} to create a new NetCDF file named @func{foo.nc} for writing:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
832
833

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
834
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
835
836
837
   ...
int streamID;
   ...
838
streamID = streamOpenWrite("foo.nc", CDI_FILETYPE_NC);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
839
840
841
842
843
844
845
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
int streamOpenWrite(const char *filename, int filetype)
{
846
847
  cdiInitialize();

848
  return streamOpen(filename, "w", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
849
850
}

851
852
853
854
855
856
static
void streamDefaultValue ( stream_t * streamptr )
{
  streamptr->self              = CDI_UNDEFID;
  streamptr->accesstype        = CDI_UNDEFID;
  streamptr->accessmode        = 0;
857
  streamptr->filetype          = CDI_FILETYPE_UNDEF;
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
  streamptr->byteorder         = CDI_UNDEFID;
  streamptr->fileID            = 0;
  streamptr->filemode          = 0;
  streamptr->numvals           = 0;
  streamptr->filename          = NULL;
  streamptr->record            = NULL;
  streamptr->varsAllocated     = 0;
  streamptr->nrecs             = 0;
  streamptr->nvars             = 0;
  streamptr->vars              = NULL;
  streamptr->ncmode            = 0;
  streamptr->curTsID           = CDI_UNDEFID;
  streamptr->rtsteps           = 0;
  streamptr->ntsteps           = CDI_UNDEFID;
  streamptr->tsteps            = NULL;
  streamptr->tstepsTableSize   = 0;
  streamptr->tstepsNextID      = 0;
  streamptr->historyID         = CDI_UNDEFID;
  streamptr->vlistID           = CDI_UNDEFID;
  streamptr->globalatts        = 0;
  streamptr->localatts         = 0;
  streamptr->unreduced         = cdiDataUnreduced;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
880
881
  streamptr->sortname          = cdiSortName > 0;
  streamptr->sortparam         = cdiSortParam > 0;
882
  streamptr->have_missval      = cdiHaveMissval;
883
  streamptr->comptype          = CDI_COMPRESS_NONE;
884
885
886
887
  streamptr->complevel         = 0;

  basetimeInit(&streamptr->basetime);

Uwe Schulzweida's avatar
Uwe Schulzweida committed
888
#ifdef HAVE_LIBNETCDF
889
890
  for ( int i = 0; i < MAX_ZAXES_PS; i++ ) streamptr->zaxisID[i]  = CDI_UNDEFID;
  for ( int i = 0; i < MAX_ZAXES_PS; i++ ) streamptr->nczvarID[i] = CDI_UNDEFID;
891

892
893
894
  for ( int i = 0; i < MAX_GRIDS_PS; i++ )
    {
      streamptr->ncgrid[i].gridID = CDI_UNDEFID;
895
896
      for (size_t j = 0; j < CDF_SIZE_ncIDs; ++j)
        streamptr->ncgrid[i].ncIDs[j] = CDI_UNDEFID;
897
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
898
899
900
901
902

  streamptr->vct.ilev          = 0;
  streamptr->vct.mlev          = 0;
  streamptr->vct.ilevID        = CDI_UNDEFID;
  streamptr->vct.mlevID        = CDI_UNDEFID;
903
#endif
904
905
906
907

  streamptr->gribContainers    = NULL;
}

908
909
static
stream_t *stream_new_entry(int resH)
910
911
912
{
  cdiInitialize(); /* ***************** make MT version !!! */

913
  stream_t *streamptr = (stream_t *) Malloc(sizeof(stream_t));
914
  streamDefaultValue ( streamptr );
915

916
917
918
919
920
921
922
  if (resH == CDI_UNDEFID)
    streamptr->self = reshPut(streamptr, &streamOps);
  else
    {
      streamptr->self = resH;
      reshReplace(resH, streamptr, &streamOps);
    }
923
924
925
926
927

  return streamptr;
}


928
void cdiStreamCloseDefaultDelegate(stream_t *streamptr, int recordBufIsToBeDeleted)
929
930
931
932
933
934
935
936
937
{
  int fileID   = streamptr->fileID;
  int filetype = streamptr->filetype;
  if ( fileID == CDI_UNDEFID )
    Warning("File %s not open!", streamptr->filename);
  else
    switch (filetype)
      {
#if  defined  (HAVE_LIBGRIB)
938
939
      case CDI_FILETYPE_GRB:
      case CDI_FILETYPE_GRB2: