stream.c 47.8 KB
Newer Older
Uwe Schulzweida's avatar
Uwe Schulzweida committed
1
2
3
4
#if defined (HAVE_CONFIG_H)
#  include "config.h"
#endif

5
6
7
8
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 600
#endif

9
#include <sys/stat.h> // struct stat
Uwe Schulzweida's avatar
Uwe Schulzweida committed
10
11
#include <ctype.h>

12
#include "binary.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
13
#include "cdi.h"
14
#include "cdi_int.h"
15
#include "cdi_cksum.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
16
#include "cdf.h"
17
18
#include "dmemory.h"
#include "error.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
19
#include "stream_grb.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
20
21
22
23
24
#include "stream_cdf.h"
#include "stream_srv.h"
#include "stream_ext.h"
#include "stream_ieg.h"
#include "file.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
25
#include "cgribex.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
26
#include "gribapi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
27
#include "vlist.h"
28
#include "serialize.h"
29
#include "resource_handle.h"
30
#include "resource_unpack.h"
31
32
#include "namespace.h"

33

34
static stream_t *stream_new_entry(int resH);
35
static void stream_delete_entry(stream_t *streamptr);
36
37
38
39
40
41
static int streamCompareP(void * streamptr1, void * streamptr2);
static void streamDestroyP(void * streamptr);
static void streamPrintP(void * streamptr, FILE * fp);
static int streamGetPackSize(void * streamptr, void *context);
static void streamPack(void * streamptr, void * buff, int size, int * position, void *context);
static int streamTxCode(void);
42

43
44
45
46
47
48
49
50
const resOps streamOps = {
  streamCompareP,
  streamDestroyP,
  streamPrintP,
  streamGetPackSize,
  streamPack,
  streamTxCode
};
Uwe Schulzweida's avatar
Uwe Schulzweida committed
51
52


53

54
55
static
int getByteorder(int byteswap)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
56
57
58
{
  int byteorder = -1;

59
  switch (HOST_ENDIANNESS)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
60
    {
61
62
63
64
65
66
67
68
69
70
    case CDI_BIGENDIAN:
      byteorder = byteswap ? CDI_LITTLEENDIAN : CDI_BIGENDIAN;
      break;
    case CDI_LITTLEENDIAN:
      byteorder = byteswap ? CDI_BIGENDIAN : CDI_LITTLEENDIAN;
      break;
    /* FIXME: does not currently adjust for PDP endianness */
    case CDI_PDPENDIAN:
    default:
      Error("unhandled endianness");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
71
    }
72
  return byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
73
74
}

75
76
// used also in CDO
int cdiGetFiletype(const char *filename, int *byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
77
78
79
{
  int filetype = CDI_EUFTYPE;
  int swap = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
80
  int version;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
81
82
  long recpos;

83
  int fileID = fileOpen(filename, "r");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
84
85
86

  if ( fileID == CDI_UNDEFID )
    {
87
      if ( strncmp(filename, "http:", 5) == 0 || strncmp(filename, "https:", 6) == 0 )
88
	return CDI_FILETYPE_NC;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
89
      else
90
	return CDI_ESYSTEM;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
91
92
    }

93
94
95
96
97
98
99
100
101
102
103
104
  char buffer[8];
  if ( fileRead(fileID, buffer, 8) != 8 )
    {
      struct stat buf;
      if ( stat(filename, &buf) == 0 )
        {
          if ( buf.st_size == 0 ) return CDI_EISEMPTY;
          if ( buf.st_mode&S_IFDIR ) return CDI_EISDIR;
        }

      return CDI_EUFTYPE;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
105
106
107

  fileRewind(fileID);

108
  if ( memcmp(buffer, "GRIB", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
109
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
110
111
112
      version = buffer[7];
      if ( version <= 1 )
	{
113
	  filetype = CDI_FILETYPE_GRB;
114
	  if ( CDI_Debug ) Message("found GRIB file = %s, version %d", filename, version);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
115
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
116
117
      else if ( version == 2 )
	{
118
	  filetype = CDI_FILETYPE_GRB2;
119
	  if ( CDI_Debug ) Message("found GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
120
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
121
    }
122
  else if ( memcmp(buffer, "CDF\001", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
123
    {
124
      filetype = CDI_FILETYPE_NC;
125
      if ( CDI_Debug ) Message("found CDF1 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
126
    }
127
  else if ( memcmp(buffer, "CDF\002", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
128
    {
129
      filetype = CDI_FILETYPE_NC2;
130
      if ( CDI_Debug ) Message("found CDF2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
131
    }
132
133
134
135
136
  else if ( memcmp(buffer, "CDF\005", 4) == 0 )
    {
      filetype = CDI_FILETYPE_NC2;
      if ( CDI_Debug ) Message("found CDF5 file = %s", filename);
    }
137
  else if ( memcmp(buffer+1, "HDF", 3) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
138
    {
139
      filetype = CDI_FILETYPE_NC4;
140
      if ( CDI_Debug ) Message("found HDF file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
141
142
143
    }
  else if ( srvCheckFiletype(fileID, &swap) )
    {
144
      filetype = CDI_FILETYPE_SRV;
145
      if ( CDI_Debug ) Message("found SRV file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
146
147
148
    }
  else if ( extCheckFiletype(fileID, &swap) )
    {
149
      filetype = CDI_FILETYPE_EXT;
150
      if ( CDI_Debug ) Message("found EXT file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
151
152
153
    }
  else if ( iegCheckFiletype(fileID, &swap) )
    {
154
      filetype = CDI_FILETYPE_IEG;
155
      if ( CDI_Debug ) Message("found IEG file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
156
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
157
  else if ( gribCheckSeek(fileID, &recpos, &version) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
158
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
159
160
      if ( version <= 1 )
	{
161
	  filetype = CDI_FILETYPE_GRB;
162
	  if ( CDI_Debug ) Message("found seeked GRIB file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
163
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
164
      else if ( version == 2 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
165
	{
166
	  filetype = CDI_FILETYPE_GRB2;
167
	  if ( CDI_Debug ) Message("found seeked GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
168
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
169
170
171
172
173
174
    }

  fileClose(fileID);

  *byteorder = getByteorder(swap);

175
  return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
176
177
178
179
180
181
182
183
}

/*
@Function  streamInqFiletype
@Title     Get the filetype

@Prototype int streamInqFiletype(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
184
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
185
186

@Description
187
The function @func{streamInqFiletype} returns the filetype of a stream.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
188
189

@Result
190
@func{streamInqFiletype} returns the type of the file format,
Uwe Schulzweida's avatar
Uwe Schulzweida committed
191
one of the set of predefined CDI file format types.
192
193
The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC}, @func{CDI_FILETYPE_NC2},
@func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
194
195
196
197
198

@EndFunction
*/
int streamInqFiletype(int streamID)
{
199
  stream_t *streamptr = stream_to_pointer(streamID);
200
  return streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
201
202
203
204
205
}


int getByteswap(int byteorder)
{
206
  int byteswap = -1;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
207

208
  switch (byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
209
    {
210
211
212
213
214
    case CDI_BIGENDIAN:
    case CDI_LITTLEENDIAN:
    case CDI_PDPENDIAN:
      byteswap = (HOST_ENDIANNESS != byteorder);
      break;
215
216
    case -1:
      break;
217
218
    default:
      Error("unexpected byteorder %d query!", byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
219
220
    }

221
  return byteswap;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
222
223
}

224
225
226
227
228
229
/*
@Function  streamDefByteorder
@Title     Define the byte order

@Prototype void streamDefByteorder(int streamID, int byteorder)
@Parameter
230
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenWrite}.
231
232
233
234
235
    @Item  byteorder The byte order of a dataset, one of the CDI constants @func{CDI_BIGENDIAN} and
                     @func{CDI_LITTLEENDIAN}.

@Description
The function @func{streamDefByteorder} defines the byte order of a binary dataset
236
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
237
238
239

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
240
241
void streamDefByteorder(int streamID, int byteorder)
{
242
  stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
243
  streamptr->byteorder = byteorder;
244
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
245
246
247
248

  switch (filetype)
    {
#if  defined  (HAVE_LIBSERVICE)
249
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
250
      {
251
	srvrec_t *srvp = (srvrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
252
253
254
255
256
257
	srvp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
258
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
259
      {
260
	extrec_t *extp = (extrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
261
262
263
264
265
266
	extp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
267
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
268
      {
269
	iegrec_t *iegp = (iegrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
270
271
272
273
274
275
	iegp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
    }
276
  reshSetStatus(streamID, &streamOps, RESH_DESYNC_IN_USE);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
277
278
}

279
280
281
282
283
284
/*
@Function  streamInqByteorder
@Title     Get the byte order

@Prototype int streamInqByteorder(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
285
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
286
287
288

@Description
The function @func{streamInqByteorder} returns the byte order of a binary dataset
289
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
290
291
292
293
294
295
296

@Result
@func{streamInqByteorder} returns the type of the byte order.
The valid CDI byte order types are @func{CDI_BIGENDIAN} and @func{CDI_LITTLEENDIAN}

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
297
298
int streamInqByteorder(int streamID)
{
299
  stream_t *streamptr = stream_to_pointer(streamID);
300
  return streamptr->byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
301
302
303
}


304
const char *streamFilesuffix(int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
305
{
306
  // static char *fileSuffix[] = {"", ".grb", ".g2", ".nc", ".nc", ".nc4", ".nc4", ".srv", ".ext", ".ieg"};
307
308
309
310
311
  /* note: the 2nd dimenstion of the fileSuffix array must be equal to or
   * larger than the length of the longest suffix (dot and \0 terminator
   * included) */
  static const char fileSuffix[][5] = {"", ".grb", ".grb", ".nc", ".nc", ".nc", ".nc", ".srv", ".ext", ".ieg"};
  int size = (int)(sizeof(fileSuffix)/sizeof(fileSuffix[0]));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
312
313

  if ( filetype > 0 && filetype < size )
314
    return fileSuffix[filetype];
Uwe Schulzweida's avatar
Uwe Schulzweida committed
315
  else
316
    return fileSuffix[0];
Uwe Schulzweida's avatar
Uwe Schulzweida committed
317
318
319
}


320
const char *streamFilename(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
321
{
322
  stream_t *streamptr = stream_to_pointer(streamID);
323
  return streamptr->filename;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
324
325
}

326
static
327
long cdiInqTimeSize(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
328
329
{
  int tsID = 0, nrecs;
330
  stream_t *streamptr = stream_to_pointer(streamID);
331
  long ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
332

333
  if ( ntsteps == (long)CDI_UNDEFID )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
334
    while ( (nrecs = streamInqTimestep(streamID, tsID++)) )
335
      ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
336

337
  return ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
338
339
}

340
static
341
int cdiInqContents(stream_t *streamptr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
342
343
{
  int status = 0;
Thomas Jahns's avatar
Thomas Jahns committed
344
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
345
346
347
348

  switch (filetype)
    {
#if  defined  (HAVE_LIBGRIB)
349
350
    case CDI_FILETYPE_GRB:
    case CDI_FILETYPE_GRB2:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
351
      {
352
        status = grbInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
353
354
355
	break;
      }
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
356
#if  defined  (HAVE_LIBSERVICE)
357
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
358
      {
359
        status = srvInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
360
361
362
363
	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
364
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
365
      {
366
        status = extInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
367
368
369
370
	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
371
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
372
      {
373
        status = iegInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
374
375
376
377
	break;
      }
#endif
#if  defined  (HAVE_LIBNETCDF)
378
379
380
381
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
382
      {
383
        status = cdfInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
384
385
386
387
388
389
	break;
      }
#endif
    default:
      {
	if ( CDI_Debug )
390
	  Message("%s support not compiled in!", strfiletype(filetype));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
391
392

	status = CDI_ELIBNAVAIL;
393
        break;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
394
395
396
397
398
      }
    }

  if ( status == 0 )
    {
Thomas Jahns's avatar
Thomas Jahns committed
399
400
401
      int vlistID = streamptr->vlistID;
      int taxisID = vlistInqTaxis(vlistID);
      if ( taxisID != CDI_UNDEFID )
402
403
404
405
406
        {
          taxis_t *taxisptr1 = &streamptr->tsteps[0].taxis;
          taxis_t *taxisptr2 = taxisPtr(taxisID);
          ptaxisCopy(taxisptr2, taxisptr1);
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
407
408
    }

409
  return status;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
410
411
}

412
int cdiStreamOpenDefaultDelegate(const char *filename, char filemode,
413
414
                                 int filetype, stream_t *streamptr,
                                 int recordBufIsToBeCreated)
415
416
417
{
  int fileID;
  switch (filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
418
419
    {
#if  defined  (HAVE_LIBGRIB)
420
    case CDI_FILETYPE_GRB:
421
#if  defined  (HAVE_LIBGRIB_API)
422
    case CDI_FILETYPE_GRB2:
423
#endif
424
      {
425
#ifndef __cplusplus
426
        fileID = gribOpen(filename, (char [2]){filemode, 0});
427
428
429
430
#else
        char temp[2] = { filemode, 0 };
        fileID = gribOpen(filename, temp);
#endif
431
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
432
433
        if (recordBufIsToBeCreated)
          {
434
            streamptr->record = (Record *) Malloc(sizeof(Record));
435
436
            streamptr->record->buffer = NULL;
          }
437
438
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
439
440
#endif
#if  defined  (HAVE_LIBSERVICE)
441
    case CDI_FILETYPE_SRV:
442
      {
443
#ifndef __cplusplus
444
        fileID = fileOpen(filename, (char [2]){filemode, 0});
445
446
447
448
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif
449
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
450
451
        if (recordBufIsToBeCreated)
          {
452
            streamptr->record = (Record *) Malloc(sizeof(Record));
453
            streamptr->record->buffer = NULL;
454
            streamptr->record->exsep  = srvNew();
455
          }
456
457
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
458
459
#endif
#if  defined  (HAVE_LIBEXTRA)
460
    case CDI_FILETYPE_EXT:
461
      {
462
#ifndef __cplusplus
463
        fileID = fileOpen(filename, (char [2]){filemode, 0});
464
465
466
467
468
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif

469
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
470
471
        if (recordBufIsToBeCreated)
          {
472
            streamptr->record = (Record *) Malloc(sizeof(Record));
473
            streamptr->record->buffer = NULL;
474
            streamptr->record->exsep  = extNew();
475
          }
476
477
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
478
479
#endif
#if  defined  (HAVE_LIBIEG)
480
    case CDI_FILETYPE_IEG:
481
      {
482
#ifndef __cplusplus
483
        fileID = fileOpen(filename, (char [2]){filemode, 0});
484
485
486
487
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif
488
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
489
490
        if (recordBufIsToBeCreated)
          {
491
            streamptr->record = (Record *) Malloc(sizeof(Record));
492
            streamptr->record->buffer = NULL;
493
            streamptr->record->exsep  = iegNew();
494
          }
495
496
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
497
498
#endif
#if  defined  (HAVE_LIBNETCDF)
499
    case CDI_FILETYPE_NC:
500
      {
501
#ifndef __cplusplus
502
        fileID = cdfOpen(filename, (char [2]){filemode, 0});
503
504
505
506
#else
        char temp[2] = { filemode, 0 };
        fileID = cdfOpen(filename, temp);
#endif
507
508
        break;
      }
509
    case CDI_FILETYPE_NC2:
510
      {
511
#ifndef __cplusplus
512
        fileID = cdfOpen64(filename, (char [2]){filemode, 0});
513
514
515
516
#else
        char temp[2] = { filemode, 0 };
        fileID = cdfOpen64(filename, temp);
#endif
517
518
        break;
      }
519
520
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
521
      {
522
#ifndef __cplusplus
523
        fileID = cdf4Open(filename, (char [2]){filemode, 0}, &filetype);
524
525
526
527
#else
        char temp[2] = { filemode, 0 };
        fileID = cdf4Open(filename, temp, &filetype);
#endif
528
        break;
529
530
531
532
533
      }
#endif
    default:
      {
        if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
534
        return CDI_ELIBNAVAIL;
535
      }
536
    }
537
538
539

  streamptr->filetype = filetype;

540
541
542
543
  return fileID;
}


544
int streamOpenID(const char *filename, char filemode, int filetype, int resH)
545
546
{
  if ( CDI_Debug )
547
    Message("Open %s mode %c file %s", strfiletype(filetype), filemode,
548
            filename?filename:"(NUL)");
549

550
  if ( ! filename || filetype < 0 ) return CDI_EINVAL;
551

552
553
  stream_t *streamptr = stream_new_entry(resH);
  int streamID = CDI_ESYSTEM;
554

555
556
557
558
  int (*streamOpenDelegate)(const char *filename, char filemode,
                            int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
    = (int (*)(const char *, char, int, stream_t *, int))
    namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
559

560
  int fileID = streamOpenDelegate(filename, filemode, filetype, streamptr, 1);
561
  if ( fileID < 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
562
563
564
565
566
    {
      streamID = fileID;
    }
  else
    {
567
568
      streamID = streamptr->self;
      if ( streamID < 0 ) return CDI_ELIMIT;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
569

570
      streamptr->filemode = filemode;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
571
572
      streamptr->filename = strdupx(filename);
      streamptr->fileID   = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
573

574
      if ( filemode == 'r' )
575
576
        {
          int vlistID = vlistCreate();
577
          if ( vlistID < 0 ) return CDI_ELIMIT;
578
579
580
581

          cdiVlistMakeInternal(vlistID);
          streamptr->vlistID = vlistID;
          /* cdiReadByteorder(streamID); */
582
          int status = cdiInqContents(streamptr);
583
584
585
586
587
588
589
590
591
592
          if ( status < 0 )
            {
              streamID = status;
            }
          else
            {
              vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
              vlistptr->ntsteps = streamptr->ntsteps;
              cdiVlistMakeImmutable(vlistID);
            }
593
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
594
    }
595

596
597
598
599
600
601
602
  if ( streamID < 0 )
    {
      Free(streamptr->record);
      stream_delete_entry(streamptr);
    }

  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
603
604
}

605
606
static
int streamOpen(const char *filename, const char *filemode, int filetype)
607
{
608
609
  if ( !filemode || strlen(filemode) != 1 ) return CDI_EINVAL;
  return streamOpenID(filename, (char)tolower(filemode[0]), filetype, CDI_UNDEFID);
610
}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
611

612
613
static
int streamOpenA(const char *filename, const char *filemode, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
614
615
{
  if ( CDI_Debug )
616
    Message("Open %s file (mode=%c); filename: %s", strfiletype(filetype), (int) *filemode, filename);
617
  if ( CDI_Debug ) printf("streamOpenA: %s\n", filename); // seg fault without this line on thunder/squall with "cdo cat x y"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
618

619
  if ( ! filename || ! filemode || filetype < 0 ) return CDI_EINVAL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
620

621
622
623
  stream_t *streamptr = stream_new_entry(CDI_UNDEFID);
  int fileID = CDI_UNDEFID;

624
  {
625
    int (*streamOpenDelegate)(const char *filename, char filemode,
626
                              int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
Thomas Jahns's avatar
Thomas Jahns committed
627
      = (int (*)(const char *, char, int, stream_t *, int))
628
      namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
629

630
    fileID = streamOpenDelegate(filename, 'r', filetype, streamptr, 1);
631
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
632

633
  if ( fileID == CDI_UNDEFID || fileID == CDI_ELIBNAVAIL || fileID == CDI_ESYSTEM ) return fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
634

635
  int streamID = streamptr->self;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
636

637
638
639
640
641
  streamptr->filemode = tolower(*filemode);
  streamptr->filename = strdupx(filename);
  streamptr->fileID   = fileID;

  streamptr->vlistID = vlistCreate();
642
  cdiVlistMakeInternal(streamptr->vlistID);
643
  /* cdiReadByteorder(streamID); */
644
  int status = cdiInqContents(streamptr);
645
  if ( status < 0 ) return status;
646
  vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
647
  vlistptr->ntsteps = (int)cdiInqTimeSize(streamID);
648
649
650
651
652

  // Needed for NetCDF4
  for ( int varID = 0; varID < vlistptr->nvars; ++varID )
    streamptr->vars[varID].defmiss = true;

653
  if ( !strcmp(filemode, "r") ) cdiVlistMakeImmutable(streamptr->vlistID);
654

655
656
  {
    void (*streamCloseDelegate)(stream_t *streamptr, int recordBufIsToBeDeleted)
657
658
      = (void (*)(stream_t *, int))
      namespaceSwitchGet(NSSWITCH_STREAM_CLOSE_BACKEND).func;
659

660
661
    streamCloseDelegate(streamptr, 0);
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
662
663
664
665

  switch (filetype)
    {
#if  defined  (HAVE_LIBGRIB)
666
    case CDI_FILETYPE_GRB:
667
#if  defined  (HAVE_LIBGRIB_API)
668
    case CDI_FILETYPE_GRB2:
669
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
670
671
      {
        fileID = gribOpen(filename, filemode);
672
        if ( fileID != CDI_UNDEFID ) gribContainersNew(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
673
674
675
676
	break;
      }
#endif
#if  defined  (HAVE_LIBSERVICE)
677
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
678
679
680
681
682
683
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
684
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
685
686
687
688
689
690
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
691
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
692
693
694
695
696
697
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBNETCDF)
698
    case CDI_FILETYPE_NC:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
699
700
      {
	fileID = cdfOpen(filename, filemode);
701
	streamptr->ncmode = 2;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
702
703
	break;
      }
704
    case CDI_FILETYPE_NC2:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
705
706
      {
	fileID = cdfOpen64(filename, filemode);
707
	streamptr->ncmode = 2;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
708
709
	break;
      }
710
711
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
712
      {
Deike Kleberg's avatar
Deike Kleberg committed
713
	fileID = cdf4Open(filename, filemode, &filetype);
714
	streamptr->ncmode = 2;
715
716
	break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
717
718
719
#endif
    default:
      {
720
	if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
721
	return CDI_ELIBNAVAIL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
722
723
724
725
726
727
      }
    }

  if ( fileID == CDI_UNDEFID )
    streamID = CDI_UNDEFID;
  else
728
    streamptr->fileID = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
729

730
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
731
732
733
734
}

/*
@Function  streamOpenRead
Uwe Schulzweida's avatar
Uwe Schulzweida committed
735
@Title     Open a dataset for reading
Uwe Schulzweida's avatar
Uwe Schulzweida committed
736

Uwe Schulzweida's avatar
Uwe Schulzweida committed
737
@Prototype int streamOpenRead(const char *path)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
738
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
739
    @Item  path  The name of the dataset to be read.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
740
741

@Description
742
The function @func{streamOpenRead} opens an existing dataset for reading.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
743
744

@Result
745
Upon successful completion @func{streamOpenRead} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
746
747
748
749
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
750
751
752
753
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
754
755
756
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
757
Here is an example using @func{streamOpenRead} to open an existing NetCDF
758
file named @func{foo.nc} for reading:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
759
760

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
761
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
762
763
764
765
766
767
768
769
770
   ...
int streamID;
   ...
streamID = streamOpenRead("foo.nc");
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
Thomas Jahns's avatar
Thomas Jahns committed
771
int streamOpenRead(const char *filename)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
772
{
773
774
  cdiInitialize();

775
  int byteorder = 0;
776
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
777

778
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
779

Thomas Jahns's avatar
Thomas Jahns committed
780
  int streamID = streamOpen(filename, "r", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
781

Uwe Schulzweida's avatar
Uwe Schulzweida committed
782
783
  if ( streamID >= 0 )
    {
784
      stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
785
786
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
787

788
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
789
790
791
792
793
}


int streamOpenAppend(const char *filename)
{
794
795
  cdiInitialize();

796
  int byteorder = 0;
797
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
798

799
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
800

801
  int streamID = streamOpenA(filename, "a", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
802

803
804
  if ( streamID >= 0 )
    {
805
      stream_t *streamptr = stream_to_pointer(streamID);
806
807
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
808

809
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
810
811
812
813
}

/*
@Function  streamOpenWrite
Uwe Schulzweida's avatar
Uwe Schulzweida committed
814
@Title     Create a new dataset
Uwe Schulzweida's avatar
Uwe Schulzweida committed
815

Uwe Schulzweida's avatar
Uwe Schulzweida committed
816
@Prototype int streamOpenWrite(const char *path, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
817
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
818
    @Item  path      The name of the new dataset.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
819
    @Item  filetype  The type of the file format, one of the set of predefined CDI file format types.
820
821
822
                     The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC},
                     @func{CDI_FILETYPE_NC2}, @func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_SRV},
                     @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
823
824

@Description
825
The function @func{streamOpenWrite} creates a new datset.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
826
@Result
827
Upon successful completion @func{streamOpenWrite} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
828
829
830
831
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
832
833
834
835
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
836
837
838
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
839
Here is an example using @func{streamOpenWrite} to create a new NetCDF file named @func{foo.nc} for writing:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
840
841

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
842
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
843
844
845
   ...
int streamID;
   ...
846
streamID = streamOpenWrite("foo.nc", CDI_FILETYPE_NC);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
847
848
849
850
851
852
853
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
int streamOpenWrite(const char *filename, int filetype)
{
854
855
  cdiInitialize();

856
  return streamOpen(filename, "w", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
857
858
}

859
860
861
862
863
864
static
void streamDefaultValue ( stream_t * streamptr )
{
  streamptr->self              = CDI_UNDEFID;
  streamptr->accesstype        = CDI_UNDEFID;
  streamptr->accessmode        = 0;
865
  streamptr->filetype          = CDI_FILETYPE_UNDEF;
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
  streamptr->byteorder         = CDI_UNDEFID;
  streamptr->fileID            = 0;
  streamptr->filemode          = 0;
  streamptr->numvals           = 0;
  streamptr->filename          = NULL;
  streamptr->record            = NULL;
  streamptr->varsAllocated     = 0;
  streamptr->nrecs             = 0;
  streamptr->nvars             = 0;
  streamptr->vars              = NULL;
  streamptr->ncmode            = 0;
  streamptr->curTsID           = CDI_UNDEFID;
  streamptr->rtsteps           = 0;
  streamptr->ntsteps           = CDI_UNDEFID;
  streamptr->tsteps            = NULL;
  streamptr->tstepsTableSize   = 0;
  streamptr->tstepsNextID      = 0;
  streamptr->historyID         = CDI_UNDEFID;
  streamptr->vlistID           = CDI_UNDEFID;
  streamptr->globalatts        = 0;
  streamptr->localatts         = 0;
  streamptr->unreduced         = cdiDataUnreduced;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
888
889
  streamptr->sortname          = cdiSortName > 0;
  streamptr->sortparam         = cdiSortParam > 0;
890
  streamptr->have_missval      = cdiHaveMissval;
891
  streamptr->comptype          = CDI_COMPRESS_NONE;
892
893
894
895
  streamptr->complevel         = 0;

  basetimeInit(&streamptr->basetime);

Uwe Schulzweida's avatar
Uwe Schulzweida committed
896
#ifdef HAVE_LIBNETCDF
897
898
  for ( int i = 0; i < MAX_ZAXES_PS; i++ ) streamptr->zaxisID[i]  = CDI_UNDEFID;
  for ( int i = 0; i < MAX_ZAXES_PS; i++ ) streamptr->nczvarID[i] = CDI_UNDEFID;
899

900
901
902
  for ( int i = 0; i < MAX_GRIDS_PS; i++ )
    {
      streamptr->ncgrid[i].gridID = CDI_UNDEFID;
903
904
      for (size_t j = 0; j < CDF_SIZE_ncIDs; ++j)
        streamptr->ncgrid[i].ncIDs[j] = CDI_UNDEFID;
905
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
906
907
908
909
910

  streamptr->vct.ilev          = 0;
  streamptr->vct.mlev          = 0;
  streamptr->vct.ilevID        = CDI_UNDEFID;
  streamptr->vct.mlevID        = CDI_UNDEFID;
911
#endif
912
913
914
915

  streamptr->gribContainers    = NULL;
}

916
917
static
stream_t *stream_new_entry(int resH)
918
919
920
{
  cdiInitialize(); /* ***************** make MT version !!! */

921
  stream_t *streamptr = (stream_t *) Malloc(sizeof(stream_t));
922
  streamDefaultValue ( streamptr );
923

924
925
926
927
928
929
930
  if (resH == CDI_UNDEFID)
    streamptr->self = reshPut(streamptr, &streamOps);
  else
    {
      streamptr->self = resH;
      reshReplace(resH, streamptr, &streamOps);
    }
931
932
933
934
935

  return streamptr;
}


936
void cdiStreamCloseDefaultDelegate(stream_t *streamptr, int recordBufIsToBeDeleted)
937
938
939
940
941
942
943
944
945
{
  int fileID   = streamptr->fileID;
  int filetype = streamptr->filetype;
  if ( fileID == CDI_UNDEFID )
    Warning("File %s not open!", streamptr->filename);
  else
    switch (filetype)
      {
#if  defined  (HAVE_LIBGRIB)
946
947
      case CDI_FILETYPE_GRB:
      case CDI_FILETYPE_GRB2:
948
949
        {
          gribClose(fileID);
950
          if ( recordBufIsToBeDeleted ) gribContainersDelete(streamptr);
951
952
953
954
          break;
        }
#endif
#if  defined  (HAVE_LIBSERVICE)
955
      case CDI_FILETYPE_SRV:
956
957
        {
          fileClose(fileID);
958
          if ( recordBufIsToBeDeleted ) srvDelete(streamptr->record->exsep);
959
960
961
962
          break;
        }
#endif
#if  defined  (HAVE_LIBEXTRA)
963
      case CDI_FILETYPE_EXT:
964
965
        {
          fileClose(fileID);
966
          if ( recordBufIsToBeDeleted ) extDelete(streamptr->record->exsep);
967
968
969
970
          break;
        }
#endif
#if  defined  (HAVE_LIBIEG)
971
      case CDI_FILETYPE_IEG:
972
973
        {
          fileClose(fileID);
974
          if ( recordBufIsToBeDeleted ) iegDelete(streamptr->record->exsep);
975
976
977
978
          break;
        }
#endif
#if  defined  (HAVE_LIBNETCDF)
979
980
981
982
      case CDI_FILETYPE_NC:
      case CDI_FILETYPE_NC2:
      case CDI_FILETYPE_NC4:
      case CDI_FILETYPE_NC4C: