stream.c 47.5 KB
Newer Older
Uwe Schulzweida's avatar
Uwe Schulzweida committed
1
2
3
4
#if defined (HAVE_CONFIG_H)
#  include "config.h"
#endif

5
6
7
8
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 600
#endif

9
#include <sys/stat.h> // struct stat
Uwe Schulzweida's avatar
Uwe Schulzweida committed
10
11
#include <ctype.h>

12
#include "binary.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
13
#include "cdi.h"
14
#include "cdi_int.h"
15
#include "cdi_cksum.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
16
#include "cdf.h"
17
18
#include "dmemory.h"
#include "error.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
19
#include "stream_grb.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
20
21
22
23
24
#include "stream_cdf.h"
#include "stream_srv.h"
#include "stream_ext.h"
#include "stream_ieg.h"
#include "file.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
25
#include "cgribex.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
26
#include "gribapi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
27
#include "vlist.h"
28
#include "serialize.h"
29
#include "resource_handle.h"
30
#include "resource_unpack.h"
31
32
#include "namespace.h"

33

34
static stream_t *stream_new_entry(int resH);
35
static void stream_delete_entry(stream_t *streamptr);
36
37
38
39
40
41
static int streamCompareP(void * streamptr1, void * streamptr2);
static void streamDestroyP(void * streamptr);
static void streamPrintP(void * streamptr, FILE * fp);
static int streamGetPackSize(void * streamptr, void *context);
static void streamPack(void * streamptr, void * buff, int size, int * position, void *context);
static int streamTxCode(void);
42

43
44
45
46
47
48
49
50
const resOps streamOps = {
  streamCompareP,
  streamDestroyP,
  streamPrintP,
  streamGetPackSize,
  streamPack,
  streamTxCode
};
Uwe Schulzweida's avatar
Uwe Schulzweida committed
51
52


53

54
55
static
int getByteorder(int byteswap)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
56
57
58
{
  int byteorder = -1;

59
  switch (HOST_ENDIANNESS)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
60
    {
61
62
63
64
65
66
67
68
69
70
    case CDI_BIGENDIAN:
      byteorder = byteswap ? CDI_LITTLEENDIAN : CDI_BIGENDIAN;
      break;
    case CDI_LITTLEENDIAN:
      byteorder = byteswap ? CDI_BIGENDIAN : CDI_LITTLEENDIAN;
      break;
    /* FIXME: does not currently adjust for PDP endianness */
    case CDI_PDPENDIAN:
    default:
      Error("unhandled endianness");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
71
    }
72
  return byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
73
74
}

75
76
// used also in CDO
int cdiGetFiletype(const char *filename, int *byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
77
78
79
{
  int filetype = CDI_EUFTYPE;
  int swap = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
80
  int version;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
81
82
  long recpos;

83
  int fileID = fileOpen(filename, "r");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
84
85
86

  if ( fileID == CDI_UNDEFID )
    {
87
      if ( strncmp(filename, "http:", 5) == 0 || strncmp(filename, "https:", 6) == 0 )
88
	return CDI_FILETYPE_NC;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
89
      else
90
	return CDI_ESYSTEM;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
91
92
    }

93
94
95
96
97
98
99
100
101
102
103
104
  char buffer[8];
  if ( fileRead(fileID, buffer, 8) != 8 )
    {
      struct stat buf;
      if ( stat(filename, &buf) == 0 )
        {
          if ( buf.st_size == 0 ) return CDI_EISEMPTY;
          if ( buf.st_mode&S_IFDIR ) return CDI_EISDIR;
        }

      return CDI_EUFTYPE;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
105
106
107

  fileRewind(fileID);

108
  if ( memcmp(buffer, "GRIB", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
109
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
110
111
112
      version = buffer[7];
      if ( version <= 1 )
	{
113
	  filetype = CDI_FILETYPE_GRB;
114
	  if ( CDI_Debug ) Message("found GRIB file = %s, version %d", filename, version);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
115
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
116
117
      else if ( version == 2 )
	{
118
	  filetype = CDI_FILETYPE_GRB2;
119
	  if ( CDI_Debug ) Message("found GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
120
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
121
    }
122
  else if ( memcmp(buffer, "CDF\001", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
123
    {
124
      filetype = CDI_FILETYPE_NC;
125
      if ( CDI_Debug ) Message("found CDF1 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
126
    }
127
  else if ( memcmp(buffer, "CDF\002", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
128
    {
129
      filetype = CDI_FILETYPE_NC2;
130
      if ( CDI_Debug ) Message("found CDF2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
131
    }
132
  else if ( memcmp(buffer+1, "HDF", 3) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
133
    {
134
      filetype = CDI_FILETYPE_NC4;
135
      if ( CDI_Debug ) Message("found HDF file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
136
137
138
    }
  else if ( srvCheckFiletype(fileID, &swap) )
    {
139
      filetype = CDI_FILETYPE_SRV;
140
      if ( CDI_Debug ) Message("found SRV file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
141
142
143
    }
  else if ( extCheckFiletype(fileID, &swap) )
    {
144
      filetype = CDI_FILETYPE_EXT;
145
      if ( CDI_Debug ) Message("found EXT file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
146
147
148
    }
  else if ( iegCheckFiletype(fileID, &swap) )
    {
149
      filetype = CDI_FILETYPE_IEG;
150
      if ( CDI_Debug ) Message("found IEG file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
151
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
152
  else if ( gribCheckSeek(fileID, &recpos, &version) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
153
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
154
155
      if ( version <= 1 )
	{
156
	  filetype = CDI_FILETYPE_GRB;
157
	  if ( CDI_Debug ) Message("found seeked GRIB file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
158
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
159
      else if ( version == 2 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
160
	{
161
	  filetype = CDI_FILETYPE_GRB2;
162
	  if ( CDI_Debug ) Message("found seeked GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
163
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
164
165
166
167
168
169
    }

  fileClose(fileID);

  *byteorder = getByteorder(swap);

170
  return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
171
172
173
174
175
176
177
178
}

/*
@Function  streamInqFiletype
@Title     Get the filetype

@Prototype int streamInqFiletype(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
179
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
180
181

@Description
182
The function @func{streamInqFiletype} returns the filetype of a stream.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
183
184

@Result
185
@func{streamInqFiletype} returns the type of the file format,
Uwe Schulzweida's avatar
Uwe Schulzweida committed
186
one of the set of predefined CDI file format types.
187
188
The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC}, @func{CDI_FILETYPE_NC2},
@func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
189
190
191
192
193

@EndFunction
*/
int streamInqFiletype(int streamID)
{
194
  stream_t *streamptr = stream_to_pointer(streamID);
195
  return streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
196
197
198
199
200
}


int getByteswap(int byteorder)
{
201
  int byteswap = -1;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
202

203
  switch (byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
204
    {
205
206
207
208
209
    case CDI_BIGENDIAN:
    case CDI_LITTLEENDIAN:
    case CDI_PDPENDIAN:
      byteswap = (HOST_ENDIANNESS != byteorder);
      break;
210
211
    case -1:
      break;
212
213
    default:
      Error("unexpected byteorder %d query!", byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
214
215
    }

216
  return byteswap;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
217
218
}

219
220
221
222
223
224
/*
@Function  streamDefByteorder
@Title     Define the byte order

@Prototype void streamDefByteorder(int streamID, int byteorder)
@Parameter
225
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenWrite}.
226
227
228
229
230
    @Item  byteorder The byte order of a dataset, one of the CDI constants @func{CDI_BIGENDIAN} and
                     @func{CDI_LITTLEENDIAN}.

@Description
The function @func{streamDefByteorder} defines the byte order of a binary dataset
231
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
232
233
234

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
235
236
void streamDefByteorder(int streamID, int byteorder)
{
237
  stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
238
  streamptr->byteorder = byteorder;
239
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
240
241
242
243

  switch (filetype)
    {
#if  defined  (HAVE_LIBSERVICE)
244
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
245
      {
246
	srvrec_t *srvp = (srvrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
247
248
249
250
251
252
	srvp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
253
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
254
      {
255
	extrec_t *extp = (extrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
256
257
258
259
260
261
	extp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
262
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
263
      {
264
	iegrec_t *iegp = (iegrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
265
266
267
268
269
270
	iegp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
    }
271
  reshSetStatus(streamID, &streamOps, RESH_DESYNC_IN_USE);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
272
273
}

274
275
276
277
278
279
/*
@Function  streamInqByteorder
@Title     Get the byte order

@Prototype int streamInqByteorder(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
280
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
281
282
283

@Description
The function @func{streamInqByteorder} returns the byte order of a binary dataset
284
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
285
286
287
288
289
290
291

@Result
@func{streamInqByteorder} returns the type of the byte order.
The valid CDI byte order types are @func{CDI_BIGENDIAN} and @func{CDI_LITTLEENDIAN}

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
292
293
int streamInqByteorder(int streamID)
{
294
  stream_t *streamptr = stream_to_pointer(streamID);
295
  return streamptr->byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
296
297
298
}


299
const char *streamFilesuffix(int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
300
{
301
  // static char *fileSuffix[] = {"", ".grb", ".g2", ".nc", ".nc", ".nc4", ".nc4", ".srv", ".ext", ".ieg"};
302
303
304
305
306
  /* note: the 2nd dimenstion of the fileSuffix array must be equal to or
   * larger than the length of the longest suffix (dot and \0 terminator
   * included) */
  static const char fileSuffix[][5] = {"", ".grb", ".grb", ".nc", ".nc", ".nc", ".nc", ".srv", ".ext", ".ieg"};
  int size = (int)(sizeof(fileSuffix)/sizeof(fileSuffix[0]));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
307
308

  if ( filetype > 0 && filetype < size )
309
    return fileSuffix[filetype];
Uwe Schulzweida's avatar
Uwe Schulzweida committed
310
  else
311
    return fileSuffix[0];
Uwe Schulzweida's avatar
Uwe Schulzweida committed
312
313
314
}


315
const char *streamFilename(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
316
{
317
  stream_t *streamptr = stream_to_pointer(streamID);
318
  return streamptr->filename;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
319
320
}

321
static
322
long cdiInqTimeSize(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
323
324
{
  int tsID = 0, nrecs;
325
  stream_t *streamptr = stream_to_pointer(streamID);
326
  long ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
327

328
  if ( ntsteps == (long)CDI_UNDEFID )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
329
    while ( (nrecs = streamInqTimestep(streamID, tsID++)) )
330
      ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
331

332
  return ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
333
334
}

335
static
336
int cdiInqContents(stream_t *streamptr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
337
338
{
  int status = 0;
Thomas Jahns's avatar
Thomas Jahns committed
339
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
340
341
342
343

  switch (filetype)
    {
#if  defined  (HAVE_LIBGRIB)
344
345
    case CDI_FILETYPE_GRB:
    case CDI_FILETYPE_GRB2:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
346
      {
347
        status = grbInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
348
349
350
	break;
      }
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
351
#if  defined  (HAVE_LIBSERVICE)
352
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
353
      {
354
        status = srvInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
355
356
357
358
	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
359
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
360
      {
361
        status = extInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
362
363
364
365
	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
366
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
367
      {
368
        status = iegInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
369
370
371
372
	break;
      }
#endif
#if  defined  (HAVE_LIBNETCDF)
373
374
375
376
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
377
      {
378
        status = cdfInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
379
380
381
382
383
384
	break;
      }
#endif
    default:
      {
	if ( CDI_Debug )
385
	  Message("%s support not compiled in!", strfiletype(filetype));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
386
387

	status = CDI_ELIBNAVAIL;
388
        break;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
389
390
391
392
393
      }
    }

  if ( status == 0 )
    {
Thomas Jahns's avatar
Thomas Jahns committed
394
395
396
      int vlistID = streamptr->vlistID;
      int taxisID = vlistInqTaxis(vlistID);
      if ( taxisID != CDI_UNDEFID )
397
398
399
400
401
        {
          taxis_t *taxisptr1 = &streamptr->tsteps[0].taxis;
          taxis_t *taxisptr2 = taxisPtr(taxisID);
          ptaxisCopy(taxisptr2, taxisptr1);
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
402
403
    }

404
  return status;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
405
406
}

407
int cdiStreamOpenDefaultDelegate(const char *filename, char filemode,
408
409
                                 int filetype, stream_t *streamptr,
                                 int recordBufIsToBeCreated)
410
411
412
{
  int fileID;
  switch (filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
413
414
    {
#if  defined  (HAVE_LIBGRIB)
415
    case CDI_FILETYPE_GRB:
416
#if  defined  (HAVE_LIBGRIB_API)
417
    case CDI_FILETYPE_GRB2:
418
#endif
419
      {
420
#ifndef __cplusplus
421
        fileID = gribOpen(filename, (char [2]){filemode, 0});
422
423
424
425
#else
        char temp[2] = { filemode, 0 };
        fileID = gribOpen(filename, temp);
#endif
426
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
427
428
        if (recordBufIsToBeCreated)
          {
429
            streamptr->record = (Record *) Malloc(sizeof(Record));
430
431
            streamptr->record->buffer = NULL;
          }
432
433
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
434
435
#endif
#if  defined  (HAVE_LIBSERVICE)
436
    case CDI_FILETYPE_SRV:
437
      {
438
#ifndef __cplusplus
439
        fileID = fileOpen(filename, (char [2]){filemode, 0});
440
441
442
443
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif
444
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
445
446
        if (recordBufIsToBeCreated)
          {
447
            streamptr->record = (Record *) Malloc(sizeof(Record));
448
            streamptr->record->buffer = NULL;
449
            streamptr->record->exsep  = srvNew();
450
          }
451
452
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
453
454
#endif
#if  defined  (HAVE_LIBEXTRA)
455
    case CDI_FILETYPE_EXT:
456
      {
457
#ifndef __cplusplus
458
        fileID = fileOpen(filename, (char [2]){filemode, 0});
459
460
461
462
463
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif

464
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
465
466
        if (recordBufIsToBeCreated)
          {
467
            streamptr->record = (Record *) Malloc(sizeof(Record));
468
            streamptr->record->buffer = NULL;
469
            streamptr->record->exsep  = extNew();
470
          }
471
472
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
473
474
#endif
#if  defined  (HAVE_LIBIEG)
475
    case CDI_FILETYPE_IEG:
476
      {
477
#ifndef __cplusplus
478
        fileID = fileOpen(filename, (char [2]){filemode, 0});
479
480
481
482
#else
        char temp[2] = { filemode, 0 };
        fileID = fileOpen(filename, temp);
#endif
483
        if ( fileID < 0 ) fileID = CDI_ESYSTEM;
484
485
        if (recordBufIsToBeCreated)
          {
486
            streamptr->record = (Record *) Malloc(sizeof(Record));
487
            streamptr->record->buffer = NULL;
488
            streamptr->record->exsep  = iegNew();
489
          }
490
491
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
492
493
#endif
#if  defined  (HAVE_LIBNETCDF)
494
    case CDI_FILETYPE_NC:
495
      {
496
#ifndef __cplusplus
497
        fileID = cdfOpen(filename, (char [2]){filemode, 0});
498
499
500
501
#else
        char temp[2] = { filemode, 0 };
        fileID = cdfOpen(filename, temp);
#endif
502
503
        break;
      }
504
    case CDI_FILETYPE_NC2:
505
      {
506
#ifndef __cplusplus
507
        fileID = cdfOpen64(filename, (char [2]){filemode, 0});
508
509
510
511
#else
        char temp[2] = { filemode, 0 };
        fileID = cdfOpen64(filename, temp);
#endif
512
513
        break;
      }
514
515
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
516
      {
517
#ifndef __cplusplus
518
        fileID = cdf4Open(filename, (char [2]){filemode, 0}, &filetype);
519
520
521
522
#else
        char temp[2] = { filemode, 0 };
        fileID = cdf4Open(filename, temp, &filetype);
#endif
523
        break;
524
525
526
527
528
      }
#endif
    default:
      {
        if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
529
        return CDI_ELIBNAVAIL;
530
      }
531
    }
532
533
534

  streamptr->filetype = filetype;

535
536
537
538
  return fileID;
}


539
int streamOpenID(const char *filename, char filemode, int filetype, int resH)
540
541
{
  if ( CDI_Debug )
542
    Message("Open %s mode %c file %s", strfiletype(filetype), filemode,
543
            filename?filename:"(NUL)");
544

545
  if ( ! filename || filetype < 0 ) return CDI_EINVAL;
546

547
548
  stream_t *streamptr = stream_new_entry(resH);
  int streamID = CDI_ESYSTEM;
549

550
551
552
553
  int (*streamOpenDelegate)(const char *filename, char filemode,
                            int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
    = (int (*)(const char *, char, int, stream_t *, int))
    namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
554

555
  int fileID = streamOpenDelegate(filename, filemode, filetype, streamptr, 1);
556
  if ( fileID < 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
557
558
559
560
561
    {
      streamID = fileID;
    }
  else
    {
562
563
      streamID = streamptr->self;
      if ( streamID < 0 ) return CDI_ELIMIT;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
564

565
      streamptr->filemode = filemode;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
566
567
      streamptr->filename = strdupx(filename);
      streamptr->fileID   = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
568

569
      if ( filemode == 'r' )
570
571
        {
          int vlistID = vlistCreate();
572
          if ( vlistID < 0 ) return CDI_ELIMIT;
573
574
575
576

          cdiVlistMakeInternal(vlistID);
          streamptr->vlistID = vlistID;
          /* cdiReadByteorder(streamID); */
577
          int status = cdiInqContents(streamptr);
578
579
580
581
582
583
584
585
586
587
          if ( status < 0 )
            {
              streamID = status;
            }
          else
            {
              vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
              vlistptr->ntsteps = streamptr->ntsteps;
              cdiVlistMakeImmutable(vlistID);
            }
588
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
589
    }
590

591
592
593
594
595
596
597
  if ( streamID < 0 )
    {
      Free(streamptr->record);
      stream_delete_entry(streamptr);
    }

  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
598
599
}

600
601
static
int streamOpen(const char *filename, const char *filemode, int filetype)
602
{
603
604
  if ( !filemode || strlen(filemode) != 1 ) return CDI_EINVAL;
  return streamOpenID(filename, (char)tolower(filemode[0]), filetype, CDI_UNDEFID);
605
}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
606

607
608
static
int streamOpenA(const char *filename, const char *filemode, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
609
610
{
  if ( CDI_Debug )
611
    Message("Open %s file (mode=%c); filename: %s", strfiletype(filetype), (int) *filemode, filename);
612
  if ( CDI_Debug ) printf("streamOpenA: %s\n", filename); // seg fault without this line on thunder/squall with "cdo cat x y"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
613

614
  if ( ! filename || ! filemode || filetype < 0 ) return CDI_EINVAL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
615

616
617
618
  stream_t *streamptr = stream_new_entry(CDI_UNDEFID);
  int fileID = CDI_UNDEFID;

619
  {
620
    int (*streamOpenDelegate)(const char *filename, char filemode,
621
                              int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
Thomas Jahns's avatar
Thomas Jahns committed
622
      = (int (*)(const char *, char, int, stream_t *, int))
623
      namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
624

625
    fileID = streamOpenDelegate(filename, 'r', filetype, streamptr, 1);
626
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
627

628
  if ( fileID == CDI_UNDEFID || fileID == CDI_ELIBNAVAIL || fileID == CDI_ESYSTEM ) return fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
629

630
  int streamID = streamptr->self;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
631

632
633
634
635
636
  streamptr->filemode = tolower(*filemode);
  streamptr->filename = strdupx(filename);
  streamptr->fileID   = fileID;

  streamptr->vlistID = vlistCreate();
637
  cdiVlistMakeInternal(streamptr->vlistID);
638
  /* cdiReadByteorder(streamID); */
639
  int status = cdiInqContents(streamptr);
640
  if ( status < 0 ) return status;
641
  vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
642
  vlistptr->ntsteps = (int)cdiInqTimeSize(streamID);
643
644
645
646
647

  // Needed for NetCDF4
  for ( int varID = 0; varID < vlistptr->nvars; ++varID )
    streamptr->vars[varID].defmiss = true;

648
  if ( !strcmp(filemode, "r") ) cdiVlistMakeImmutable(streamptr->vlistID);
649

650
651
  {
    void (*streamCloseDelegate)(stream_t *streamptr, int recordBufIsToBeDeleted)
652
653
      = (void (*)(stream_t *, int))
      namespaceSwitchGet(NSSWITCH_STREAM_CLOSE_BACKEND).func;
654

655
656
    streamCloseDelegate(streamptr, 0);
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
657
658
659
660

  switch (filetype)
    {
#if  defined  (HAVE_LIBGRIB)
661
    case CDI_FILETYPE_GRB:
662
#if  defined  (HAVE_LIBGRIB_API)
663
    case CDI_FILETYPE_GRB2:
664
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
665
666
      {
        fileID = gribOpen(filename, filemode);
667
        if ( fileID != CDI_UNDEFID ) gribContainersNew(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
668
669
670
671
	break;
      }
#endif
#if  defined  (HAVE_LIBSERVICE)
672
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
673
674
675
676
677
678
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBEXTRA)
679
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
680
681
682
683
684
685
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBIEG)
686
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
687
688
689
690
691
692
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
#if  defined  (HAVE_LIBNETCDF)
693
    case CDI_FILETYPE_NC:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
694
695
      {
	fileID = cdfOpen(filename, filemode);
696
	streamptr->ncmode = 2;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
697
698
	break;
      }
699
    case CDI_FILETYPE_NC2:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
700
701
      {
	fileID = cdfOpen64(filename, filemode);
702
	streamptr->ncmode = 2;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
703
704
	break;
      }
705
706
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
707
      {
Deike Kleberg's avatar
Deike Kleberg committed
708
	fileID = cdf4Open(filename, filemode, &filetype);
709
	streamptr->ncmode = 2;
710
711
	break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
712
713
714
#endif
    default:
      {
715
	if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
716
	return CDI_ELIBNAVAIL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
717
718
719
720
721
722
      }
    }

  if ( fileID == CDI_UNDEFID )
    streamID = CDI_UNDEFID;
  else
723
    streamptr->fileID = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
724

725
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
726
727
728
729
}

/*
@Function  streamOpenRead
Uwe Schulzweida's avatar
Uwe Schulzweida committed
730
@Title     Open a dataset for reading
Uwe Schulzweida's avatar
Uwe Schulzweida committed
731

Uwe Schulzweida's avatar
Uwe Schulzweida committed
732
@Prototype int streamOpenRead(const char *path)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
733
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
734
    @Item  path  The name of the dataset to be read.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
735
736

@Description
737
The function @func{streamOpenRead} opens an existing dataset for reading.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
738
739

@Result
740
Upon successful completion @func{streamOpenRead} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
741
742
743
744
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
745
746
747
748
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
749
750
751
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
752
Here is an example using @func{streamOpenRead} to open an existing NetCDF
753
file named @func{foo.nc} for reading:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
754
755

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
756
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
757
758
759
760
761
762
763
764
765
   ...
int streamID;
   ...
streamID = streamOpenRead("foo.nc");
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
Thomas Jahns's avatar
Thomas Jahns committed
766
int streamOpenRead(const char *filename)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
767
{
768
769
  cdiInitialize();

770
  int byteorder = 0;
771
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
772

773
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
774

Thomas Jahns's avatar
Thomas Jahns committed
775
  int streamID = streamOpen(filename, "r", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
776

Uwe Schulzweida's avatar
Uwe Schulzweida committed
777
778
  if ( streamID >= 0 )
    {
779
      stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
780
781
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
782

783
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
784
785
786
787
788
}


int streamOpenAppend(const char *filename)
{
789
790
  cdiInitialize();

791
  int byteorder = 0;
792
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
793

794
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
795

796
  int streamID = streamOpenA(filename, "a", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
797

798
799
  if ( streamID >= 0 )
    {
800
      stream_t *streamptr = stream_to_pointer(streamID);
801
802
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
803

804
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
805
806
807
808
}

/*
@Function  streamOpenWrite
Uwe Schulzweida's avatar
Uwe Schulzweida committed
809
@Title     Create a new dataset
Uwe Schulzweida's avatar
Uwe Schulzweida committed
810

Uwe Schulzweida's avatar
Uwe Schulzweida committed
811
@Prototype int streamOpenWrite(const char *path, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
812
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
813
    @Item  path      The name of the new dataset.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
814
    @Item  filetype  The type of the file format, one of the set of predefined CDI file format types.
815
816
817
                     The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC},
                     @func{CDI_FILETYPE_NC2}, @func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_SRV},
                     @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
818
819

@Description
820
The function @func{streamOpenWrite} creates a new datset.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
821
@Result
822
Upon successful completion @func{streamOpenWrite} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
823
824
825
826
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
827
828
829
830
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
831
832
833
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
834
Here is an example using @func{streamOpenWrite} to create a new NetCDF file named @func{foo.nc} for writing:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
835
836

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
837
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
838
839
840
   ...
int streamID;
   ...
841
streamID = streamOpenWrite("foo.nc", CDI_FILETYPE_NC);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
842
843
844
845
846
847
848
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
int streamOpenWrite(const char *filename, int filetype)
{
849
850
  cdiInitialize();

851
  return streamOpen(filename, "w", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
852
853
}

854
855
856
857
858
859
static
void streamDefaultValue ( stream_t * streamptr )
{
  streamptr->self              = CDI_UNDEFID;
  streamptr->accesstype        = CDI_UNDEFID;
  streamptr->accessmode        = 0;
860
  streamptr->filetype          = CDI_FILETYPE_UNDEF;
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
  streamptr->byteorder         = CDI_UNDEFID;
  streamptr->fileID            = 0;
  streamptr->filemode          = 0;
  streamptr->numvals           = 0;
  streamptr->filename          = NULL;
  streamptr->record            = NULL;
  streamptr->varsAllocated     = 0;
  streamptr->nrecs             = 0;
  streamptr->nvars             = 0;
  streamptr->vars              = NULL;
  streamptr->ncmode            = 0;
  streamptr->curTsID           = CDI_UNDEFID;
  streamptr->rtsteps           = 0;
  streamptr->ntsteps           = CDI_UNDEFID;
  streamptr->tsteps            = NULL;
  streamptr->tstepsTableSize   = 0;
  streamptr->tstepsNextID      = 0;
  streamptr->historyID         = CDI_UNDEFID;
  streamptr->vlistID           = CDI_UNDEFID;
  streamptr->globalatts        = 0;
  streamptr->localatts         = 0;
  streamptr->unreduced         = cdiDataUnreduced;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
883
884
  streamptr->sortname          = cdiSortName > 0;
  streamptr->sortparam         = cdiSortParam > 0;
885
  streamptr->have_missval      = cdiHaveMissval;
886
  streamptr->comptype          = CDI_COMPRESS_NONE;
887
888
889
890
  streamptr->complevel         = 0;

  basetimeInit(&streamptr->basetime);

Uwe Schulzweida's avatar
Uwe Schulzweida committed
891
#ifdef HAVE_LIBNETCDF
892
893
  for ( int i = 0; i < MAX_ZAXES_PS; i++ ) streamptr->zaxisID[i]  = CDI_UNDEFID;
  for ( int i = 0; i < MAX_ZAXES_PS; i++ ) streamptr->nczvarID[i] = CDI_UNDEFID;
894

895
896
897
  for ( int i = 0; i < MAX_GRIDS_PS; i++ )
    {
      streamptr->ncgrid[i].gridID = CDI_UNDEFID;
898
899
      for (size_t j = 0; j < CDF_SIZE_ncIDs; ++j)
        streamptr->ncgrid[i].ncIDs[j] = CDI_UNDEFID;
900
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
901
902
903
904
905

  streamptr->vct.ilev          = 0;
  streamptr->vct.mlev          = 0;
  streamptr->vct.ilevID        = CDI_UNDEFID;
  streamptr->vct.mlevID        = CDI_UNDEFID;
906
#endif
907
908
909
910

  streamptr->gribContainers    = NULL;
}

911
912
static
stream_t *stream_new_entry(int resH)
913
914
915
{
  cdiInitialize(); /* ***************** make MT version !!! */

916
  stream_t *streamptr = (stream_t *) Malloc(sizeof(stream_t));
917
  streamDefaultValue ( streamptr );
918

919
920
921
922
923
924
925
  if (resH == CDI_UNDEFID)
    streamptr->self = reshPut(streamptr, &streamOps);
  else
    {
      streamptr->self = resH;
      reshReplace(resH, streamptr, &streamOps);
    }
926
927
928
929
930

  return streamptr;
}


931
void cdiStreamCloseDefaultDelegate(stream_t *streamptr, int recordBufIsToBeDeleted)
932
933
934
935
936
937
938
939
940
{
  int fileID   = streamptr->fileID;
  int filetype = streamptr->filetype;
  if ( fileID == CDI_UNDEFID )
    Warning("File %s not open!", streamptr->filename);
  else
    switch (filetype)
      {
#if  defined  (HAVE_LIBGRIB)
941
942
      case CDI_FILETYPE_GRB:
      case CDI_FILETYPE_GRB2:
943
944
        {
          gribClose(fileID);
945