stream.c 50.2 KB
Newer Older
1 2
#ifdef  HAVE_CONFIG_H
#include "config.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
3 4
#endif

5 6 7 8
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE 600
#endif

9
#include <sys/stat.h> // struct stat
Uwe Schulzweida's avatar
Uwe Schulzweida committed
10 11
#include <ctype.h>

12
#include "binary.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
13
#include "cdi.h"
14
#include "cdi_int.h"
15
#include "cdi_cksum.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
16
#include "cdf.h"
17 18
#include "dmemory.h"
#include "error.h"
19
#include "stream_cgribex.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
20
#include "stream_grb.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
21 22 23 24 25
#include "stream_cdf.h"
#include "stream_srv.h"
#include "stream_ext.h"
#include "stream_ieg.h"
#include "file.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
26
#include "cgribex.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
27
#include "gribapi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
28
#include "vlist.h"
29
#include "serialize.h"
30
#include "resource_handle.h"
31
#include "resource_unpack.h"
32
#include "namespace.h"
33
#include "async_worker.h"
34

35

36
static stream_t *stream_new_entry(int resH);
37
static void stream_delete_entry(stream_t *streamptr);
38 39 40 41 42 43
static int streamCompareP(void * streamptr1, void * streamptr2);
static void streamDestroyP(void * streamptr);
static void streamPrintP(void * streamptr, FILE * fp);
static int streamGetPackSize(void * streamptr, void *context);
static void streamPack(void * streamptr, void * buff, int size, int * position, void *context);
static int streamTxCode(void);
44

45 46 47 48 49 50 51 52
const resOps streamOps = {
  streamCompareP,
  streamDestroyP,
  streamPrintP,
  streamGetPackSize,
  streamPack,
  streamTxCode
};
Uwe Schulzweida's avatar
Uwe Schulzweida committed
53 54


55

56 57
static
int getByteorder(int byteswap)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
58 59 60
{
  int byteorder = -1;

61
  switch (HOST_ENDIANNESS)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
62
    {
63 64 65 66 67 68 69 70 71 72
    case CDI_BIGENDIAN:
      byteorder = byteswap ? CDI_LITTLEENDIAN : CDI_BIGENDIAN;
      break;
    case CDI_LITTLEENDIAN:
      byteorder = byteswap ? CDI_BIGENDIAN : CDI_LITTLEENDIAN;
      break;
    /* FIXME: does not currently adjust for PDP endianness */
    case CDI_PDPENDIAN:
    default:
      Error("unhandled endianness");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
73
    }
74
  return byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
75 76
}

77 78
// used also in CDO
int cdiGetFiletype(const char *filename, int *byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
79 80 81
{
  int filetype = CDI_EUFTYPE;
  int swap = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
82
  int version;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
83 84
  long recpos;

85
  int fileID = fileOpen(filename, "r");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
86 87 88

  if ( fileID == CDI_UNDEFID )
    {
89
      if ( strncmp(filename, "http:", 5) == 0 || strncmp(filename, "https:", 6) == 0 )
90
	return CDI_FILETYPE_NC;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
91
      else
92
	return CDI_ESYSTEM;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
93
    }
94
  else if ( fileID == -2 ) return CDI_ETMOF;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
95

96 97 98 99 100 101 102 103 104 105 106 107
  char buffer[8];
  if ( fileRead(fileID, buffer, 8) != 8 )
    {
      struct stat buf;
      if ( stat(filename, &buf) == 0 )
        {
          if ( buf.st_size == 0 ) return CDI_EISEMPTY;
          if ( buf.st_mode&S_IFDIR ) return CDI_EISDIR;
        }

      return CDI_EUFTYPE;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
108 109 110

  fileRewind(fileID);

111
  if ( memcmp(buffer, "GRIB", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
112
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
113 114 115
      version = buffer[7];
      if ( version <= 1 )
	{
116
	  filetype = CDI_FILETYPE_GRB;
117
	  if ( CDI_Debug ) Message("found GRIB file = %s, version %d", filename, version);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
118
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
119 120
      else if ( version == 2 )
	{
121
	  filetype = CDI_FILETYPE_GRB2;
122
	  if ( CDI_Debug ) Message("found GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
123
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
124
    }
125
  else if ( memcmp(buffer, "CDF\001", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
126
    {
127
      filetype = CDI_FILETYPE_NC;
128
      if ( CDI_Debug ) Message("found CDF1 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
129
    }
130
  else if ( memcmp(buffer, "CDF\002", 4) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
131
    {
132
      filetype = CDI_FILETYPE_NC2;
133
      if ( CDI_Debug ) Message("found CDF2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
134
    }
135 136
  else if ( memcmp(buffer, "CDF\005", 4) == 0 )
    {
137
      filetype = CDI_FILETYPE_NC5;
138 139
      if ( CDI_Debug ) Message("found CDF5 file = %s", filename);
    }
140
  else if ( memcmp(buffer+1, "HDF", 3) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
141
    {
142
      filetype = CDI_FILETYPE_NC4;
143
      if ( CDI_Debug ) Message("found HDF file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
144
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
145
#ifdef  HAVE_LIBSERVICE
Uwe Schulzweida's avatar
Uwe Schulzweida committed
146 147
  else if ( srvCheckFiletype(fileID, &swap) )
    {
148
      filetype = CDI_FILETYPE_SRV;
149
      if ( CDI_Debug ) Message("found SRV file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
150
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
151 152
#endif
#ifdef  HAVE_LIBEXTRA
Uwe Schulzweida's avatar
Uwe Schulzweida committed
153 154
  else if ( extCheckFiletype(fileID, &swap) )
    {
155
      filetype = CDI_FILETYPE_EXT;
156
      if ( CDI_Debug ) Message("found EXT file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
157
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
158 159
#endif
#ifdef  HAVE_LIBIEG
Uwe Schulzweida's avatar
Uwe Schulzweida committed
160 161
  else if ( iegCheckFiletype(fileID, &swap) )
    {
162
      filetype = CDI_FILETYPE_IEG;
163
      if ( CDI_Debug ) Message("found IEG file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
164
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
165 166
#endif
#ifdef HAVE_LIBGRIB
Uwe Schulzweida's avatar
Uwe Schulzweida committed
167
  else if ( gribCheckSeek(fileID, &recpos, &version) == 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
168
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
169 170
      if ( version <= 1 )
	{
171
	  filetype = CDI_FILETYPE_GRB;
172
	  if ( CDI_Debug ) Message("found seeked GRIB file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
173
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
174
      else if ( version == 2 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
175
	{
176
	  filetype = CDI_FILETYPE_GRB2;
177
	  if ( CDI_Debug ) Message("found seeked GRIB2 file = %s", filename);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
178
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
179
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
180
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
181 182 183 184 185

  fileClose(fileID);

  *byteorder = getByteorder(swap);

186
  return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
187 188 189 190 191 192 193 194
}

/*
@Function  streamInqFiletype
@Title     Get the filetype

@Prototype int streamInqFiletype(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
195
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
196 197

@Description
198
The function @func{streamInqFiletype} returns the filetype of a stream.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
199 200

@Result
201
@func{streamInqFiletype} returns the type of the file format,
Uwe Schulzweida's avatar
Uwe Schulzweida committed
202
one of the set of predefined CDI file format types.
203
The valid CDI file format types are @func{CDI_FILETYPE_GRB}, @func{CDI_FILETYPE_GRB2}, @func{CDI_FILETYPE_NC}, @func{CDI_FILETYPE_NC2},
204
@func{CDI_FILETYPE_NC4}, @func{CDI_FILETYPE_NC4C}, @func{CDI_FILETYPE_NC5}, @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} and @func{CDI_FILETYPE_IEG}.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
205 206 207 208 209

@EndFunction
*/
int streamInqFiletype(int streamID)
{
210
  stream_t *streamptr = stream_to_pointer(streamID);
211
  return streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
212 213 214 215 216
}


int getByteswap(int byteorder)
{
217
  int byteswap = -1;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
218

219
  switch (byteorder)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
220
    {
221 222 223 224 225
    case CDI_BIGENDIAN:
    case CDI_LITTLEENDIAN:
    case CDI_PDPENDIAN:
      byteswap = (HOST_ENDIANNESS != byteorder);
      break;
226 227
    case -1:
      break;
228 229
    default:
      Error("unexpected byteorder %d query!", byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
230 231
    }

232
  return byteswap;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
233 234
}

235 236 237

void streamDefNumWorker(int streamID, int numWorker)
{
238 239 240 241 242
  if (numWorker > 0)
    {
      stream_t *streamptr = stream_to_pointer(streamID);
      streamptr->numWorker = numWorker;
    }
243 244 245
}


246 247 248 249 250 251
/*
@Function  streamDefByteorder
@Title     Define the byte order

@Prototype void streamDefByteorder(int streamID, int byteorder)
@Parameter
252
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenWrite}.
253 254 255 256 257
    @Item  byteorder The byte order of a dataset, one of the CDI constants @func{CDI_BIGENDIAN} and
                     @func{CDI_LITTLEENDIAN}.

@Description
The function @func{streamDefByteorder} defines the byte order of a binary dataset
258
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
259 260 261

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
262 263
void streamDefByteorder(int streamID, int byteorder)
{
264
  stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
265
  streamptr->byteorder = byteorder;
266
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
267 268 269

  switch (filetype)
    {
270
#ifdef  HAVE_LIBSERVICE
271
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
272
      {
273
	srvrec_t *srvp = (srvrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
274 275 276 277 278
	srvp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
279
#ifdef  HAVE_LIBEXTRA
280
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
281
      {
282
	extrec_t *extp = (extrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
283 284 285 286 287
	extp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
288
#ifdef  HAVE_LIBIEG
289
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
290
      {
291
	iegrec_t *iegp = (iegrec_t*) streamptr->record->exsep;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
292 293 294 295 296 297
	iegp->byteswap = getByteswap(byteorder);

	break;
      }
#endif
    }
298

299
  reshSetStatus(streamID, &streamOps, RESH_DESYNC_IN_USE);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
300 301
}

302 303 304 305 306 307
/*
@Function  streamInqByteorder
@Title     Get the byte order

@Prototype int streamInqByteorder(int streamID)
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
308
    @Item  streamID  Stream ID, from a previous call to @fref{streamOpenRead} or @fref{streamOpenWrite}.
309 310 311

@Description
The function @func{streamInqByteorder} returns the byte order of a binary dataset
312
with the file format type @func{CDI_FILETYPE_SRV}, @func{CDI_FILETYPE_EXT} or @func{CDI_FILETYPE_IEG}.
313 314 315 316 317 318 319

@Result
@func{streamInqByteorder} returns the type of the byte order.
The valid CDI byte order types are @func{CDI_BIGENDIAN} and @func{CDI_LITTLEENDIAN}

@EndFunction
*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
320 321
int streamInqByteorder(int streamID)
{
322
  stream_t *streamptr = stream_to_pointer(streamID);
323
  return streamptr->byteorder;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
324 325 326
}


327
const char *streamFilesuffix(int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
328
{
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
  static const char *noSuffix  = "";
  static const char *ncSuffix  = ".nc";
  static const char *grbSuffix = ".grb";
  static const char *srvSuffix = ".srv";
  static const char *extSuffix = ".ext";
  static const char *iegSuffix = ".ieg";

  switch (filetype)
    {
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
    case CDI_FILETYPE_NC5:   return ncSuffix;
    case CDI_FILETYPE_GRB:
    case CDI_FILETYPE_GRB2:  return grbSuffix;
    case CDI_FILETYPE_SRV:   return srvSuffix;
    case CDI_FILETYPE_EXT:   return extSuffix;
    case CDI_FILETYPE_IEG:   return iegSuffix;
    default:                 return noSuffix;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
350 351 352
}


353
const char *streamFilename(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
354
{
355
  stream_t *streamptr = stream_to_pointer(streamID);
356
  return streamptr->filename;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
357 358
}

359
static
360
long cdiInqTimeSize(int streamID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
361 362
{
  int tsID = 0, nrecs;
363
  stream_t *streamptr = stream_to_pointer(streamID);
364
  long ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
365

366
  if ( ntsteps == (long)CDI_UNDEFID )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
367
    while ( (nrecs = streamInqTimestep(streamID, tsID++)) )
368
      ntsteps = streamptr->ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
369

370
  return ntsteps;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
371 372
}

373
static
374
int cdiInqContents(stream_t *streamptr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
375 376
{
  int status = 0;
Thomas Jahns's avatar
Thomas Jahns committed
377
  int filetype = streamptr->filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
378 379 380

  switch (filetype)
    {
381
#ifdef  HAVE_LIBGRIB
382 383
    case CDI_FILETYPE_GRB:
    case CDI_FILETYPE_GRB2:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
384
      {
385
        status = grbInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
386 387 388
	break;
      }
#endif
389
#ifdef  HAVE_LIBSERVICE
390
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
391
      {
392
        status = srvInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
393 394 395
	break;
      }
#endif
396
#ifdef  HAVE_LIBEXTRA
397
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
398
      {
399
        status = extInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
400 401 402
	break;
      }
#endif
403
#ifdef  HAVE_LIBIEG
404
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
405
      {
406
        status = iegInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
407 408 409
	break;
      }
#endif
410
#ifdef  HAVE_LIBNETCDF
411 412 413 414
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
415
    case CDI_FILETYPE_NC5:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
416
      {
417
        status = cdfInqContents(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
418 419 420 421 422 423
	break;
      }
#endif
    default:
      {
	if ( CDI_Debug )
424
	  Message("%s support not compiled in!", strfiletype(filetype));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
425 426

	status = CDI_ELIBNAVAIL;
427
        break;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
428 429 430 431 432
      }
    }

  if ( status == 0 )
    {
Thomas Jahns's avatar
Thomas Jahns committed
433 434 435
      int vlistID = streamptr->vlistID;
      int taxisID = vlistInqTaxis(vlistID);
      if ( taxisID != CDI_UNDEFID )
436 437 438 439 440
        {
          taxis_t *taxisptr1 = &streamptr->tsteps[0].taxis;
          taxis_t *taxisptr2 = taxisPtr(taxisID);
          ptaxisCopy(taxisptr2, taxisptr1);
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
441 442
    }

443
  return status;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
444 445
}

446 447

int cdiStreamOpenDefaultDelegate(const char *filename, char filemode, int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
448 449
{
  int fileID;
450 451
  char temp[2] = { filemode, 0 };

452
  switch (filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
453
    {
454
#if defined(HAVE_LIBGRIB) && (defined(HAVE_LIBCGRIBEX) || defined(HAVE_LIBGRIB_API))
455
    case CDI_FILETYPE_GRB:
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
      {
        fileID = gribOpen(filename, temp);
        if ( fileID < 0 ) return CDI_ESYSTEM;
        if (recordBufIsToBeCreated)
          {
            streamptr->record = (Record *) Malloc(sizeof(Record));
            streamptr->record->buffer = NULL;
#ifdef  HAVE_LIBCGRIBEX
            streamptr->record->cgribexp = cgribexNew();
#else
            streamptr->record->cgribexp = NULL;
#endif
          }
        break;
      }
471
#ifdef  HAVE_LIBGRIB_API
472
    case CDI_FILETYPE_GRB2:
473
      {
474
        fileID = gribOpen(filename, temp);
475
        if ( fileID < 0 ) return CDI_ESYSTEM;
476 477
        if (recordBufIsToBeCreated)
          {
478
            streamptr->record = (Record *) Malloc(sizeof(Record));
479 480
            streamptr->record->buffer = NULL;
          }
481 482
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
483
#endif
484
#endif
485
#ifdef  HAVE_LIBSERVICE
486
    case CDI_FILETYPE_SRV:
487
      {
488
        fileID = fileOpen(filename, temp);
489
        if ( fileID < 0 ) return CDI_ESYSTEM;
490 491
        if (recordBufIsToBeCreated)
          {
492
            streamptr->record = (Record *) Malloc(sizeof(Record));
493
            streamptr->record->buffer = NULL;
494
            streamptr->record->exsep = srvNew();
495
          }
496 497
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
498
#endif
499
#ifdef  HAVE_LIBEXTRA
500
    case CDI_FILETYPE_EXT:
501
      {
502
        fileID = fileOpen(filename, temp);
503
        if ( fileID < 0 ) return CDI_ESYSTEM;
504 505
        if (recordBufIsToBeCreated)
          {
506
            streamptr->record = (Record *) Malloc(sizeof(Record));
507
            streamptr->record->buffer = NULL;
508
            streamptr->record->exsep = extNew();
509
          }
510 511
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
512
#endif
513
#ifdef  HAVE_LIBIEG
514
    case CDI_FILETYPE_IEG:
515
      {
516
        fileID = fileOpen(filename, temp);
517
        if ( fileID < 0 ) return CDI_ESYSTEM;
518 519
        if (recordBufIsToBeCreated)
          {
520
            streamptr->record = (Record *) Malloc(sizeof(Record));
521
            streamptr->record->buffer = NULL;
522
            streamptr->record->exsep = iegNew();
523
          }
524 525
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
526
#endif
527
#ifdef  HAVE_LIBNETCDF
528 529
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
530
    case CDI_FILETYPE_NC5:
531
      {
532
        fileID = cdfOpen(filename, temp, filetype);
533 534
        break;
      }
535 536
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
537
      {
538
        fileID = cdf4Open(filename, temp, &filetype);
539
        break;
540 541 542 543 544
      }
#endif
    default:
      {
        if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
545
        return CDI_ELIBNAVAIL;
546
      }
547
    }
548 549 550

  streamptr->filetype = filetype;

551 552 553 554
  return fileID;
}


555
int streamOpenID(const char *filename, char filemode, int filetype, int resH)
556 557
{
  if ( CDI_Debug )
558
    Message("Open %s mode %c file %s", strfiletype(filetype), filemode, filename?filename:"(NUL)");
559

560
  if ( ! filename || filetype < 0 ) return CDI_EINVAL;
561

562 563
  stream_t *streamptr = stream_new_entry(resH);
  int streamID = CDI_ESYSTEM;
564

565 566 567 568
  int (*streamOpenDelegate)(const char *filename, char filemode,
                            int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
    = (int (*)(const char *, char, int, stream_t *, int))
    namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
569

570
  int fileID = streamOpenDelegate(filename, filemode, filetype, streamptr, 1);
571
  if ( fileID < 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
572 573 574 575 576
    {
      streamID = fileID;
    }
  else
    {
577 578
      streamID = streamptr->self;
      if ( streamID < 0 ) return CDI_ELIMIT;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
579

580
      streamptr->filemode = filemode;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
581 582
      streamptr->filename = strdupx(filename);
      streamptr->fileID   = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
583

584
      if ( filemode == 'r' )
585 586
        {
          int vlistID = vlistCreate();
587
          if ( vlistID < 0 ) return CDI_ELIMIT;
588 589 590 591

          cdiVlistMakeInternal(vlistID);
          streamptr->vlistID = vlistID;
          /* cdiReadByteorder(streamID); */
592
          int status = cdiInqContents(streamptr);
593 594 595 596 597 598 599 600 601 602
          if ( status < 0 )
            {
              streamID = status;
            }
          else
            {
              vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
              vlistptr->ntsteps = streamptr->ntsteps;
              cdiVlistMakeImmutable(vlistID);
            }
603
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
604
    }
605

606 607 608 609 610 611 612
  if ( streamID < 0 )
    {
      Free(streamptr->record);
      stream_delete_entry(streamptr);
    }

  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
613 614
}

615 616
static
int streamOpen(const char *filename, const char *filemode, int filetype)
617
{
618 619
  if ( !filemode || strlen(filemode) != 1 ) return CDI_EINVAL;
  return streamOpenID(filename, (char)tolower(filemode[0]), filetype, CDI_UNDEFID);
620
}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
621

622 623
static
int streamOpenA(const char *filename, const char *filemode, int filetype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
624 625
{
  if ( CDI_Debug )
626
    Message("Open %s file (mode=%c); filename: %s", strfiletype(filetype), (int) *filemode, filename);
627
  if ( CDI_Debug ) printf("streamOpenA: %s\n", filename); // seg fault without this line on thunder/squall with "cdo cat x y"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
628

629
  if ( ! filename || ! filemode || filetype < 0 ) return CDI_EINVAL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
630

631 632 633
  stream_t *streamptr = stream_new_entry(CDI_UNDEFID);
  int fileID = CDI_UNDEFID;

634
  {
635
    int (*streamOpenDelegate)(const char *filename, char filemode,
636
                              int filetype, stream_t *streamptr, int recordBufIsToBeCreated)
Thomas Jahns's avatar
Thomas Jahns committed
637
      = (int (*)(const char *, char, int, stream_t *, int))
638
      namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func;
639

640
    fileID = streamOpenDelegate(filename, 'r', filetype, streamptr, 1);
641
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
642

643
  if ( fileID == CDI_UNDEFID || fileID == CDI_ELIBNAVAIL || fileID == CDI_ESYSTEM ) return fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
644

645
  int streamID = streamptr->self;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
646

647 648 649 650 651
  streamptr->filemode = tolower(*filemode);
  streamptr->filename = strdupx(filename);
  streamptr->fileID   = fileID;

  streamptr->vlistID = vlistCreate();
652
  cdiVlistMakeInternal(streamptr->vlistID);
653
  /* cdiReadByteorder(streamID); */
654
  int status = cdiInqContents(streamptr);
655
  if ( status < 0 ) return status;
656
  vlist_t *vlistptr = vlist_to_pointer(streamptr->vlistID);
657
  vlistptr->ntsteps = (int)cdiInqTimeSize(streamID);
658 659 660 661 662

  // Needed for NetCDF4
  for ( int varID = 0; varID < vlistptr->nvars; ++varID )
    streamptr->vars[varID].defmiss = true;

663
  if ( !strcmp(filemode, "r") ) cdiVlistMakeImmutable(streamptr->vlistID);
664

665 666
  {
    void (*streamCloseDelegate)(stream_t *streamptr, int recordBufIsToBeDeleted)
667 668
      = (void (*)(stream_t *, int))
      namespaceSwitchGet(NSSWITCH_STREAM_CLOSE_BACKEND).func;
669

670 671
    streamCloseDelegate(streamptr, 0);
  }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
672 673 674

  switch (filetype)
    {
675
#if defined(HAVE_LIBGRIB) && (defined(HAVE_LIBCGRIBEX) || defined(HAVE_LIBGRIB_API))
676
    case CDI_FILETYPE_GRB:
677
#ifdef  HAVE_LIBGRIB_API
678
    case CDI_FILETYPE_GRB2:
679
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
680 681
      {
        fileID = gribOpen(filename, filemode);
682
        if ( fileID != CDI_UNDEFID ) gribContainersNew(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
683 684 685
	break;
      }
#endif
686
#ifdef  HAVE_LIBSERVICE
687
    case CDI_FILETYPE_SRV:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
688 689 690 691 692
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
693
#ifdef  HAVE_LIBEXTRA
694
    case CDI_FILETYPE_EXT:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
695 696 697 698 699
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
700
#ifdef  HAVE_LIBIEG
701
    case CDI_FILETYPE_IEG:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
702 703 704 705 706
      {
        fileID = fileOpen(filename, filemode);
	break;
      }
#endif
707
#ifdef  HAVE_LIBNETCDF
708 709
    case CDI_FILETYPE_NC:
    case CDI_FILETYPE_NC2:
710
    case CDI_FILETYPE_NC5:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
711
      {
712
	fileID = cdfOpen(filename, filemode, filetype);
713
	streamptr->ncmode = 2;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
714 715
	break;
      }
716 717
    case CDI_FILETYPE_NC4:
    case CDI_FILETYPE_NC4C:
718
      {
Deike Kleberg's avatar
Deike Kleberg committed
719
	fileID = cdf4Open(filename, filemode, &filetype);
720
	streamptr->ncmode = 2;
721 722
	break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
723 724 725
#endif
    default:
      {
726
	if ( CDI_Debug ) Message("%s support not compiled in!", strfiletype(filetype));
727
	return CDI_ELIBNAVAIL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
728 729 730 731 732 733
      }
    }

  if ( fileID == CDI_UNDEFID )
    streamID = CDI_UNDEFID;
  else
734
    streamptr->fileID = fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
735

736
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
737 738 739 740
}

/*
@Function  streamOpenRead
Uwe Schulzweida's avatar
Uwe Schulzweida committed
741
@Title     Open a dataset for reading
Uwe Schulzweida's avatar
Uwe Schulzweida committed
742

Uwe Schulzweida's avatar
Uwe Schulzweida committed
743
@Prototype int streamOpenRead(const char *path)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
744
@Parameter
Deike Kleberg's avatar
Deike Kleberg committed
745
    @Item  path  The name of the dataset to be read.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
746 747

@Description
748
The function @func{streamOpenRead} opens an existing dataset for reading.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
749 750

@Result
751
Upon successful completion @func{streamOpenRead} returns an identifier to the
Uwe Schulzweida's avatar
Uwe Schulzweida committed
752 753 754 755
open stream. Otherwise, a negative number with the error status is returned.

@Errors
@List
Deike Kleberg's avatar
Deike Kleberg committed
756 757 758 759
   @Item  CDI_ESYSTEM     Operating system error.
   @Item  CDI_EINVAL      Invalid argument.
   @Item  CDI_EUFILETYPE  Unsupported file type.
   @Item  CDI_ELIBNAVAIL  Library support not compiled in.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
760 761 762
@EndList

@Example
Uwe Schulzweida's avatar
Uwe Schulzweida committed
763
Here is an example using @func{streamOpenRead} to open an existing NetCDF
764
file named @func{foo.nc} for reading:
Uwe Schulzweida's avatar
Uwe Schulzweida committed
765 766

@Source
Uwe Schulzweida's avatar
Uwe Schulzweida committed
767
#include "cdi.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
768 769 770 771 772 773 774 775 776
   ...
int streamID;
   ...
streamID = streamOpenRead("foo.nc");
if ( streamID < 0 ) handle_error(streamID);
   ...
@EndSource
@EndFunction
*/
Thomas Jahns's avatar
Thomas Jahns committed
777
int streamOpenRead(const char *filename)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
778
{
779 780
  cdiInitialize();

781
  int byteorder = 0;
782
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
783

784
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
785

Thomas Jahns's avatar
Thomas Jahns committed
786
  int streamID = streamOpen(filename, "r", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
787

Uwe Schulzweida's avatar
Uwe Schulzweida committed
788 789
  if ( streamID >= 0 )
    {
790
      stream_t *streamptr = stream_to_pointer(streamID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
791 792
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
793

794
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
795 796 797 798 799
}


int streamOpenAppend(const char *filename)
{
800 801
  cdiInitialize();

802
  int byteorder = 0;
803
  int filetype = cdiGetFiletype(filename, &byteorder);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
804

805
  if ( filetype < 0 ) return filetype;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
806

807
  int streamID = streamOpenA(filename, "a", filetype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
808

809 810
  if ( streamID >= 0 )
    {
811
      stream_t *streamptr = stream_to_pointer(streamID);
812 813
      streamptr->byteorder = byteorder;
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
814

815
  return streamID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
816 817 818 819
}

/*
@Function  streamOpenWrite
Uwe Schulzweida's avatar
Uwe Schulzweida committed
820
@Title     Create a new dataset
Uwe Schulzweida's avatar
Uwe Schulzweida committed
821

Uwe Schulzweida's avatar
Uwe Schulzweida committed
822
@Prototype int streamOpenWrite(const char *path, int filetype)