stream_gribapi.c 82.9 KB
Newer Older
Uwe Schulzweida's avatar
Uwe Schulzweida committed
1
2
3
4
#if defined (HAVE_CONFIG_H)
#  include "config.h"
#endif

5
#include <limits.h>
Uwe Schulzweida's avatar
Uwe Schulzweida committed
6
7
#include <stdio.h>

Uwe Schulzweida's avatar
Uwe Schulzweida committed
8
#include "dmemory.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
9
#include "cdi.h"
10
#include "cdi_int.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
11
#include "file.h"
12
13
#include "gribapi_utilities.h"
#include "stream_grb.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
14
15
16
#include "varscan.h"
#include "datetime.h"
#include "vlist.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
17
#include "stream_grb.h"
18
#include "calendar.h"
19
#include "subtype.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
20
21
22


#if  defined  (HAVE_LIBGRIB_API)
23
#  include "cgribex.h"      /* gribGetSize, gribRead, gribGetZip, GRIB1_LTYPE_99 */
Uwe Schulzweida's avatar
Uwe Schulzweida committed
24
#  include "gribapi.h"
25
26

#  include <grib_api.h>
Uwe Schulzweida's avatar
Uwe Schulzweida committed
27
28
29
30
#endif

extern int cdiInventoryMode;

Thomas Jahns's avatar
Thomas Jahns committed
31
#if  defined  (HAVE_LIBGRIB_API)
32
static const var_tile_t dummy_tiles = { -1, -1, -1, -1, -1, -1 };
Thomas Jahns's avatar
Thomas Jahns committed
33
#endif
34

Uwe Schulzweida's avatar
Uwe Schulzweida committed
35
typedef struct {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
36
  int param;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
37
38
39
  int level1;
  int level2;
  int ltype;
40
  int tsteptype;
41
  char name[32];
42
43
44

  var_tile_t tiles;

45
} compvar2_t;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
46
47
48
49


#if  defined  (HAVE_LIBGRIB_API)
static
50
int gribapiGetZaxisType(long editionNumber, int grib_ltype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
51
{
Uwe Schulzweida's avatar
Uwe Schulzweida committed
52
53
54
  int zaxistype = ZAXIS_GENERIC;

  if ( editionNumber <= 1 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
55
    {
56
      zaxistype = grib1ltypeToZaxisType(grib_ltype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
57
58
59
    }
  else
    {
60
      zaxistype = grib2ltypeToZaxisType(grib_ltype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
61
62
63
64
65
    }

  return (zaxistype);
}

66
static
67
int getTimeunits(long unitsOfTime)
68
69
{
  int timeunits = -1;
70
71
72
73
74
75
76
77
78
79
80
81
82

  switch (unitsOfTime)
    {
    case 13:  timeunits = TUNIT_SECOND;  break;
    case  0:  timeunits = TUNIT_MINUTE;  break;
    case  1:  timeunits = TUNIT_HOUR;    break;
    case 10:  timeunits = TUNIT_3HOURS;  break;
    case 11:  timeunits = TUNIT_6HOURS;  break;
    case 12:  timeunits = TUNIT_12HOURS; break;
    case  2:  timeunits = TUNIT_DAY;     break;
    default:  timeunits = TUNIT_HOUR;    break;
    }

83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  return (timeunits);
}

static
double timeunit_factor(int tu1, int tu2)
{
  double factor = 1;

  if ( tu2 == TUNIT_HOUR )
    {
      switch (tu1)
        {
        case TUNIT_SECOND:  factor = 3600;   break;
        case TUNIT_MINUTE:  factor = 60;     break;
        case TUNIT_HOUR:    factor = 1;      break;
        case TUNIT_3HOURS:  factor = 1./3;   break;
        case TUNIT_6HOURS:  factor = 1./6;   break;
        case TUNIT_12HOURS: factor = 1./12;  break;
        case TUNIT_DAY:     factor = 1./24;  break;
        }
    }

  return (factor);
}

static
int gribapiGetTimeUnits(grib_handle *gh)
{
  int timeunits = -1;
112
  long unitsOfTime = -1;
113

114
  grib_get_long(gh, "indicatorOfUnitOfTimeRange", &unitsOfTime);
115

116
  GRIB_CHECK(my_grib_set_long(gh, "stepUnits", unitsOfTime), 0);
117

118
  timeunits = getTimeunits(unitsOfTime);
119
120
121
122

  return (timeunits);
}

123
124
125
126
static
int gribapiGetEndStep(grib_handle *gh, int startStep, int timeunits)
{
  int endStep = startStep;
127
  int timeunits2 = timeunits;
128

129
130
  long unitsOfTime;
  int status = grib_get_long(gh, "stepUnits", &unitsOfTime);
131
132
  if ( status == 0 ) timeunits2 = getTimeunits(unitsOfTime);
  //timeunits2 = gribapiGetTimeUnits(gh);
133

134
  long lpar;
135
136
137
  status = grib_get_long(gh, "endStep", &lpar);

  if ( status == 0 )
138
    endStep = (int) (((double)lpar * timeunit_factor(timeunits, timeunits2)) + 0.5);
139
140
141
142
143
  // printf("%d %d %d %d %d %g\n", startStep, endStep, lpar, timeunits, timeunits2, timeunit_factor(timeunits, timeunits2));

  return (endStep);
}

144
145
146
147
148
149
150
static
void gribapiGetDataDateTime(grib_handle *gh, int *datadate, int *datatime)
{
  long lpar;

  GRIB_CHECK(grib_get_long(gh, "dataDate", &lpar), 0);
  *datadate = (int) lpar;
151
  GRIB_CHECK(grib_get_long(gh, "dataTime", &lpar), 0);  //FIXME: This looses the seconds in GRIB2 files.
152
153
154
  *datatime = (int) lpar*100;
}

155
156
157
static
void gribapiSetDataDateTime(grib_handle *gh, int datadate, int datatime)
{
158
159
  GRIB_CHECK(my_grib_set_long(gh, "dataDate", datadate), 0);
  GRIB_CHECK(my_grib_set_long(gh, "dataTime", datatime/100), 0);
160
161
}

162
static
163
int gribapiGetValidityDateTime(grib_handle *gh, int *vdate, int *vtime)
164
{
165
  int rdate, rtime;
166
  int timeUnits, startStep = 0, endStep;
167
168
  int tstepRange = 0;
  int range;
169
  int status;
170
  long lpar;
171
172
  long sigofrtime = 3;

173
  if ( gribEditionNumber(gh) > 1 )
174
175
176
    {
      GRIB_CHECK(grib_get_long(gh, "significanceOfReferenceTime", &sigofrtime), 0);
    }
177
178
179
180
  else
    {
      GRIB_CHECK(grib_get_long(gh, "timeRangeIndicator", &sigofrtime), 0);
    }
181

182
  if ( sigofrtime == 3 )        //XXX: This looks like a bug to me, because timeRangeIndicator == 3 does not seem to have the same meaning as significanceOfReferenceTime == 3. I would recommend replacing this condition with `if(!gribapiTimeIsFC())`.
183
    {
184
      gribapiGetDataDateTime(gh, vdate, vtime);
185
186
187
    }
  else
    {
188
189
      gribapiGetDataDateTime(gh, &rdate, &rtime);

190
191
      status = grib_get_long(gh, "forecastTime", &lpar);
      if ( status == 0 ) startStep = (int) lpar;
192
      timeUnits = gribapiGetTimeUnits(gh);
193
      endStep = gribapiGetEndStep(gh, startStep, timeUnits);
194
195
196
197
198
199
200
201
202

      range = endStep - startStep;

      if ( range > 0 )
	{
	  if ( startStep == 0 ) tstepRange = -1;
	  else                  tstepRange =  1;
	}

203
204
205
206
      {
	static int lprint = TRUE;
	extern int grib_calendar;
	int ryear, rmonth, rday, rhour, rminute, rsecond;
207
208
209
	int julday, secofday;
	int64_t time_period = endStep;
        int64_t addsec;
210
211
212
213

	cdiDecodeDate(rdate, &ryear, &rmonth, &rday);
	cdiDecodeTime(rtime, &rhour, &rminute, &rsecond);

214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
        if ( rday > 0 )
          {
            encode_caldaysec(grib_calendar, ryear, rmonth, rday, rhour, rminute, rsecond, &julday, &secofday);

            addsec = 0;
            switch ( timeUnits )
              {
              case TUNIT_SECOND:  addsec =         time_period; break;
              case TUNIT_MINUTE:  addsec =    60 * time_period; break;
              case TUNIT_HOUR:    addsec =  3600 * time_period; break;
              case TUNIT_3HOURS:  addsec = 10800 * time_period; break;
              case TUNIT_6HOURS:  addsec = 21600 * time_period; break;
              case TUNIT_12HOURS: addsec = 43200 * time_period; break;
              case TUNIT_DAY:     addsec = 86400 * time_period; break;
              default:
                if ( lprint )
                  {
                    Warning("Time unit %d unsupported", timeUnits);
                    lprint = FALSE;
                  }
                break;
              }

            julday_add_seconds(addsec, &julday, &secofday);

            decode_caldaysec(grib_calendar, julday, secofday, &ryear, &rmonth, &rday, &rhour, &rminute, &rsecond);
          }
241

242
243
244
	*vdate = cdiEncodeDate(ryear, rmonth, rday);
	*vtime = cdiEncodeTime(rhour, rminute, rsecond);
      }
245
    }
246
247

  return (tstepRange);
248
249
}

250
static
251
void grib1GetLevel(grib_handle *gh, int *leveltype, int *lbounds, int *level1, int *level2)
252
{
253
  *leveltype = 0;
254
255
256
  *lbounds   = 0;
  *level1    = 0;
  *level2    = 0;
257

258
  long lpar;
259
  if(!grib_get_long(gh, "indicatorOfTypeOfLevel", &lpar))       //1 byte
260
    {
261
      *leveltype = (int) lpar;
262

263
      switch (*leveltype)
264
265
266
267
	{
	case GRIB1_LTYPE_SIGMA_LAYER:
	case GRIB1_LTYPE_HYBRID_LAYER:
	case GRIB1_LTYPE_LANDDEPTH_LAYER:
268
	  { *lbounds = 1; break; }
269
	}
270

271
272
273
274
275
276
277
278
      if ( *lbounds )
	{
	  GRIB_CHECK(grib_get_long(gh, "topLevel", &lpar), 0);  //1 byte
	  *level1 = (int)lpar;
	  GRIB_CHECK(grib_get_long(gh, "bottomLevel", &lpar), 0);       //1 byte
	  *level2 = (int)lpar;
	}
      else
279
	{
280
          double dlevel;
281
	  GRIB_CHECK(grib_get_double(gh, "level", &dlevel), 0); //2 byte
282
283
	  if ( *leveltype == 100 ) dlevel *= 100;
	  if ( dlevel < -2.e9 || dlevel > 2.e9 ) dlevel = 0;
284
	  if ( *leveltype == GRIB1_LTYPE_99 ) *leveltype = 100;
285

286
287
288
	  *level1 = (int) dlevel;
	  *level2 = 0;
	}
289
    }
290
291
}

292
293
294
static
double grib2ScaleFactor(long factor)
{
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
  switch(factor)
    {
      case GRIB_MISSING_LONG: return 1;
      case 0: return 1;
      case 1: return 0.1;
      case 2: return 0.01;
      case 3: return 0.001;
      case 4: return 0.0001;
      case 5: return 0.00001;
      case 6: return 0.000001;
      case 7: return 0.0000001;
      case 8: return 0.00000001;
      case 9: return 0.000000001;
      default: return 0;
    }
}

static
int calcLevel(int level_sf, long factor, long level)
{
  double result = 0;
Thomas Jahns's avatar
Thomas Jahns committed
316
  if(level != GRIB_MISSING_LONG) result = (double)level*grib2ScaleFactor(factor);
317
318
  if(level_sf) result *= level_sf;
  return (int)result;
319
320
}

321
static
322
323
void grib2GetLevel(grib_handle *gh, int *leveltype1, int *leveltype2, int *lbounds, int *level1, 
                   int *level2, int *level_sf, int *level_unit)
324
{
325
326
327
  int status;
  long lpar;
  long factor;
328

329
330
  *leveltype1 = 0;
  *leveltype2 = -1;
331
332
333
334
335
  *lbounds    = 0;
  *level1     = 0;
  *level2     = 0;
  *level_sf   = 0;
  *level_unit = 0;
336

337
  status = grib_get_long(gh, "typeOfFirstFixedSurface", &lpar); //1 byte
338
  if ( status == 0 )
339
    {
340
      long llevel;
341

342
      *leveltype1 = (int) lpar;
343

344
      status = grib_get_long(gh, "typeOfSecondFixedSurface", &lpar); //1 byte
345
      /* FIXME: assert(lpar >= INT_MIN && lpar <= INT_MAX) */
346
      if ( status == 0 ) *leveltype2 = (int)lpar;
347

348
      if ( *leveltype1 != 255 && *leveltype2 != 255 && *leveltype2 > 0 ) *lbounds = 1;
349
      switch(*leveltype1)
350
        {
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
          case GRIB2_LTYPE_REFERENCE:
            if(*leveltype2 == 1) *lbounds = 0;
            break;

          case GRIB2_LTYPE_LANDDEPTH:
            *level_sf = 1000;
            *level_unit = CDI_UNIT_M;
            break;

          case GRIB2_LTYPE_ISOBARIC:
            *level_sf = 1000;
            *level_unit = CDI_UNIT_PA;
            break;

          case GRIB2_LTYPE_SIGMA:
            *level_sf = 1000;
            *level_unit = 0;
            break;
369
        }
370

371
372
373
      GRIB_CHECK(grib_get_long(gh, "scaleFactorOfFirstFixedSurface", &factor), 0);      //1 byte
      GRIB_CHECK(grib_get_long(gh, "scaledValueOfFirstFixedSurface", &llevel), 0);      //4 byte
      *level1 = calcLevel(*level_sf, factor, llevel);
374

375
376
377
378
379
      if ( *lbounds )
        {
          GRIB_CHECK(grib_get_long(gh, "scaleFactorOfSecondFixedSurface", &factor), 0); //1 byte
          GRIB_CHECK(grib_get_long(gh, "scaledValueOfSecondFixedSurface", &llevel), 0); //4 byte
          *level2 = calcLevel(*level_sf, factor, llevel);
380
        }
381
382
    }
}
383

384
static
385
void gribGetLevel(grib_handle *gh, int* leveltype1, int* leveltype2, int* lbounds, int* level1, int* level2, int* level_sf, int* level_unit, var_tile_t* tiles)
386
387
388
389
390
391
392
393
394
395
396
{
  if ( gribEditionNumber(gh) <= 1 )
    {
      grib1GetLevel(gh, leveltype1, lbounds, level1, level2);
      *leveltype2 = -1;
      *level_sf = 0;
      *level_unit = 0;
    }
  else
    {
      grib2GetLevel(gh, leveltype1, leveltype2, lbounds, level1, level2, level_sf, level_unit);
397
398

      /* read in tiles attributes (if there are any) */
Thomas Jahns's avatar
Thomas Jahns committed
399
400
401
402
403
404
      tiles->tileindex = (int)gribGetLongDefault(gh, cdiSubtypeAttributeName[SUBTYPE_ATT_TILEINDEX], -1);
      tiles->totalno_of_tileattr_pairs = (int)gribGetLongDefault(gh, cdiSubtypeAttributeName[SUBTYPE_ATT_TOTALNO_OF_TILEATTR_PAIRS], -1);
      tiles->tileClassification = (int)gribGetLongDefault(gh, cdiSubtypeAttributeName[SUBTYPE_ATT_TILE_CLASSIFICATION], -1);
      tiles->numberOfTiles = (int)gribGetLongDefault(gh, cdiSubtypeAttributeName[SUBTYPE_ATT_NUMBER_OF_TILES], -1);
      tiles->numberOfAttributes = (int)gribGetLongDefault(gh, cdiSubtypeAttributeName[SUBTYPE_ATT_NUMBER_OF_ATTR], -1);
      tiles->attribute = (int)gribGetLongDefault(gh, cdiSubtypeAttributeName[SUBTYPE_ATT_TILEATTRIBUTE], -1);
405
    }
406
407
}

408
409
410
411
412
static
void gribapiGetString(grib_handle *gh, const char *key, char *string, size_t length)
{
  string[0] = 0;

413
414
415
416
417
418
  int ret = grib_get_string(gh, key, string, &length);
  if (ret != 0)
    {
      fprintf(stderr, "grib_get_string(gh, \"%s\", ...) failed!\n", key);
      GRIB_CHECK(ret, 0);
    }
419
420
421
422
  if      ( length == 8 && memcmp(string, "unknown", length) == 0 ) string[0] = 0;
  else if ( length == 2 && memcmp(string, "~", length)       == 0 ) string[0] = 0;
}

423
#if  defined  (HAVE_LIBGRIB_API)
424
static
425
void gribapiAddRecord(stream_t * streamptr, int param, grib_handle *gh,
426
                      size_t recsize, off_t position, int datatype, int comptype, const char *varname,
427
                      int leveltype1, int leveltype2, int lbounds, int level1, int level2, int level_sf, int level_unit,
428
                      const var_tile_t *tiles, int lread_additional_keys)
429
430
431
432
433
434
435
436
437
438
439
440
{
  int zaxistype;
  int gridID = CDI_UNDEFID, varID;
  int levelID = 0;
  int tsID, recID;
  int numavg;
  int tsteptype;
  record_t *record;
  grid_t grid;
  int vlistID;
  long lpar;
  int status;
441
  char stdname[CDI_MAX_NAME], longname[CDI_MAX_NAME], units[CDI_MAX_NAME];
442
  size_t vlen;
443
  long ens_index = 0, ens_count = 0, ens_forecast_type = 0;
444

Uwe Schulzweida's avatar
Uwe Schulzweida committed
445
  vlistID = streamptr->vlistID;
446
  tsID    = streamptr->curTsID;
447
  recID   = recordNewEntry(streamptr, tsID);
448
449
450
451
452
453
  record  = &streamptr->tsteps[tsID].records[recID];

  tsteptype = gribapiGetTsteptype(gh);
  // numavg  = ISEC1_AvgNum;
  numavg  = 0;

454
  // fprintf(stderr, "param %d %d %d %d\n", param, level1, level2, leveltype1);
455

456
457
458
459
460
  (*record).size      = recsize;
  (*record).position  = position;
  (*record).param     = param;
  (*record).ilevel    = level1;
  (*record).ilevel2   = level2;
461
  (*record).ltype     = leveltype1;
462
  (*record).tsteptype = tsteptype;
463
464
  if ( tiles ) (*record).tiles = *tiles;
  else         (*record).tiles = dummy_tiles;
465
466
467
468
469
470
471
472
473
474
475
476

  //FIXME: This may leave the variable name unterminated (which is the behavior that I found in the code).
  //       I don't know precisely how this field is used, so I did not change this behavior to avoid regressions,
  //       but I think that it would be better to at least add a line
  //
  //           record->varname[sizeof(record->varname) - 1] = 0;`
  //
  //       after the `strncpy()` call.
  //
  //       I would consider using strdup() (that requires POSIX-2008 compliance, though), or a similar homebrew approach.
  //       I. e. kick the fixed size array and allocate enough space, whatever that may be.
  strncpy(record->varname, varname, sizeof(record->varname));
477
478

  gribapiGetGrid(gh, &grid);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
479

480
  gridID = varDefGrid(vlistID, &grid, 0);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
481

482
  zaxistype = gribapiGetZaxisType(gribEditionNumber(gh), leveltype1);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
483

484
  switch (zaxistype)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
485
    {
486
487
488
    case ZAXIS_HYBRID:
    case ZAXIS_HYBRID_HALF:
      {
Thomas Jahns's avatar
Thomas Jahns committed
489
        size_t vctsize;
490
491
        size_t dummy;
        double *vctptr;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
492

493
        GRIB_CHECK(grib_get_long(gh, "NV", &lpar), 0);
Thomas Jahns's avatar
Thomas Jahns committed
494
495
        /* FIXME: assert(lpar >= 0) */
        vctsize = (size_t)lpar;
496
497
        if ( vctsize > 0 )
          {
498
            vctptr = (double *) Malloc(vctsize*sizeof(double));
Thomas Jahns's avatar
Thomas Jahns committed
499
            dummy = vctsize;
500
            GRIB_CHECK(grib_get_double_array(gh, "pv", vctptr, &dummy), 0);
Thomas Jahns's avatar
Thomas Jahns committed
501
            varDefVCT(vctsize, vctptr);
502
            Free(vctptr);
503
504
505
506
507
          }
        break;
      }
    case ZAXIS_REFERENCE:
      {
508
        unsigned char uuid[CDI_UUID_SIZE];
509
        long ltmp;
510
        long nhlev, nvgrid;
511
512

        GRIB_CHECK(grib_get_long(gh, "NV", &lpar), 0);
513
        if ( lpar != 6 )
514
515
516
          {
            fprintf(stderr, "Warning ...\n");
          }
517
        GRIB_CHECK(grib_get_long(gh, "nlev", &ltmp), 0);
518
        nhlev = ltmp;
519
520
        GRIB_CHECK(grib_get_long(gh, "numberOfVGridUsed", &ltmp), 0);
        nvgrid = ltmp;
521
        size_t len = (size_t)CDI_UUID_SIZE;
522
523
        memset(uuid, 0, CDI_UUID_SIZE);
        GRIB_CHECK(grib_get_bytes(gh, "uuidOfVGrid", uuid, &len), 0);
524
        varDefZAxisReference((int) nhlev, (int) nvgrid, uuid);
525
526
        break;
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
527
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
528

529
  // if ( datatype > 32 ) datatype = DATATYPE_PACK32;
530
  if ( datatype <  0 ) datatype = DATATYPE_PACK;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
531

532
  stdname[0] = 0;
533
534
535
  longname[0] = 0;
  units[0] = 0;

536
  if ( varname[0] != 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
537
    {
538
      vlen = CDI_MAX_NAME;
539
      gribapiGetString(gh, "name", longname, vlen);
540
      vlen = CDI_MAX_NAME;
541
      gribapiGetString(gh, "units", units, vlen);
542
543
544
545
546

      {
        vlen = CDI_MAX_NAME;
        status = grib_get_string(gh, "cfName", stdname, &vlen);
        if ( status != 0 || vlen <= 1 ) stdname[0] = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
547
        else if ( strncmp(stdname, "unknown", 7) == 0 ) stdname[0] = 0;
548
      }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
549
    }
550
  // fprintf(stderr, "param %d name %s %s %s\n", param, name, longname, units);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
551

552
553
  /* add the previously read record data to the (intermediate) list of records */
  int tile_index = -1;
554
  varAddRecord(recID, param, gridID, zaxistype, lbounds, level1, level2, level_sf, level_unit,
555
	       datatype, &varID, &levelID, tsteptype, numavg, leveltype1, leveltype2,
556
	       varname, stdname, longname, units, tiles, &tile_index);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
557

558
559
  (*record).varID   = (short)varID;
  (*record).levelID = (short)levelID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
560

561
  varDefCompType(varID, comptype);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
562

563
564
  /*
    Get the ensemble Info from the grib-2 Tables and update the intermediate datastructure.
565
    Further update to the "vlist" is handled in the same way as for GRIB-1 by "cdi_generate_vars"
566
  */
567
568
569
570
571
572
573
  status = grib_get_long(gh, "typeOfEnsembleForecast", &ens_forecast_type );
  if ( status == 0 )
    {
      GRIB_CHECK(grib_get_long(gh, "numberOfForecastsInEnsemble", &ens_count ), 0);
      GRIB_CHECK(grib_get_long(gh, "perturbationNumber", &ens_index ), 0);
    }

574
575
576
  if ( ens_index > 0 )
    varDefEnsembleInfo(varID, (int)ens_index, (int)ens_count, (int)ens_forecast_type);

577
578
579
580
581
  long typeOfGeneratingProcess = 0;
  status = grib_get_long(gh, "typeOfGeneratingProcess", &typeOfGeneratingProcess);
  if ( status == 0 )
    varDefTypeOfGeneratingProcess(varID, (int) typeOfGeneratingProcess);

582
583
584
585
586
  long productDefinitionTemplate = 0;
  status = grib_get_long(gh, "productDefinitionTemplateNumber", &productDefinitionTemplate);
  if ( status == 0 )
    varDefProductDefinitionTemplate(varID, (int) productDefinitionTemplate);

587
588
589
590
  int    i;
  long   lval;
  double dval;

591
592
593
594
595
596
597
598
599
  if (lread_additional_keys)
    for ( i = 0; i < cdiNAdditionalGRIBKeys; i++ )
      {
        /* note: if the key is not defined, we do not throw an error! */
        if ( grib_get_long(gh, cdiAdditionalGRIBKeys[i], &lval) == 0 )
          varDefOptGribInt(varID, tile_index, lval, cdiAdditionalGRIBKeys[i]);
        if ( grib_get_double(gh, cdiAdditionalGRIBKeys[i], &dval) == 0 )
          varDefOptGribDbl(varID, tile_index, dval, cdiAdditionalGRIBKeys[i]);
      }
600

Uwe Schulzweida's avatar
Uwe Schulzweida committed
601
602
  if ( varInqInst(varID) == CDI_UNDEFID )
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
603
604
605
606
607
      long center, subcenter;
      int instID;
      GRIB_CHECK(grib_get_long(gh, "centre", &center), 0);
      GRIB_CHECK(grib_get_long(gh, "subCentre", &subcenter), 0);
      instID    = institutInq((int)center, (int)subcenter, NULL, NULL);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
608
      if ( instID == CDI_UNDEFID )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
609
	instID = institutDef((int)center, (int)subcenter, NULL, NULL);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
610
611
612
613
614
615
      varDefInst(varID, instID);
    }

  if ( varInqModel(varID) == CDI_UNDEFID )
    {
      int modelID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
616
      long processID;
617
618
619
      status = grib_get_long(gh, "generatingProcessIdentifier", &processID);
      if ( status == 0 )
	{
620
621
          /* FIXME: assert(processID >= INT_MIN && processID <= INT_MAX) */
	  modelID = modelInq(varInqInst(varID), (int)processID, NULL);
622
	  if ( modelID == CDI_UNDEFID )
623
	    modelID = modelDef(varInqInst(varID), (int)processID, NULL);
624
625
	  varDefModel(varID, modelID);
	}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
626
627
628
629
    }

  if ( varInqTable(varID) == CDI_UNDEFID )
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
630
      int pdis, pcat, pnum;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
631

Uwe Schulzweida's avatar
Uwe Schulzweida committed
632
      cdiDecodeParam(param, &pnum, &pcat, &pdis);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
633

Uwe Schulzweida's avatar
Uwe Schulzweida committed
634
635
636
637
      if ( pdis == 255 )
	{
	  int tableID;
	  int tabnum = pcat;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
638

Uwe Schulzweida's avatar
Uwe Schulzweida committed
639
	  tableID = tableInq(varInqModel(varID), tabnum, NULL);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
640

Uwe Schulzweida's avatar
Uwe Schulzweida committed
641
642
643
	  if ( tableID == CDI_UNDEFID )
	    tableID = tableDef(varInqModel(varID), tabnum, NULL);
	  varDefTable(varID, tableID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
644
645
646
647
648
649
650
	}
    }

  streamptr->tsteps[tsID].nallrecs++;
  streamptr->nrecs++;

  if ( CDI_Debug )
651
    Message("varID = %d  param = %d  zaxistype = %d  gridID = %d  levelID = %d",
Uwe Schulzweida's avatar
Uwe Schulzweida committed
652
	    varID, param, zaxistype, gridID, levelID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
653
}
654
#endif
655

656
657
static compvar2_t gribapiVarSet(int param, int level1, int level2, int leveltype, 
                                int tsteptype, char *name, var_tile_t tiles_data)
658
659
{
  compvar2_t compVar;
660
661
662
  size_t maxlen = sizeof(compVar.name);
  size_t len = strlen(name);
  if ( len > maxlen ) len = maxlen;
663

664
665
666
667
668
  compVar.param     = param;
  compVar.level1    = level1;
  compVar.level2    = level2;
  compVar.ltype     = leveltype;
  compVar.tsteptype = tsteptype;
669
670
  memset(compVar.name, 0, maxlen);
  memcpy(compVar.name, name, len);
671

672
  compVar.tiles = tiles_data;
673
674
  return (compVar);
}
675
#endif
676

677
#ifdef HAVE_LIBGRIB_API
678
static
679
int gribapiVarCompare(compvar2_t compVar, record_t record, int flag)
680
681
{
  compvar2_t compVar0;
682
  size_t maxlen = sizeof(compVar.name);
683

684
685
686
687
688
  compVar0.param     = record.param;
  compVar0.level1    = record.ilevel;
  compVar0.level2    = record.ilevel2;
  compVar0.ltype     = record.ltype;
  compVar0.tsteptype = record.tsteptype;
689
  memcpy(compVar0.name, record.varname, maxlen);
690

691
692
693
694
695
696
  if ( flag == 0 )
    {
      if ( compVar0.tsteptype == TSTEP_INSTANT  && compVar.tsteptype == TSTEP_INSTANT3 ) compVar0.tsteptype = TSTEP_INSTANT3;
      if ( compVar0.tsteptype == TSTEP_INSTANT3 && compVar.tsteptype == TSTEP_INSTANT  ) compVar0.tsteptype = TSTEP_INSTANT;
    }

697
698
  compVar0.tiles = record.tiles;

699
  int rstatus = memcmp(&compVar0, &compVar, sizeof(compvar2_t));
700
701
702

  return (rstatus);
}
Uwe Schulzweida's avatar
Uwe Schulzweida committed
703

704
static void ensureBufferSize(size_t requiredSize, size_t* curSize, unsigned char **buffer) {
705
706
707
  if ( *curSize < requiredSize )
    {
      *curSize = requiredSize;
708
      *buffer = (unsigned char *) Realloc(*buffer, *curSize);
709
710
711
    }
}

712
static
Thomas Jahns's avatar
Thomas Jahns committed
713
grib_handle* gribapiGetDiskRepresentation(size_t recsize, size_t* buffersize, unsigned char** gribbuffer, int* outDatatype, int* outCompressionType, long* outUnzipsize)
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
{
  int lieee = FALSE;

  grib_handle* gh = grib_handle_new_from_message(NULL, (void *) *gribbuffer, recsize);
  if(gribEditionNumber(gh) > 1)
    {
      size_t len = 256;
      char typeOfPacking[256];

      if ( grib_get_string(gh, "packingType", typeOfPacking, &len) == 0 )
        {
          // fprintf(stderr, "packingType %d %s\n", len, typeOfPacking);
          if      ( strncmp(typeOfPacking, "grid_jpeg", len) == 0 ) *outCompressionType = COMPRESS_JPEG;
          else if ( strncmp(typeOfPacking, "grid_ccsds", len) == 0 ) *outCompressionType = COMPRESS_SZIP;
          else if ( strncmp(typeOfPacking, "grid_ieee", len) == 0 ) lieee = TRUE;
        }
    }
  else
    {
Thomas Jahns's avatar
Thomas Jahns committed
733
      if( gribGetZip((long)recsize, *gribbuffer, outUnzipsize) > 0 )
734
735
        {
          *outCompressionType = COMPRESS_SZIP;
Thomas Jahns's avatar
Thomas Jahns committed
736
          ensureBufferSize((size_t)*outUnzipsize + 100, buffersize, gribbuffer);
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
        }
      else
        {
          *outCompressionType = COMPRESS_NONE;
        }
    }

  if ( lieee )
    {
      *outDatatype = DATATYPE_FLT64;
      long precision;
      int status = grib_get_long(gh, "precision", &precision);
      if ( status == 0 && precision == 1 ) *outDatatype = DATATYPE_FLT32;
    }
  else
    {
      *outDatatype = DATATYPE_PACK;
      long bitsPerValue;
      if ( grib_get_long(gh, "bitsPerValue", &bitsPerValue) == 0 )
        {
          if ( bitsPerValue > 0 && bitsPerValue <= 32 ) *outDatatype = (int)bitsPerValue;
        }
    }
  return gh;
}
762
#endif
763

764
#if  defined  (HAVE_LIBGRIB_API)
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
typedef enum { CHECKTIME_OK, CHECKTIME_SKIP, CHECKTIME_STOP, CHECKTIME_INCONSISTENT } checkTimeResult;
static checkTimeResult checkTime(stream_t* streamptr, compvar2_t compVar, const DateTime* verificationTime, const DateTime* expectedVTime) {
  //First determine whether the current record exists already.
  int recID = 0;
  for ( ; recID < streamptr->nrecs; recID++ )
    {
      if ( gribapiVarCompare(compVar, streamptr->tsteps[0].records[recID], 1) == 0 ) break;
    }
  int recordExists = recID < streamptr->nrecs;

  //Then we need to know whether the verification time is consistent.
  int consistentTime = !memcmp(verificationTime, expectedVTime, sizeof(*verificationTime));

  //Finally, we make a decision.
  if ( cdiInventoryMode == 1 )
    {
      if ( recordExists ) return CHECKTIME_STOP;
      if ( !consistentTime ) return CHECKTIME_INCONSISTENT;
    }
  else
    {
      if ( !consistentTime ) return CHECKTIME_STOP;
      if ( recordExists ) return CHECKTIME_SKIP;
    }
  return CHECKTIME_OK;
}
791
#endif
792
793
794
795
796
797
798
799

#define gribWarning(text, nrecs, timestep, varname, param, level1, level2) do \
  { \
    char paramstr[32]; \
    cdiParamToString(param, paramstr, sizeof(paramstr)); \
    Warning("Record %2d (name=%s id=%s lev1=%d lev2=%d) timestep %d: %s", nrecs, varname, paramstr, level1, level2, timestep, text); \
  } \
while(0)
800

801
#if  defined  (HAVE_LIBGRIB_API)
802
int gribapiScanTimestep1(stream_t * streamptr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
803
804
805
{
  off_t recpos = 0;
  unsigned char *gribbuffer = NULL;
806
  size_t buffersize = 0;
807
808
  DateTime datetime0 = { .date = 10101, .time = 0 };
  int nrecs_scanned = 0;        //Only used for debug output.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
809
  int warn_time = TRUE;
810
  // int warn_numavg = TRUE;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
811
812
813
814
815
  int rdate = 0, rtime = 0, tunit = 0, fcast = 0;
  grib_handle *gh = NULL;

  streamptr->curTsID = 0;

816
817
  int tsID  = tstepsNewEntry(streamptr);
  taxis_t *taxis = &streamptr->tsteps[tsID].taxis;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
818
819

  if ( tsID != 0 )
820
    Error("Internal problem! tstepsNewEntry returns %d", tsID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
821

822
  int fileID = streamptr->fileID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
823

Thomas Jahns's avatar
Thomas Jahns committed
824
  unsigned nrecs = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
825
826
  while ( TRUE )
    {
827
828
      int level1 = 0, level2 = 0;
      size_t recsize = (size_t)gribGetSize(fileID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
829
830
831
      recpos  = fileGetPos(fileID);

      if ( recsize == 0 )
832
833
834
835
836
        {
          streamptr->ntsteps = 1;
          break;
        }
      ensureBufferSize(recsize, &buffersize, &gribbuffer);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
837

838
839
      size_t readsize = recsize;
      int rstatus = gribRead(fileID, gribbuffer, &readsize);        //Search for next 'GRIB', read the following record, and position file offset after it.
Uwe Schulzweida's avatar
Uwe Schulzweida committed
840
      if ( rstatus ) break;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
841

842
      int datatype, comptype = 0;
843
844
      long unzipsize;
      gh = gribapiGetDiskRepresentation(recsize, &buffersize, &gribbuffer, &datatype, &comptype, &unzipsize);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
845

846
      nrecs_scanned++;
847
      GRIB_CHECK(my_grib_set_double(gh, "missingValue", cdiDefaultMissval), 0);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
848

849
850
      int param = gribapiGetParam(gh);
      int leveltype1 = -1, leveltype2 = -1, lbounds, level_sf, level_unit;
851
      var_tile_t tiles = dummy_tiles;
852
      gribGetLevel(gh, &leveltype1, &leveltype2, &lbounds, &level1, &level2, &level_sf, &level_unit, &tiles);
853

854
      char varname[256];
855
856
      varname[0] = 0;
      gribapiGetString(gh, "shortName", varname, sizeof(varname));
857

858
859
      int tsteptype = gribapiGetTsteptype(gh);
      int vdate = 0, vtime = 0;
860
      gribapiGetValidityDateTime(gh, &vdate, &vtime);
861
      DateTime datetime = { .date = vdate, .time = vtime };
862
      /*
863
      printf("%d %d %d\n", vdate, vtime, leveltype1);
864
      */
865
866

      if( datetime0.date == 10101 && datetime0.time == 0 )
867
        {
868
          if( datetimeCmp(datetime, datetime0) || !nrecs )       //Do we really need this condition? I have included it in order not to change the number of times gribapiGetDataDateTime() etc. get called. But if those are sideeffect-free, this condition should be removed.
869
870
871
872
873
874
875
876
            {
              datetime0 = datetime;

              gribapiGetDataDateTime(gh, &rdate, &rtime);

              fcast = gribapiTimeIsFC(gh);
              if ( fcast ) tunit = gribapiGetTimeUnits(gh);
            }
877
        }
878
879

      if(nrecs)
880
        {
881
          checkTimeResult result = checkTime(streamptr, gribapiVarSet(param, level1, level2, leveltype1, tsteptype, varname, tiles), &datetime, &datetime0);
882
          if(result == CHECKTIME_STOP)
883
            {
884
              break;
885
            }
886
          else if(result == CHECKTIME_SKIP)
887
            {
888
889
              gribWarning("Parameter already exist, skipped!", nrecs_scanned, tsID+1, varname, param, level1, level2);
              continue;
890
            }
891
892
893
894
895
896
          else if(result == CHECKTIME_INCONSISTENT && warn_time)
            {
              gribWarning("Inconsistent verification time!", nrecs_scanned, tsID+1, varname, param, level1, level2);
              warn_time = FALSE;
            }
          assert(result == CHECKTIME_OK || result == CHECKTIME_INCONSISTENT);
897
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
898
899
      /*
      if ( ISEC1_AvgNum )
900
901
902
903
904
905
906
907
908
909
910
911
        {
          if (  taxis->numavg && warn_numavg && (taxis->numavg != ISEC1_AvgNum) )
            {
              Message("Change numavg from %d to %d not allowed!",
                      taxis->numavg, ISEC1_AvgNum);
              warn_numavg = FALSE;
            }
          else
            {
              taxis->numavg = ISEC1_AvgNum;
            }
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
912
913
914
915
      */
      nrecs++;

      if ( CDI_Debug )
916
917
918
        {
          char paramstr[32];
          cdiParamToString(param, paramstr, sizeof(paramstr));
Thomas Jahns's avatar
Thomas Jahns committed
919
          Message("%4u %8d name=%s id=%s ltype=%d lev1=%d lev2=%d vdate=%d vtime=%d",
920
                nrecs, (int)recpos, varname, paramstr, leveltype1, level1, level2, vdate, vtime);
921
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
922

923
924
      var_tile_t *ptiles = NULL;
      if ( memcmp(&tiles, &dummy_tiles, sizeof(var_tile_t)) != 0 ) ptiles = &tiles;
925
      gribapiAddRecord(streamptr, param, gh, recsize, recpos, datatype, comptype, varname,
926
                       leveltype1, leveltype2, lbounds, level1, level2, level_sf, level_unit, ptiles, 1);
927
928
929

      grib_handle_delete(gh);
      gh = NULL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
930
931
    }

932
933
  if ( gh ) grib_handle_delete(gh);

Uwe Schulzweida's avatar
Uwe Schulzweida committed
934
935
  streamptr->rtsteps = 1;

936
937
  if ( nrecs == 0 ) return (CDI_EUFSTRUCT);

938
  cdi_generate_vars(streamptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
939

940
  int taxisID = -1;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
941
942
943
944
945
946
947
948
949
950
951
952
953
954
  if ( fcast )
    {
      taxisID = taxisCreate(TAXIS_RELATIVE);
      taxis->type  = TAXIS_RELATIVE;
      taxis->rdate = rdate;
      taxis->rtime = rtime;
      taxis->unit  = tunit;
    }
  else
    {
      taxisID = taxisCreate(TAXIS_ABSOLUTE);
      taxis->type  = TAXIS_ABSOLUTE;
    }

955
956
  taxis->vdate = (int)datetime0.date;
  taxis->vtime = (int)datetime0.time;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
957

958
  int vlistID = streamptr->vlistID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
959
960
  vlistDefTaxis(vlistID, taxisID);

961
  int nrecords = streamptr->tsteps[0].nallrecs;
962
  if ( nrecords < streamptr->tsteps[0].recordSize )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
963
964
965
    {
      streamptr->tsteps[0].recordSize = nrecords;
      streamptr->tsteps[0].records =
966
        (record_t *) Realloc(streamptr->tsteps[0].records, (size_t)nrecords*sizeof(record_t));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
967
968
    }

969
  streamptr->tsteps[0].recIDs = (int *) Malloc((size_t)nrecords*sizeof(int));
Uwe Schulzweida's avatar
Uwe Schulzweida committed
970
  streamptr->tsteps[0].nrecs = nrecords;
Uwe Schulzweida's avatar