Erroneous behaviour setting GRIB2 bitsPerValue
Hi,
Recently, I've encountered a problem regarding the creation of GRIB2 files and, in particular, the setting of the bitsPerValue information. One might argue if the following problem is caused by an erroneous behaviour of the CDI or the underlying ECCodes library:
To reproduce the problem, I've created a branch ex_bitsPerValue where I modified the example program examples/cdi_write.c accordingly (commit 2f5f90b9). Here, the CDI library creates a multi-level variable "QC" and sets
vlistDefVarDatatype(vlistID, varID2, 16);We can see that this information is forwarded to the ECCodes library:
CDI_DEBUG=1 CDI_GRIBAPI_DEBUG=1 ./cdi_write
cdiGetenvInt : set CDI_GRIBAPI_DEBUG to 1
vlistCreate : create vlistID = 26
vlistDefVarTiles : gridID = 24 zaxisID = 25 timetype = 1
taxisCreate : taxistype: 1
taxisCreate : taxisID: 27
streamOpenID : Open GRIB2 mode w file example.grb
vlistDuplicate : call to vlistDuplicate
vlistCreate : create vlistID = 29
vlistCopy : call to vlistCopy, vlistIDs 26 -> 29
stream_new_var : gridID = 24 zaxisID = 25
stream_new_var : varID 0: create 1 tiles with 5 level(s), zaxisID=25
stream_new_var : streamptr->vars[varID].recordTable[isub].recordID[0]=-1
grib_set_long( grib_handle* h, "grib2LocalSectionPresent", 0)
grib_set_long( grib_handle* h, "numberOfValues", 0)
cdiStreamDefTimestep_: streamID = 28 tsID = 0
cdiStreamWriteVar_: streamID = 28 varID = 0
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "typeOfGeneratingProcess", 0)
grib_set_long( grib_handle* h, "backgroundProcess", 0)
grib_set_long( grib_handle* h, "productDefinitionTemplateNumber", 0)
grib_set_string( grib_handle* h, "stepType", "instant")
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850101)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_string( grib_handle* h, "shortName", "QC")
grib_set_long( grib_handle* h, "shapeOfTheEarth", 0)
grib_set_long( grib_handle* h, "bitsPerValue", 16)
grib_set_string( grib_handle* h, "packingType", "grid_simple")
grib_set_long( grib_handle* h, "numberOfValues", 72)
grib_set_string( grib_handle* h, "gridType", "regular_ll")
grib_set_long( grib_handle* h, "Ni", 12)
grib_set_double( grib_handle* h, "longitudeOfFirstGridPointInDegrees", 0.000000)
grib_set_double( grib_handle* h, "longitudeOfLastGridPointInDegrees", 330.000000)
grib_set_double( grib_handle* h, "iDirectionIncrementInDegrees", 30.000000)
grib_set_long( grib_handle* h, "Nj", 6)
grib_set_double( grib_handle* h, "latitudeOfFirstGridPointInDegrees", -75.000000)
grib_set_double( grib_handle* h, "latitudeOfLastGridPointInDegrees", 75.000000)
grib_set_long( grib_handle* h, "iScansNegatively", 0)
grib_set_long( grib_handle* h, "jScansPositively", 1)
grib_set_double( grib_handle* h, "jDirectionIncrementInDegrees", 30.000000)
grib_set_long( grib_handle* h, "typeOfFirstFixedSurface", 100)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 101300)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850101)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 92500)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850101)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 85000)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850101)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 50000)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850101)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 20000)
cdiStreamDefTimestep_: streamID = 28 tsID = 1
cdiStreamWriteVar_: streamID = 28 varID = 0
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850102)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 101300)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850102)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 92500)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850102)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 85000)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850102)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 50000)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850102)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 20000)
cdiStreamDefTimestep_: streamID = 28 tsID = 2
cdiStreamWriteVar_: streamID = 28 varID = 0
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850103)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 101300)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850103)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 92500)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850103)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 85000)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850103)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 50000)
grb_write_var_slice: gridID = 24 zaxisID = 25
grib_set_long( grib_handle* h, "significanceOfReferenceTime", 0)
grib_set_long( grib_handle* h, "stepRange", 0)
grib_set_long( grib_handle* h, "dataDate", 19850103)
grib_set_long( grib_handle* h, "hour", 12)
grib_set_long( grib_handle* h, "minute", 0)
grib_set_long( grib_handle* h, "second", 0)
grib_set_long( grib_handle* h, "scaleFactorOfFirstFixedSurface", 0)
grib_set_long( grib_handle* h, "scaledValueOfFirstFixedSurface", 20000)
streamClose : streamID = 28 filename = example.grb
vlist_delete : call to vlist_delete, vlistID = 29
stream_delete_entry: Removed idx 28 from stream list
vlist_delete : call to vlist_delete, vlistID = 26However, what's special about my setup is the fact that the first levels of my example are constant.
Therefore we see in the example output bitsPerValue=0 - I interpret this as "all values are equal, therefore GRIB2 encodes this in a trivial way".
But when we write out a mixture of constant and non-constant records, we get a problem: ECCodes obviously recognizes the first levels as constant (bitsPerValue=0) and does not apply the original CDI setting bitsPerValue=16. But this information is not restored when the non-constant QC levels are written: We get bitsPerValue=24 for these levels instead of 24 bits.
More precisely: If we enable the code section
// Init var2
for ( size_t i = 0; i < nlon*nlat*2; i++ ) var2[i] = 0.;
for ( size_t i = nlon*nlat*2; i < nlon*nlat*nlev; i++ ) var2[i] = sin((double)i); i.e. a mixture of constant and non-constant levels, we get
$ /sw/rhel6-x64/eccodes/eccodes-2.5.0-intel14/bin/grib_ls -P bitsPerValue example.grb
example.grb
bitsPerValue edition centre date dataType gridType stepRange typeOfLevel level shortName packingType
0 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 1013 qc grid_simple
0 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 925 qc grid_simple
24 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 850 qc grid_simple
24 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 500 qc grid_simple
24 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 200 qc grid_simple
0 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 1013 qc grid_simple
0 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 925 qc grid_simple
24 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 850 qc grid_simple
24 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 500 qc grid_simple
24 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 200 qc grid_simple
0 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 1013 qc grid_simple
0 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 925 qc grid_simple
24 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 850 qc grid_simple
24 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 500 qc grid_simple
24 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 200 qc grid_simple On the other hand, if we have all levels non-constant:
// Init var2
for ( size_t i = 0; i < nlon*nlat*nlev; i++ ) var2[i] = sin((double)i); we get
$ /sw/rhel6-x64/eccodes/eccodes-2.5.0-intel14/bin/grib_ls -P bitsPerValue example.grb
example.grb
bitsPerValue edition centre date dataType gridType stepRange typeOfLevel level shortName packingType
16 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 1013 qc grid_simple
16 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 925 qc grid_simple
16 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 850 qc grid_simple
16 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 500 qc grid_simple
16 2 ecmf 19850101 af regular_ll 0 isobaricInhPa 200 qc grid_simple
16 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 1013 qc grid_simple
16 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 925 qc grid_simple
16 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 850 qc grid_simple
16 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 500 qc grid_simple
16 2 ecmf 19850102 af regular_ll 0 isobaricInhPa 200 qc grid_simple
16 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 1013 qc grid_simple
16 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 925 qc grid_simple
16 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 850 qc grid_simple
16 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 500 qc grid_simple
16 2 ecmf 19850103 af regular_ll 0 isobaricInhPa 200 qc grid_simple