Skip to content
GitLab
Commit 1fa8e69b authored by Thomas Jahns's avatar Thomas Jahns 🤸
Browse files

Improve robustness of checksumming tests. 

* This way re-ordering of GRIB records does not change checksums.
parent 16811848
Hide whitespace changes
Inline Side-by-side
tests/stream_cksum.c
View file @ 1fa8e69b
......@@ -2,6 +2,7 @@
# include "config.h"
#endif
#include <stdbool.h>
#include <stdio.h>
#include "cdi.h"
......@@ -9,25 +10,30 @@
#include "stream_cksum.h"
#include "dmemory.h"
const char *zaxisNamePtr(int zaxistype);
struct cksum_table *
cksum_stream(const char *fname, size_t *table_len)
{
int taxisID, vlistID, streamID;
int nvars, ngrids, nzaxis;
int i;
int nvars;
uint32_t *checksum_state = NULL;
struct var_size_t
enum directionZ {
DIRECTION_DOWN=1, DIRECTION_NONE, DIRECTION_UP
};
struct var_desc_t
{
int x, y, z;
enum directionZ zDirection;
int code;
size_t chars;
} *varSize = NULL;
} *varDesc = NULL;
size_t var_size_max_chars = 0;
double *buf = NULL;
struct cksum_table *file_vars = NULL;
do {
// Open the dataset
streamID = streamOpenRead(fname);
int streamID = streamOpenRead(fname);
if ( streamID < 0 )
{
fprintf(stderr, "Cannot open data input file %s: %s\n",
......@@ -37,11 +43,13 @@ cksum_stream(const char *fname, size_t *table_len)
}
// Get the variable list of the dataset
vlistID = streamInqVlist(streamID);
int vlistID = streamInqVlist(streamID);
int fileType = streamInqFiletype(streamID);
bool isLegacyFile = fileType == CDI_FILETYPE_SRV
|| fileType == CDI_FILETYPE_EXT;
nvars = vlistNvars(vlistID);
ngrids = vlistNgrids(vlistID);
nzaxis = vlistNzaxis(vlistID);
int ngrids = vlistNgrids(vlistID);
int nzaxis = vlistNzaxis(vlistID);
if (nzaxis < 0 || ngrids < 0)
{
fprintf(stderr, "Error in grid/zaxis count query %d:%d\n",
......@@ -50,58 +58,97 @@ cksum_stream(const char *fname, size_t *table_len)
break;
}
checksum_state = (uint32_t *) Calloc((size_t)nvars, sizeof (checksum_state[0]));
varSize = (struct var_size_t *)malloc((size_t)nvars * sizeof (varSize[0]));
varDesc = (struct var_desc_t *)malloc((size_t)nvars * sizeof (varDesc[0]));
for (i = 0; i < nvars; ++i)
for (int varIdx = 0; varIdx < nvars; ++varIdx)
{
int grid = vlistInqVarGrid(vlistID, i), gridType;
int zaxis = vlistInqVarZaxis(vlistID, i);
if (grid == CDI_UNDEFID || zaxis == CDI_UNDEFID)
int grid = vlistInqVarGrid(vlistID, varIdx), gridType,
varCode = vlistInqVarCode(vlistID, varIdx);
varDesc[varIdx].code = varCode;
int zaxisID = vlistInqVarZaxis(vlistID, varIdx);
if (grid == CDI_UNDEFID || zaxisID == CDI_UNDEFID)
{
fputs("error in axis/grid inquiry\n", stderr);
nvars = -1;
break;
}
if ((varSize[i].z = zaxisInqSize(zaxis)) <= 0)
{
fputs("invalid Z-axis found\n", stderr);
nvars = -1;
break;
}
if ((gridType = gridInqType(grid)) != GRID_LONLAT
&& gridType != GRID_GENERIC)
{
fprintf(stderr, "unexpected non-lonlat grid found: %d\n",
gridType);
nvars = -1;
break;
}
if ((varSize[i].x = gridInqXsize(grid)) < 0)
{
fprintf(stderr, "invalid X-size found: %d\n", varSize[i].x);
nvars = -1;
break;
}
if (varSize[i].x == 0) varSize[i].x = 1;
if ((varSize[i].y = gridInqYsize(grid)) < 0)
{
fprintf(stderr, "invalid Y-size found: %d\n", varSize[i].y);
nvars = -1;
int zSize;
if ((zSize = varDesc[varIdx].z = zaxisInqSize(zaxisID)) <= 0)
{
fputs("invalid Z-axis found\n", stderr);
nvars = -1;
break;
}
if (isLegacyFile)
varDesc[varIdx].zDirection = DIRECTION_NONE;
else if (zSize > 1)
{
double lev[2];
for (int levIdx = 0; levIdx < 2; ++levIdx)
lev[levIdx] = zaxisInqLevel(zaxisID, levIdx);
int zaxistype = zaxisInqType(zaxisID);
switch (zaxistype)
{
case ZAXIS_PRESSURE:
if (lev[0] < lev[1])
varDesc[varIdx].zDirection = DIRECTION_DOWN;
else if (lev[1] < lev[0])
varDesc[varIdx].zDirection = DIRECTION_UP;
else
{
fprintf(stderr,
"unexpected level ordering on z-Axis for variable"
" code %d found: lev[0]=%g, lev[1]=%g\n",
varCode, lev[0], lev[1]);
nvars = -1;
}
break;
default:
fprintf(stderr, "unexpected type of z-Axis for variable"
" code %d found: %s\n",
varCode, zaxisNamePtr(zaxistype));
nvars = -1;
}
}
else
varDesc[varIdx].zDirection = DIRECTION_NONE;
if (nvars == -1)
break;
}
if (varSize[i].y == 0) varSize[i].y = 1;
varSize[i].chars = (size_t)varSize[i].x * (size_t)varSize[i].y
* (size_t)varSize[i].z * sizeof (buf[0]);
if (var_size_max_chars < varSize[i].chars)
var_size_max_chars = varSize[i].chars;
}
if ((gridType = gridInqType(grid)) != GRID_LONLAT
&& gridType != GRID_GENERIC)
{
fprintf(stderr, "unexpected non-lonlat grid found: %d\n",
gridType);
nvars = -1;
break;
}
if ((varDesc[varIdx].x = gridInqXsize(grid)) < 0)
{
fprintf(stderr, "invalid X-size found: %d\n", varDesc[varIdx].x);
nvars = -1;
break;
}
if (varDesc[varIdx].x == 0) varDesc[varIdx].x = 1;
if ((varDesc[varIdx].y = gridInqYsize(grid)) < 0)
{
fprintf(stderr, "invalid Y-size found: %d\n", varDesc[varIdx].y);
nvars = -1;
break;
}
if (varDesc[varIdx].y == 0) varDesc[varIdx].y = 1;
varDesc[varIdx].chars
= (size_t)varDesc[varIdx].x * (size_t)varDesc[varIdx].y
* (size_t)varDesc[varIdx].z * sizeof (buf[0]);
if (var_size_max_chars < varDesc[varIdx].chars)
var_size_max_chars = varDesc[varIdx].chars;
}
buf = (double*)malloc(var_size_max_chars);
if (nvars == -1)
break;
// Get the Time axis from the variable list
taxisID = vlistInqTaxis(vlistID);
int taxisID = vlistInqTaxis(vlistID);
int tsID = 0;
// Inquire the time step
......@@ -110,24 +157,39 @@ cksum_stream(const char *fname, size_t *table_len)
// Get the verification date and time
int vdatetime[2] = { taxisInqVtime(taxisID), taxisInqVdate(taxisID) };
// Read var1 and var2
for (i = 0; i < nvars; ++i)
for (int varIdx = 0; varIdx < nvars; ++varIdx)
{
int nmiss;
streamReadVar(streamID, i, buf, &nmiss);
memcrc_r(checksum_state + i, (const unsigned char *)vdatetime, sizeof (vdatetime));
memcrc_r(checksum_state + i, (const unsigned char *)buf,
varSize[i].chars);
streamReadVar(streamID, varIdx, buf, &nmiss);
memcrc_r(checksum_state + varIdx, (const unsigned char *)vdatetime,
sizeof (vdatetime));
if (varDesc[varIdx].zDirection == DIRECTION_UP
|| varDesc[varIdx].zDirection == DIRECTION_NONE)
memcrc_r(checksum_state + varIdx, (const unsigned char *)buf,
varDesc[varIdx].chars);
else
{
size_t nlev = (size_t)varDesc[varIdx].z,
charsPerLev = (size_t)varDesc[varIdx].x
* (size_t)varDesc[varIdx].y * sizeof (buf[0]);
for (size_t lev = 0; lev < nlev; ++lev)
memcrc_r(checksum_state + varIdx,
(const unsigned char *)buf + (nlev - lev - 1)
* charsPerLev, charsPerLev);
}
}
++tsID;
}
file_vars = (struct cksum_table *)malloc((size_t)nvars * sizeof (file_vars[0]));
for (i = 0; i < nvars; ++i)
file_vars = (struct cksum_table *)malloc((size_t)nvars
* sizeof (file_vars[0]));
for (int varIdx = 0; varIdx < nvars; ++varIdx)
{
file_vars[i].code = vlistInqVarCode(vlistID, i);
file_vars[i].cksum
= memcrc_finish(checksum_state + i,
(off_t)((varSize[i].chars + sizeof (int) * 2) * (size_t)tsID));
file_vars[varIdx].code = varDesc[varIdx].code;
file_vars[varIdx].cksum
= memcrc_finish(checksum_state + varIdx,
(off_t)((varDesc[varIdx].chars + sizeof (int) * 2)
* (size_t)tsID));
}
// Close the input stream
streamClose(streamID);
......@@ -136,10 +198,19 @@ cksum_stream(const char *fname, size_t *table_len)
// free resources
free(checksum_state);
free(varSize);
free(varDesc);
free(buf);
*table_len = (size_t)nvars;
return file_vars;
}
/*
* Local Variables:
* c-file-style: "Java"
* c-basic-offset: 2
* indent-tabs-mode: nil
* show-trailing-whitespace: t
* require-trailing-newline: t
* End:
*/
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Imprint | Privacy Policy