Skip to content
GitLab
Commit d44900bc authored by Thomas Jahns's avatar Thomas Jahns 🤸
Browse files

Extract stream checksumming into separate file for tests.

parent 458693e3
Hide whitespace changes
Inline Side-by-side
.gitattributes
View file @ d44900bc
......@@ -309,6 +309,8 @@ tests/pio_cksum_mpinonb.in -text
tests/pio_cksum_writer.in -text
tests/pio_write.c -text
tests/pio_write_run.in -text
tests/stream_cksum.c -text
tests/stream_cksum.h -text
tests/test_cksum_extra.in -text
tests/test_cksum_grib.in -text
tests/test_cksum_ieg.in -text
......
tests/Makefile.am
View file @ d44900bc
......@@ -10,6 +10,7 @@ test_grib_SOURCES = test_grib.c
cksum_write_SOURCES = cksum_write.c cksum.c cksum.h
cksum_read_SOURCES = cksum_read.c \
var_cksum.c var_cksum.h \
stream_cksum.c stream_cksum.h \
cksum.c cksum.h \
ensure_array_size.h ensure_array_size.c
pio_write_SOURCES = pio_write.c cksum.h cksum.c
......
tests/Makefile.in
View file @ d44900bc
......@@ -66,7 +66,8 @@ CONFIG_CLEAN_FILES = test_cksum_grib test_cksum_nc test_cksum_nc2 \
pio_cksum_fpguard pio_cksum_asynch pio_cksum_writer
CONFIG_CLEAN_VPATH_FILES =
am_cksum_read_OBJECTS = cksum_read.$(OBJEXT) var_cksum.$(OBJEXT) \
cksum.$(OBJEXT) ensure_array_size.$(OBJEXT)
stream_cksum.$(OBJEXT) cksum.$(OBJEXT) \
ensure_array_size.$(OBJEXT)
cksum_read_OBJECTS = $(am_cksum_read_OBJECTS)
cksum_read_LDADD = $(LDADD)
cksum_read_DEPENDENCIES =
......@@ -291,6 +292,7 @@ test_grib_SOURCES = test_grib.c
cksum_write_SOURCES = cksum_write.c cksum.c cksum.h
cksum_read_SOURCES = cksum_read.c \
var_cksum.c var_cksum.h \
stream_cksum.c stream_cksum.h \
cksum.c cksum.h \
ensure_array_size.h ensure_array_size.c
......@@ -394,6 +396,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/cksum_write.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ensure_array_size.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pio_write.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stream_cksum.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test_grib.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/var_cksum.Po@am__quote@
......
tests/cksum_read.c
View file @ d44900bc
......@@ -7,17 +7,9 @@
#include <stdlib.h>
#include <stdio.h>
#include "cdi.h"
#include "cksum.h"
#include "ensure_array_size.h"
#include "pio_util.h"
#include "var_cksum.h"
struct varSize
{
int x, y, z;
size_t chars;
};
#include "stream_cksum.h"
static struct cksum_table *
read_table(const char *table_fname, size_t *table_len)
......@@ -47,136 +39,6 @@ read_table(const char *table_fname, size_t *table_len)
}
static struct cksum_table *
cksum_stream(const char *fname, size_t *table_len)
{
int taxisID, vlistID, streamID, tsID;
int vdate, vtime, nvars, ngrids, nzaxis;
int i;
uint32_t *checksum_state = NULL;
struct varSize *varSize = NULL;
size_t var_size_max_chars = 0;
double *buf = NULL;
struct cksum_table *file_vars;
do {
// Open the dataset
streamID = streamOpenRead(fname);
if ( streamID < 0 )
{
fprintf(stderr, "Cannot open data input file %s: %s\n",
fname, cdiStringError(streamID));
nvars = -1;
break;
}
// Get the variable list of the dataset
vlistID = streamInqVlist(streamID);
nvars = vlistNvars(vlistID);
ngrids = vlistNgrids(vlistID);
nzaxis = vlistNzaxis(vlistID);
if (nzaxis < 0 || ngrids < 0)
{
fprintf(stderr, "Error in grid/zaxis count query %d:%d\n",
ngrids, nzaxis);
nvars = -1;
break;
}
checksum_state = xcalloc(nvars, sizeof (checksum_state[0]));
varSize = xmalloc(nvars * sizeof (varSize[0]));
for (i = 0; i < nvars; ++i)
{
int grid = vlistInqVarGrid(vlistID, i), gridType;
int zaxis = vlistInqVarZaxis(vlistID, i);
if (grid == CDI_UNDEFID || zaxis == CDI_UNDEFID)
{
fputs("error in axis/grid inquiry\n", stderr);
nvars = -1;
break;
}
if ((varSize[i].z = zaxisInqSize(zaxis)) <= 0)
{
fputs("invalid Z-axis found\n", stderr);
nvars = -1;
break;
}
if ((gridType = gridInqType(grid)) != GRID_LONLAT
&& gridType != GRID_GENERIC)
{
fprintf(stderr, "unexpected non-lonlat grid found: %d\n",
gridType);
nvars = -1;
break;
}
if ((varSize[i].x = gridInqXsize(grid)) < 0)
{
fprintf(stderr, "invalid X-size found: %d\n", varSize[i].x);
nvars = -1;
break;
}
if (varSize[i].x == 0) varSize[i].x = 1;
if ((varSize[i].y = gridInqYsize(grid)) < 0)
{
fprintf(stderr, "invalid Y-size found: %d\n", varSize[i].y);
nvars = -1;
break;
}
if (varSize[i].y == 0) varSize[i].y = 1;
varSize[i].chars = (size_t)varSize[i].x * varSize[i].y * varSize[i].z
* sizeof (buf[0]);
if (var_size_max_chars < varSize[i].chars)
var_size_max_chars = varSize[i].chars;
}
buf = xmalloc(var_size_max_chars);
if (nvars == -1)
break;
// Get the Time axis from the variable list
taxisID = vlistInqTaxis(vlistID);
tsID = 0;
// Inquire the time step
while (streamInqTimestep(streamID, tsID))
{
// Get the verification date and time
vdate = taxisInqVdate(taxisID);
vtime = taxisInqVtime(taxisID);
// Read var1 and var2
for (i = 0; i < nvars; ++i)
{
int nmiss;
streamReadVar(streamID, i, buf, &nmiss);
memcrc_r(checksum_state + i, (const unsigned char *)buf,
varSize[i].chars);
}
++tsID;
}
file_vars = xmalloc(nvars * sizeof (file_vars[0]));
for (i = 0; i < nvars; ++i)
{
file_vars[i].code = vlistInqVarCode(vlistID, i);
file_vars[i].cksum = memcrc_finish(checksum_state + i,
(off_t)varSize[i].chars * tsID);
}
// Close the input stream
streamClose(streamID);
} while (0);
// free resources
free(checksum_state);
free(varSize);
free(buf);
*table_len = nvars;
return file_vars;
}
int main(int argc, char *argv[])
{
char *fname = "example.grb", *table_fname = "example.cksum";
......
tests/stream_cksum.c 0 → 100644
View file @ d44900bc
#include "cdi.h"
#include "cksum.h"
#include "stream_cksum.h"
#include "pio_util.h"
struct cksum_table *
cksum_stream(const char *fname, size_t *table_len)
{
int taxisID, vlistID, streamID, tsID;
int vdate, vtime, nvars, ngrids, nzaxis;
int i;
uint32_t *checksum_state = NULL;
struct
{
int x, y, z;
size_t chars;
} *varSize = NULL;
size_t var_size_max_chars = 0;
double *buf = NULL;
struct cksum_table *file_vars;
do {
// Open the dataset
streamID = streamOpenRead(fname);
if ( streamID < 0 )
{
fprintf(stderr, "Cannot open data input file %s: %s\n",
fname, cdiStringError(streamID));
nvars = -1;
break;
}
// Get the variable list of the dataset
vlistID = streamInqVlist(streamID);
nvars = vlistNvars(vlistID);
ngrids = vlistNgrids(vlistID);
nzaxis = vlistNzaxis(vlistID);
if (nzaxis < 0 || ngrids < 0)
{
fprintf(stderr, "Error in grid/zaxis count query %d:%d\n",
ngrids, nzaxis);
nvars = -1;
break;
}
checksum_state = xcalloc(nvars, sizeof (checksum_state[0]));
varSize = xmalloc(nvars * sizeof (varSize[0]));
for (i = 0; i < nvars; ++i)
{
int grid = vlistInqVarGrid(vlistID, i), gridType;
int zaxis = vlistInqVarZaxis(vlistID, i);
if (grid == CDI_UNDEFID || zaxis == CDI_UNDEFID)
{
fputs("error in axis/grid inquiry\n", stderr);
nvars = -1;
break;
}
if ((varSize[i].z = zaxisInqSize(zaxis)) <= 0)
{
fputs("invalid Z-axis found\n", stderr);
nvars = -1;
break;
}
if ((gridType = gridInqType(grid)) != GRID_LONLAT
&& gridType != GRID_GENERIC)
{
fprintf(stderr, "unexpected non-lonlat grid found: %d\n",
gridType);
nvars = -1;
break;
}
if ((varSize[i].x = gridInqXsize(grid)) < 0)
{
fprintf(stderr, "invalid X-size found: %d\n", varSize[i].x);
nvars = -1;
break;
}
if (varSize[i].x == 0) varSize[i].x = 1;
if ((varSize[i].y = gridInqYsize(grid)) < 0)
{
fprintf(stderr, "invalid Y-size found: %d\n", varSize[i].y);
nvars = -1;
break;
}
if (varSize[i].y == 0) varSize[i].y = 1;
varSize[i].chars = (size_t)varSize[i].x * varSize[i].y * varSize[i].z
* sizeof (buf[0]);
if (var_size_max_chars < varSize[i].chars)
var_size_max_chars = varSize[i].chars;
}
buf = xmalloc(var_size_max_chars);
if (nvars == -1)
break;
// Get the Time axis from the variable list
taxisID = vlistInqTaxis(vlistID);
tsID = 0;
// Inquire the time step
while (streamInqTimestep(streamID, tsID))
{
// Get the verification date and time
vdate = taxisInqVdate(taxisID);
vtime = taxisInqVtime(taxisID);
// Read var1 and var2
for (i = 0; i < nvars; ++i)
{
int nmiss;
streamReadVar(streamID, i, buf, &nmiss);
memcrc_r(checksum_state + i, (const unsigned char *)buf,
varSize[i].chars);
}
++tsID;
}
file_vars = xmalloc(nvars * sizeof (file_vars[0]));
for (i = 0; i < nvars; ++i)
{
file_vars[i].code = vlistInqVarCode(vlistID, i);
file_vars[i].cksum = memcrc_finish(checksum_state + i,
(off_t)varSize[i].chars * tsID);
}
// Close the input stream
streamClose(streamID);
} while (0);
// free resources
free(checksum_state);
free(varSize);
free(buf);
*table_len = nvars;
return file_vars;
}
tests/stream_cksum.h 0 → 100644
View file @ d44900bc
#ifndef STREAM_CKSUM_H
#define STREAM_CKSUM_H
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "var_cksum.h"
struct cksum_table *
cksum_stream(const char *fname, size_t *table_len);
#endif
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Imprint | Privacy Policy