Commit 7da7338b authored by Uwe Schulzweida's avatar Uwe Schulzweida
Browse files

cdiwrite update

parent 36afec99
......@@ -22,9 +22,71 @@
#include "pstream.h"
static
const char *filetypestr(int filetype)
{
switch ( filetype )
{
case FILETYPE_GRB: return ("GRIB"); break;
case FILETYPE_GRB2: return ("GRIB2"); break;
case FILETYPE_NC: return ("netCDF"); break;
case FILETYPE_NC2: return ("netCDF2"); break;
case FILETYPE_NC4: return ("netCDF4"); break;
case FILETYPE_NC4C: return ("netCDF4 classic"); break;
case FILETYPE_SRV: return ("SERVICE"); break;
case FILETYPE_EXT: return ("EXTRA"); break;
case FILETYPE_IEG: return ("IEG"); break;
default: return ("");
}
}
static
const char *datatypestr(int datatype)
{
static char str[20];
str[0] = 0;
sprintf(str, "%d bit packed", datatype);
if ( datatype == DATATYPE_PACK ) return ("P0");
else if ( datatype > 0 && datatype <= 32 ) return (str);
else if ( datatype == DATATYPE_CPX32 ) return ("C32");
else if ( datatype == DATATYPE_CPX64 ) return ("C64");
else if ( datatype == DATATYPE_FLT32 ) return ("32 bit floats");
else if ( datatype == DATATYPE_FLT64 ) return ("64 bit floats");
else if ( datatype == DATATYPE_INT8 ) return ("I8");
else if ( datatype == DATATYPE_INT16 ) return ("I16");
else if ( datatype == DATATYPE_INT32 ) return ("I32");
else if ( datatype == DATATYPE_UINT8 ) return ("U8");
else if ( datatype == DATATYPE_UINT16 ) return ("U16");
else if ( datatype == DATATYPE_UINT32 ) return ("U32");
else return ("");
}
static
off_t filesize(const char *filename)
{
FILE *fp;
off_t pos = 0;
fp = fopen(filename, "r");
if ( fp == NULL )
{
fprintf(stderr, "Open failed on %s\n", filename);
}
else
{
fseek(fp, 0L, SEEK_END);
pos = ftello(fp);
}
return pos;
}
void *CDIwrite(void *argument)
{
int nvars = 1, nlevs = 0, ntimesteps = 0;
int nvars = 10, nlevs = 0, ntimesteps = 30;
char *defaultgrid = "global_.2";
int operatorID;
int streamID;
int tsID, varID, levelID;
......@@ -33,28 +95,35 @@ void *CDIwrite(void *argument)
int vlistID;
int gridID = -1, zaxisID, taxisID;
int vdate, vtime, julday;
int filetype, datatype;
off_t fsize;
unsigned int seed = 1;
const char *gridfile;
double file_size, data_size = 0;
double tw;
double *levels = NULL;
double ***vars = NULL;
extern int timer_write;
srand(seed);
cdoInitialize(argument);
operatorInputArg("grid, <nlevs, <ntimesteps, <nvars>>>");
if ( operatorArgc() < 1 ) cdoAbort("Too few arguments!");
if ( cdoVerbose ) cdoPrint("parameter: <grid, <nlevs, <ntimesteps, <nvars>>>>");
// operatorInputArg("<grid, <nlevs, <ntimesteps, <nvars>>>>");
if ( operatorArgc() > 4 ) cdoAbort("Too many arguments!");
gridfile = operatorArgv()[0];
gridID = cdoDefineGrid(gridfile);
gridfile = defaultgrid;
if ( operatorArgc() >= 1 ) gridfile = operatorArgv()[0];
if ( operatorArgc() >= 2 ) nlevs = atol(operatorArgv()[1]);
if ( operatorArgc() >= 3 ) ntimesteps = atol(operatorArgv()[2]);
if ( operatorArgc() >= 4 ) nvars = atol(operatorArgv()[3]);
srand(seed);
gridID = cdoDefineGrid(gridfile);
gridsize = gridInqSize(gridID);
zaxisID = zaxisCreate(ZAXIS_SURFACE, 1);
zaxisID = zaxisCreate(ZAXIS_SURFACE, 1);
if ( cdoVerbose )
{
......@@ -98,7 +167,10 @@ void *CDIwrite(void *argument)
streamDefVlist(streamID, vlistID);
julday = date_to_julday(CALENDAR_PROLEPTIC, 10101);
filetype = streamInqFiletype(streamID);
datatype = vlistInqVarDatatype(vlistID, 0);
julday = date_to_julday(CALENDAR_PROLEPTIC, 19870101);
for ( tsID = 0; tsID < ntimesteps; tsID++ )
{
......@@ -115,12 +187,23 @@ void *CDIwrite(void *argument)
{
streamDefRecord(streamID, varID, levelID);
streamWriteRecord(streamID, vars[varID][levelID], 0);
data_size += gridsize*8;
}
}
}
streamClose(streamID);
tw = timer_val(timer_write);
data_size /= 1024.*1024.*1024.;
cdoPrint("Wrote %.1f GB of 64 bit floats to %s %s", data_size, datatypestr(datatype), filetypestr(filetype));
fsize = filesize(cdoStreamName(0));
file_size = fsize;
file_size /= 1024.*1024.*1024.;
cdoPrint("Wrote %.1f GB in %.1f seconds, total %.1f MB/s", file_size, tw, 1024*file_size/tw);
vlistDestroy(vlistID);
for ( varID = 0; varID < nvars; varID++ )
......
......@@ -1033,11 +1033,12 @@ int main(int argc, char *argv[])
}
else
{
timer_total = timer_new("total");
timer_read = timer_new("read");
timer_write = timer_new("write");
if ( cdoTimer )
{
timer_total = timer_new("total");
timer_read = timer_new("read");
timer_write = timer_new("write");
timer_remap = timer_new("remap");
timer_remap_sort = timer_new("remap sort");
timer_remap_con = timer_new("remap con");
......@@ -1045,11 +1046,11 @@ int main(int argc, char *argv[])
timer_remap_con3 = timer_new("remap con3");
}
if ( cdoTimer ) timer_start(timer_total);
timer_start(timer_total);
operatorModule(operatorName)(argument);
if ( cdoTimer ) timer_stop(timer_total);
timer_stop(timer_total);
if ( cdoTimer ) timer_report();
}
......
......@@ -80,6 +80,7 @@ int timer_new(const char *text);
void timer_report(void);
void timer_start(int it);
void timer_stop(int it);
double timer_val(int it);
void operatorInputArg(const char *enter);
int operatorArgc(void);
......
......@@ -673,9 +673,9 @@ int pstreamOpenWrite(const char *argument, int filetype)
#if defined (HAVE_LIBPTHREAD)
pthread_mutex_lock(&streamOpenWriteMutex);
#endif
if ( cdoTimer ) timer_start(timer_write);
timer_start(timer_write);
fileID = streamOpenWrite(argument, filetype);
if ( cdoTimer ) timer_stop(timer_write);
timer_stop(timer_write);
#if defined (HAVE_LIBPTHREAD)
pthread_mutex_unlock(&streamOpenWriteMutex);
#endif
......@@ -755,9 +755,9 @@ int pstreamOpenAppend(const char *argument)
if ( PSTREAM_Debug ) Message("file %s", argument);
if ( cdoTimer ) timer_start(timer_write);
timer_start(timer_write);
fileID = streamOpenAppend(argument);
if ( cdoTimer ) timer_stop(timer_write);
timer_stop(timer_write);
if ( fileID < 0 ) cdiError(fileID, "Open failed on %s", argument);
/*
cdoInqHistory(fileID);
......@@ -901,9 +901,9 @@ int pstreamInqVlist(int pstreamID)
{
extern int cdoDefaultTimeType;
if ( cdoTimer ) timer_start(timer_read);
timer_start(timer_read);
vlistID = streamInqVlist(pstreamptr->fileID);
if ( cdoTimer ) timer_stop(timer_read);
timer_stop(timer_read);
if ( cdoDefaultTimeType != CDI_UNDEFID )
taxisDefType(vlistInqTaxis(vlistID), cdoDefaultTimeType);
......@@ -1048,9 +1048,9 @@ void pstreamDefVlist(int pstreamID, int vlistID)
pstreamDefVarlist(pstreamptr, vlistID);
if ( cdoTimer ) timer_start(timer_write);
timer_start(timer_write);
streamDefVlist(pstreamptr->fileID, vlistID);
if ( cdoTimer ) timer_stop(timer_write);
timer_stop(timer_write);
}
}
......@@ -1067,9 +1067,9 @@ int pstreamInqRecord(int pstreamID, int *varID, int *levelID)
else
#endif
{
if ( cdoTimer ) timer_start(timer_read);
timer_start(timer_read);
streamInqRecord(pstreamptr->fileID, varID, levelID);
if ( cdoTimer ) timer_stop(timer_read);
timer_stop(timer_read);
}
return (0);
......@@ -1092,9 +1092,9 @@ void pstreamDefRecord(int pstreamID, int varID, int levelID)
else
#endif
{
if ( cdoTimer ) timer_start(timer_write);
timer_start(timer_write);
streamDefRecord(pstreamptr->fileID, varID, levelID);
if ( cdoTimer ) timer_stop(timer_write);
timer_stop(timer_write);
}
}
......@@ -1113,9 +1113,9 @@ void pstreamReadRecord(int pstreamID, double *data, int *nmiss)
else
#endif
{
if ( cdoTimer ) timer_start(timer_read);
timer_start(timer_read);
streamReadRecord(pstreamptr->fileID, data, nmiss);
if ( cdoTimer ) timer_stop(timer_read);
timer_stop(timer_read);
}
}
......@@ -1202,14 +1202,14 @@ void pstreamWriteRecord(int pstreamID, double *data, int nmiss)
#endif
{
int varID = pstreamptr->varID;
if ( cdoTimer ) timer_start(timer_write);
timer_start(timer_write);
if ( pstreamptr->varlist )
if ( pstreamptr->varlist[varID].check_datarange )
pstreamCheckDatarange(pstreamptr, varID, data, nmiss);
streamWriteRecord(pstreamptr->fileID, data, nmiss);
if ( cdoTimer ) timer_stop(timer_write);
timer_stop(timer_write);
}
}
......@@ -1231,9 +1231,9 @@ int pstreamInqTimestep(int pstreamID, int tsID)
if ( pstreamptr->mfiles ) tsID -= pstreamptr->tsID0;
if ( cdoTimer ) timer_start(timer_read);
timer_start(timer_read);
nrecs = streamInqTimestep(pstreamptr->fileID, tsID);
if ( cdoTimer ) timer_stop(timer_read);
timer_stop(timer_read);
if ( nrecs == 0 && pstreamptr->mfiles &&
(pstreamptr->nfiles < pstreamptr->mfiles) )
......@@ -1259,10 +1259,10 @@ int pstreamInqTimestep(int pstreamID, int tsID)
#endif
if ( cdoVerbose ) cdoPrint("Continuation file: %s", filename);
if ( cdoTimer ) timer_start(timer_read);
timer_start(timer_read);
fileID = streamOpenRead(filename);
vlistIDnew = streamInqVlist(fileID);
if ( cdoTimer ) timer_stop(timer_read);
timer_stop(timer_read);
vlistCompare(vlistIDold, vlistIDnew, CMP_HRD);
vlistDestroy(vlistIDold);
......@@ -1276,9 +1276,9 @@ int pstreamInqTimestep(int pstreamID, int tsID)
pstreamptr->name = filename;
pstreamptr->fileID = fileID;
if ( cdoTimer ) timer_start(timer_read);
timer_start(timer_read);
nrecs = streamInqTimestep(pstreamptr->fileID, 0);
if ( cdoTimer ) timer_stop(timer_read);
timer_stop(timer_read);
}
if ( tsID == 0 && cdoDefaultTimeType != CDI_UNDEFID )
......@@ -1316,11 +1316,11 @@ void pstreamDefTimestep(int pstreamID, int tsID)
taxisDefType(taxisID, cdoDefaultTimeType);
}
if ( cdoTimer ) timer_start(timer_write);
timer_start(timer_write);
/* don't use sync -> very slow on GPFS */
// if ( tsID > 0 ) streamSync(pstreamptr->fileID);
streamDefTimestep(pstreamptr->fileID, tsID);
if ( cdoTimer ) timer_stop(timer_write);
timer_stop(timer_write);
}
}
......
......@@ -220,7 +220,7 @@ void timer_check(int it)
fprintf(rt_unit, "timer: invalid timer id %d\n", it);
}
static
double timer_val(int it)
{
double val, dt;
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment