diff --git a/ChangeLog b/ChangeLog index f83160e45476694e8a4eaa767a1b5e80d859b649..c268565f80941b7511f17ee5ae1f794bad6e4d14 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2023-12-28 Uwe Schulzweida + + * Add environment variable CDI_LOCK_IO to lock IO access + 2023-11-22 Uwe Schulzweida * calc_chunk_cache_size: wrong result for 3D data (bug fix) @@ -183,13 +187,13 @@ 2022-10-17 Uwe Schulzweida - * Added environment variable CDI_SHUFFLE to set shuffle option to NetCDF4 deflation compression + * Add environment variable CDI_SHUFFLE to set shuffle option to NetCDF4 deflation compression 2022-10-16 Uwe Schulzweida - * Improved read performance of temporal chunked NetCDF4 data - * Added environment variable CDI_CHUNK_CACHE to set the NetCDF4 chunk cache size - * Added environment variable CDI_CHUNK_CACHE_MAX to set the maximum chunk cache size + * Improve read performance of temporal chunked NetCDF4 data + * Add environment variable CDI_CHUNK_CACHE to set the NetCDF4 chunk cache size + * Add environment variable CDI_CHUNK_CACHE_MAX to set the maximum chunk cache size 2022-10-14 Uwe Schulzweida diff --git a/src/cdi_int.c b/src/cdi_int.c index befd869165e4a4a5082947e8229dfb508ad38c6d..9c3c4fdf6369d23ce4677c7440db47682b086388 100644 --- a/src/cdi_int.c +++ b/src/cdi_int.c @@ -16,6 +16,11 @@ #include "cgribex.h" #endif +#ifdef HAVE_LIBPTHREAD +#include <pthread.h> +pthread_mutex_t CDI_IO_Mutex = PTHREAD_MUTEX_INITIALIZER; +#endif + int CDI_Default_Calendar = CALENDAR_PROLEPTIC; int CDI_Default_InstID = CDI_UNDEFID; @@ -77,6 +82,7 @@ bool CDI_gribapi_debug = false; bool CDI_gribapi_grib1 = false; int cdiDefaultLeveltype = -1; int cdiDataUnreduced = 0; +int CDI_Lock_IO = 0; int cdiSortName = 0; int cdiSortParam = 0; int cdiHaveMissval = 0; @@ -337,6 +343,9 @@ cdiInitialize(void) value = cdi_getenv_int("CDI_ECCODES_GRIB1"); if (value >= 0) cdiSetEccodesGrib1((bool) value); + value = cdi_getenv_int("CDI_LOCK_IO"); + if (value >= 0) CDI_Lock_IO = (int) value; + value = cdi_getenv_int("CDI_READ_CELL_CORNERS"); if (value >= 0) CDI_Read_Cell_Corners = (int) value; @@ -430,7 +439,7 @@ cdiInitialize(void) envstr = getenv("CDI_QUERY_ABORT"); if (envstr) { - const int ival = atoi(envstr); + int ival = atoi(envstr); if (ival == 0 || ival == 1) { CDI_Query_Abort = ival; @@ -441,7 +450,7 @@ cdiInitialize(void) envstr = getenv("CDI_VERSION_INFO"); if (envstr) { - const int ival = atoi(envstr); + int ival = atoi(envstr); if (ival == 0 || ival == 1) { CDI_Version_Info = ival; @@ -452,7 +461,7 @@ cdiInitialize(void) envstr = getenv("CDI_CONVERT_CUBESPHERE"); if (envstr) { - const int ival = atoi(envstr); + int ival = atoi(envstr); if (ival == 0 || ival == 1) { CDI_Convert_Cubesphere = ival; @@ -489,7 +498,7 @@ cdiInitialize(void) const char * strfiletype(int filetype) { - const int size = (int) (sizeof(Filetypes) / sizeof(char *)); + int size = (int) (sizeof(Filetypes) / sizeof(char *)); return (filetype > 0 && filetype < size) ? Filetypes[filetype] : Filetypes[0]; } @@ -498,6 +507,7 @@ cdiDefGlobal(const char *string, int value) { // clang-format off if (str_is_equal(string, "REGULARGRID") ) cdiDataUnreduced = value; + else if (str_is_equal(string, "LOCKIO") ) CDI_Lock_IO = (bool) value; else if (str_is_equal(string, "ECCODES_DEBUG") ) CDI_gribapi_debug = (bool) value; else if (str_is_equal(string, "ECCODES_GRIB1") ) cdiSetEccodesGrib1((bool) value); else if (str_is_equal(string, "SORTNAME") ) cdiSortName = value; diff --git a/src/cdi_int.h b/src/cdi_int.h index fd7316ad1c13751274dd474fba173995b5856277..b9a9bca3b76c6215b180469e72c39797d14b5954 100644 --- a/src/cdi_int.h +++ b/src/cdi_int.h @@ -25,6 +25,16 @@ #include "cdi.h" #include "cdf_config.h" +#ifdef HAVE_LIBPTHREAD +#include <pthread.h> +extern pthread_mutex_t CDI_IO_Mutex; +#define CDI_IO_LOCK() pthread_mutex_lock(&CDI_IO_Mutex) +#define CDI_IO_UNLOCK() pthread_mutex_unlock(&CDI_IO_Mutex) +#else +#define CDI_IO_LOCK() +#define CDI_IO_UNLOCK() +#endif + // Base file types #define CDI_FILETYPE_GRIB 100 // File type GRIB @@ -370,6 +380,7 @@ extern int CDI_Netcdf_Chunksizehint; extern int CDI_ChunkType; extern int CDI_Test; extern int CDI_Split_Ltype105; +extern int CDI_Lock_IO; extern int cdiDataUnreduced; extern int cdiSortName; extern int cdiSortParam; diff --git a/src/stream.c b/src/stream.c index 48cad842a4ff7c02452b920f61e80545ec66e4ab..ed465f75fd4f7967628256566bd832244db4a4ad 100644 --- a/src/stream.c +++ b/src/stream.c @@ -663,6 +663,8 @@ streamOpenID(const char *filename, char filemode, int filetype, int resH) stream_t *streamptr = stream_new_entry(resH); int streamID = CDI_ESYSTEM; + if (CDI_Lock_IO) CDI_IO_LOCK(); + int (*streamOpenDelegate)(const char *filename, char filemode, int filetype, stream_t *streamptr, int recordBufIsToBeCreated) = (int (*)(const char *, char, int, stream_t *, int)) namespaceSwitchGet(NSSWITCH_STREAM_OPEN_BACKEND).func; @@ -684,6 +686,8 @@ streamOpenID(const char *filename, char filemode, int filetype, int resH) streamptr->fileID = fileID; } + if (CDI_Lock_IO) CDI_IO_UNLOCK(); + return streamID; } @@ -1287,10 +1291,14 @@ streamClose(int streamID) { stream_t *streamptr = stream_to_pointer(streamID); + if (CDI_Lock_IO) CDI_IO_LOCK(); + if (CDI_Debug) Message("streamID = %d filename = %s", streamID, streamptr->filename); streamDestroy(streamptr); reshRemove(streamID, &streamOps); if (CDI_Debug) Message("Removed stream %d from stream list", streamID); + + if (CDI_Lock_IO) CDI_IO_UNLOCK(); } void @@ -1418,9 +1426,16 @@ int streamDefTimestep(int streamID, int tsID) { stream_t *streamptr = stream_to_pointer(streamID); + + if (CDI_Lock_IO) CDI_IO_LOCK(); + int (*myStreamDefTimestep_)(stream_t * streamptr, int tsID) = (int (*)(stream_t *, int)) namespaceSwitchGet(NSSWITCH_STREAM_DEF_TIMESTEP_).func; - return myStreamDefTimestep_(streamptr, tsID); + int status = myStreamDefTimestep_(streamptr, tsID); + + if (CDI_Lock_IO) CDI_IO_UNLOCK(); + + return status; } int @@ -1476,6 +1491,8 @@ streamInqTimestep(int streamID, int tsID) if (CDI_Debug) Message("streamID = %d tsID = %d filetype = %d", streamID, tsID, filetype); + if (CDI_Lock_IO) CDI_IO_LOCK(); + switch (cdiBaseFiletype(filetype)) { #ifdef HAVE_LIBGRIB @@ -1527,6 +1544,8 @@ streamInqTimestep(int streamID, int tsID) } } + if (CDI_Lock_IO) CDI_IO_UNLOCK(); + int taxisID = vlistInqTaxis(vlistID); if (taxisID == -1) Error("Timestep undefined for fileID = %d", streamID); @@ -1564,8 +1583,12 @@ All further vlist changes have to use the vlist object returned by streamInqVlis void streamDefVlist(int streamID, int vlistID) { + if (CDI_Lock_IO) CDI_IO_LOCK(); + void (*myStreamDefVlist)(int streamID, int vlistID) = (void (*)(int, int)) namespaceSwitchGet(NSSWITCH_STREAM_DEF_VLIST_).func; myStreamDefVlist(streamID, vlistID); + + if (CDI_Lock_IO) CDI_IO_UNLOCK(); } // The single image implementation of streamDefVlist diff --git a/src/stream_read.c b/src/stream_read.c index e42dd0327dbcf45a4b9d05d097c1bacc723b7867..2e81fb3c8a4d1806b3ce46e7aed1d997c72e46b7 100644 --- a/src/stream_read.c +++ b/src/stream_read.c @@ -230,6 +230,8 @@ stream_read_record(int streamID, int memtype, void *data, size_t *nmiss) stream_t *streamptr = stream_to_pointer(streamID); + if (CDI_Lock_IO) CDI_IO_LOCK(); + *nmiss = 0; switch (cdiBaseFiletype(streamptr->filetype)) @@ -251,6 +253,8 @@ stream_read_record(int streamID, int memtype, void *data, size_t *nmiss) #endif default: Error("%s support not compiled in!", strfiletype(streamptr->filetype)); } + + if (CDI_Lock_IO) CDI_IO_UNLOCK(); } void diff --git a/src/stream_write.c b/src/stream_write.c index 226db5f9c1ca15fe8c10e8752e5f273cfd19180c..ef8d9aa2bc7249d209b2589e6025c20d105095e8 100644 --- a/src/stream_write.c +++ b/src/stream_write.c @@ -279,6 +279,8 @@ stream_write_record(int streamID, int memtype, const void *data, SizeType nmiss) stream_t *streamptr = stream_to_pointer(streamID); + if (CDI_Lock_IO) CDI_IO_LOCK(); + switch (cdiBaseFiletype(streamptr->filetype)) { #ifdef HAVE_LIBGRIB @@ -298,6 +300,8 @@ stream_write_record(int streamID, int memtype, const void *data, SizeType nmiss) #endif default: Error("%s support not compiled in!", strfiletype(streamptr->filetype)); } + + if (CDI_Lock_IO) CDI_IO_UNLOCK(); } /*