From 74cbbea7898aeac81d27ba3529fc6f2a6ffcc127 Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Thu, 24 Oct 2024 13:52:30 +0200
Subject: [PATCH 1/8] lock access to NCZARR in CDI_Threadsafe mode
---
ChangeLog | 1 +
src/stream.c | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 7d1c5ab2e..6ad18d979 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,6 +5,7 @@
2024-10-24 Uwe Schulzweida
+ * lock access to NCZARR in CDI_Threadsafe mode
* Renamed interface functions streamXXXRecord to streamXXXField
2024-10-22 Uwe Schulzweida
diff --git a/src/stream.c b/src/stream.c
index 8ebd6be67..7264d3535 100644
--- a/src/stream.c
+++ b/src/stream.c
@@ -674,7 +674,8 @@ streamOpenID(const char *filename, char filemode, int filetype, int resH)
#ifndef HAVE_LIBPTHREAD
Error("CDI threadsafe failed, pthread support not compiled in!");
#endif
- if (filetype == CDI_FILETYPE_NC4 || filetype == CDI_FILETYPE_NC4C) streamptr->lockIO = true;
+ if (filetype == CDI_FILETYPE_NC4 || filetype == CDI_FILETYPE_NC4C || filetype == CDI_FILETYPE_NCZARR)
+ streamptr->lockIO = true;
}
#endif
--
GitLab
From f4dd7a7226d4c3edba4ebaeb472b8fd62f570b75 Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Tue, 29 Oct 2024 13:34:54 +0100
Subject: [PATCH 2/8] streamOpenID: changed Error to Warning
---
src/stream.c | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/src/stream.c b/src/stream.c
index 7264d3535..8da402e54 100644
--- a/src/stream.c
+++ b/src/stream.c
@@ -671,11 +671,17 @@ streamOpenID(const char *filename, char filemode, int filetype, int resH)
#ifndef HAVE_NC4HDF5_THREADSAFE
if (CDI_Threadsafe)
{
-#ifndef HAVE_LIBPTHREAD
- Error("CDI threadsafe failed, pthread support not compiled in!");
-#endif
+#ifdef HAVE_LIBPTHREAD
if (filetype == CDI_FILETYPE_NC4 || filetype == CDI_FILETYPE_NC4C || filetype == CDI_FILETYPE_NCZARR)
streamptr->lockIO = true;
+#else
+ static bool lwarn = true;
+ if (lwarn)
+ {
+ lwarn = false;
+ Warning("CDI threadsafe failed, pthread support not compiled in!");
+ }
+#endif
}
#endif
--
GitLab
From d8ca5126d62deb33c0604fa8c6e4c1181a602296 Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Tue, 29 Oct 2024 14:02:15 +0100
Subject: [PATCH 3/8] cdf_read_timesteps: wrf_read_timestep() also if units
missing
---
src/stream_cdf_i.c | 24 +++++++++++++-----------
1 file changed, 13 insertions(+), 11 deletions(-)
diff --git a/src/stream_cdf_i.c b/src/stream_cdf_i.c
index a58121a27..dd32e6e2b 100644
--- a/src/stream_cdf_i.c
+++ b/src/stream_cdf_i.c
@@ -4270,7 +4270,7 @@ cdfVerifyVars(int nvars, ncvar_t *ncvars
}
static CdiDateTime
-wrf_read_timestep(int fileID, int nctimevarid, int tsID)
+wrf_read_timestep(int fileID, int nctimevarid, size_t tsID)
{
enum
{
@@ -4278,7 +4278,7 @@ wrf_read_timestep(int fileID, int nctimevarid, int tsID)
dateTimeSepPos = 19,
dateTimeStrSize = 128,
};
- size_t start[2] = { (size_t) tsID, 0 }, count[2] = { 1, dateTimeSepPos };
+ size_t start[2] = { tsID, 0 }, count[2] = { 1, dateTimeSepPos };
char stvalue[dateTimeStrSize];
stvalue[0] = 0;
cdf_get_vara_text(fileID, nctimevarid, start, count, stvalue);
@@ -4324,7 +4324,7 @@ cdf_read_timesteps(size_t numTimesteps, stream_t *streamptr, taxis_t *taxis0)
int fileID = streamptr->fileID;
int nctimevarid = streamptr->basetime.ncvarid;
int nctimeboundsid = streamptr->basetime.ncvarboundsid;
- bool hasTimesteps = (nctimevarid != CDI_UNDEFID && streamptr->basetime.hasUnits);
+ bool hasTimesteps = (nctimevarid != CDI_UNDEFID /*&& streamptr->basetime.hasUnits*/);
int *ncStepIndices = (int *) Malloc(numTimesteps * sizeof(int));
for (size_t tsID = 0; tsID < numTimesteps; ++tsID) ncStepIndices[tsID] = (int) tsID;
@@ -4337,10 +4337,9 @@ cdf_read_timesteps(size_t numTimesteps, stream_t *streamptr, taxis_t *taxis0)
if (streamptr->basetime.isWRF)
{
- for (size_t tsID = 0; tsID < numTimesteps; ++tsID)
- vDateTimeList[tsID] = wrf_read_timestep(fileID, nctimevarid, (int) tsID);
+ for (size_t tsID = 0; tsID < numTimesteps; ++tsID) vDateTimeList[tsID] = wrf_read_timestep(fileID, nctimevarid, tsID);
}
- else
+ else if (streamptr->basetime.hasUnits)
{
double *timevarBuffer = (double *) Malloc(numTimesteps * sizeof(double));
cdf_get_var_double(fileID, nctimevarid, timevarBuffer);
@@ -4348,6 +4347,10 @@ cdf_read_timesteps(size_t numTimesteps, stream_t *streamptr, taxis_t *taxis0)
vDateTimeList[tsID] = cdi_decode_timeval(get_timevalue(fileID, nctimevarid, tsID, timevarBuffer), taxis0);
if (timevarBuffer) Free(timevarBuffer);
}
+ else
+ {
+ hasTimesteps = false;
+ }
}
// process query information if available
@@ -4383,6 +4386,7 @@ cdf_read_timesteps(size_t numTimesteps, stream_t *streamptr, taxis_t *taxis0)
}
if (ncStepIndices) Free(ncStepIndices);
+ if (vDateTimeList) Free(vDateTimeList);
if (hasTimesteps)
{
@@ -4421,8 +4425,6 @@ cdf_read_timesteps(size_t numTimesteps, stream_t *streamptr, taxis_t *taxis0)
}
}
}
-
- if (vDateTimeList) Free(vDateTimeList);
}
}
@@ -4503,7 +4505,7 @@ find_coordinates_vars(int ndims, ncdim_t *ncdims, int nvars, ncvar_t *ncvars)
// set time dependent data vars
static void
-find_varying_data_vars(int timedimid, int nvars, ncvar_t *ncvars)
+find_varying_data_vars1d(int timedimid, int nvars, ncvar_t *ncvars)
{
for (int ncvarid = 0; ncvarid < nvars; ncvarid++)
{
@@ -4637,9 +4639,9 @@ cdfInqContents(stream_t *streamptr)
if (CDI_Convert_Cubesphere) check_cube_sphere(vlistID, nvars, ncvars, ncdims);
- if (CDI_Debug) cdf_print_vars(ncvars, nvars, "find_varying_data_vars");
+ if (CDI_Debug) cdf_print_vars(ncvars, nvars, "find_varying_data_vars1d");
- find_varying_data_vars(timedimid, nvars, ncvars);
+ find_varying_data_vars1d(timedimid, nvars, ncvars);
// find time vars
bool timeHasUnits = false;
--
GitLab
From 8dcba40c5169d68ec7448801a7994a05afc954be Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Mon, 4 Nov 2024 11:47:44 +0100
Subject: [PATCH 4/8] reset chunktype if z and t are chunked
---
ChangeLog | 4 ++++
src/stream_cdf_i.c | 10 ++++++----
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 6ad18d979..f7eb7ff3d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,10 @@
* using CGRIBEX library version 2.3.1
* Version 2.5.0 released
+2024-11-04 Uwe Schulzweida
+
+ * reset chunktype if z and t are chunked
+
2024-10-24 Uwe Schulzweida
* lock access to NCZARR in CDI_Threadsafe mode
diff --git a/src/stream_cdf_i.c b/src/stream_cdf_i.c
index dd32e6e2b..29c21e994 100644
--- a/src/stream_cdf_i.c
+++ b/src/stream_cdf_i.c
@@ -2075,19 +2075,21 @@ grid_set_chunktype(grid_t *grid, ncvar_t *ncvar)
if (ncvar->isChunked)
{
int ndims = ncvar->ndims;
+ size_t chunkSizeAllDims = 1;
+ for (int i = 0; i < ndims; ++i) chunkSizeAllDims *= ncvar->chunks[i];
+ size_t dimN = ncvar->chunks[ndims - 1];
if (grid->type == GRID_UNSTRUCTURED)
{
- size_t chunkSize = ncvar->chunks[ndims - 1];
+ size_t chunkSize = (chunkSizeAllDims == dimN) ? dimN : 0;
ncvar->chunkType = (chunkSize == grid->size) ? CDI_CHUNK_GRID : CDI_CHUNK_AUTO;
if (ncvar->chunkType == CDI_CHUNK_AUTO && chunkSize > 1) ncvar->chunkSize = (int) chunkSize;
}
else
{
- if (grid->x.size > 1 && grid->y.size > 1 && ndims > 1 && grid->x.size == ncvar->chunks[ndims - 1]
- && grid->y.size == ncvar->chunks[ndims - 2])
+ if (grid->x.size > 1 && grid->y.size > 1 && ndims > 1 && grid->x.size == dimN && grid->y.size == ncvar->chunks[ndims - 2])
ncvar->chunkType = CDI_CHUNK_GRID;
- else if (grid->x.size > 1 && grid->x.size == ncvar->chunks[ndims - 1])
+ else if (grid->x.size > 1 && grid->x.size == dimN && chunkSizeAllDims == dimN)
ncvar->chunkType = CDI_CHUNK_LINES;
else
ncvar->chunkType = CDI_CHUNK_AUTO;
--
GitLab
From 4ca3e1e1c30ce1879598031de38f04fae8bc4f9d Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Mon, 18 Nov 2024 20:27:13 +0100
Subject: [PATCH 5/8] cdf_time_dimid: check_dimids failed (bug fix)
---
src/stream_cdf_i.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/stream_cdf_i.c b/src/stream_cdf_i.c
index 29c21e994..67a210971 100644
--- a/src/stream_cdf_i.c
+++ b/src/stream_cdf_i.c
@@ -544,8 +544,8 @@ cdf_time_dimid(int fileID, int ndims, ncdim_t *ncdims, int nvars, ncvar_t *ncvar
if (dimid0 != CDI_UNDEFID && check_dimids[dimid0] == false)
{
- check_dimids[dimid0] = true;
if (ncdims[dimid0].ncvarid != CDI_UNDEFID && ncdims[dimid0].ncvarid != varid) continue;
+ check_dimids[dimid0] = true;
char sbuf[CDI_MAX_NAME];
for (int iatt = 0; iatt < ncvar->nattsNC; ++iatt)
--
GitLab
From 7ad2b632334dd041c6b50b2f5741f8fc92e56137 Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Sat, 23 Nov 2024 11:21:13 +0100
Subject: [PATCH 6/8] Added function find_dimtypes()
---
src/stream_cdf_i.c | 59 +++++++++++++++++++++++-----------------------
1 file changed, 30 insertions(+), 29 deletions(-)
diff --git a/src/stream_cdf_i.c b/src/stream_cdf_i.c
index 67a210971..748939062 100644
--- a/src/stream_cdf_i.c
+++ b/src/stream_cdf_i.c
@@ -1702,6 +1702,31 @@ verify_vars_attr(int nvars, ncvar_t *ncvars, ncdim_t *ncdims)
}
}
+static void
+find_dimtypes(ncvar_t *ncvars, ncvar_t *ncvar, bool *plxdim, bool *plydim, bool *plzdim, int *plcdim)
+{
+ bool lxdim = false, lydim = false, lzdim = false /*, ltdim = false */;
+ int lcdim = 0;
+ int ndims = ncvar->ndims;
+ for (int i = 0; i < ndims; i++)
+ {
+ int dimtype = ncvar->dimtypes[i];
+ lxdim |= (dimtype == X_AXIS);
+ lydim |= (dimtype == Y_AXIS);
+ lzdim |= (dimtype == Z_AXIS);
+ if (ncvar->cvarids[i] != CDI_UNDEFID) lcdim++;
+ // ltdim |= (dimtype == T_AXIS);
+ }
+
+ if (!lxdim && ncvar->xvarid != CDI_UNDEFID && ncvars[ncvar->xvarid].ndims == 0) lxdim = true;
+ if (!lydim && ncvar->yvarid != CDI_UNDEFID && ncvars[ncvar->yvarid].ndims == 0) lydim = true;
+
+ *plxdim = lxdim;
+ *plydim = lydim;
+ *plzdim = lzdim;
+ *plcdim = lcdim;
+}
+
static void
cdf_set_dimtype(int nvars, ncvar_t *ncvars, ncdim_t *ncdims)
{
@@ -1733,22 +1758,10 @@ cdf_set_dimtype(int nvars, ncvar_t *ncvars, ncdim_t *ncdims)
if (ncvar->varStatus == DataVar)
{
bool lxdim = false, lydim = false, lzdim = false /* , ltdim = false */;
- int lcdim = 0;
- int ndims = ncvar->ndims;
- for (int i = 0; i < ndims; i++)
- {
- int dimtype = ncvar->dimtypes[i];
- lxdim |= (dimtype == X_AXIS);
- lydim |= (dimtype == Y_AXIS);
- lzdim |= (dimtype == Z_AXIS);
- if (ncvar->cvarids[i] != CDI_UNDEFID) lcdim++;
- // ltdim |= (dimtype == T_AXIS);
- }
-
+ int lcdim;
+ find_dimtypes(ncvars, ncvar, &lxdim, &lydim, &lzdim, &lcdim);
int allcdims = lcdim;
-
- if (!lxdim && ncvar->xvarid != CDI_UNDEFID && ncvars[ncvar->xvarid].ndims == 0) lxdim = true;
- if (!lydim && ncvar->yvarid != CDI_UNDEFID && ncvars[ncvar->yvarid].ndims == 0) lydim = true;
+ int ndims = ncvar->ndims;
if (lxdim && (lydim || ncvar->gridtype == GRID_UNSTRUCTURED))
for (int i = ndims - 1; i >= 0; i--)
@@ -1796,21 +1809,9 @@ cdf_set_dimtype(int nvars, ncvar_t *ncvars, ncdim_t *ncdims)
{
bool lxdim = false, lydim = false, lzdim = false /*, ltdim = false */;
int lcdim = 0;
- int ndims = ncvar->ndims;
- for (int i = 0; i < ndims; i++)
- {
- int dimtype = ncvar->dimtypes[i];
- lxdim |= (dimtype == X_AXIS);
- lydim |= (dimtype == Y_AXIS);
- lzdim |= (dimtype == Z_AXIS);
- if (ncvar->cvarids[i] != CDI_UNDEFID) lcdim++;
- // ltdim |= (dimtype == T_AXIS);
- }
-
+ find_dimtypes(ncvars, ncvar, &lxdim, &lydim, &lzdim, &lcdim);
int allcdims = lcdim;
-
- if (!lxdim && ncvar->xvarid != CDI_UNDEFID && ncvars[ncvar->xvarid].ndims == 0) lxdim = true;
- if (!lydim && ncvar->yvarid != CDI_UNDEFID && ncvars[ncvar->yvarid].ndims == 0) lydim = true;
+ int ndims = ncvar->ndims;
// if ( ndims > 1 )
for (int i = ndims - 1; i >= 0; i--)
--
GitLab
From d91eca87846a7d2f1fb0720ecec5583b93f0fade Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Sat, 23 Nov 2024 14:27:47 +0100
Subject: [PATCH 7/8] NetCDF: improved HealPIX support
---
ChangeLog | 6 +++++-
src/stream_cdf_i.c | 12 +++++++++++-
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index f7eb7ff3d..60d0a39cd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,8 +1,12 @@
-2024-11-?? Uwe Schulzweida
+2024-11-28 Uwe Schulzweida
* using CGRIBEX library version 2.3.1
* Version 2.5.0 released
+2024-11-23 Uwe Schulzweida
+
+ * NetCDF: improved HealPIX support
+
2024-11-04 Uwe Schulzweida
* reset chunktype if z and t are chunked
diff --git a/src/stream_cdf_i.c b/src/stream_cdf_i.c
index 748939062..9b7565a01 100644
--- a/src/stream_cdf_i.c
+++ b/src/stream_cdf_i.c
@@ -55,6 +55,7 @@ typedef struct
int ncid;
int varStatus;
bool ignoreVar;
+ bool isHealPIX;
bool isCubeSphere;
bool isCharAxis;
bool isXaxis;
@@ -588,6 +589,7 @@ init_ncvars(int nvars, ncvar_t *ncvars, int ncid)
ncvar->ncid = ncid;
ncvar->varStatus = UndefVar;
ncvar->ignoreVar = false;
+ ncvar->isHealPIX = false;
ncvar->isCubeSphere = false;
ncvar->isCharAxis = false;
ncvar->isXaxis = false;
@@ -1486,6 +1488,14 @@ scan_vars_attr(int nvars, ncvar_t *ncvars, int ndims, ncdim_t *ncdims, int model
{
ncvar->gmapid = nc_gmap_id;
cdf_set_var(&ncvars[ncvar->gmapid], CoordVar);
+ int nc_gmap_varid = ncvars[ncvar->gmapid].ncid;
+ if (cdfCheckAttText(nc_gmap_varid, nc_gmap_id, "grid_mapping_name"))
+ {
+ char gridMappingName[CDI_MAX_NAME];
+ cdfGetAttText(nc_gmap_varid, nc_gmap_id, "grid_mapping_name", CDI_MAX_NAME, gridMappingName);
+ // if (str_is_equal(gridMappingName, "healpix")) ncvars[ncvar->gmapid].isHealPIX = true;
+ if (str_is_equal(gridMappingName, "healpix")) ncvar->isHealPIX = true;
+ }
}
else
Warning("%s - %s", nc_strerror(status), attstring);
@@ -1830,7 +1840,7 @@ cdf_set_dimtype(int nvars, ncvar_t *ncvars, ncdim_t *ncdims)
dimtype = X_AXIS;
lxdim = true;
}
- else if (!lydim && ncvar->gridtype != GRID_UNSTRUCTURED)
+ else if (!lydim && ncvar->gridtype != GRID_UNSTRUCTURED && ncvar->isHealPIX == false)
// else if ( !lydim && ! (ncvars[ncvar->xvarid].dimids[0] == ncvars[ncvar->yvarid].dimids[0] &&
// ncvars[ncvar->xvarid].ndims == 1 && ncvars[ncvar->yvarid].ndims == 1))
{
--
GitLab
From c59834a769ba989e8b4f3b6b762ca09d340bfa7a Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Mon, 25 Nov 2024 09:06:33 +0100
Subject: [PATCH 8/8] Docu update
---
NEWS | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/NEWS b/NEWS
index 358f410f5..d21b4942d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,26 @@
CDI NEWS
--------
+Version 2.5.0 (28 Nov 2024):
+
+ Changes:
+ * NetCDF: improved HealPIX support
+ * reset chunktype if z and t are chunked
+ * lock access to NCZARR in CDI_Threadsafe mode
+
+Version 2.4.4 (19 Sep 2024):
+
+ Fixed bugs:
+ * absolute timeaxis failed with timeunits seconds [Bug #11916]
+
+Version 2.4.2 (21 Jun 2024):
+
+ Changes:
+ * NetCDF4: improved calculation of input chunk cache size
+ * NetCDF4: switched off output chunk cache
+ Fixed bugs:
+ * cdiInqContents: missing lock of non thread-safe netCDF4/HDF5 calls (bug fix)
+
Version 2.4.1 (21 May 2024):
Changes:
--
GitLab