From 10cbeb0f2061b2e0f009a2d7dfedd2c8422d07ae Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Wed, 29 Jan 2025 15:02:39 +0100
Subject: [PATCH] Added CDI_KEY_CHUNKSIZE_DIMZ
---
ChangeLog | 1 +
src/cdf.c | 3 ++-
src/cdf_int.c | 22 ++++++++++++++++++++++
src/cdf_int.h | 3 +++
src/cdf_write.c | 44 ++++++++++++++++++++++++++++----------------
src/cdi.h | 1 +
src/stream_cdf_i.c | 22 ----------------------
7 files changed, 57 insertions(+), 39 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index a4e64a7f2..3a32e93be 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,7 @@
2025-01-29 Uwe Schulzweida
+ * Added CDI_KEY_CHUNKSIZE_DIMZ
* Added CDI_KEY_CHUNKSIZE_DIMT
2025-01-14 Uwe Schulzweida
diff --git a/src/cdf.c b/src/cdf.c
index 78e7fff9f..43b1df25f 100644
--- a/src/cdf.c
+++ b/src/cdf.c
@@ -200,7 +200,7 @@ static void
cdfCloseFile(int fileID)
{
- if (CDF_Debug) Message( "Closing cdf file: %d", fileID);
+ if (CDF_Debug) Message("Closing cdf file: %d", fileID);
cdf_close(fileID);
}
@@ -209,6 +209,7 @@ cdfClose(int fileID)
{
cdfCloseFile(fileID);
}
+
#endif
/*
* Local Variables:
diff --git a/src/cdf_int.c b/src/cdf_int.c
index e12ec4d9d..868449b5c 100644
--- a/src/cdf_int.c
+++ b/src/cdf_int.c
@@ -800,6 +800,28 @@ cdf_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp
}
#endif
+size_t
+cdf_xtype_to_numbytes(nc_type xtype)
+{
+ size_t numBytes = 8;
+
+ // clang-format off
+ if (xtype == NC_BYTE ) numBytes = 1;
+ else if (xtype == NC_CHAR ) numBytes = 1;
+ else if (xtype == NC_SHORT ) numBytes = 2;
+ else if (xtype == NC_INT ) numBytes = 4;
+ else if (xtype == NC_FLOAT ) numBytes = 4;
+#ifdef HAVE_NETCDF4
+ else if (xtype == NC_UBYTE ) numBytes = 1;
+ else if (xtype == NC_USHORT) numBytes = 2;
+ else if (xtype == NC_LONG ) numBytes = 4;
+ else if (xtype == NC_UINT ) numBytes = 4;
+#endif
+ // clang-format on
+
+ return numBytes;
+}
+
#endif
/*
* Local Variables:
diff --git a/src/cdf_int.h b/src/cdf_int.h
index 062931eea..6afae6030 100644
--- a/src/cdf_int.h
+++ b/src/cdf_int.h
@@ -99,6 +99,9 @@ int cdi_nc_enddef_serial(int ncid, int streamID);
int cdi_nc__enddef_serial(int ncid, int streamID, size_t hdr_pad, size_t v_align, size_t v_minfree, size_t r_align);
typedef int (*cdi_nc_enddef_funcp)(int ncid, int streamID);
typedef int (*cdi_nc__enddef_funcp)(int ncid, int streamID, size_t hdr_pad, size_t v_align, size_t v_minfree, size_t r_align);
+
+size_t cdf_xtype_to_numbytes(nc_type xtype);
+
#endif
#endif /* CDF_INT_H */
diff --git a/src/cdf_write.c b/src/cdf_write.c
index 9fc92f4a5..e3314948a 100644
--- a/src/cdf_write.c
+++ b/src/cdf_write.c
@@ -747,8 +747,9 @@ calc_chunksize_x(int chunkType, long chunkSize, size_t xsize, bool isReg2dGrid)
static int
cdfDefineDimsAndChunks(const stream_t *streamptr, int varID, int xid, int yid, int zid, size_t gridsize, const int dimorder[3],
- int dims[4], bool useChunks, size_t chunks[4], char axis[5], size_t *piax)
+ int dims[4], bool useChunks, size_t chunks[4], char axis[5], size_t *piax, size_t *pchunkCacheSize)
{
+ size_t chunkCacheSize = 1;
int fileID = streamptr->fileID;
int vlistID = streamptr->vlistID;
@@ -757,21 +758,22 @@ cdfDefineDimsAndChunks(const stream_t *streamptr, int varID, int xid, int yid, i
for (int i = 0; i < 4; ++i) chunks[i] = 0;
- size_t xsize = 0, ysize = 0;
+ size_t xsize = 0, ysize = 0, zsize = 0;
if (xid != CDI_UNDEFID) cdf_inq_dimlen(fileID, xid, &xsize);
if (yid != CDI_UNDEFID) cdf_inq_dimlen(fileID, yid, &ysize);
+ if (zid != CDI_UNDEFID) cdf_inq_dimlen(fileID, zid, &zsize);
int timetype = vlistInqVarTimetype(vlistID, varID);
if (vlistHasTime(vlistID) && timetype != TIME_CONSTANT)
{
- int chunkSize = 0;
- cdiInqKeyInt(vlistID, CDI_GLOBAL, CDI_KEY_CHUNKSIZE_DIMT, &chunkSize);
- printf("chunkSize %d\n", chunkSize);
+ int chunkSizeDimT = 0;
+ cdiInqKeyInt(vlistID, CDI_GLOBAL, CDI_KEY_CHUNKSIZE_DIMT, &chunkSizeDimT);
int tid = streamptr->basetime.ncdimid;
if (tid == CDI_UNDEFID) Error("Internal problem, time undefined!");
axis[iax++] = 'T';
- chunks[ndims] = (chunkSize > 0) ? chunkSize : 1;
+ chunks[ndims] = (chunkSizeDimT > 0) ? chunkSizeDimT : 1;
dims[ndims] = tid;
+ chunkCacheSize *= chunks[ndims];
ndims++;
}
@@ -792,21 +794,27 @@ cdfDefineDimsAndChunks(const stream_t *streamptr, int varID, int xid, int yid, i
{
if (dimorder[id] == 3 && zid != CDI_UNDEFID)
{
+ int chunkSizeDimZ = 0;
+ cdiInqKeyInt(vlistID, varID, CDI_KEY_CHUNKSIZE_DIMZ, &chunkSizeDimZ);
+ if (chunkSizeDimZ == -1) chunkSizeDimZ = zsize;
axis[iax++] = 'Z';
- chunks[ndims] = 1;
+ chunks[ndims] = (chunkSizeDimZ > 0) ? chunkSizeDimZ : 1;
dims[ndims] = zid;
+ chunkCacheSize *= (chunkCacheSize > 1) ? zsize : chunks[ndims];
ndims++;
}
else if (dimorder[id] == 2 && yid != CDI_UNDEFID)
{
chunks[ndims] = calc_chunksize_y(chunkType, gridsize, xsize, ysize);
dims[ndims] = yid;
+ chunkCacheSize *= ysize;
ndims++;
}
else if (dimorder[id] == 1 && xid != CDI_UNDEFID)
{
chunks[ndims] = calc_chunksize_x(chunkType, chunkSize, xsize, isReg2dGrid);
dims[ndims] = xid;
+ chunkCacheSize *= xsize;
ndims++;
}
}
@@ -815,6 +823,7 @@ cdfDefineDimsAndChunks(const stream_t *streamptr, int varID, int xid, int yid, i
fprintf(stderr, "useChunks %d chunkType %d chunkSize %d chunks %zu %zu %zu %zu\n", useChunks, chunkType, chunkSize, chunks[0],
chunks[1], chunks[2], chunks[3]);
+ *pchunkCacheSize = chunkCacheSize;
*piax = iax;
return ndims;
}
@@ -924,14 +933,15 @@ cdfGenVarname(int fileID, char name[CDI_MAX_NAME], int pnum, int pcat, int *pdis
}
static void
-cdfDefVarChunkCache(int fileID, int ncvarID)
+cdfDefVarChunkCache(int fileID, int ncvarID, size_t chunkCacheSize)
{
size_t size = 0;
size_t nelems = 0;
float preemption = 0;
- if (CDI_Chunk_Cache > 0 && nc_get_var_chunk_cache(fileID, ncvarID, &size, &nelems, &preemption) == NC_NOERR)
+ if (nc_get_var_chunk_cache(fileID, ncvarID, &size, &nelems, &preemption) == NC_NOERR)
{
- size = (size_t) CDI_Chunk_Cache;
+ if (chunkCacheSize > size) size = chunkCacheSize;
+ if (CDI_Chunk_Cache > 0) size = (size_t) CDI_Chunk_Cache;
}
nc_set_var_chunk_cache(fileID, ncvarID, size, nelems, preemption);
@@ -982,8 +992,9 @@ cdfDefVar(stream_t *streamptr, int varID)
char axis[5];
int dims[4];
size_t chunks[4];
- int ndims
- = cdfDefineDimsAndChunks(streamptr, varID, xid, yid, zid, (size_t) gridsize, dimorder, dims, useChunks, chunks, axis, &iax);
+ size_t chunkCacheSize = 1;
+ int ndims = cdfDefineDimsAndChunks(streamptr, varID, xid, yid, zid, (size_t) gridsize, dimorder, dims, useChunks, chunks, axis,
+ &iax, &chunkCacheSize);
char name[CDI_MAX_NAME];
int length = CDI_MAX_NAME;
@@ -1006,19 +1017,20 @@ cdfDefVar(stream_t *streamptr, int varID)
else
cdfGenVarname(fileID, name, pnum, pcat, &pdis, &code);
- int dtype = vlistInqVarDatatype(vlistID, varID);
- nc_type xtype = cdfDefDatatype(dtype, streamptr);
-
if (streamptr->ncmode == 2)
{
cdf_redef(fileID);
streamptr->ncmode = 1;
}
+ int dtype = vlistInqVarDatatype(vlistID, varID);
+ nc_type xtype = cdfDefDatatype(dtype, streamptr);
+
int ncvarID = -1;
cdf_def_var(fileID, name, xtype, ndims, dims, &ncvarID);
- cdfDefVarChunkCache(fileID, ncvarID);
+ chunkCacheSize *= cdf_xtype_to_numbytes(xtype);
+ cdfDefVarChunkCache(fileID, ncvarID, chunkCacheSize);
#ifdef HAVE_NETCDF4
#ifdef NC_QUANTIZE_BITROUND
diff --git a/src/cdi.h b/src/cdi.h
index 8f62f5859..9216a9396 100644
--- a/src/cdi.h
+++ b/src/cdi.h
@@ -847,6 +847,7 @@ SizeType gridInqYCvals(int gridID, char *ycvals[]);
#define CDI_KEY_CHUNKTYPE 965 // ChunkType: CDI_CHUNK_AUTO/CDI_CHUNK_GRID/CDI_CHUNK_LINES
#define CDI_KEY_CHUNKSIZE 966 // ChunkSize
#define CDI_KEY_CHUNKSIZE_DIMT 967 // ChunkSize time dimension
+#define CDI_KEY_CHUNKSIZE_DIMZ 968 // ChunkSize zaxis dimension
// Floating point keys
#define CDI_KEY_MISSVAL 701 // Missing value
diff --git a/src/stream_cdf_i.c b/src/stream_cdf_i.c
index ca2ffdfa3..10136c99d 100644
--- a/src/stream_cdf_i.c
+++ b/src/stream_cdf_i.c
@@ -3533,28 +3533,6 @@ cdf_define_institut_and_model_id(int vlistID, int varID)
if (varTableID != CDI_UNDEFID) vlistDefVarTable(vlistID, varID, varTableID);
}
-static size_t
-cdf_xtype_to_numbytes(int xtype)
-{
- size_t numBytes = 8;
-
- // clang-format off
- if (xtype == NC_BYTE ) numBytes = 1;
- else if (xtype == NC_CHAR ) numBytes = 1;
- else if (xtype == NC_SHORT ) numBytes = 2;
- else if (xtype == NC_INT ) numBytes = 4;
- else if (xtype == NC_FLOAT ) numBytes = 4;
-#ifdef HAVE_NETCDF4
- else if (xtype == NC_UBYTE ) numBytes = 1;
- else if (xtype == NC_USHORT) numBytes = 2;
- else if (xtype == NC_LONG ) numBytes = 4;
- else if (xtype == NC_UINT ) numBytes = 4;
-#endif
- // clang-format on
-
- return numBytes;
-}
-
static inline size_t
size_of_dim_chunks(size_t n, size_t c)
{
--
GitLab