diff --git a/ChangeLog b/ChangeLog index df102bc3d34281296ba32d8611d8a3c2a854d600..55498e93346702a3b64704d379bdcae010b44c93 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2023-01-24 Uwe Schulzweida + + * NetCDF4: added support for Zstandard compression + 2023-01-06 Uwe Schulzweida * vlistDefVarDatatype: refactor handling of missing value (bug fix) diff --git a/app/cdi.c b/app/cdi.c index d61095c294bfe8e310f931720ca07e07271491f8..2a474099f9a325bbd0fc8ccecfd1764dd578fac5 100644 --- a/app/cdi.c +++ b/app/cdi.c @@ -185,6 +185,7 @@ usage(void) fprintf(stderr, " -z szip SZIP compression of GRIB1 records\n"); fprintf(stderr, " jpeg JPEG compression of GRIB2 records\n"); fprintf(stderr, " zip[_1-9] Deflate compression of netCDF4 variables\n"); + fprintf(stderr, " zstd[_1-19] zstandard compression of netCDF4 variables\n"); fprintf(stderr, "\n"); fprintf(stderr, " Report bugs to <https://code.mpimet.mpg.de/projects/cdi>\n"); } @@ -725,6 +726,11 @@ defineCompress(const char *arg) comptype = CDI_COMPRESS_ZIP; complevel = (len == 5 && arg[3] == '_' && isdigit(arg[4])) ? atoi(&arg[4]) : 1; } + else if (strncmp(arg, "zstd", 4) == 0) + { + comptype = CDI_COMPRESS_ZSTD; + complevel = (len >=6 && len <= 7 && arg[4] == '_' && isdigit(arg[5])) ? atoi(&arg[5]) : 1; + } else fprintf(stderr, "%s compression unsupported!\n", arg); } diff --git a/src/cdf_write.c b/src/cdf_write.c index 7018eb1cc64cc14ac9722225ab7d7d650086bac0..b04589e967610257ddad55ffc62c9bd84d7a171b 100644 --- a/src/cdf_write.c +++ b/src/cdf_write.c @@ -13,17 +13,19 @@ #include "vlist.h" #include "vlist_var.h" +#include <netcdf_filter.h> + void -cdfDefVarDeflate(int ncid, int ncvarID, int deflateLevel) +cdfDefVarDeflate(int ncid, int ncvarID, int compLevel) { #ifdef HAVE_NETCDF4 // Set chunking, shuffle, and deflate. int shuffle = (CDI_Shuffle > 0), deflate = 1; - if (deflateLevel < 1 || deflateLevel > 9) deflateLevel = 1; + if (compLevel < 1 || compLevel > 9) compLevel = 1; int status; - if ((status = nc_def_var_deflate(ncid, ncvarID, shuffle, deflate, deflateLevel))) + if ((status = nc_def_var_deflate(ncid, ncvarID, shuffle, deflate, compLevel))) { Error("nc_def_var_deflate failed; %s", nc_strerror(status)); } @@ -37,6 +39,27 @@ cdfDefVarDeflate(int ncid, int ncvarID, int deflateLevel) #endif } +void +cdfDefVarZstd(int ncid, int ncvarID, int compLevel) +{ +#ifdef HAVE_NETCDF4 + if (compLevel < 1 || compLevel > 19) compLevel = 1; + + int status; + if ((status = nc_def_var_zstandard(ncid, ncvarID, compLevel))) + { + Error("nc_def_var_zstandard failed; %s", nc_strerror(status)); + } +#else + static bool lwarn = true; + if (lwarn) + { + lwarn = false; + Warning("zstd compression failed, NetCDF4 not available!"); + } +#endif +} + void cdfDefVarSzip(int ncid, int ncvarID, int pixels_per_block) { @@ -441,6 +464,23 @@ cdfDefVarCompression(const stream_t *streamptr, int ncvarID, nc_type xtype) } } } + else if (streamptr->comptype == CDI_COMPRESS_ZSTD) + { + if (streamptr->filetype == CDI_FILETYPE_NC4 || streamptr->filetype == CDI_FILETYPE_NC4C + || streamptr->filetype == CDI_FILETYPE_NCZARR) + { + cdfDefVarZstd(streamptr->fileID, ncvarID, streamptr->complevel); + } + else + { + static bool lwarn = true; + if (lwarn) + { + lwarn = false; + Warning("SZIP compression is only available for NetCDF4!"); + } + } + } else if (streamptr->comptype == CDI_COMPRESS_SZIP) { if (streamptr->filetype == CDI_FILETYPE_NC4 || streamptr->filetype == CDI_FILETYPE_NC4C @@ -849,7 +889,7 @@ cdfDefVar(stream_t *streamptr, int varID) cdiDecodeParam(param, &pnum, &pcat, &pdis); int gridID = vlistInqVarGrid(vlistID, varID); - size_t gridsize = gridInqSize(gridID); + SizeType gridsize = gridInqSize(gridID); int gridtype = gridInqType(gridID); int gridindex = nc_grid_index(streamptr, gridID); int xid = (gridtype != GRID_TRAJECTORY) ? streamptr->ncgrid[gridindex].ncIDs[CDF_DIMID_X] : CDI_UNDEFID; diff --git a/src/cdi.h b/src/cdi.h index df02a520327fbc7dedc1ae12185f84ea868d5597..3215ccaeb2caa62a627e286492ea7565901d93b0 100644 --- a/src/cdi.h +++ b/src/cdi.h @@ -96,6 +96,7 @@ extern "C" { #define CDI_COMPRESS_AEC 2 #define CDI_COMPRESS_ZIP 3 #define CDI_COMPRESS_JPEG 4 +#define CDI_COMPRESS_ZSTD 5 // external data types