From 3a14ee562b245a057dd8ff5a012cb5aea820ce3f Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 13 Jan 2025 13:37:36 +0100 Subject: [PATCH 01/15] removed unused files --- src/cdo_benchmark.h | 28 --------------- src/namelistdump.cc | 88 --------------------------------------------- 2 files changed, 116 deletions(-) delete mode 100644 src/cdo_benchmark.h delete mode 100644 src/namelistdump.cc diff --git a/src/cdo_benchmark.h b/src/cdo_benchmark.h deleted file mode 100644 index bcfa64d9c..000000000 --- a/src/cdo_benchmark.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef CDO_BENCHMARK_H -#define CDO_BENCHMARK_H -#define DBE2 -#ifdef DBE2 - -#include <iostream> -#include <chrono> - -template <typename func> -void -cdo_benchmark(func lambda) -{ - auto start = std::chrono::steady_clock::now(); - lambda(); - auto end = std::chrono::steady_clock::now(); - auto elapsedm = std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count(); - // time end - std::cout << "time: " << elapsedm << " ns" << std::endl; -} - -#define CDO_BENCH(x) cdo_benchmark(x) - -#else -#define CDO_BENCH(x) \ - std::cout << "benchmark found, please remove. " << __FILE__ << ": " << __LINE__ << std::endl; \ - x() -#endif -#endif diff --git a/src/namelistdump.cc b/src/namelistdump.cc deleted file mode 100644 index 1ca4f6474..000000000 --- a/src/namelistdump.cc +++ /dev/null @@ -1,88 +0,0 @@ -/* - This file is part of CDO. CDO is a collection of Operators to manipulate and analyse Climate model Data. - - Author: Uwe Schulzweida - -*/ - -#include <cstdio> -#include <cstdlib> -#include <cstring> -#include <cerrno> -#include <sys/stat.h> - -#include "namelist.h" - -int -main(int argc, char *argv[]) -{ - if (argc != 2) - { - fprintf(stderr, "Usage: %s namelist\n", argv[0]); - return -1; - } - - const char *filename = argv[1]; - printf("Parse namelist %s:\n", filename); - - struct stat sbuf; - size_t filesize = (stat(filename, &sbuf) == 0) ? sbuf.st_size : 0; - - if (filesize == 0) - { - fprintf(stderr, "Empty table file: %s\n", filename); - return -1; - } - - FILE *fp = std::fopen(filename, "r"); - if (fp == nullptr) - { - fprintf(stderr, "Open failed on %s: %s\n", filename, strerror(errno)); - return -1; - } - - char *buffer = (char *) std::malloc(filesize); - size_t nitems = fread(buffer, 1, filesize, fp); - - std::fclose(fp); - - if (nitems != filesize) - { - fprintf(stderr, "Read failed on %s!\n", filename); - std::free(buffer); - return -1; - } - - NamelistParser p; - - NamelistError status = p.parse(buffer, filesize); - printf("Processed number of lines: %lu\n", p.lineno - 1); - if (status != NamelistError::UNDEFINED) - { - switch (status) - { - case NamelistError::INVAL: - fprintf(stderr, "Namelist error: Invalid character in %s (line=%lu character='%c')!\n", filename, p.lineno, - buffer[p.pos]); - break; - case NamelistError::PART: - fprintf(stderr, "Namelist error: End of string not found in %s (line=%lu)!\n", filename, p.lineno); - break; - case NamelistError::INKEY: - fprintf(stderr, "Namelist error: Invalid keyword in %s (line=%lu)!\n", filename, p.lineno); - break; - case NamelistError::INTYP: - fprintf(stderr, "Namelist error: Invalid keyword type in %s (line=%lu)!\n", filename, p.lineno); - break; - case NamelistError::INOBJ: fprintf(stderr, "Namelist error: Invalid object in %s (line=%lu)!\n", filename, p.lineno); break; - case NamelistError::EMKEY: fprintf(stderr, "Namelsit error: Emtry key name in %s (line=%lu)!\n", filename, p.lineno); break; - default: fprintf(stderr, "Namelsit error in %s (line=%lu)!\n", filename, p.lineno); break; - } - } - - p.dump(buffer); - - std::free(buffer); - - return 0; -} -- GitLab From 247efd6a0931867fbba9dee6f7c180be35a66b48 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 13 Jan 2025 14:27:34 +0100 Subject: [PATCH 02/15] fixed Module Templates not compiling --- src/Templates.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Templates.cc b/src/Templates.cc index e05f52ab1..6bbc77dc6 100644 --- a/src/Templates.cc +++ b/src/Templates.cc @@ -9,6 +9,8 @@ #include "process_int.h" +// THIS IS A TEMPLATE FOR LATER USE +class Template { CdoStreamID streamID1; int taxisID1; @@ -84,6 +86,7 @@ public: } }; +class Template2 { size_t numMissVals; -- GitLab From 87465f9e2ea3a339b0a96749c28e0b923ccdf6ea Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 13 Jan 2025 14:28:22 +0100 Subject: [PATCH 03/15] leftover vlistNumFields replaced --- src/cdo_varlist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdo_varlist.h b/src/cdo_varlist.h index d6836171a..780a3641c 100644 --- a/src/cdo_varlist.h +++ b/src/cdo_varlist.h @@ -67,7 +67,7 @@ public: VarList(int _vlistID) : vlistID(_vlistID) { cdoVars_init(vars, _vlistID); - m_maxFields = vlistNumFields(_vlistID); + m_maxFields = vlistNumGrids(_vlistID); m_numSteps = vlistNtsteps(_vlistID); m_numConstVars = num_const_vars(vars); m_numVaryingVars = num_varying_vars(vars); -- GitLab From a38b349e1e2baf6d25d5d3251001430c9c124c67 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 13 Jan 2025 14:28:52 +0100 Subject: [PATCH 04/15] added missing files, sort --- src/CMakeLists.txt | 298 ++++++++++++++++++++++++--------------------- 1 file changed, 160 insertions(+), 138 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 84b73839b..334b73d6d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,4 @@ list( APPEND cdolib_src_files - after_dvtrans.cc after_fctrans.cc after_namelist.cc after_sptrans.cc @@ -9,6 +8,119 @@ list( APPEND cdolib_src_files bitinformation.cc bitinformation.h cdi_lockedIO.cc + cellsearch_reg2d.cc + cellsearch_reg2d.h + cellsearch_spherepart.h + cellsearch_unstruct.h + cellsearch_utils.h + grid_cellsearch.cc + grid_cellsearch.h + mpim_grid/gridreference.cc + mpim_grid/gridreference.h + mpim_grid/mpim_grid.cc + mpim_grid/mpim_grid.h + mpmo.cc + mpmo.h + mpmo_color.cc + mpmo_color.h + namelist.cc + namelist.h + nanoflann.hpp + node.cc + node.h + oper_args.cc + oper_args.h + operator_help.cc + operator_help.h + par_io.cc + par_io.h + param_conversion.cc + param_conversion.h + parse_literals.cc + parse_literals.h + parser.cc + parser.h + percentiles.cc + percentiles.h + percentiles_hist.cc + percentiles_hist.h + pipe.cc + pipe.h + pipeStream.cc + pipeStream.h + pmlist.cc + pmlist.h + point.h + pointsearch_full.h + pointsearch_healpix.h + pointsearch_kdtree.h + pointsearch_nanoflann.h + pointsearch_reg2d.cc + pointsearch_reg2d.h + pointsearch_spherepart.h + pointsearch_unstruct.h + pointsearch_utils.h + printinfo.cc + printinfo.h + process.cc + process.h + processManager.cc + processManager.h + process_int.cc + process_int.h + progress.cc + progress.h + pthread_debug.cc + pthread_debug.h + region.cc + region.h + remap.h + remap_bicubic.cc + remap_bilinear.cc + remap_cell_search.cc + remap_conserv.cc + remap_knn.cc + remap_point_search.cc + remap_scrip_io.cc + remap_search_reg2d.cc + remap_store_link.cc + remap_store_link.h + remap_utils.cc + remap_utils.h + remap_vars.cc + remap_vars.h + remaplib.cc + selboxinfo.h + sellist.cc + sellist.h + specspace.cc + specspace.h + statistic.cc + statistic.h + stdnametable.cc + stdnametable.h + table.cc + table.h + transform.h + util_date.h + util_fileextensions.cc + util_fileextensions.h + util_files.cc + util_files.h + util_string.cc + util_string.h + util_wildcards.cc + util_wildcards.h + varray.cc + varray.h + vector3d.h + verifygrid.h + vertical_interp.cc + vertical_interp.h + vertint_util.cc + vertint_util.h + zaxis_print.cc + after_dvtrans.cc cdi_lockedIO.h cdi_uuid.h cdoStream.cc @@ -20,21 +132,23 @@ list( APPEND cdolib_src_files cdo_data.h cdo_default_values.cc cdo_default_values.h - cdo_features.cc - cdo_features.h + cdo_exception.h cdo_fctrans.cc cdo_fctrans.h + cdo_features.cc + cdo_features.h cdo_fft.cc cdo_fft.h cdo_fftw3.cc cdo_fftw3.h cdo_fill.cc - cdo_exception.h cdo_fill.h cdo_getopt.cc cdo_getopt.h cdo_history.cc cdo_history.h + cdo_magics_mapper.cc + cdo_magics_mapper.h cdo_math.cc cdo_math.h cdo_module.cc @@ -57,6 +171,7 @@ list( APPEND cdolib_src_files cdo_season.h cdo_settings.cc cdo_settings.h + cdo_stepstat.h cdo_syntax_error.cc cdo_syntax_error.h cdo_task.cc @@ -68,11 +183,7 @@ list( APPEND cdolib_src_files cdo_vlist.h cdo_zaxis.cc cdo_zaxis.h - cellsearch_reg2d.cc - cellsearch_reg2d.h - cellsearch_spherepart.h - cellsearch_unstruct.h - cellsearch_utils.h + cdotest.cc cf_interface.h cfortran.h cfortran.h @@ -82,6 +193,7 @@ list( APPEND cdolib_src_files commandline.cc commandline.h compare.h + config.h const.h constants.cc constants.h @@ -121,17 +233,15 @@ list( APPEND cdolib_src_files field_functions.h field_memory.cc field_meridional.cc + field_trend.cc + field_trend.h field_vinterp.cc field_vinterp.h field_zonal.cc - field_trend.cc - field_trend.h fieldc.cc fieldc_complex.cc fileStream.cc fileStream.h - memoryStream.cc - memoryStream.h fill_1d.cc fill_1d.h gaussian_latitudes.c @@ -144,8 +254,6 @@ list( APPEND cdolib_src_files grid_from_name.cc grid_gme.cc grid_icosphere.cc - grid_cellsearch.cc - grid_cellsearch.h grid_pointsearch.cc grid_pointsearch.h grid_print.cc @@ -166,7 +274,11 @@ list( APPEND cdolib_src_files knndata.h libncl.h listbuffer.h + magics_template_parser.cc + magics_template_parser.h matrix_view.h + memoryStream.cc + memoryStream.h merge_axis.cc merge_axis.h module_info.cc @@ -176,115 +288,19 @@ list( APPEND cdolib_src_files mpim_grid/grid_convert.h mpim_grid/grid_healpix.cc mpim_grid/grid_healpix.h + mpim_grid/grid_options.cc + mpim_grid/grid_options.h mpim_grid/grid_proj.cc mpim_grid/grid_proj.h mpim_grid/grid_rot.cc mpim_grid/grid_rot.h - mpim_grid/gridreference.cc - mpim_grid/gridreference.h - mpim_grid/mpim_grid.cc - mpim_grid/mpim_grid.h - mpmo.cc - mpmo.h - mpmo_color.cc - mpmo_color.h - namelist.cc - namelist.h - nanoflann.hpp - operator_help.cc - operator_help.h - oper_args.cc - oper_args.h - par_io.cc - par_io.h - param_conversion.cc - param_conversion.h - parse_literals.cc - parse_literals.h - percentiles.cc - percentiles.h - percentiles_hist.cc - percentiles_hist.h - pipe.cc - pipe.h - pipeStream.cc - pipeStream.h - pmlist.cc - pmlist.h - printinfo.cc - printinfo.h - point.h - pointsearch_reg2d.cc - pointsearch_reg2d.h - pointsearch_healpix.h - pointsearch_unstruct.h - pointsearch_kdtree.h - pointsearch_nanoflann.h - pointsearch_spherepart.h - pointsearch_full.h - pointsearch_utils.h - process.cc - process.h - processManager.cc - processManager.h - parser.h - parser.cc - node.cc - node.h - process_int.cc - process_int.h - progress.cc - progress.h - pthread_debug.cc - pthread_debug.h - region.h - region.cc - remap.h - remap_bicubic.cc - remap_bilinear.cc - remap_cell_search.cc - remap_conserv.cc - remap_knn.cc - remap_point_search.cc - remap_scrip_io.cc - remap_search_reg2d.cc - remap_store_link.cc - remap_store_link.h - remap_utils.cc - remap_utils.h - remap_vars.cc - remap_vars.h - remaplib.cc - selboxinfo.h - sellist.cc - sellist.h - specspace.cc - specspace.h - statistic.cc - statistic.h - stdnametable.cc - stdnametable.h - table.cc - table.h - transform.h - util_fileextensions.cc - util_fileextensions.h - util_files.cc - util_files.h - util_string.cc - util_string.h - util_wildcards.cc - util_wildcards.h - util_date.h - varray.cc - varray.h - vector3d.h - verifygrid.h - vertical_interp.cc - vertical_interp.h - vertint_util.h - vertint_util.cc - zaxis_print.cc + remap_gradients.cc + remap_grid.h + remap_method_conserv.cc + remap_method_conserv.h + remap_stat.cc + results_template_parser.cc + results_template_parser.h ) list( APPEND cdolib_src_files @@ -339,23 +355,22 @@ list( APPEND cdo_src_files ) list( APPEND cdo_operators_src_files - Adisit.cc Afterburner.cc Arith.cc Arithc.cc Arithdays.cc Arithlat.cc Bitrounding.cc - Cat.cc CDIread.cc CDItest.cc CDIwrite.cc - Change.cc - Change_e5slm.cc - Cloudlayer.cc CMOR.cc CMOR_lite.cc CMOR_table.cc + Cat.cc + Change.cc + Change_e5slm.cc + Cloudlayer.cc Collgrid.cc Command.cc Comp.cc @@ -377,15 +392,14 @@ list( APPEND cdo_operators_src_files Distgrid.cc Duplicate.cc EOFs.cc - Eof3d.cc - EcaIndices.cc EcaEtccdi.cc - Echam5ini.cc + EcaIndices.cc Enlarge.cc Enlargegrid.cc Ensstat.cc Ensstat3.cc Ensval.cc + Eof3d.cc Eofcoeff.cc Eofcoeff3d.cc EstFreq.cc @@ -411,8 +425,8 @@ list( APPEND cdo_operators_src_files Importamsr.cc Importbinary.cc Importcmsaf.cc - Importobs.cc Importfv3grid.cc + Importobs.cc Info.cc Input.cc Intgrid.cc @@ -451,8 +465,8 @@ list( APPEND cdo_operators_src_files Regres.cc Remapeta.cc Remapgrid.cc - Remapweights.cc Remapstat.cc + Remapweights.cc Replace.cc Replacevalues.cc Rhopot.cc @@ -462,12 +476,12 @@ list( APPEND cdo_operators_src_files Samplegrid.cc Samplegridicon.cc Seascount.cc + Seasmonstat.cc Seaspctl.cc Seasstat.cc - Seasmonstat.cc Selbox.cc - Selgridcell.cc Select.cc + Selgridcell.cc Selmulti.cc Seloperator.cc Selrec.cc @@ -488,8 +502,8 @@ list( APPEND cdo_operators_src_files Settime.cc Setzaxis.cc Shiftxy.cc - Showinfo.cc Showattribute.cc + Showinfo.cc Sinfo.cc Smooth.cc Sort.cc @@ -505,6 +519,7 @@ list( APPEND cdo_operators_src_files Tee.cc Test.cc Tests.cc + Templates.cc Timcount.cc Timcumsum.cc Timfillmiss.cc @@ -525,28 +540,28 @@ list( APPEND cdo_operators_src_files Vargen.cc Varrms.cc Varsstat.cc + Verifygrid.cc + Verifyweights.cc + Vertcum.cc Vertfillmiss.cc Vertintap.cc Vertintgh.cc Vertintml.cc Vertintzs.cc Vertstat.cc - Vertcum.cc Vertwind.cc - Verifygrid.cc - Verifyweights.cc Wct.cc Wind.cc WindTrans.cc Writegrid.cc Writerandom.cc - Yeararith.cc - Yearmonstat.cc Ydayarith.cc Ydaypctl.cc Ydaystat.cc Ydrunpctl.cc Ydrunstat.cc + Yeararith.cc + Yearmonstat.cc Yhourarith.cc Yhourstat.cc Ymonarith.cc @@ -556,6 +571,13 @@ list( APPEND cdo_operators_src_files Yseaspctl.cc Yseasstat.cc Zonstat.cc + Adisit.cc + Echam5ini.cc + Maggraph.cc + Magplot.cc + Magvector.cc + Setfilter.cc + Splitsel.cc ) add_library(operators ${cdo_operators_src_files}) -- GitLab From 0692c42a74341a886b3166de817e9a056092f851 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Wed, 15 Jan 2025 13:21:12 +0100 Subject: [PATCH 05/15] renamed threading variables for max omp threads and choosen num of threads --- src/Ensstat.cc | 2 +- src/Ensstat3.cc | 4 ++-- src/Fillmiss.cc | 2 +- src/Filter.cc | 2 +- src/Fourier.cc | 2 +- src/Gridboxstat.cc | 4 ++-- src/Intlevel.cc | 2 +- src/Intlevel3d.cc | 2 +- src/Remapstat.cc | 10 +++++----- src/Runpctl.cc | 2 +- src/Samplegridicon.cc | 2 +- src/Smooth.cc | 2 +- src/Timfillmiss.cc | 2 +- src/Timsort.cc | 2 +- src/Tstepcount.cc | 2 +- src/Vertfillmiss.cc | 2 +- src/after_fctrans.cc | 2 +- src/after_sptrans.cc | 20 ++++++++++---------- src/bitinformation.cc | 4 ++-- src/cdo.cc | 4 ++-- src/cdo_fctrans.cc | 12 ++++++------ src/cdo_fftw3.cc | 12 ++++++------ src/cdo_options.cc | 3 ++- src/cdo_options.h | 3 ++- src/cdo_settings.cc | 6 +++--- src/cdo_settings.h | 2 +- src/cdo_task.cc | 2 +- src/cellsearch_spherepart.h | 6 +++--- src/fileStream.cc | 4 ++-- src/hetaeta.cc | 6 +++--- src/pointsearch_kdtree.h | 2 +- src/process.cc | 2 +- src/remap_conserv.cc | 24 ++++++++++++------------ src/remap_knn.cc | 12 ++++++------ src/remap_store_link.cc | 2 +- src/remap_vars.cc | 4 ++-- 36 files changed, 89 insertions(+), 87 deletions(-) diff --git a/src/Ensstat.cc b/src/Ensstat.cc index 1507c3122..4272e1ff2 100644 --- a/src/Ensstat.cc +++ b/src/Ensstat.cc @@ -237,7 +237,7 @@ public: auto task = Options::CDO_task ? std::make_unique<cdo::Task>() : nullptr; const int numTasks = Options::CDO_task ? 2 : 1; - FieldVector workFields(Threading::ompNumThreads); + FieldVector workFields(Threading::ompNumMaxThreads); for (auto &work : workFields) work.resize(numFiles); FieldVector fieldVector[2]; diff --git a/src/Ensstat3.cc b/src/Ensstat3.cc index 1b29f821b..f2cccf695 100644 --- a/src/Ensstat3.cc +++ b/src/Ensstat3.cc @@ -158,8 +158,8 @@ public: /* ("first touch strategy") */ /* --> #pragma omp parallel for ... */ /* *************************************************** */ - field = FieldVector(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + field = FieldVector(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { field[i].resize(numFiles); field[i].weightv.resize(numFiles); diff --git a/src/Fillmiss.cc b/src/Fillmiss.cc index a9c935f7c..1a500ce5b 100644 --- a/src/Fillmiss.cc +++ b/src/Fillmiss.cc @@ -312,7 +312,7 @@ setmisstodis(Varray<T1> &vIn, Varray<T2> &vOut, int gridID, size_t numMissVals, if (nv != nvals) cdo_abort("Internal problem, number of valid values differ!"); std::vector<KnnData> knnDataMem; - for (int i = 0; i < Threading::ompNumThreads; ++i) knnDataMem.push_back(KnnData(numNeighbors)); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) knnDataMem.push_back(KnnData(numNeighbors)); cdo::timer timer; diff --git a/src/Filter.cc b/src/Filter.cc index f2dbc52d0..79688d23a 100644 --- a/src/Filter.cc +++ b/src/Filter.cc @@ -272,7 +272,7 @@ public: if (Options::cdoVerbose) cdo_print("Allocate %zu array%s over %zu steps: size=%zu Bytes", numArrays, numArrays > 1 ? "s" : "", nts, allocatedMem); - std::vector<FilterMemory> fourierMemory(Threading::ompNumThreads); + std::vector<FilterMemory> fourierMemory(Threading::ompNumMaxThreads); if (useFFTW) { diff --git a/src/Fourier.cc b/src/Fourier.cc index 4812d9d0d..ae68101af 100644 --- a/src/Fourier.cc +++ b/src/Fourier.cc @@ -237,7 +237,7 @@ public: int nts = tsID; - std::vector<FourierMemory> fourierMemory(Threading::ompNumThreads); + std::vector<FourierMemory> fourierMemory(Threading::ompNumMaxThreads); if (use_fftw) { diff --git a/src/Gridboxstat.cc b/src/Gridboxstat.cc index 6aad078d6..5a0c8ee84 100644 --- a/src/Gridboxstat.cc +++ b/src/Gridboxstat.cc @@ -231,8 +231,8 @@ gridbox_stat(const Field &field1, Field &field2, size_t xinc, size_t yinc, int s auto useWeight = !field1.weightv.empty(); auto boxsize = xinc * yinc; - FieldVector fields(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + FieldVector fields(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { fields[i].resize(boxsize); if (useWeight) fields[i].weightv.resize(boxsize); diff --git a/src/Intlevel.cc b/src/Intlevel.cc index bb8c82335..58bbb7ba3 100644 --- a/src/Intlevel.cc +++ b/src/Intlevel.cc @@ -211,7 +211,7 @@ static void vert_gen_weights3d1d(bool extrapolate, size_t gridsize, int nlev1, const Varray<T> &xlev1, int nlev2, const std::vector<double> &lev2, Varray<int> &xlev_idx, Varray<float> &xlev_wgt) { - auto nthreads = Threading::ompNumThreads; + auto nthreads = Threading::ompNumMaxThreads; Varray2D<double> lev1p2(nthreads, Varray<double>(nlev1 + 2)); Varray2D<float> lev_wgt(nthreads, Varray<float>(nlev2)); Varray2D<int> lev_idx(nthreads, Varray<int>(nlev2)); diff --git a/src/Intlevel3d.cc b/src/Intlevel3d.cc index aab6a491f..2dadae8dd 100644 --- a/src/Intlevel3d.cc +++ b/src/Intlevel3d.cc @@ -39,7 +39,7 @@ static void vert_gen_weights3d(bool expol, size_t gridsize, int nlev1, const Varray<float> &xlev1, int nlev2, const Varray<float> &xlev2, Varray<int> &xlev_idx, Varray<float> &xlev_wgt) { - auto nthreads = Threading::ompNumThreads; + auto nthreads = Threading::ompNumMaxThreads; Varray2D<double> lev1p2(nthreads, Varray<double>(nlev1 + 2)); Varray2D<double> lev2(nthreads, Varray<double>(nlev2)); Varray2D<float> lev_wgt(nthreads, Varray<float>(nlev2)); diff --git a/src/Remapstat.cc b/src/Remapstat.cc index fdc785498..888817f4e 100644 --- a/src/Remapstat.cc +++ b/src/Remapstat.cc @@ -519,8 +519,8 @@ gen_mapdata(int gridID1, int gridID2) auto maxIndex = gridsize1; if (gridsize1 > 1000000) maxIndex /= 4; Vmask vmask(gridsize1, 0); - Varray2D<size_t> indices_2D(Threading::ompNumThreads, Varray<size_t>(maxIndex)); - Varray2D<double> dist_2D(Threading::ompNumThreads, Varray<double>(maxIndex)); + Varray2D<size_t> indices_2D(Threading::ompNumMaxThreads, Varray<size_t>(maxIndex)); + Varray2D<double> dist_2D(Threading::ompNumMaxThreads, Varray<double>(maxIndex)); StatInfo statInfo; @@ -560,10 +560,10 @@ gen_mapdata(int gridID1, int gridID2) } // if (Options::cdoVerbose) printf("%zu numIndices %zu nvalues %zu maxdist %g\n", i+1, numIndices, nvalues, maxdist); - if (Options::cdoVerbose && Threading::ompNumThreads == 1) statInfo.add(numIndices, nvalues, searchRadius); + if (Options::cdoVerbose && Threading::ompNumMaxThreads == 1) statInfo.add(numIndices, nvalues, searchRadius); } - if (Options::cdoVerbose && Threading::ompNumThreads == 1) statInfo.print(); + if (Options::cdoVerbose && Threading::ompNumMaxThreads == 1) statInfo.print(); if (Options::cdoVerbose) check_vmask(vmask); if (Options::cdoVerbose) cdo_print("Point search qnearest: %.2f seconds", timer.elapsed()); @@ -608,7 +608,7 @@ remap_kernel(int operfunc, const Varray<size_t> &indices, size_t &numMissVals2, static void remap_field(const Varray2D<size_t> &mapdata, const Field &field1, Field &field2, int operfunc) { - std::vector<Field> fields(Threading::ompNumThreads); + std::vector<Field> fields(Threading::ompNumMaxThreads); auto gridsize2 = gridInqSize(field2.grid); cdo::timer timer; diff --git a/src/Runpctl.cc b/src/Runpctl.cc index 0756b5193..4c5b8f40d 100644 --- a/src/Runpctl.cc +++ b/src/Runpctl.cc @@ -28,7 +28,7 @@ runpctl(double pn, int ndates, size_t gridsize, Varray<T> &v2, double mv, const { T missval = mv; size_t numMissVals = 0; - Varray2D<T> array_2D(Threading::ompNumThreads, Varray<T>(ndates)); + Varray2D<T> array_2D(Threading::ompNumMaxThreads, Varray<T>(ndates)); #ifdef _OPENMP #pragma omp parallel for default(shared) schedule(dynamic) diff --git a/src/Samplegridicon.cc b/src/Samplegridicon.cc index ae8645f38..5c2189bac 100644 --- a/src/Samplegridicon.cc +++ b/src/Samplegridicon.cc @@ -311,7 +311,7 @@ compute_child_from_bounds(CellIndex &cellindex2, Varray<double> &grid_center_lon constexpr int MaxSearch = 128; std::vector<KnnData> knnDataMem; - for (int i = 0; i < Threading::ompNumThreads; ++i) knnDataMem.push_back(KnnData(MaxSearch)); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) knnDataMem.push_back(KnnData(MaxSearch)); cellindex2.child.resize(MAX_CHILDS * ncells2); auto &child2 = cellindex2.child; diff --git a/src/Smooth.cc b/src/Smooth.cc index f4483f034..9da7e42aa 100644 --- a/src/Smooth.cc +++ b/src/Smooth.cc @@ -72,7 +72,7 @@ smooth(int gridID, double mv, const Varray<T1> &array1, Varray<T2> &array2, cons knnParams.weightR = spoint.weightR; std::vector<KnnData> knnDataList; - for (int i = 0; i < Threading::ompNumThreads; ++i) knnDataList.push_back(KnnData(knnParams)); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) knnDataList.push_back(KnnData(knnParams)); cdo::timer timer; diff --git a/src/Timfillmiss.cc b/src/Timfillmiss.cc index 0a0cd0a5f..25a4cda34 100644 --- a/src/Timfillmiss.cc +++ b/src/Timfillmiss.cc @@ -195,7 +195,7 @@ public: timeValues[tsID] = julianDate_to_double(calendar, dtlist.get_vDateTime(tsID), dtlist.get_vDateTime(0)); } - dataValues2D = Varray2D<double>(Threading::ompNumThreads); + dataValues2D = Varray2D<double>(Threading::ompNumMaxThreads); for (auto &dataValues : dataValues2D) dataValues.resize(numSteps); for (int varID = 0; varID < numVars; ++varID) { step(varID); } diff --git a/src/Timsort.cc b/src/Timsort.cc index a2e56c862..119bb8e15 100644 --- a/src/Timsort.cc +++ b/src/Timsort.cc @@ -104,7 +104,7 @@ public: int nts = tsID; - std::vector<Field> fields(Threading::ompNumThreads); + std::vector<Field> fields(Threading::ompNumMaxThreads); for (int varID = 0; varID < numVars; ++varID) { diff --git a/src/Tstepcount.cc b/src/Tstepcount.cc index c7796796c..35c76835e 100644 --- a/src/Tstepcount.cc +++ b/src/Tstepcount.cc @@ -120,7 +120,7 @@ public: int nts = tsID; - std::vector<Field> fields(Threading::ompNumThreads); + std::vector<Field> fields(Threading::ompNumMaxThreads); for (int varID = 0; varID < numVars; ++varID) { diff --git a/src/Vertfillmiss.cc b/src/Vertfillmiss.cc index 72d3a0616..d15f56784 100644 --- a/src/Vertfillmiss.cc +++ b/src/Vertfillmiss.cc @@ -113,7 +113,7 @@ public: auto numLevels = var.nlevels; auto missval = var.missval; - Varray2D<double> dataValues2D(Threading::ompNumThreads); + Varray2D<double> dataValues2D(Threading::ompNumMaxThreads); for (auto &dataValues : dataValues2D) dataValues.resize(numLevels); Varray<double> levelValues(numLevels); diff --git a/src/after_fctrans.cc b/src/after_fctrans.cc index 2b7830c26..18b978b3a 100644 --- a/src/after_fctrans.cc +++ b/src/after_fctrans.cc @@ -2013,7 +2013,7 @@ fc2gp(const double *restrict trig, const long *restrict ifax, const double *rest long nvexx = lot - (nblox - 1) * NFFT; const long nvex0 = nvexx; - const long nthmax = (nblox < Threading::ompNumThreads) ? nblox : Threading::ompNumThreads; + const long nthmax = (nblox < Threading::ompNumMaxThreads) ? nblox : Threading::ompNumMaxThreads; const long nvals = lot * jump; Varray<double> wfc(nvals); diff --git a/src/after_sptrans.cc b/src/after_sptrans.cc index 07a30cfbf..63a1e2765 100644 --- a/src/after_sptrans.cc +++ b/src/after_sptrans.cc @@ -126,15 +126,15 @@ after_legini_full(long ntr, long nlat, double *restrict poli, double *restrict p auto needHnm = (pdev != nullptr) || (pol2 != nullptr); #ifdef _OPENMP - Varray2D<double> pnm_2(Threading::ompNumThreads); - Varray2D<double> hnm_2(Threading::ompNumThreads); - Varray2D<double> ztemp1_2(Threading::ompNumThreads); - Varray2D<double> ztemp2_2(Threading::ompNumThreads); - for (long i = 0; i < Threading::ompNumThreads; ++i) pnm_2[i].resize(dimsp); + Varray2D<double> pnm_2(Threading::ompNumMaxThreads); + Varray2D<double> hnm_2(Threading::ompNumMaxThreads); + Varray2D<double> ztemp1_2(Threading::ompNumMaxThreads); + Varray2D<double> ztemp2_2(Threading::ompNumMaxThreads); + for (long i = 0; i < Threading::ompNumMaxThreads; ++i) pnm_2[i].resize(dimsp); if (needHnm) - for (long i = 0; i < Threading::ompNumThreads; ++i) hnm_2[i].resize(dimsp); - for (long i = 0; i < Threading::ompNumThreads; ++i) ztemp1_2[i].resize(twowaves); - for (long i = 0; i < Threading::ompNumThreads; ++i) ztemp2_2[i].resize(twowaves); + for (long i = 0; i < Threading::ompNumMaxThreads; ++i) hnm_2[i].resize(dimsp); + for (long i = 0; i < Threading::ompNumMaxThreads; ++i) ztemp1_2[i].resize(twowaves); + for (long i = 0; i < Threading::ompNumMaxThreads; ++i) ztemp2_2[i].resize(twowaves); #else Varray<double> pnm(dimsp), hnm, ztemp1(twowaves), ztemp2(twowaves); if (needHnm) hnm.resize(dimsp); @@ -197,7 +197,7 @@ sp2fc(const double *sa, double *fa, const double *poli, long nlev, long nlat, lo cumindex[0] = 0; for (long jmm = 1; jmm < ntp1; jmm++) cumindex[jmm] = cumindex[jmm - 1] + (ntp1 - jmm + 1); - if (nlev >= Threading::ompNumThreads) + if (nlev >= Threading::ompNumMaxThreads) { #ifdef _OPENMP #pragma omp parallel for default(shared) @@ -268,7 +268,7 @@ fc2sp(const double *fa, double *sa, const double *poli, long nlev, long nlat, lo cumindex[0] = 0; for (long jmm = 1; jmm < ntp1; jmm++) cumindex[jmm] = cumindex[jmm - 1] + (ntp1 - jmm + 1); - if (nlev >= Threading::ompNumThreads) + if (nlev >= Threading::ompNumMaxThreads) { #ifdef _OPENMP #pragma omp parallel for default(shared) diff --git a/src/bitinformation.cc b/src/bitinformation.cc index e12c4212c..53158a4c5 100644 --- a/src/bitinformation.cc +++ b/src/bitinformation.cc @@ -134,7 +134,7 @@ bitpair_count(float *A, float *B, size_t n) uint32_t *Buint = (uint32_t *) B; #ifdef CDO - std::vector<BitpairCounters> BC(Threading::ompNumThreads); + std::vector<BitpairCounters> BC(Threading::ompNumMaxThreads); #else std::vector<BitpairCounters> BC(1); #endif @@ -158,7 +158,7 @@ bitpair_count(float *A, float *B, size_t n) #ifdef CDO #ifdef _OPENMP - for (int m = 1; m < Threading::ompNumThreads; ++m) + for (int m = 1; m < Threading::ompNumMaxThreads; ++m) { for (int k = 0; k < NBITS; ++k) for (int j = 0; j < 2; ++j) diff --git a/src/cdo.cc b/src/cdo.cc index 62addce2d..9502e107a 100644 --- a/src/cdo.cc +++ b/src/cdo.cc @@ -887,7 +887,7 @@ main(int argc, char *argv[]) cdo::set_external_proj_func(); cdo::set_stacksize(67108864); // 64MB cdo::set_coresize(Options::coresize); - cdo::setup_openMP(CDO_numThreads); + cdo::setup_openMP(Threading::ompNumUserRequestedThreads); if (cdo::dbg()) cdo_print_debug_info(); @@ -930,7 +930,7 @@ main(int argc, char *argv[]) if (Options::Timer) timer_report(allTimers); } - if (CDO_Rusage) cdo::features::print_rusage(); + if (Options::CDO_Rusage) cdo::features::print_rusage(); return Options::cdoExitStatus; } diff --git a/src/cdo_fctrans.cc b/src/cdo_fctrans.cc index 2b552c34d..b6cb55a84 100644 --- a/src/cdo_fctrans.cc +++ b/src/cdo_fctrans.cc @@ -29,9 +29,9 @@ fc2gp(const double *fc, double *gp, long nlat, long nlon, long nlev, long nfc) fftw_plan plan; }; - std::vector<FourierMemory> ompmem(Threading::ompNumThreads); + std::vector<FourierMemory> ompmem(Threading::ompNumMaxThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { ompmem[i].in_fft = fftw_alloc_complex(nlon / 2 + 1); ompmem[i].out_fft = (double *) fftw_malloc(nlon * sizeof(double)); @@ -72,7 +72,7 @@ fc2gp(const double *fc, double *gp, long nlat, long nlon, long nlev, long nfc) } } - for (int i = 0; i < Threading::ompNumThreads; ++i) + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { fftw_free(ompmem[i].in_fft); fftw_free(ompmem[i].out_fft); @@ -96,9 +96,9 @@ gp2fc(const double *gp, double *fc, long nlat, long nlon, long nlev, long nfc) fftw_plan plan; }; - std::vector<FourierMemory> ompmem(Threading::ompNumThreads); + std::vector<FourierMemory> ompmem(Threading::ompNumMaxThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { ompmem[i].in_fft = (double *) fftw_malloc(nlon * sizeof(double)); ompmem[i].out_fft = fftw_alloc_complex(nlon / 2 + 1); @@ -134,7 +134,7 @@ gp2fc(const double *gp, double *fc, long nlat, long nlon, long nlev, long nfc) } } - for (int i = 0; i < Threading::ompNumThreads; ++i) + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { fftw_free(ompmem[i].in_fft); fftw_free(ompmem[i].out_fft); diff --git a/src/cdo_fftw3.cc b/src/cdo_fftw3.cc index bd7af926d..9bdec59a8 100644 --- a/src/cdo_fftw3.cc +++ b/src/cdo_fftw3.cc @@ -23,9 +23,9 @@ fourier2grid(int gridID1, const Varray<double> &array1, Varray<double> &array2) fftw_plan plan; }; - std::vector<FourierMemory> ompmem(Threading::ompNumThreads); + std::vector<FourierMemory> ompmem(Threading::ompNumMaxThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { ompmem[i].in_fft = fftw_alloc_complex(nlon); ompmem[i].out_fft = (double *) fftw_malloc(nlon * sizeof(double)); @@ -55,7 +55,7 @@ fourier2grid(int gridID1, const Varray<double> &array1, Varray<double> &array2) for (size_t ilon = 0; ilon < nlon; ++ilon) array2[ilat * nlon + ilon] = out_fft[ilon]; } - for (int i = 0; i < Threading::ompNumThreads; ++i) + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { fftw_free(ompmem[i].in_fft); fftw_free(ompmem[i].out_fft); @@ -79,9 +79,9 @@ grid2fourier(int gridID1, const Varray<double> &array1, int gridID2, Varray<doub fftw_plan plan; }; - std::vector<FourierMemory> ompmem(Threading::ompNumThreads); + std::vector<FourierMemory> ompmem(Threading::ompNumMaxThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { ompmem[i].in_fft = (double *) fftw_malloc(nlon * sizeof(double)); ompmem[i].out_fft = fftw_alloc_complex(nlon); @@ -111,7 +111,7 @@ grid2fourier(int gridID1, const Varray<double> &array1, int gridID2, Varray<doub } } - for (int i = 0; i < Threading::ompNumThreads; ++i) + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { fftw_free(ompmem[i].in_fft); fftw_free(ompmem[i].out_fft); diff --git a/src/cdo_options.cc b/src/cdo_options.cc index 3f93a6053..59edac371 100644 --- a/src/cdo_options.cc +++ b/src/cdo_options.cc @@ -96,7 +96,8 @@ const char *cdoExpName = nullptr; namespace Threading { -int ompNumThreads = 1; +int ompNumMaxThreads = 1; +int ompNumUserRequestedThreads = 0; bool cdoLockIO = false; } // namespace Threading diff --git a/src/cdo_options.h b/src/cdo_options.h index 06588a43e..bb0e52924 100644 --- a/src/cdo_options.h +++ b/src/cdo_options.h @@ -88,7 +88,8 @@ extern bool REMAP_genweights; namespace Threading { -extern int ompNumThreads; +extern int ompNumMaxThreads; +extern int ompNumUserRequestedThreads; extern bool cdoLockIO; } // namespace Threading diff --git a/src/cdo_settings.cc b/src/cdo_settings.cc index 98c00d1b9..4989ab7b7 100644 --- a/src/cdo_settings.cc +++ b/src/cdo_settings.cc @@ -384,13 +384,13 @@ setup_openMP(int numThreads) if (numThreads <= 0) numThreads = 1; omp_set_num_threads(numThreads); - Threading::ompNumThreads = omp_get_max_threads(); + Threading::ompNumMaxThreads = omp_get_max_threads(); if (omp_get_max_threads() > omp_get_num_procs()) fprintf(stderr, "Warning: Number of OMP threads=%d is greater than number of Cores=%d!\n", omp_get_max_threads(), omp_get_num_procs()); - if (Threading::ompNumThreads < numThreads) - fprintf(stderr, "Warning: omp_get_max_threads() returns %d!\n", Threading::ompNumThreads); + if (Threading::ompNumMaxThreads < numThreads) + fprintf(stderr, "Warning: omp_get_max_threads() returns %d!\n", Threading::ompNumMaxThreads); if (cdo::dbg()) cdo::features::print_openmp_info(); diff --git a/src/cdo_settings.h b/src/cdo_settings.h index 0d28b82b4..f89e8d147 100644 --- a/src/cdo_settings.h +++ b/src/cdo_settings.h @@ -25,7 +25,7 @@ void set_chunktype(const std::string &arg); void evaluate_color_options(const std::string &arg); -void setup_openMP(int CDO_numThreads); +void setup_openMP(int ompNumUserRequestedThreads); }; // namespace cdo diff --git a/src/cdo_task.cc b/src/cdo_task.cc index 9a8d640ea..d41a59200 100644 --- a/src/cdo_task.cc +++ b/src/cdo_task.cc @@ -19,7 +19,7 @@ void Task::task(cdo::Task *taskInfo) { #if defined(_OPENMP) - omp_set_num_threads(Threading::ompNumThreads); // Has to be called for every thread! + omp_set_num_threads(Threading::ompNumMaxThreads); // Has to be called for every thread! #endif // cond.wait mutex must be locked before we can wait diff --git a/src/cellsearch_spherepart.h b/src/cellsearch_spherepart.h index b7855c6f2..b5669b995 100644 --- a/src/cellsearch_spherepart.h +++ b/src/cellsearch_spherepart.h @@ -77,8 +77,8 @@ private: create(size_t numCells, size_t numCorners, const Varray<double> &cornerLons, const Varray<double> &cornerLats) { Varray<enum yac_edge_type> edgeTypes(numCorners, YAC_GREAT_CIRCLE_EDGE); - Varray<yac_grid_cell> cells(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + Varray<yac_grid_cell> cells(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { cells[i].coordinates_xyz = new double[numCorners][3]; cells[i].edge_type = edgeTypes.data(); @@ -114,7 +114,7 @@ private: m_yacCellSearch = yac_bnd_sphere_part_search_new(m_bndCircles, numCells); - for (int i = 0; i < Threading::ompNumThreads; ++i) delete[] cells[i].coordinates_xyz; + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) delete[] cells[i].coordinates_xyz; } }; diff --git a/src/fileStream.cc b/src/fileStream.cc index f1e1a491a..a55664a3e 100644 --- a/src/fileStream.cc +++ b/src/fileStream.cc @@ -209,8 +209,8 @@ FileStream::def_vlist(int p_vlistID) if (Options::VersionInfo) cdiDefAttTxt(p_vlistID, CDI_GLOBAL, "CDO", (int) strlen(cdo_comment()), cdo_comment()); #ifdef _OPENMP - if (Threading::ompNumThreads > 1) - cdiDefAttInt(p_vlistID, CDI_GLOBAL, "cdo_openmp_thread_number", CDI_DATATYPE_INT32, 1, &Threading::ompNumThreads); + if (Threading::ompNumMaxThreads > 1) + cdiDefAttInt(p_vlistID, CDI_GLOBAL, "cdo_openmp_thread_number", CDI_DATATYPE_INT32, 1, &Threading::ompNumMaxThreads); #endif defDatarangeList(p_vlistID); diff --git a/src/hetaeta.cc b/src/hetaeta.cc index 08f955821..838699807 100644 --- a/src/hetaeta.cc +++ b/src/hetaeta.cc @@ -423,7 +423,7 @@ hetaeta(bool ltq, int ngp, const Vmask &imiss, int nlev1, const double *ah1, con long nlev1p1 = nlev1 + 1; long nlev2p1 = nlev2 + 1; - auto nthreads = Threading::ompNumThreads; + auto nthreads = Threading::ompNumMaxThreads; Varray2D<double> ph1(nthreads, Varray<double>(nlev1p1)); Varray2D<double> lnph1(nthreads, Varray<double>(nlev1p1)); Varray2D<double> fi1(nthreads, Varray<double>(nlev1p1)); @@ -576,7 +576,7 @@ template void hetaeta(bool ltq, int ngp, const Vmask &imiss, int nlev1, const do #ifdef TEST_HETAETA // g++ -DTEST_HETAETA -I../libcdi/src -std=c++20 -g -Wall -fopenmp hetaeta.cc -int Threading::ompNumThreads = 2; +int Threading::ompNumMaxThreads = 2; int cdo_omp_get_thread_num() @@ -594,7 +594,7 @@ main(int argc, char *argv[]) constexpr int NGP = 512; #ifdef _OPENMP - omp_set_num_threads(Threading::ompNumThreads); + omp_set_num_threads(Threading::ompNumMaxThreads); #endif printf("NGP = %d\n", NGP); diff --git a/src/pointsearch_kdtree.h b/src/pointsearch_kdtree.h index 8c8908654..498abbc69 100644 --- a/src/pointsearch_kdtree.h +++ b/src/pointsearch_kdtree.h @@ -132,7 +132,7 @@ private: // if (Options::cdoVerbose) cdo_print("BBOX: min=%g/%g/%g max=%g/%g/%g", min[0], min[1], min[2], max[0], max[1], max[2]); - m_kdtree = kd_buildTree(pointlist.data(), n, min, max, 3, Threading::ompNumThreads); + m_kdtree = kd_buildTree(pointlist.data(), n, min, max, 3, Threading::ompNumMaxThreads); // if (m_kdtree == nullptr) cdo_abort("kd_buildTree failed!"); } }; diff --git a/src/process.cc b/src/process.cc index 383c27456..d3555a61f 100644 --- a/src/process.cc +++ b/src/process.cc @@ -370,7 +370,7 @@ void Process::cdo_initialize() { #ifdef _OPENMP - omp_set_num_threads(Threading::ompNumThreads); // Has to be called for every module (pthread)! + omp_set_num_threads(Threading::ompNumMaxThreads); // Has to be called for every module (pthread)! #endif #ifdef HAVE_LIBPTHREAD threadID = pthread_self(); diff --git a/src/remap_conserv.cc b/src/remap_conserv.cc index 37a6b72bf..fe7fcd105 100644 --- a/src/remap_conserv.cc +++ b/src/remap_conserv.cc @@ -452,11 +452,11 @@ remap_conserv_weights(RemapSearch &remapSearch, RemapVars &rv) } } - Varray<GridCell> tgtGridCell2(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) gridcell_init_yac(tgtGridCell2[i], tgtNumCorners, tgtEdgeType); + Varray<GridCell> tgtGridCell2(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) gridcell_init_yac(tgtGridCell2[i], tgtNumCorners, tgtEdgeType); - Varray<CellSearch> cellSearch2(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + Varray<CellSearch> cellSearch2(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { cellSearch2[i].numCorners = srcNumCorners; cellSearch2[i].edgeType = srcEdgeType; @@ -474,7 +474,7 @@ remap_conserv_weights(RemapSearch &remapSearch, RemapVars &rv) size_t numSearchCellsStat[3] = { 0, 100000, 0 }; - Varray<Varray<size_t>> indices2(Threading::ompNumThreads); + Varray<Varray<size_t>> indices2(Threading::ompNumMaxThreads); // Loop over target grid cells @@ -560,7 +560,7 @@ remap_conserv_weights(RemapSearch &remapSearch, RemapVars &rv) } // Finished with all cells: deallocate search arrays - for (auto ompthID = 0; ompthID < Threading::ompNumThreads; ++ompthID) + for (auto ompthID = 0; ompthID < Threading::ompNumMaxThreads; ++ompthID) { cellSearch2[ompthID].free(); gridcell_free_yac(tgtGridCell2[ompthID]); @@ -652,11 +652,11 @@ remap_conserv(const Varray<T1> &srcArray, Varray<T2> &tgtArray, double srcMissva } } - Varray<GridCell> tgtGridCell2(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) gridcell_init_yac(tgtGridCell2[i], tgtNumCorners, tgtEdgeType); + Varray<GridCell> tgtGridCell2(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) gridcell_init_yac(tgtGridCell2[i], tgtNumCorners, tgtEdgeType); - Varray<CellSearch> cellSearch2(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) + Varray<CellSearch> cellSearch2(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) { cellSearch2[i].numCorners = srcNumCorners; cellSearch2[i].edgeType = srcEdgeType; @@ -671,7 +671,7 @@ remap_conserv(const Varray<T1> &srcArray, Varray<T2> &tgtArray, double srcMissva size_t numSearchCellsStat[3] = { 0, 100000, 0 }; - Varray<Varray<size_t>> indices2(Threading::ompNumThreads); + Varray<Varray<size_t>> indices2(Threading::ompNumMaxThreads); // Loop over target grid cells @@ -749,7 +749,7 @@ remap_conserv(const Varray<T1> &srcArray, Varray<T2> &tgtArray, double srcMissva // Finished with all cells: deallocate search arrays - for (auto ompthID = 0; ompthID < Threading::ompNumThreads; ++ompthID) + for (auto ompthID = 0; ompthID < Threading::ompNumMaxThreads; ++ompthID) { cellSearch2[ompthID].free(); gridcell_free_yac(tgtGridCell2[ompthID]); diff --git a/src/remap_knn.cc b/src/remap_knn.cc index f2fd4326a..00d68d0c1 100644 --- a/src/remap_knn.cc +++ b/src/remap_knn.cc @@ -36,8 +36,8 @@ remap_knn_weights(const KnnParams &knnParams, RemapSearch &rsearch, RemapVars &r weight_links_alloc(knnParams.k, tgtGridSize, weightLinks); std::vector<KnnData> knnDataList; - knnDataList.reserve(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) knnDataList.push_back(KnnData(knnParams)); + knnDataList.reserve(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) knnDataList.push_back(KnnData(knnParams)); cdo::timer timer; @@ -116,8 +116,8 @@ remap_knn(const Varray<T1> &srcArray, Varray<T2> &tgtArray, double srcMissval, s if (numMissVals) remap_set_mask(srcArray, srcGridSize, numMissVals, srcMissval, srcGridMask); std::vector<KnnData> knnDataList; - knnDataList.reserve(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) knnDataList.push_back(KnnData(knnParams)); + knnDataList.reserve(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) knnDataList.push_back(KnnData(knnParams)); cdo::timer timer; @@ -190,8 +190,8 @@ intgrid_knn(KnnParams knnParams, const Field &field1, Field &field2) if (field1.numMissVals) remap_set_mask(srcArray, srcGridSize, field1.numMissVals, srcMissval, srcGridMask); std::vector<KnnData> knnDataList; - knnDataList.reserve(Threading::ompNumThreads); - for (int i = 0; i < Threading::ompNumThreads; ++i) knnDataList.push_back(KnnData(knnParams)); + knnDataList.reserve(Threading::ompNumMaxThreads); + for (int i = 0; i < Threading::ompNumMaxThreads; ++i) knnDataList.push_back(KnnData(knnParams)); remap_search_init(mapType, remap.search, remap.srcGrid, remap.tgtGrid); diff --git a/src/remap_store_link.cc b/src/remap_store_link.cc index d69de6f3f..3d02d7a9a 100644 --- a/src/remap_store_link.cc +++ b/src/remap_store_link.cc @@ -119,7 +119,7 @@ weight_links_to_remap_links(int doAlloc, size_t tgtGridSize, std::vector<WeightL auto &tgtCellIndices = rv.tgtCellIndices; auto &weights = rv.weights; - auto useLinksArrays = (Threading::ompNumThreads > 1 && rv.numLinksPerValue == -1 && tgtGridSize > cdoMinLoopSize); + auto useLinksArrays = (Threading::ompNumMaxThreads > 1 && rv.numLinksPerValue == -1 && tgtGridSize > cdoMinLoopSize); if (useLinksArrays) rv.linksOffset.resize(tgtGridSize); if (useLinksArrays) rv.linksPerValue.resize(tgtGridSize); diff --git a/src/remap_vars.cc b/src/remap_vars.cc index 4acb4b421..c155eb06e 100644 --- a/src/remap_vars.cc +++ b/src/remap_vars.cc @@ -305,8 +305,8 @@ remap_laf(const Varray<T1> &srcArray, Varray<T2> &tgtArray, double tgtMissval, s auto max_cls = get_max_index(numLinks, tgtSize, tgtIndices); #ifdef _OPENMP - Varray2D<T1> src_cls2(Threading::ompNumThreads, Varray<T1>(max_cls)); - Varray2D<double> src_weights2(Threading::ompNumThreads, Varray<double>(max_cls)); + Varray2D<T1> src_cls2(Threading::ompNumMaxThreads, Varray<T1>(max_cls)); + Varray2D<double> src_weights2(Threading::ompNumMaxThreads, Varray<double>(max_cls)); #else Varray<T1> src_cls(max_cls); Varray<double> src_weights(max_cls); -- GitLab From 25343d7e7a537c1b762ffa201b61be59e96c76ff Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Wed, 15 Jan 2025 13:54:13 +0100 Subject: [PATCH 06/15] cdo.cc cleanup and distribution --- src/Makefile.am | 2 + src/cdo.cc | 498 +---------------------------------------- src/cdo_def_options.cc | 482 +++++++++++++++++++++++++++++++++++++++ src/cdo_def_options.h | 6 + src/cdo_options.cc | 1 + src/cdo_options.h | 1 + src/grid_pointsearch.h | 3 + src/table.cc | 17 ++ src/table.h | 3 +- 9 files changed, 520 insertions(+), 493 deletions(-) create mode 100644 src/cdo_def_options.cc create mode 100644 src/cdo_def_options.h diff --git a/src/Makefile.am b/src/Makefile.am index c01e042ea..1f4cb5532 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -19,6 +19,8 @@ libcdo_la_SOURCES = after_dvtrans.cc \ cdo_cmor.h \ cdo_data.cc \ cdo_data.h \ + cdo_def_options.cc \ + cdo_def_options.h \ cdo_default_values.cc \ cdo_default_values.h \ cdo_features.cc \ diff --git a/src/cdo.cc b/src/cdo.cc index 9502e107a..dc712f3ab 100644 --- a/src/cdo.cc +++ b/src/cdo.cc @@ -22,35 +22,26 @@ #include <cdi.h> -#include "cpp_lib.h" +//#include "cpp_lib.h" #include "cdo_timer.h" #include "cdo_getopt.h" #include "cdo_settings.h" #include "cdo_rlimit.h" -#include "mpim_grid.h" -#include "griddes.h" #include "cdo_default_values.h" #include "param_conversion.h" #include "progress.h" #include "module_info.h" -#include "percentiles.h" #include "util_wildcards.h" -#include "util_string.h" #include "process_int.h" #include "processManager.h" -#include "cdo_options.h" #include "commandline.h" #include "mpmo_color.h" #include "cdo_output.h" #include "cdo_features.h" -#include "cdo_zaxis.h" -#include "table.h" -#include "datetime.h" -#include "grid_cellsearch.h" #include "cdo_pthread.h" -#include "institution.h" #include "parser.h" #include "factory.h" +#include "cdo_def_options.h" static ProcessManager g_processManager; @@ -62,16 +53,12 @@ cdo_exit(std::string msg = "") exit(EXIT_FAILURE); } -static int CDO_numThreads = 0; -static int CDO_Rusage = 0; static bool applyDryRun = false; #ifdef HIRLAM_EXTENSIONS extern "C" void streamGrbDefDataScanningMode(int scanmode); #endif -void set_pointsearch_method(const std::string &methodStr); - static void cdo_display_syntax_help(const std::string &help, FILE *p_target) { @@ -262,22 +249,6 @@ cdo_print_debug_info() print_pthread_info(); } -static std::string -predefined_tables(int p_padding) -{ - const char *name; - constexpr int id_padding = 4; - int padding = p_padding + id_padding; - int numTables = tableInqNumber(); - std::string tables{ "Predefined tables: " }; - for (int id = 0; id < numTables; id++) - { - if (id % 7 == 6) tables += std::string("\n") + std::string(padding, ' '); - if ((name = tableInqNamePtr(id))) tables += std::string(name); - if (id < numTables - 1) tables += ","; - } - return tables; -} static void create_options_from_envvars() @@ -293,184 +264,7 @@ create_options_from_envvars() CLIOptions::option_from_envvar("CDO_TEST"); } -static void -setup_cli_options() -{ - CLIOptions::option("envvars") - ->add_effect([&]() { CLIOptions::print_envvars = true; }) - ->aborts_program(true) - ->set_category("Info") - ->add_help("Prints the environment variables of CDO."); - - CLIOptions::option("settings") - ->add_effect([&]() { CLIOptions::print_settings = true; }) - ->aborts_program(true) - ->set_category("Info") - ->add_help("Prints the settings of CDO."); - - CLIOptions::option("debug", "d") - ->add_effect([&]() { - unsigned cdoDebugLevel = 0; - unsigned cdiDebugLevel = 0; - cdo::parse_debug_arguments({ "1" }, cdoDebugLevel, cdiDebugLevel); - - cdiDebug(cdiDebugLevel); - cdo::set_debug(cdoDebugLevel); - cdo::features::version(); - }) - ->set_category("Output") - ->add_help("Pring all available debug messages"); - - CLIOptions::option("scoped_debug", "D") - ->describe_argument("comma seperated scopes") - ->set_category("Output") - ->on_empty_argument([]() { - std::cerr << "No debug level given please choose: " << std::endl; - print_debug_options(); - exit(EXIT_SUCCESS); - }) - ->add_effect([&](const std::string &argument) { - auto [success, tokens] = tokenize_comma_seperated_int_list(argument); - if (tokens.empty()) - { - print_debug_options(); - exit(EXIT_SUCCESS); - } - - unsigned cdoDebugLevel = 0; - unsigned cdiDebugLevel = 0; - cdo::parse_debug_arguments(tokens, cdoDebugLevel, cdiDebugLevel); - - cdiDebug(cdiDebugLevel); - cdo::set_debug(cdoDebugLevel); - - cdo::features::version(); - }) - ->add_help("Multiple scopes simultaneously possible. Use this option without arguments to get a list of possible scopes"); - - CLIOptions::option("worker") - ->describe_argument("num") - ->add_effect([&](const std::string &argument) { Options::numStreamWorker = parameter_to_int(argument); }) - ->set_category("Multi Threading") - ->add_help("Number of worker to decode/decompress GRIB records."); - - CLIOptions::option("precision") - ->describe_argument("float_digits[,double_digits]") - ->add_effect([&](const std::string &argument) { cdo::set_digits(argument); }) - ->set_category("Numeric") - ->add_help("Precision to use in displaying floating-point data (default: 7,15)."); - - CLIOptions::option("percentile") - ->describe_argument("method") - ->set_category("Numeric") - ->add_effect([&](const std::string &argument) { percentile_set_method(argument); }) - ->add_help("Methods: nrank, nist, rtype8, <NumPy method (linear|lower|higher|nearest|...)>"); - - CLIOptions::option("netcdf_hdr_pad") - ->describe_argument("nbr") - ->add_effect([&](const std::string &argument) { - int netcdf_hdr_pad = parameter_to_bytes(argument); - if (netcdf_hdr_pad >= 0) cdo::netcdf_hdr_pad = netcdf_hdr_pad; - }) - ->add_help("Pad NetCDF output header with nbr bytes."); - - CLIOptions::option("use_fftw") - ->describe_argument("true|false") - ->add_effect([&](const std::string &argument) { Options::Use_FFTW = (int) parameter_to_bool(argument); }) - ->add_help("Sets fftw usage."); - - CLIOptions::option("config") - ->describe_argument("all|all-json|<specific_feature_name>") - ->add_effect([&](const std::string &argument) { cdo::features::print_config(argument); }) - ->on_empty_argument([&]() { cdo::features::print_argument_options(); }) - ->aborts_program(true) - ->set_category("Info") - ->add_help("Prints all features and the enabled status.", "Use option <all> to see explicit feature names."); - - CLIOptions::option("pointsearchmethod") - ->set_internal(true) - ->describe_argument("<kdtree|nanoflann|spherepart|full>") - ->set_category("Search Methods") - ->add_effect([&](const std::string &argument) { set_pointsearch_method(argument); }) - ->add_help("Sets the point search method."); - - CLIOptions::option("gridsearchradius") - ->describe_argument("degrees[0..180]") - ->set_category("Search Methods") - ->add_effect([&](const std::string &argument) { - auto fval = radius_str_to_deg(argument); - if (fval < 0 || fval > 180) cdo_abort("%s=%g out of bounds (0-180 deg)!", "gridsearchradius", fval); - cdo_set_search_radius(fval); - }) - ->add_help("Sets the grid search radius (0-180 deg)."); - - CLIOptions::option("remap_weights") - ->describe_argument("0|1") - ->add_effect([&](const std::string &argument) { - auto intarg = parameter_to_int(argument); - if (intarg != 0 && intarg != 1) cdo_abort("Unsupported value for option --remap_weights %d [0/1]", intarg); - Options::REMAP_genweights = intarg; - }) - ->add_help("Generate remap weights (default: 1)."); - - CLIOptions::option("no_remap_weights") - ->add_effect([&]() { Options::REMAP_genweights = 0; }) - ->add_help("Switch off generation of remap weights."); - - CLIOptions::option("enableexcept") - ->describe_argument("except") - ->set_category("Numeric") - ->add_effect([&](const std::string &argument) { - auto except = cdo::evaluate_except_options(argument); - if (except < 0) cdo_abort("option --%s: unsupported argument: %s", "enableexcept", argument); - cdo::set_feenableexcept(except); - if (signal(SIGFPE, cdo::signal_handler) == SIG_ERR) cdo_warning("can't catch SIGFPE!"); - }) - ->add_help("Set individual floating-point traps ", "(DIVBYZERO, INEXACT, INVALID, OVERFLOW, UNDERFLOW, ALL_EXCEPT)"); - - CLIOptions::option("timestat_date") - ->describe_argument("srcdate") - ->add_effect([&](const std::string &argument) { set_timestat_date(argument); }) - ->add_help("Target timestamp (temporal statistics): ", "first, middle, midhigh or last source timestep."); - - CLIOptions::option("ignore_time_bounds") - ->add_effect([&]() { - extern bool CDO_Ignore_Time_Bounds; - CDO_Ignore_Time_Bounds = true; - }) - ->add_help("Ignores time bounds for time range statistics."); - - CLIOptions::option("use_time_bounds") - ->add_effect([&]() { - extern bool CDO_Use_Time_Bounds; - CDO_Use_Time_Bounds = true; - }) - ->add_help("Enables use of timebounds."); - - CLIOptions::option("cmor")->add_effect([&]() { Options::CMOR_Mode = 1; })->add_help("CMOR conform NetCDF output."); - - CLIOptions::option("reduce_dim")->add_effect([&]() { Options::CDO_Reduce_Dim = 1; })->add_help("Reduce NetCDF dimensions."); - - CLIOptions::option("float") - ->add_effect([&]() { Options::CDO_Memtype = MemType::Float; }) - ->set_category("Numeric") - ->add_help("Using single precision floats for data in memory."); - - CLIOptions::option("single") - ->add_effect([&]() { Options::CDO_Memtype = MemType::Float; }) - ->set_category("Numeric") - ->add_help("Using single precision floats for data in memory."); - - CLIOptions::option("double") - ->add_effect([&]() { Options::CDO_Memtype = MemType::Double; }) - ->set_category("Numeric") - ->add_help("Using double precision floats for data in memory."); - - CLIOptions::option("rusage") - ->add_effect([&]() { CDO_Rusage = 1; }) - ->add_help("Print information about resource utilization.") - ->set_category("Info"); - +static void setup_cli_options(){ CLIOptions::option("attribs") ->describe_argument("arbitrary|filesOnly|onlyFirst|noOutput|obase") ->aborts_program(true) @@ -504,32 +298,17 @@ setup_cli_options() } }) ->add_help("Prints list of operators."); - CLIOptions::option("operators_no_output") ->aborts_program(true) ->add_effect([&]() { print_operator_attributes("noOutput"); }) ->set_category("Info") ->add_help("Prints all operators which produce no output."); - - CLIOptions::option("pedantic")->add_effect([&]() { MpMO::enable_pedantic(true); })->add_help("Warnings count as errors."); - CLIOptions::option("color", "C") ->describe_argument("auto|no|all") ->add_effect([&](const std::string &argument) { cdo::evaluate_color_options(argument); }) ->set_category("Output") ->add_help("Set behaviour of colorized output messages."); - - CLIOptions::option("eccodes") - ->add_effect([&]() { cdiDefGlobal("ECCODES_GRIB1", true); }) - ->set_category("Format Specific") - ->add_help("Use ecCodes to decode/encode GRIB1 messages."); - - CLIOptions::option("format", "f") - ->describe_argument("grb1|grb2|nc1|nc2|nc4|nc4c|nc5|nczarr|srv|ext|ieg") - ->add_effect([&](const std::string &argument) { cdo::set_default_filetype(argument); }) - ->add_help("Format of the output file."); - - CLIOptions::option("help", "h") + CLIOptions::option("help", "h") ->describe_argument("operator") ->add_effect([&](const std::string &operator_name) { cdo_print_help(operator_name); }) ->on_empty_argument([]() { cdo_usage(stdout); }) @@ -537,245 +316,14 @@ setup_cli_options() ->set_category("Help") ->add_help("Shows either help information for the given operator or the usage of CDO."); - CLIOptions::option("history") - ->add_effect([&]() { Options::CDO_Append_History = true; }) - ->set_category("History") - ->add_help("Do append to NetCDF \"history\" global attribute."); - - CLIOptions::option("no_history") - ->add_effect([&]() { Options::CDO_Append_History = false; }) - ->set_category("History") - ->add_help("Do not append to NetCDF \"history\" global attribute."); - - CLIOptions::option("version", "V") - ->add_effect([&]() { cdo::features::version(); }) - ->aborts_program(true) - ->set_category("Info") - ->add_help("Print the version number."); - - CLIOptions::option("dryrun", "A")->add_effect([&]() { applyDryRun = true; })->add_help("Dry run that shows processed CDO call."); - - CLIOptions::option("absolute_taxis", "a") - ->add_effect([&]() { - if (CdoDefault::TaxisType == TAXIS_RELATIVE) - cdo_abort("option --%s: can't be combined with option --%s", "absolute_taxis (-a)", "relative_taxis (-r)"); - CdoDefault::TaxisType = TAXIS_ABSOLUTE; - }) - ->add_help("Generate an absolute time axis."); - - CLIOptions::option("force")->add_effect([&]() { Options::force = true; })->add_help("Forcing a CDO process."); - - CLIOptions::option("fast") - ->set_internal(true) - ->add_effect([&]() { - Options::fast = true; - cdiDefGlobal("NETCDF_LAZY_GRID_LOAD", true); - }) - ->add_help("If available, use a faster method even if it requires more memory."); - - // clang-format off - CLIOptions::option("default_datatype", "b") - ->describe_argument("nbits") - ->set_category("Numeric") - ->add_effect([&](const std::string &argument) { cdo::set_default_datatype(argument); }) - ->add_help("Set the number of bits for the output precision", - " I8|I16|I32|F32|F64 for nc1,nc2,nc4,nc4c,nc5,nczarr;", - " U8|U16|U32 for nc4,nc4c,nc5;", - " F32|F64 for grb2,srv,ext,ieg;", - " P1 - P24 for grb1,grb2"); - // clang-format on - - CLIOptions::option("check_data_range", "c") - ->add_effect([&]() { Options::CheckDatarange = true; }) - ->add_help("Enables checks for data overflow."); - - CLIOptions::option("grid", "g") - ->describe_argument("grid") - ->add_effect([&](const std::string &argument) { cdo_set_grids(argument); }) - ->add_help("Set default grid name or file. Available grids: ", - "F<XXX>, t<RES>, tl<RES>, r<NX>x<NY>, global_<DXY>, zonal_<DY>, gme<NI>, lon=<LON>/lat=<LAT>, hpz<ZOOM>"); - - CLIOptions::option("institution", "i") - ->describe_argument("institute_name") - ->add_effect([&](const std::string &argument) { define_institution(argument); }) - ->add_help("Sets institution name."); - - CLIOptions::option("chunktype", "k") - ->describe_argument("auto|grid|lines") - - ->set_category("Format Specific") - ->add_effect([&](const std::string &argument) { cdo::set_chunktype(argument); }) - ->add_help("NetCDF4 chunk type: auto, grid or lines."); - - CLIOptions::option("chunksize") - ->describe_argument("size") - ->set_category("Format Specific") - ->add_effect([&](const std::string &argument) { - int chunkSize = parameter_to_bytes(argument); - if (chunkSize >= 0) Options::cdoChunkSize = chunkSize; - }) - ->add_help("NetCDF4 chunk size."); - - CLIOptions::option("lock_io", "L")->add_effect([&]() { Threading::cdoLockIO = true; })->add_help("Lock IO (sequential access)."); - - CLIOptions::option("zaxis", "l") - ->describe_argument("zaxis") - ->add_effect([&](const std::string &argument) { cdo_set_zaxes(argument); }) - ->add_help("Set default zaxis name or file."); - - CLIOptions::option("set_missval", "m") - ->describe_argument("missval") - ->add_effect([&](const std::string &argument) { cdiDefMissval(std::stof(argument)); }) - ->add_help("Set the missing value of non NetCDF files (default: " + get_scientific(cdiInqMissval()) + ")."); - - CLIOptions::option("has_missval", "M") - ->add_effect([&]() { cdiDefGlobal("HAVE_MISSVAL", true); }) - ->add_help("Set HAS_MISSVAL to true."); - - CLIOptions::option("varnames", "n") - ->set_internal(true) - ->describe_argument("<varname| file>") - ->add_effect([&](const std::string &argument) { Options::cdoVarnames = split_string(argument, ","); }) - ->add_help("Set default varnames or file."); - CLIOptions::option("overwrite", "O") ->add_effect([&]() { Options::cdoOverwriteMode = true; }) ->add_help("Overwrite existing output file, if checked."); - CLIOptions::option("num_threads", "P") - ->describe_argument("nthreads") - ->add_effect([&](const std::string &argument) { CDO_numThreads = parameter_to_int(argument); }) - ->set_category("Multi Threading") - ->add_help("Set number of OpenMP threads."); - - CLIOptions::option("parrallel_read", "p") - ->set_internal(true) - ->add_effect([&]() { - Options::CDO_Parallel_Read = true; - Options::CDO_task = true; - }) - ->set_category("Multi Threading") - ->add_help("Enables parallel read."); - - CLIOptions::option("sortname", "Q") - ->add_effect([&]() { cdiDefGlobal("SORTNAME", true); }) - ->set_category("Format Specific") - ->add_help("Alphanumeric sorting of NetCDF parameter names."); - - CLIOptions::option("seed") - ->describe_argument("seed") - ->set_category("Numeric") - ->add_effect([&](const std::string &argument) { - int intarg = parameter_to_int(argument); - if (intarg < 0) cdo_abort("Unsupported value for option --seed %d [>=0]", intarg); - Options::Random_Seed = intarg; - }) - ->add_help("Seed for a new sequence of pseudo-random numbers. <seed> must be >= 0"); - - CLIOptions::option("regular", "R") - ->add_effect([&]() { - Options::cdoRegulargrid = true; - cdiDefGlobal("REGULARGRID", true); - }) - ->set_category("CGRIBEX") - ->add_help("Convert GRIB1 data from global reduced to regular Gaussian grid (cgribex only)."); - - CLIOptions::option("relative_taxis", "r") - ->add_effect([&]() { - if (CdoDefault::TaxisType == TAXIS_ABSOLUTE) - cdo_abort("option --%s: can't be combined with option --%s", "relative_taxis (-r)", "absolute_taxis (-a)"); - CdoDefault::TaxisType = TAXIS_RELATIVE; - }) - ->add_help("Generate a relative time axis."); - - CLIOptions::option("cdo_diagnostic", "S") - ->add_effect([&]() { Options::cdoDiag = true; }) - ->add_help("Create an extra output stream for the module TIMSTAT. This stream", - "contains the number of non missing values for each output period."); - - CLIOptions::option("silent", "s") - ->add_effect([&]() { - Options::silentMode = true; - MpMO::enable_silent_mode(Options::silentMode); - }) - ->set_category("Output") - ->add_help("Silent mode."); - - CLIOptions::option("timer", "T")->add_effect([&]() { Options::Timer = true; })->add_help("Enable timer."); - - CLIOptions::option("table", "t") - ->describe_argument("codetab") - ->set_category("CGRIBEX") - ->add_effect([&](const std::string &argument) { CdoDefault::TableID = cdo::define_table(argument); }) - ->add_help("Set GRIB1 default parameter code table name or file (cgribex only).", predefined_tables(CLIOptions::padding)); - CLIOptions::option("interactive", "u") ->add_effect([&]() { Options::cdoInteractive = true; }) ->add_help("Enable CDO interactive mode."); - CLIOptions::option("verbose", "v") - ->add_effect([&]() { - Options::cdoVerbose = true; - MpMO::enable_verbose(true); - CLIOptions::print_envvars = true; - gridEnableVerbose(Options::cdoVerbose); - }) - ->add_help("Print extra details for some operators."); - - CLIOptions::option("disable_warnings", "w") - ->add_effect([&]() { // disable warning messages - MpMO::enable_warnings(false); - extern int _Verbose; // CDI Warnings - _Verbose = 0; - }) - ->set_category("Output") - ->add_help("Disable warning messages."); - - CLIOptions::option("par_io", "X") - ->set_internal(true) - ->add_effect([&]() { - Options::cdoParIO = true; // multi threaded I/O - }) - ->add_help("Enables multithreaded I/O.") - ->set_category("Multi Threading"); - - CLIOptions::option("shuffle") - ->add_effect([&]() { Options::cdoShuffle = true; }) - ->set_category("Compression") - ->add_help("Specify shuffling of variable data bytes before compression (NetCDF)"); - - CLIOptions::option("compress", "Z") - ->add_effect([&]() { Options::cdoCompress = true; }) - ->set_category("Compression") - ->add_help("Enables compression. Default = SZIP"); - - CLIOptions::option("filter", "F") - ->describe_argument("filterspec") - ->add_effect([&](const std::string &argument) { cdo::set_filterspec(argument); }) - ->set_category("Compression") - ->add_help("NetCDF4 filter specification"); - - CLIOptions::option("compression_type", "z") - ->describe_argument("aec|jpeg|zip[_1-9]|zstd[1-19]") - ->set_category("Compression") - ->add_effect([&](const std::string &argument) { cdo::set_compression_type(argument); }) - ->add_help("aec AEC compression of GRIB2 records", "jpeg JPEG compression of GRIB2 records", - "zip[_1-9] Deflate compression of NetCDF4 variables", "zstd[_1-19] Zstandard compression of NetCDF4 variables"); - - CLIOptions::option("nsb") - ->set_internal(true) - ->describe_argument("1-23") - ->add_effect([&](const std::string &argument) { Options::nsb = parameter_to_int(argument); }) - ->set_category("Numeric") - ->add_help("Number of significant bits used for bit-rounding."); - - CLIOptions::option("show_available_options") - ->set_internal(true) - ->aborts_program(true) - ->set_category("Info") - ->add_effect([&]() { CLIOptions::print_available_options(); }) - ->add_help("Shows all available optins and prints all shortforms, only internal use for testing."); - CLIOptions::option("argument_groups") ->aborts_program(true) ->add_help("Explanation and Examples for subgrouping operators with [ ] syntax") @@ -788,43 +336,8 @@ setup_cli_options() ->add_effect([&]() { cdo_display_syntax_help(Parser::apply_help, stderr); }) ->set_category("Help"); - CLIOptions::option("sortparam")->add_effect([]() { cdiDefGlobal("SORTPARAM", true); }); - -#ifdef HIRLAM_EXTENSIONS - CLIOptions::option("Dkext") - ->describe_argument("debLev") - ->set_category("Hirlam Extension") - ->add_effect([&](const std::string &argument) { - auto extDebugVal = parameter_to_int(argument); - if (extDebugVal > 0) - { - extern int cdiDebugExt; - cdoDebugExt = extDebugVal; - cdiDebugExt = extDebugVal; - } - }) - ->add_help("Setting debugLevel for extensions."); - - CLIOptions::option("outputGribDataScanningMode") - ->describe_argument("mode") - ->set_category("Hirlam Extension") - ->add_effect([&](const std::string &argument) { - auto scanningModeValue = parameter_to_int(argument); - if (cdoDebugExt) printf("scanningModeValue=%d\n", scanningModeValue); + CLIOptions::option("dryrun", "A")->add_effect([&]() { applyDryRun = true; })->add_help("Dry run that shows processed CDO call."); - if ((scanningModeValue == 0) || (scanningModeValue == 64) || (scanningModeValue == 96)) - { - streamGrbDefDataScanningMode(scanningModeValue); // -1: not used; allowed modes: <0, - // 64, 96>; Default is 64 - } - else - { - cdo_warning("Warning: %d not in allowed modes: <0, 64, 96>; Using default: 64\n", scanningModeValue); - streamGrbDefDataScanningMode(64); - } - }) - ->add_help("Setting grib scanning mode for data in output file <0, 64, 96>.", "Default is 64"); -#endif // HIRLAM_EXTENSIONS } static void @@ -868,6 +381,7 @@ main(int argc, char *argv[]) create_options_from_envvars(); CLIOptions::get_env_vars(); + setup_options(); setup_cli_options(); auto CDO_optind = CLIOptions::parse(std::vector<std::string>(argv, argv + argc)); diff --git a/src/cdo_def_options.cc b/src/cdo_def_options.cc new file mode 100644 index 000000000..425e22b83 --- /dev/null +++ b/src/cdo_def_options.cc @@ -0,0 +1,482 @@ +#include <csignal> +#include "cdo_def_options.h" +#include "cdo_getopt.h" +#include "percentiles.h" +#include "cdo_options.h" +#include "cdo_default_values.h" +#include "util_string.h" +#include "cdo_features.h" +#include "griddes.h" +#include "cdo_output.h" +#include "param_conversion.h" +#include "cdo_settings.h" +#include "cdi.h" +#include "datetime.h" +#include "table.h" +#include "mpim_grid/mpim_grid.h" +#include "grid_pointsearch.h" +#include "institution.h" +#include "cdo_zaxis.h" + +void +setup_options() +{ + CLIOptions::option("envvars") + ->add_effect([&]() { CLIOptions::print_envvars = true; }) + ->aborts_program(true) + ->set_category("Info") + ->add_help("Prints the environment variables of CDO."); + + CLIOptions::option("settings") + ->add_effect([&]() { CLIOptions::print_settings = true; }) + ->aborts_program(true) + ->set_category("Info") + ->add_help("Prints the settings of CDO."); + + CLIOptions::option("debug", "d") + ->add_effect([&]() { + unsigned cdoDebugLevel = 0; + unsigned cdiDebugLevel = 0; + cdo::parse_debug_arguments({ "1" }, cdoDebugLevel, cdiDebugLevel); + + cdiDebug(cdiDebugLevel); + cdo::set_debug(cdoDebugLevel); + cdo::features::version(); + }) + ->set_category("Output") + ->add_help("Pring all available debug messages"); + + CLIOptions::option("scoped_debug", "D") + ->describe_argument("comma seperated scopes") + ->set_category("Output") + ->on_empty_argument([]() { + std::cerr << "No debug level given please choose: " << std::endl; + print_debug_options(); + exit(EXIT_SUCCESS); + }) + ->add_effect([&](const std::string &argument) { + auto [success, tokens] = tokenize_comma_seperated_int_list(argument); + if (tokens.empty()) + { + print_debug_options(); + exit(EXIT_SUCCESS); + } + + unsigned cdoDebugLevel = 0; + unsigned cdiDebugLevel = 0; + cdo::parse_debug_arguments(tokens, cdoDebugLevel, cdiDebugLevel); + + cdiDebug(cdiDebugLevel); + cdo::set_debug(cdoDebugLevel); + + cdo::features::version(); + }) + ->add_help("Multiple scopes simultaneously possible. Use this option without arguments to get a list of possible scopes"); + + CLIOptions::option("worker") + ->describe_argument("num") + ->add_effect([&](const std::string &argument) { Options::numStreamWorker = parameter_to_int(argument); }) + ->set_category("Multi Threading") + ->add_help("Number of worker to decode/decompress GRIB records."); + + CLIOptions::option("precision") + ->describe_argument("float_digits[,double_digits]") + ->add_effect([&](const std::string &argument) { cdo::set_digits(argument); }) + ->set_category("Numeric") + ->add_help("Precision to use in displaying floating-point data (default: 7,15)."); + + CLIOptions::option("percentile") + ->describe_argument("method") + ->set_category("Numeric") + ->add_effect([&](const std::string &argument) { percentile_set_method(argument); }) + ->add_help("Methods: nrank, nist, rtype8, <NumPy method (linear|lower|higher|nearest|...)>"); + + CLIOptions::option("netcdf_hdr_pad") + ->describe_argument("nbr") + ->add_effect([&](const std::string &argument) { + int netcdf_hdr_pad = parameter_to_bytes(argument); + if (netcdf_hdr_pad >= 0) cdo::netcdf_hdr_pad = netcdf_hdr_pad; + }) + ->add_help("Pad NetCDF output header with nbr bytes."); + + CLIOptions::option("use_fftw") + ->describe_argument("true|false") + ->add_effect([&](const std::string &argument) { Options::Use_FFTW = (int) parameter_to_bool(argument); }) + ->add_help("Sets fftw usage."); + + CLIOptions::option("config") + ->describe_argument("all|all-json|<specific_feature_name>") + ->add_effect([&](const std::string &argument) { cdo::features::print_config(argument); }) + ->on_empty_argument([&]() { cdo::features::print_argument_options(); }) + ->aborts_program(true) + ->set_category("Info") + ->add_help("Prints all features and the enabled status.", "Use option <all> to see explicit feature names."); + + CLIOptions::option("pointsearchmethod") + ->set_internal(true) + ->describe_argument("<kdtree|nanoflann|spherepart|full>") + ->set_category("Search Methods") + ->add_effect([&](const std::string &argument) { set_pointsearch_method(argument); }) + ->add_help("Sets the point search method."); + + CLIOptions::option("gridsearchradius") + ->describe_argument("degrees[0..180]") + ->set_category("Search Methods") + ->add_effect([&](const std::string &argument) { + auto fval = radius_str_to_deg(argument); + if (fval < 0 || fval > 180) cdo_abort("%s=%g out of bounds (0-180 deg)!", "gridsearchradius", fval); + cdo_set_search_radius(fval); + }) + ->add_help("Sets the grid search radius (0-180 deg)."); + + CLIOptions::option("remap_weights") + ->describe_argument("0|1") + ->add_effect([&](const std::string &argument) { + auto intarg = parameter_to_int(argument); + if (intarg != 0 && intarg != 1) cdo_abort("Unsupported value for option --remap_weights %d [0/1]", intarg); + Options::REMAP_genweights = intarg; + }) + ->add_help("Generate remap weights (default: 1)."); + + CLIOptions::option("no_remap_weights") + ->add_effect([&]() { Options::REMAP_genweights = 0; }) + ->add_help("Switch off generation of remap weights."); + + CLIOptions::option("enableexcept") + ->describe_argument("except") + ->set_category("Numeric") + ->add_effect([&](const std::string &argument) { + auto except = cdo::evaluate_except_options(argument); + if (except < 0) cdo_abort("option --%s: unsupported argument: %s", "enableexcept", argument); + cdo::set_feenableexcept(except); + if (signal(SIGFPE, cdo::signal_handler) == SIG_ERR) cdo_warning("can't catch SIGFPE!"); + }) + ->add_help("Set individual floating-point traps ", "(DIVBYZERO, INEXACT, INVALID, OVERFLOW, UNDERFLOW, ALL_EXCEPT)"); + + CLIOptions::option("timestat_date") + ->describe_argument("srcdate") + ->add_effect([&](const std::string &argument) { set_timestat_date(argument); }) + ->add_help("Target timestamp (temporal statistics): ", "first, middle, midhigh or last source timestep."); + + CLIOptions::option("ignore_time_bounds") + ->add_effect([&]() { + extern bool CDO_Ignore_Time_Bounds; + CDO_Ignore_Time_Bounds = true; + }) + ->add_help("Ignores time bounds for time range statistics."); + + CLIOptions::option("use_time_bounds") + ->add_effect([&]() { + extern bool CDO_Use_Time_Bounds; + CDO_Use_Time_Bounds = true; + }) + ->add_help("Enables use of timebounds."); + + CLIOptions::option("cmor")->add_effect([&]() { Options::CMOR_Mode = 1; })->add_help("CMOR conform NetCDF output."); + + CLIOptions::option("reduce_dim")->add_effect([&]() { Options::CDO_Reduce_Dim = 1; })->add_help("Reduce NetCDF dimensions."); + + CLIOptions::option("float") + ->add_effect([&]() { Options::CDO_Memtype = MemType::Float; }) + ->set_category("Numeric") + ->add_help("Using single precision floats for data in memory."); + + CLIOptions::option("single") + ->add_effect([&]() { Options::CDO_Memtype = MemType::Float; }) + ->set_category("Numeric") + ->add_help("Using single precision floats for data in memory."); + + CLIOptions::option("double") + ->add_effect([&]() { Options::CDO_Memtype = MemType::Double; }) + ->set_category("Numeric") + ->add_help("Using double precision floats for data in memory."); + + CLIOptions::option("rusage") + ->add_effect([&]() { Options::CDO_Rusage = 1; }) + ->add_help("Print information about resource utilization.") + ->set_category("Info"); + + CLIOptions::option("pedantic")->add_effect([&]() { MpMO::enable_pedantic(true); })->add_help("Warnings count as errors."); + + CLIOptions::option("eccodes") + ->add_effect([&]() { cdiDefGlobal("ECCODES_GRIB1", true); }) + ->set_category("Format Specific") + ->add_help("Use ecCodes to decode/encode GRIB1 messages."); + + CLIOptions::option("format", "f") + ->describe_argument("grb1|grb2|nc1|nc2|nc4|nc4c|nc5|nczarr|srv|ext|ieg") + ->add_effect([&](const std::string &argument) { cdo::set_default_filetype(argument); }) + ->add_help("Format of the output file."); + + + CLIOptions::option("history") + ->add_effect([&]() { Options::CDO_Append_History = true; }) + ->set_category("History") + ->add_help("Do append to NetCDF \"history\" global attribute."); + + CLIOptions::option("no_history") + ->add_effect([&]() { Options::CDO_Append_History = false; }) + ->set_category("History") + ->add_help("Do not append to NetCDF \"history\" global attribute."); + + CLIOptions::option("version", "V") + ->add_effect([&]() { cdo::features::version(); }) + ->aborts_program(true) + ->set_category("Info") + ->add_help("Print the version number."); + + CLIOptions::option("absolute_taxis", "a") + ->add_effect([&]() { + if (CdoDefault::TaxisType == TAXIS_RELATIVE) + cdo_abort("option --%s: can't be combined with option --%s", "absolute_taxis (-a)", "relative_taxis (-r)"); + CdoDefault::TaxisType = TAXIS_ABSOLUTE; + }) + ->add_help("Generate an absolute time axis."); + + CLIOptions::option("force")->add_effect([&]() { Options::force = true; })->add_help("Forcing a CDO process."); + + CLIOptions::option("fast") + ->set_internal(true) + ->add_effect([&]() { + Options::fast = true; + cdiDefGlobal("NETCDF_LAZY_GRID_LOAD", true); + }) + ->add_help("If available, use a faster method even if it requires more memory."); + + // clang-format off + CLIOptions::option("default_datatype", "b") + ->describe_argument("nbits") + ->set_category("Numeric") + ->add_effect([&](const std::string &argument) { cdo::set_default_datatype(argument); }) + ->add_help("Set the number of bits for the output precision", + " I8|I16|I32|F32|F64 for nc1,nc2,nc4,nc4c,nc5,nczarr;", + " U8|U16|U32 for nc4,nc4c,nc5;", + " F32|F64 for grb2,srv,ext,ieg;", + " P1 - P24 for grb1,grb2"); + // clang-format on + + CLIOptions::option("check_data_range", "c") + ->add_effect([&]() { Options::CheckDatarange = true; }) + ->add_help("Enables checks for data overflow."); + + CLIOptions::option("grid", "g") + ->describe_argument("grid") + ->add_effect([&](const std::string &argument) { cdo_set_grids(argument); }) + ->add_help("Set default grid name or file. Available grids: ", + "F<XXX>, t<RES>, tl<RES>, r<NX>x<NY>, global_<DXY>, zonal_<DY>, gme<NI>, lon=<LON>/lat=<LAT>, hpz<ZOOM>"); + + CLIOptions::option("institution", "i") + ->describe_argument("institute_name") + ->add_effect([&](const std::string &argument) { define_institution(argument); }) + ->add_help("Sets institution name."); + + CLIOptions::option("chunktype", "k") + ->describe_argument("auto|grid|lines") + + ->set_category("Format Specific") + ->add_effect([&](const std::string &argument) { cdo::set_chunktype(argument); }) + ->add_help("NetCDF4 chunk type: auto, grid or lines."); + + CLIOptions::option("chunksize") + ->describe_argument("size") + ->set_category("Format Specific") + ->add_effect([&](const std::string &argument) { + int chunkSize = parameter_to_bytes(argument); + if (chunkSize >= 0) Options::cdoChunkSize = chunkSize; + }) + ->add_help("NetCDF4 chunk size."); + + CLIOptions::option("lock_io", "L")->add_effect([&]() { Threading::cdoLockIO = true; })->add_help("Lock IO (sequential access)."); + + CLIOptions::option("zaxis", "l") + ->describe_argument("zaxis") + ->add_effect([&](const std::string &argument) { cdo_set_zaxes(argument); }) + ->add_help("Set default zaxis name or file."); + + CLIOptions::option("set_missval", "m") + ->describe_argument("missval") + ->add_effect([&](const std::string &argument) { cdiDefMissval(std::stof(argument)); }) + ->add_help("Set the missing value of non NetCDF files (default: " + get_scientific(cdiInqMissval()) + ")."); + + CLIOptions::option("has_missval", "M") + ->add_effect([&]() { cdiDefGlobal("HAVE_MISSVAL", true); }) + ->add_help("Set HAS_MISSVAL to true."); + + CLIOptions::option("varnames", "n") + ->set_internal(true) + ->describe_argument("<varname| file>") + ->add_effect([&](const std::string &argument) { Options::cdoVarnames = split_string(argument, ","); }) + ->add_help("Set default varnames or file."); + + + CLIOptions::option("num_threads", "P") + ->describe_argument("nthreads") + ->add_effect([&](const std::string &argument) { Threading::ompNumUserRequestedThreads = parameter_to_int(argument); }) + ->set_category("Multi Threading") + ->add_help("Set number of OpenMP threads."); + + CLIOptions::option("parrallel_read", "p") + ->set_internal(true) + ->add_effect([&]() { + Options::CDO_Parallel_Read = true; + Options::CDO_task = true; + }) + ->set_category("Multi Threading") + ->add_help("Enables parallel read."); + + CLIOptions::option("sortname", "Q") + ->add_effect([&]() { cdiDefGlobal("SORTNAME", true); }) + ->set_category("Format Specific") + ->add_help("Alphanumeric sorting of NetCDF parameter names."); + + CLIOptions::option("seed") + ->describe_argument("seed") + ->set_category("Numeric") + ->add_effect([&](const std::string &argument) { + int intarg = parameter_to_int(argument); + if (intarg < 0) cdo_abort("Unsupported value for option --seed %d [>=0]", intarg); + Options::Random_Seed = intarg; + }) + ->add_help("Seed for a new sequence of pseudo-random numbers. <seed> must be >= 0"); + + CLIOptions::option("regular", "R") + ->add_effect([&]() { + Options::cdoRegulargrid = true; + cdiDefGlobal("REGULARGRID", true); + }) + ->set_category("CGRIBEX") + ->add_help("Convert GRIB1 data from global reduced to regular Gaussian grid (cgribex only)."); + + CLIOptions::option("relative_taxis", "r") + ->add_effect([&]() { + if (CdoDefault::TaxisType == TAXIS_ABSOLUTE) + cdo_abort("option --%s: can't be combined with option --%s", "relative_taxis (-r)", "absolute_taxis (-a)"); + CdoDefault::TaxisType = TAXIS_RELATIVE; + }) + ->add_help("Generate a relative time axis."); + + CLIOptions::option("cdo_diagnostic", "S") + ->add_effect([&]() { Options::cdoDiag = true; }) + ->add_help("Create an extra output stream for the module TIMSTAT. This stream", + "contains the number of non missing values for each output period."); + + CLIOptions::option("silent", "s") + ->add_effect([&]() { + Options::silentMode = true; + MpMO::enable_silent_mode(Options::silentMode); + }) + ->set_category("Output") + ->add_help("Silent mode."); + + CLIOptions::option("timer", "T")->add_effect([&]() { Options::Timer = true; })->add_help("Enable timer."); + + CLIOptions::option("table", "t") + ->describe_argument("codetab") + ->set_category("CGRIBEX") + ->add_effect([&](const std::string &argument) { CdoDefault::TableID = cdo::define_table(argument); }) + ->add_help("Set GRIB1 default parameter code table name or file (cgribex only).", cdo::predefined_tables(CLIOptions::padding)); + + + CLIOptions::option("sortparam")->add_effect([]() { cdiDefGlobal("SORTPARAM", true); }); + + CLIOptions::option("verbose", "v") + ->add_effect([&]() { + Options::cdoVerbose = true; + MpMO::enable_verbose(true); + CLIOptions::print_envvars = true; + gridEnableVerbose(Options::cdoVerbose); + }) + ->add_help("Print extra details for some operators."); + + CLIOptions::option("disable_warnings", "w") + ->add_effect([&]() { // disable warning messages + MpMO::enable_warnings(false); + extern int _Verbose; // CDI Warnings + _Verbose = 0; + }) + ->set_category("Output") + ->add_help("Disable warning messages."); + + CLIOptions::option("par_io", "X") + ->set_internal(true) + ->add_effect([&]() { + Options::cdoParIO = true; // multi threaded I/O + }) + ->add_help("Enables multithreaded I/O.") + ->set_category("Multi Threading"); + + CLIOptions::option("shuffle") + ->add_effect([&]() { Options::cdoShuffle = true; }) + ->set_category("Compression") + ->add_help("Specify shuffling of variable data bytes before compression (NetCDF)"); + + CLIOptions::option("compress", "Z") + ->add_effect([&]() { Options::cdoCompress = true; }) + ->set_category("Compression") + ->add_help("Enables compression. Default = SZIP"); + + CLIOptions::option("filter", "F") + ->describe_argument("filterspec") + ->add_effect([&](const std::string &argument) { cdo::set_filterspec(argument); }) + ->set_category("Compression") + ->add_help("NetCDF4 filter specification"); + + CLIOptions::option("compression_type", "z") + ->describe_argument("aec|jpeg|zip[_1-9]|zstd[1-19]") + ->set_category("Compression") + ->add_effect([&](const std::string &argument) { cdo::set_compression_type(argument); }) + ->add_help("aec AEC compression of GRIB2 records", "jpeg JPEG compression of GRIB2 records", + "zip[_1-9] Deflate compression of NetCDF4 variables", "zstd[_1-19] Zstandard compression of NetCDF4 variables"); + + CLIOptions::option("nsb") + ->set_internal(true) + ->describe_argument("1-23") + ->add_effect([&](const std::string &argument) { Options::nsb = parameter_to_int(argument); }) + ->set_category("Numeric") + ->add_help("Number of significant bits used for bit-rounding."); + + CLIOptions::option("show_available_options") + ->set_internal(true) + ->aborts_program(true) + ->set_category("Info") + ->add_effect([&]() { CLIOptions::print_available_options(); }) + ->add_help("Shows all available optins and prints all shortforms, only internal use for testing."); + + + +#ifdef HIRLAM_EXTENSIONS + CLIOptions::option("Dkext") + ->describe_argument("debLev") + ->set_category("Hirlam Extension") + ->add_effect([&](const std::string &argument) { + auto extDebugVal = parameter_to_int(argument); + if (extDebugVal > 0) + { + extern int cdiDebugExt; + cdoDebugExt = extDebugVal; + cdiDebugExt = extDebugVal; + } + }) + ->add_help("Setting debugLevel for extensions."); + + CLIOptions::option("outputGribDataScanningMode") + ->describe_argument("mode") + ->set_category("Hirlam Extension") + ->add_effect([&](const std::string &argument) { + auto scanningModeValue = parameter_to_int(argument); + if (cdoDebugExt) printf("scanningModeValue=%d\n", scanningModeValue); + + if ((scanningModeValue == 0) || (scanningModeValue == 64) || (scanningModeValue == 96)) + { + streamGrbDefDataScanningMode(scanningModeValue); // -1: not used; allowed modes: <0, + // 64, 96>; Default is 64 + } + else + { + cdo_warning("Warning: %d not in allowed modes: <0, 64, 96>; Using default: 64\n", scanningModeValue); + streamGrbDefDataScanningMode(64); + } + }) + ->add_help("Setting grib scanning mode for data in output file <0, 64, 96>.", "Default is 64"); +#endif // HIRLAM_EXTENSIONS +} diff --git a/src/cdo_def_options.h b/src/cdo_def_options.h new file mode 100644 index 000000000..27d3b2b7d --- /dev/null +++ b/src/cdo_def_options.h @@ -0,0 +1,6 @@ +#ifndef CDO_DEF_OPTIONS_H +#define CDO_DEF_OPTIONS_H + +void setup_options(); + +#endif diff --git a/src/cdo_options.cc b/src/cdo_options.cc index 59edac371..94fecf5da 100644 --- a/src/cdo_options.cc +++ b/src/cdo_options.cc @@ -52,6 +52,7 @@ int cdoCompType = CDI_COMPRESS_NONE; int cdoCompLevel = 0; bool cdoInteractive = false; bool cdoVerbose = false; +int CDO_Rusage = 0; int cdoExitStatus = 0; bool Timer = false; diff --git a/src/cdo_options.h b/src/cdo_options.h index bb0e52924..008d65e8a 100644 --- a/src/cdo_options.h +++ b/src/cdo_options.h @@ -48,6 +48,7 @@ extern int cdoCompType; extern int cdoCompLevel; extern bool cdoInteractive; extern bool cdoVerbose; +extern int CDO_Rusage; extern bool cdoProcessInfo; extern int cdoExitStatus; extern bool Timer; diff --git a/src/grid_pointsearch.h b/src/grid_pointsearch.h index 43c511943..f56b15b14 100644 --- a/src/grid_pointsearch.h +++ b/src/grid_pointsearch.h @@ -55,6 +55,9 @@ public: PointsearchUnstruct unstruct; }; + +void set_pointsearch_method(const std::string &methodStr); + void grid_search_point_unstruct(GridPointsearch &gps, const PointLonLat &pointLL, KnnData &knnData); void grid_search_point_smooth(GridPointsearch &gps, const PointLonLat &pointLL, KnnData &knnData); diff --git a/src/table.cc b/src/table.cc index b5fd1fbf2..7c48d2be3 100644 --- a/src/table.cc +++ b/src/table.cc @@ -40,4 +40,21 @@ define_table(const std::string &tablearg) return tableID; } +std::string +predefined_tables(int p_padding) +{ + const char *name; + constexpr int id_padding = 4; + int padding = p_padding + id_padding; + int numTables = tableInqNumber(); + std::string tables{ "Predefined tables: " }; + for (int id = 0; id < numTables; id++) + { + if (id % 7 == 6) tables += std::string("\n") + std::string(padding, ' '); + if ((name = tableInqNamePtr(id))) tables += std::string(name); + if (id < numTables - 1) tables += ","; + } + return tables; +} + } // namespace cdo diff --git a/src/table.h b/src/table.h index 6630436b4..07d53b8da 100644 --- a/src/table.h +++ b/src/table.h @@ -7,7 +7,8 @@ namespace cdo { int define_table(const std::string &tablearg); +std::string predefined_tables(int p_padding); -} +} // namespace cdo #endif -- GitLab From a73d5817f6a7b0a8e515e8dfc1c77bd8343604d9 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Wed, 15 Jan 2025 13:59:57 +0100 Subject: [PATCH 07/15] added cdo_def_options.h to cdolib interface --- include/cdo.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/cdo.h b/include/cdo.h index cc19cc3e6..1539884fe 100644 --- a/include/cdo.h +++ b/include/cdo.h @@ -5,6 +5,7 @@ #include <node.h> #include <parser.h> #include <factory.h> +#include <cdo_def_options.h> #include <cdo_node_attach_exception.h> #include <cdo_exception.h> #include <cdo_module.h> -- GitLab From 278bf75ced1972645200c10c8dcf221b851e26a9 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Wed, 15 Jan 2025 14:02:12 +0100 Subject: [PATCH 08/15] added cdo_def_options to CMakeLists.txt --- src/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 334b73d6d..3c7f3e3cc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -130,6 +130,8 @@ list( APPEND cdolib_src_files cdo_cmor.h cdo_data.cc cdo_data.h + cdo_def_options.cc + cdo_def_options.h cdo_default_values.cc cdo_default_values.h cdo_exception.h -- GitLab From 2528cedad79f7ca35c2ce4c2ee63178a44256184 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 20 Jan 2025 10:17:06 +0100 Subject: [PATCH 09/15] added file type to debug message --- src/memoryStream.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memoryStream.cc b/src/memoryStream.cc index 4d19acf48..a37725ffc 100644 --- a/src/memoryStream.cc +++ b/src/memoryStream.cc @@ -68,7 +68,7 @@ MemoryStream::open_write(int p_filetype) set_compression(m_fileID, CDI_FILETYPE_NC4); m_filetype = CDI_FILETYPE_NC4; - Debug(FILE_STREAM, "finished open_write"); + Debug(FILE_STREAM, "finished open_write with filetype set to: %d", inqFileType()); return m_fileID; } -- GitLab From 57819e9431d70719d856d7fcc1ad398824e9d4c8 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 20 Jan 2025 10:17:45 +0100 Subject: [PATCH 10/15] fixed node type check being in the wrong place --- src/processManager.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/processManager.cc b/src/processManager.cc index 2d0aa173f..900e7a50e 100644 --- a/src/processManager.cc +++ b/src/processManager.cc @@ -141,8 +141,6 @@ ProcessManager::buildProcessTree(std::vector<std::shared_ptr<Node>> roots) Debug(PROCESS, "adding out files to %s", node->oper); Debug(PROCESS, "node type = %d", n->type); if (n->type == Node::NodeType::OUTFILE) { first_process->add_file_out_stream(n->oper); } - else if (n->type == Node::NodeType::OUT_MEM_BUFFER) { first_process->add_mem_out_stream(n->ncid); } - else { cdo_abort("Unkown stream type"); } } } -- GitLab From 401650daf38fc47ba5be04582b1caedfe8629ad8 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 20 Jan 2025 10:18:14 +0100 Subject: [PATCH 11/15] readded bandit tests in CMakeLists.txt --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0e0a5545c..d2de7728c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,2 +1,2 @@ -add_subdirectory(bandit_tests) add_subdirectory(pytest) +add_subdirectory(bandit_tests) -- GitLab From c190c80569c905bc2953980117f2ddc0855d1aa4 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 20 Jan 2025 10:18:47 +0100 Subject: [PATCH 12/15] removed test_operator_args from CMakeLists.txt, unused feature --- test/bandit_tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bandit_tests/CMakeLists.txt b/test/bandit_tests/CMakeLists.txt index 72845b4b8..65e74144c 100644 --- a/test/bandit_tests/CMakeLists.txt +++ b/test/bandit_tests/CMakeLists.txt @@ -14,7 +14,7 @@ add_executable(test_param_conversion_test param_conversion_test.cc) add_executable(test_pmlist pmlist.cc) add_executable(test_util_string util_string.cc) add_executable(test_wildcards wildcards.cc) -add_executable(test_operator_args operator_args.cc) +#add_executable(test_operator_args operator_args.cc) set(labels_test_wildcards ENABLE_ON_HAVE_WORDEXP) -- GitLab From 50604d5ef8a99e153c2e169ec947cda967e2d961 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 20 Jan 2025 10:19:19 +0100 Subject: [PATCH 13/15] added minimum version of netCDF to CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f797f03af..b4f889d67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,7 @@ ELSE() ENDIF() -find_package(netCDF REQUIRED) +find_package(netCDF 4.0 REQUIRED) set(netcdf_flag HAVE_LIBNETCDF) if (${netCDF_FOUND}) add_compile_definitions(${netcdf_flag}=${netCDF_FOUND}) -- GitLab From 808511b6ea126af4d2388f6bd15bcd3097f29df4 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 20 Jan 2025 10:46:12 +0100 Subject: [PATCH 14/15] fixed shadow warning for function argument optarg --- src/datetime.cc | 12 ++++++------ src/datetime.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/datetime.cc b/src/datetime.cc index e82cf5cae..e260cc719 100644 --- a/src/datetime.cc +++ b/src/datetime.cc @@ -37,18 +37,18 @@ time_units_cstr(TimeUnits timeUnit) } void -set_timestat_date(const std::string &optarg) +set_timestat_date(const std::string &p_optarg) { TimeStat timestatdate = TimeStat::UNDEF; // clang-format off - if (optarg == "first") timestatdate = TimeStat::FIRST; - else if (optarg == "last") timestatdate = TimeStat::LAST; - else if (optarg == "middle") timestatdate = TimeStat::MEAN; - else if (optarg == "midhigh") timestatdate = TimeStat::MIDHIGH; + if (p_optarg == "first") timestatdate = TimeStat::FIRST; + else if (p_optarg == "last") timestatdate = TimeStat::LAST; + else if (p_optarg == "middle") timestatdate = TimeStat::MEAN; + else if (p_optarg == "midhigh") timestatdate = TimeStat::MIDHIGH; // clang-format on - if (timestatdate == TimeStat::UNDEF) cdo_abort("option --%s: unsupported argument: %s", "timestat_date", optarg); + if (timestatdate == TimeStat::UNDEF) cdo_abort("option --%s: unsupported argument: %s", "timestat_date", p_optarg); CDO_Timestat_Date = timestatdate; } diff --git a/src/datetime.h b/src/datetime.h index ec510c7de..61cda014e 100644 --- a/src/datetime.h +++ b/src/datetime.h @@ -111,7 +111,7 @@ private: }; CdiDateTime datetime_avg(int calendar, int ndates, const std::vector<CdiDateTime> &cdiDateTimes); -void set_timestat_date(const std::string &optarg); +void set_timestat_date(const std::string &p_optarg); void adjust_month_and_year(int &month, int &year); -- GitLab From 8695bace2e7734bf694ad1dffcb9cc5865365130 Mon Sep 17 00:00:00 2001 From: Oliver Heidmann <oliver.heidmann@mpimet.mpg.de> Date: Mon, 20 Jan 2025 12:13:02 +0100 Subject: [PATCH 15/15] fixed wring function being used --- src/cdo_varlist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdo_varlist.h b/src/cdo_varlist.h index 780a3641c..d6836171a 100644 --- a/src/cdo_varlist.h +++ b/src/cdo_varlist.h @@ -67,7 +67,7 @@ public: VarList(int _vlistID) : vlistID(_vlistID) { cdoVars_init(vars, _vlistID); - m_maxFields = vlistNumGrids(_vlistID); + m_maxFields = vlistNumFields(_vlistID); m_numSteps = vlistNtsteps(_vlistID); m_numConstVars = num_const_vars(vars); m_numVaryingVars = num_varying_vars(vars); -- GitLab