From adc9ada30aa3014a4dc539dbdc1c973adc86a9d9 Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Tue, 4 Mar 2025 11:25:27 +0100
Subject: [PATCH 1/2] Detrend: failed if missing_value is between 0 and
numSteps (bug fix)
---
ChangeLog | 4 ++++
libcdi | 2 +-
src/Detrend.cc | 12 ++++++------
src/Trendarith.cc | 43 ++++++++++++++++++++++++++++---------------
src/field_trend.cc | 22 +++++++++-------------
5 files changed, 48 insertions(+), 35 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index add4402e3..b4cd3f098 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,10 @@
* Using YAC version 3.4.0
* Version 2.5.1 release
+2025-03-04 Uwe Schulzweida
+
+ * Detrend: failed if missing_value is between 0 and numSteps (bug fix)
+
2025-03-03 Uwe Schulzweida
* New operator: air_density
diff --git a/libcdi b/libcdi
index 34bec0e7a..8dbf0a5b3 160000
--- a/libcdi
+++ b/libcdi
@@ -1 +1 @@
-Subproject commit 34bec0e7aae27fe52d28d879baedd052e7e437fe
+Subproject commit 8dbf0a5b3427ca421f488041bbfa032e2e4d21dd
diff --git a/src/Detrend.cc b/src/Detrend.cc
index 3cea78432..1bf09713d 100644
--- a/src/Detrend.cc
+++ b/src/Detrend.cc
@@ -124,9 +124,9 @@ public:
auto ¶mB = work[1][varID][levelID].vec_d;
auto &sumj = work[0][varID][levelID].vec_d;
auto &sumjj = work[1][varID][levelID].vec_d;
- const auto &sumjx = work[2][varID][levelID].vec_d;
- const auto &sumx = work[3][varID][levelID].vec_d;
- const auto &zn = work[4][varID][levelID].vec_d;
+ auto const &sumjx = work[2][varID][levelID].vec_d;
+ auto const &sumx = work[3][varID][levelID].vec_d;
+ auto const &zn = work[4][varID][levelID].vec_d;
auto trend_kernel = [&](auto i, auto is_EQ) {
auto temp1 = SUBM(sumjx[i], DIVM(MULM(sumj[i], sumx[i]), zn[i]));
@@ -151,13 +151,13 @@ public:
auto numVars = varList.numVars();
for (int varID = 0; varID < numVars; ++varID)
{
- const auto &var = varList.vars[varID];
+ auto const &var = varList.vars[varID];
if (var.isConstant) continue;
for (int levelID = 0; levelID < var.nlevels; ++levelID)
{
auto &field = varsData[varID][levelID];
- const auto ¶mA = work[0][varID][levelID];
- const auto ¶mB = work[1][varID][levelID];
+ auto const ¶mA = work[0][varID][levelID];
+ auto const ¶mB = work[1][varID][levelID];
sub_trend(zj, field, paramA, paramB);
}
}
diff --git a/src/Trendarith.cc b/src/Trendarith.cc
index e873a860d..83caee17b 100644
--- a/src/Trendarith.cc
+++ b/src/Trendarith.cc
@@ -16,6 +16,7 @@
#include "process_int.h"
#include "cdo_vlist.h"
+#include "cdo_omp.h"
#include "datetime.h"
#include "pmlist.h"
#include "param_conversion.h"
@@ -30,12 +31,18 @@ add_trend(double zj, Varray<T> &v1, const Varray<double> &v2, const Varray<doubl
auto missval1 = mv;
auto missval2 = mv;
- auto add_kernel = [&](auto i, auto is_EQ) { return ADDM(v1[i], ADDM(v2[i], MULM(v3[i], zj))); };
+ auto add_kernel = [&](auto is_EQ) {
+#ifdef _OPENMP
+#pragma omp parallel for if (n > cdoMinLoopSize) default(shared) schedule(static)
+#endif
+ for (size_t i = 0; i < n; ++i)
+ {
+ auto tmp = (is_EQ(v2[i], mv) || is_EQ(v3[i], mv)) ? mv : (v2[i] + v3[i] * zj);
+ v1[i] = ADDM(v1[i], tmp);
+ }
+ };
- if (std::isnan(missval1))
- for (size_t i = 0; i < n; ++i) v1[i] = add_kernel(i, fp_is_equal);
- else
- for (size_t i = 0; i < n; ++i) v1[i] = add_kernel(i, is_equal);
+ std::isnan(mv) ? add_kernel(fp_is_equal) : add_kernel(is_equal);
}
static void
@@ -52,12 +59,18 @@ sub_trend(double zj, Varray<T> &v1, const Varray<double> &v2, const Varray<doubl
auto missval1 = mv;
auto missval2 = mv;
- auto sub_kernel = [&](auto i, auto is_EQ) { return SUBM(v1[i], ADDM(v2[i], MULM(v3[i], zj))); };
+ auto sub_kernel = [&](auto is_EQ) {
+#ifdef _OPENMP
+#pragma omp parallel for if (n > cdoMinLoopSize) default(shared) schedule(static)
+#endif
+ for (size_t i = 0; i < n; ++i)
+ {
+ auto tmp = (is_EQ(v2[i], mv) || is_EQ(v3[i], mv)) ? mv : (v2[i] + v3[i] * zj);
+ v1[i] = SUBM(v1[i], tmp);
+ }
+ };
- if (std::isnan(missval1))
- for (size_t i = 0; i < n; ++i) v1[i] = sub_kernel(i, fp_is_equal);
- else
- for (size_t i = 0; i < n; ++i) v1[i] = sub_kernel(i, is_equal);
+ std::isnan(mv) ? sub_kernel(fp_is_equal) : sub_kernel(is_equal);
}
static void
@@ -125,7 +138,6 @@ public:
bool tstepIsEqual;
VarList varList1;
- FieldVector2D vars2, vars3;
public:
void
@@ -161,7 +173,12 @@ public:
streamID4 = cdo_open_write(3);
cdo_def_vlist(streamID4, vlistID4);
+ }
+ void
+ run() override
+ {
+ FieldVector2D vars2, vars3;
field2D_init(vars2, varList1, FIELD_VEC);
field2D_init(vars3, varList1, FIELD_VEC);
@@ -182,11 +199,7 @@ public:
cdo_read_field(streamID3, vars3[varID][levelID]);
}
}
- }
- void
- run() override
- {
auto calendar = taxisInqCalendar(taxisID1);
CheckTimeIncr checkTimeIncr;
JulianDate julianDate0;
diff --git a/src/field_trend.cc b/src/field_trend.cc
index c27770e00..e2cac1fd3 100644
--- a/src/field_trend.cc
+++ b/src/field_trend.cc
@@ -64,22 +64,18 @@ sub_trend(double zj, Varray<T> &v1, const Varray<double> &v2, const Varray<doubl
auto missval1 = mv;
auto missval2 = mv;
- auto sub_kernel = [&](auto i, auto is_EQ) { return SUBM(v1[i], ADDM(v2[i], MULM(v3[i], zj))); };
-
- if (std::isnan(missval1))
- {
-#ifdef _OPENMP
-#pragma omp parallel for if (len > cdoMinLoopSize) default(shared) schedule(static)
-#endif
- for (size_t i = 0; i < len; ++i) { v1[i] = sub_kernel(i, fp_is_equal); }
- }
- else
- {
+ auto sub_kernel = [&](auto is_EQ) {
#ifdef _OPENMP
#pragma omp parallel for if (len > cdoMinLoopSize) default(shared) schedule(static)
#endif
- for (size_t i = 0; i < len; ++i) { v1[i] = sub_kernel(i, is_equal); }
- }
+ for (size_t i = 0; i < len; ++i)
+ {
+ auto tmp = (is_EQ(v2[i], mv) || is_EQ(v3[i], mv)) ? mv : (v2[i] + v3[i] * zj);
+ v1[i] = SUBM(v1[i], tmp);
+ }
+ };
+
+ std::isnan(mv) ? sub_kernel(fp_is_equal) : sub_kernel(is_equal);
}
void
--
GitLab
From e2e5bda78a78a67a0d112b5b83fcebf052a3baeb Mon Sep 17 00:00:00 2001
From: Uwe Schulzweida <uwe.schulzweida@mpimet.mpg.de>
Date: Tue, 4 Mar 2025 14:47:05 +0100
Subject: [PATCH 2/2] Renamed CDO_Parallel_Read to CDO_Async_Read
---
src/Detrend.cc | 2 +-
src/Diff.cc | 2 +-
src/Info.cc | 2 +-
src/Longinfo.cc | 2 +-
src/Timstat.cc | 3 +--
src/Trend.cc | 2 +-
src/cdo.cc | 9 +++++++++
src/cdo_def_options.cc | 4 ++--
src/cdo_options.cc | 2 +-
src/cdo_options.h | 2 +-
10 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/src/Detrend.cc b/src/Detrend.cc
index 1bf09713d..1eb9a3ea9 100644
--- a/src/Detrend.cc
+++ b/src/Detrend.cc
@@ -181,7 +181,7 @@ public:
void
run() override
{
- auto runAsync = (Options::CDO_Parallel_Read > 0);
+ auto runAsync = (Options::CDO_Async_Read > 0);
auto task = runAsync ? std::make_unique<cdo::Task>() : nullptr;
auto calendar = taxisInqCalendar(taxisID1);
diff --git a/src/Diff.cc b/src/Diff.cc
index 54e0a7e6f..be4c10a4b 100644
--- a/src/Diff.cc
+++ b/src/Diff.cc
@@ -395,7 +395,7 @@ public:
void
run() override
{
- auto runAsync = (Options::CDO_Parallel_Read > 0);
+ auto runAsync = (Options::CDO_Async_Read > 0);
auto task = runAsync ? std::make_unique<cdo::Task>() : nullptr;
auto numTasks = runAsync ? 2 : 1;
diff --git a/src/Info.cc b/src/Info.cc
index ffc0f4ce9..a3de0bad1 100644
--- a/src/Info.cc
+++ b/src/Info.cc
@@ -540,7 +540,7 @@ public:
auto numVars = varList.numVars();
if (numVars == 0) continue;
- auto runAsync = (Options::CDO_Parallel_Read > 0);
+ auto runAsync = (Options::CDO_Async_Read > 0);
auto task = runAsync ? std::make_unique<cdo::Task>() : nullptr;
auto numTasks = runAsync ? 2 : 1;
diff --git a/src/Longinfo.cc b/src/Longinfo.cc
index b7270393e..af2b47012 100644
--- a/src/Longinfo.cc
+++ b/src/Longinfo.cc
@@ -190,7 +190,7 @@ public:
void
run() override
{
- auto runAsync = (Options::CDO_Parallel_Read > 0);
+ auto runAsync = (Options::CDO_Async_Read > 0);
auto task = runAsync ? std::make_unique<cdo::Task>() : nullptr;
auto numTasks = runAsync ? 2 : 1;
diff --git a/src/Timstat.cc b/src/Timstat.cc
index 5e8c01284..d862ba32a 100644
--- a/src/Timstat.cc
+++ b/src/Timstat.cc
@@ -501,8 +501,7 @@ public:
void
run() override
{
- auto runAsync = (Options::CDO_Parallel_Read > 0);
- runAsync ? run_async() : run_sync();
+ Options::CDO_Async_Read ? run_async() : run_sync();
}
void
diff --git a/src/Trend.cc b/src/Trend.cc
index 925efaf44..bf71e2da8 100644
--- a/src/Trend.cc
+++ b/src/Trend.cc
@@ -270,7 +270,7 @@ public:
void
run() override
{
- auto runAsync = (Options::CDO_Parallel_Read > 0);
+ auto runAsync = (Options::CDO_Async_Read > 0);
runAsync ? run_async() : run_sync();
}
diff --git a/src/cdo.cc b/src/cdo.cc
index 00947d323..d1360a7ca 100644
--- a/src/cdo.cc
+++ b/src/cdo.cc
@@ -153,6 +153,15 @@ get_env_vars()
->add_default("false")
->add_help("'true' test new features [default: false].");
+ CLIOptions::envvar("CDO_ASYNC_READ")
+ ->add_effect([&](const std::string &envstr) {
+ Options::CDO_Async_Read = parameter_to_bool(envstr);
+ Options::CDO_task = Options::CDO_Async_Read;
+ })
+ ->describe_argument("true|false")
+ ->add_default("false")
+ ->add_help("'true' asyncronous read of input files [default: true].");
+
CLIOptions::envvar("CDO_CORESIZE")
->add_effect([&](const std::string &envstr) { Options::coresize = parameter_to_long(envstr); })
->describe_argument("max. core dump size")
diff --git a/src/cdo_def_options.cc b/src/cdo_def_options.cc
index d5031cb84..4beee3b7f 100644
--- a/src/cdo_def_options.cc
+++ b/src/cdo_def_options.cc
@@ -354,10 +354,10 @@ setup_options()
->set_category("Multi Threading")
->add_help("Set number of OpenMP threads.");
- CLIOptions::option("parrallel_read", "p")
+ CLIOptions::option("p")
->set_internal(true)
->add_effect([&]() {
- Options::CDO_Parallel_Read = true;
+ Options::CDO_Async_Read = true;
Options::CDO_task = true;
})
->set_category("Multi Threading")
diff --git a/src/cdo_options.cc b/src/cdo_options.cc
index 3c422e570..cae421494 100644
--- a/src/cdo_options.cc
+++ b/src/cdo_options.cc
@@ -69,7 +69,7 @@ int CMOR_Mode = false;
bool cdoDiag = false;
MemType CDO_Memtype(MemType::Native);
-bool CDO_Parallel_Read = false;
+bool CDO_Async_Read = false;
int CDO_Reduce_Dim = false;
int CDO_Append_History = true;
diff --git a/src/cdo_options.h b/src/cdo_options.h
index c0252216d..bf5bbfeb8 100644
--- a/src/cdo_options.h
+++ b/src/cdo_options.h
@@ -67,7 +67,7 @@ extern bool cdoDiag;
extern MemType CDO_Memtype;
-extern bool CDO_Parallel_Read;
+extern bool CDO_Async_Read;
extern bool CDO_task;
extern int CDO_Reduce_Dim;
--
GitLab