cdo.cc 66.7 KB
Newer Older
Uwe Schulzweida's avatar
Uwe Schulzweida committed
1
/*
Uwe Schulzweida's avatar
Uwe Schulzweida committed
2
  This file is part of CDO. CDO is a collection of Operators to
Uwe Schulzweida's avatar
Uwe Schulzweida committed
3
4
  manipulate and analyse Climate model Data.

Uwe Schulzweida's avatar
Uwe Schulzweida committed
5
  Copyright (C) 2003-2020 Uwe Schulzweida, <uwe.schulzweida AT mpimet.mpg.de>
Uwe Schulzweida's avatar
Uwe Schulzweida committed
6
7
8
9
10
11
12
13
14
15
16
17
  See COPYING file for copying and redistribution conditions.

  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; version 2 of the License.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
*/

18
#ifdef HAVE_CONFIG_H
19
#include "config.h"
20
21
#endif

Uwe Schulzweida's avatar
Uwe Schulzweida committed
22
23
24
25
#ifdef _OPENMP
#include <omp.h>
#endif

26
#include <iostream>
27
#include <algorithm>
28
#include <vector>
Oliver Heidmann's avatar
Oliver Heidmann committed
29

Uwe Schulzweida's avatar
Uwe Schulzweida committed
30
#ifdef HAVE_EXECINFO_H
Uwe Schulzweida's avatar
Uwe Schulzweida committed
31
32
33
#include <execinfo.h>
#endif

Uwe Schulzweida's avatar
Uwe Schulzweida committed
34
#include <fenv.h>
Uwe Schulzweida's avatar
Uwe Schulzweida committed
35
#include <sys/stat.h>
36
#include <unistd.h> /* sysconf, gethostname */
Uwe Schulzweida's avatar
Uwe Schulzweida committed
37
#include <cstring>
Oliver Heidmann's avatar
Oliver Heidmann committed
38
#include <csignal>
Uwe Schulzweida's avatar
Uwe Schulzweida committed
39

Ralf Mueller's avatar
Ralf Mueller committed
40
#include <cdi.h>
Uwe Schulzweida's avatar
Uwe Schulzweida committed
41
42
43
44

#include "cdo_getopt.h"
#include "cdo_rlimit.h"
#include "cdo_task.h"
Oliver Heidmann's avatar
Oliver Heidmann committed
45
#include <mpim_grid.h>
46
#include <griddes.h>
47
#include "cdo_default_values.h"
48
#include "cdo_cdi_wrapper.h"
49
#include "param_conversion.h"
50
#include "progress.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
51

52
#ifdef HAVE_LIBPTHREAD
Uwe Schulzweida's avatar
Uwe Schulzweida committed
53
54
55
#include "pthread_debug.h"
#endif

56
#include "module_list.h"
57
#include "percentiles.h"
58
#include "util_wildcards.h"
59
#include "util_string.h"
Oliver Heidmann's avatar
Oliver Heidmann committed
60
#include "process_int.h"
61
#include "cdo_options.h"
62
#include "timer.h"
63
#include "commandline.h"
64
65
#include "mpmo_color.h"
#include "cdo_output.h"
66
#include "cdo_features.h"
67
#include "cdo_zaxis.h"
Oliver Heidmann's avatar
Oliver Heidmann committed
68
#include "compare.h"
Oliver Heidmann's avatar
Oliver Heidmann committed
69
#include "dmemory.h"
70
#include "table.h"
71
#include "datetime.h"
Oliver Heidmann's avatar
Oliver Heidmann committed
72
#include "remap_grid_cell_search.h"
Oliver Heidmann's avatar
Oliver Heidmann committed
73
#include "cdo_pthread.h"
74
#include "institution.h"
75
#include "cdo_apply.h"
76

77
#ifndef VERSION
78
#define VERSION "0.0.1"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
79
80
#endif

81
#define MAX_NUM_VARNAMES 256
82

83
static ProcessManager g_processManager;
84

Uwe Schulzweida's avatar
Uwe Schulzweida committed
85
86
void
cdoExit()
87
{
88
  g_processManager.killProcesses();
89
  exit(EXIT_FAILURE);
90
91
}

92
static int Debug = 0;
93
static int DebugLevel = 0;
94
95
96
static int Version = 0;
static int Help = 0;
static int numThreads = 0;
97
static int timer_total;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
98
99
static int CDO_netcdf_hdr_pad = 0;
static int CDO_Rusage = 0;
100

Uwe Schulzweida's avatar
Uwe Schulzweida committed
101
extern "C" void streamGrbDefDataScanningMode(int scanmode);
102

103
void setPointSearchMethod(const std::string &methodstr);
104

105
#define ITSME (cstrIsEqual(cdo::Username, "\x6d\x32\x31\x34\x30\x30\x33"))
Uwe Schulzweida's avatar
Uwe Schulzweida committed
106

107
static void
Uwe Schulzweida's avatar
Uwe Schulzweida committed
108
cdo_stackframe()
Uwe Schulzweida's avatar
Uwe Schulzweida committed
109
110
111
{
#if defined HAVE_EXECINFO_H && defined HAVE_BACKTRACE
  void *callstack[32];
112
  const int frames = backtrace(callstack, 32);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
113
114
115
  char **messages = backtrace_symbols(callstack, frames);

  fprintf(stderr, "[bt] Execution path:\n");
116
117
  if (messages)
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
118
      for (int i = 0; i < frames; ++i) fprintf(stderr, "[bt] %s\n", messages[i]);
119
120
      free(messages);
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
121
122
#endif
}
123
#if defined HAVE_FEENABLEEXCEPT
124
static int
Uwe Schulzweida's avatar
Uwe Schulzweida committed
125
cdo_feenableexcept(const int excepts)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
126
{
127
128
  int feenableexcept(int);
  int old_excepts = feenableexcept(excepts);
129
  return old_excepts;
130
}
131
#else
132
133
134
static int
cdo_feenableexcept(const int excepts)
{
Uwe Schulzweida's avatar
Uwe Schulzweida committed
135
  static fenv_t fenv;
136
  int old_excepts = -1;  // previous masks
137
  if (fegetenv(&fenv)) return -1;
138

139
#if defined(HAVE_FENV_T___CONTROL) && defined(HAVE_FENV_T___MXCSR)
140
  const unsigned new_excepts = ((unsigned) excepts) & FE_ALL_EXCEPT;
141
  old_excepts = (int) (fenv.__control & FE_ALL_EXCEPT);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
142

143
144
  // unmask
  fenv.__control &= ~new_excepts;
145
  fenv.__mxcsr &= ~(new_excepts << 7);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
146
147
#endif

Uwe Schulzweida's avatar
Uwe Schulzweida committed
148
  return (fesetenv(&fenv) ? -1 : old_excepts);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
149
}
150
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
151

152
static void
Uwe Schulzweida's avatar
Uwe Schulzweida committed
153
cdoSignalHandler(const int signo)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
154
{
155
  if (signo == SIGFPE)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
156
157
158
159
160
161
    {
      cdo_stackframe();
      cdoAbort("floating-point exception!");
    }
}

162
static void
Uwe Schulzweida's avatar
Uwe Schulzweida committed
163
cdoSetDigits(const char *const arg)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
164
165
{
  char *ptr1 = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
166
  if (arg != 0 && (int) strlen(arg) > 0 && arg[0] != ',') Options::CDO_flt_digits = (int) strtol(arg, &ptr1, 10);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
167

Uwe Schulzweida's avatar
Uwe Schulzweida committed
168
169
  if (Options::CDO_flt_digits < 1 || Options::CDO_flt_digits > 20)
    cdoAbort("Unreasonable value for float significant digits: %d", Options::CDO_flt_digits);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
170

171
  if (ptr1 && *ptr1 == ',')
Uwe Schulzweida's avatar
Uwe Schulzweida committed
172
173
    {
      char *ptr2 = 0;
174
175
176
      Options::CDO_dbl_digits = (int) strtol(ptr1 + 1, &ptr2, 10);
      if (ptr2 == ptr1 + 1 || Options::CDO_dbl_digits < 1 || Options::CDO_dbl_digits > 20)
        cdoAbort("Unreasonable value for double significant digits: %d", Options::CDO_dbl_digits);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
177
178
179
    }
}

180
static void
Uwe Schulzweida's avatar
Uwe Schulzweida committed
181
cdo_version()
Uwe Schulzweida's avatar
Uwe Schulzweida committed
182
{
Uwe Schulzweida's avatar
Uwe Schulzweida committed
183
184
185
  const int filetypes[] = { CDI_FILETYPE_SRV, CDI_FILETYPE_EXT, CDI_FILETYPE_IEG, CDI_FILETYPE_GRB,  CDI_FILETYPE_GRB2,
                            CDI_FILETYPE_NC,  CDI_FILETYPE_NC2, CDI_FILETYPE_NC4, CDI_FILETYPE_NC4C, CDI_FILETYPE_NC5 };
  const char *typenames[] = { "srv", "ext", "ieg", "grb1", "grb2", "nc1", "nc2", "nc4", "nc4c", "nc5" };
Uwe Schulzweida's avatar
Uwe Schulzweida committed
186

187
  fprintf(stderr, "%s\n", cdo::Version);
188
189
#ifdef SYSTEM_TYPE
  fprintf(stderr, "System: %s\n", SYSTEM_TYPE);
190
#endif
191
#ifdef CXX_COMPILER
Uwe Schulzweida's avatar
Uwe Schulzweida committed
192
  fprintf(stderr, "CXX Compiler: %s\n", CXX_COMPILER);
193
#ifdef CXX_VERSION
Uwe Schulzweida's avatar
Uwe Schulzweida committed
194
195
  fprintf(stderr, "CXX version : %s\n", CXX_VERSION);
#endif
196
#endif
197
#ifdef C_COMPILER
Uwe Schulzweida's avatar
Uwe Schulzweida committed
198
  fprintf(stderr, "C Compiler: %s\n", C_COMPILER);
199
#ifdef C_VERSION
Uwe Schulzweida's avatar
Uwe Schulzweida committed
200
  fprintf(stderr, "C version : %s\n", C_VERSION);
201
202
#endif
#endif
203
#ifdef F77_COMPILER
204
  fprintf(stderr, "F77 Compiler: %s\n", F77_COMPILER);
205
#ifdef F77_VERSION
206
207
  fprintf(stderr, "F77 version : %s\n", F77_VERSION);
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
208
209
210
211
#endif

  printFeatures();
  printLibraries();
212

Uwe Schulzweida's avatar
Uwe Schulzweida committed
213
  fprintf(stderr, "Filetypes: ");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
214
  set_text_color(stderr, BRIGHT, GREEN);
215
216
  for (size_t i = 0; i < sizeof(filetypes) / sizeof(int); ++i)
    if (cdiHaveFiletype(filetypes[i])) fprintf(stderr, "%s ", typenames[i]);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
217
  reset_text_color(stderr);
218
219
  fprintf(stderr, "\n");

Uwe Schulzweida's avatar
Uwe Schulzweida committed
220
221
222
223
  cdiPrintVersion();
  fprintf(stderr, "\n");
}

224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
static void
cdo_variableInputs()
{
  set_text_color(stderr, BRIGHT, BLUE);
  fprintf(stderr, "#==============================================================================#\n");
  reset_text_color(stderr);
  fprintf(stderr, "    For operators with variable number of inputs:\n");
  fprintf(stderr, "    Brackets can be used for grouping input to the right operator.\n");
  reset_text_color(stderr);
  fprintf(stderr, "    example:\n");
  fprintf(stderr, "       -add -select,x=0 [ file1 -add -topo -file2 ] -merge [ file3 file4 ] out\n");
  set_text_color(stderr, BRIGHT, BLUE);
  fprintf(stderr, "#==============================================================================#\n");
  reset_text_color(stderr);
}

240
static void
Uwe Schulzweida's avatar
Uwe Schulzweida committed
241
cdo_usage()
Uwe Schulzweida's avatar
Uwe Schulzweida committed
242
{
243
  const char *name;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
244

Uwe Schulzweida's avatar
Uwe Schulzweida committed
245
  /*  fprintf(stderr, "%s\n", CDO_version);*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
246
  /*  fprintf(stderr, "\n");*/
Uwe Schulzweida's avatar
Uwe Schulzweida committed
247
  fprintf(stderr, "usage : cdo  [Options]  Operator1  [-Operator2  [-OperatorN]]\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
248
249
  fprintf(stderr, "\n");
  fprintf(stderr, "  Options:\n");
250
  set_text_color(stderr, BLUE);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
251
  fprintf(stderr, "    -a             Generate an absolute time axis\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
252
  fprintf(stderr, "    -b <nbits>     Set the number of bits for the output precision\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
253
254
  fprintf(stderr, "                   (I8/I16/I32/F32/F64 for nc1/nc2/nc4/nc4c/nc5; U8/U16/U32 for nc4/nc4c/nc5;"
                  " F32/F64 for grb2/srv/ext/ieg; P1 - P24 for grb1/grb2)\n");
255
  fprintf(stderr, "                   Add L or B to set the byteorder to Little or Big endian\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
256
  fprintf(stderr, "    --cmor         CMOR conform NetCDF output\n");
257
  fprintf(stderr, "    -C, --color    Set behaviour of colorized output messages <auto,no,all>\n");
258
  fprintf(stderr, "    --eccodes      Use ecCodes to decode/encode GRIB1 messages\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
259
  fprintf(stderr, "    --enableexcept <except>\n");
260
  fprintf(stderr, "                   Set individual floating-point traps "
261
                  "(DIVBYZERO, INEXACT, INVALID, OVERFLOW, UNDERFLOW, ALL_EXCEPT)\n");
262
  fprintf(stderr, "    -f, --format <format>\n");
263
  fprintf(stderr, "                   Format of the output file. (grb1/grb2/nc1/nc2/nc4/nc4c/nc5/srv/ext/ieg)\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
264
  fprintf(stderr, "    -g <grid>      Set default grid name or file. Available grids: \n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
265
266
  fprintf(stderr,
          "                   F<XXX>, t<RES>, tl<RES>, global_<DXY>, r<NX>x<NY>, g<NX>x<NY>, gme<NI>, lon=<LON>/lat=<LAT>\n");
267
  fprintf(stderr, "    -h, --help     Help information for the perators\n");
268
  fprintf(stderr, "    --no_history   Do not append to NetCDF \"history\" global attribute\n");
269
  fprintf(stderr, "    --netcdf_hdr_pad, --hdr_pad, --header_pad <nbr>\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
270
  fprintf(stderr, "                   Pad NetCDF output header with nbr bytes\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
271
272
273
  /*
  fprintf(stderr, "    -i <inst>      Institution name/file\n");
  fprintf(stderr, "                   Predefined instituts: ");
274
  for ( int id = 0; id < institutInqNumber; id++ )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
275
276
277
278
279
    if ( (name = institutInqNamePtr(id)) )
      fprintf(stderr, " %s", name);
  fprintf(stderr, "\n");
  */
  /* fprintf(stderr, "    -l <level>     Level file\n"); */
Uwe Schulzweida's avatar
Uwe Schulzweida committed
280
  fprintf(stderr, "    -k <chunktype> NetCDF4 chunk type: auto, grid or lines\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
281
  fprintf(stderr, "    -L             Lock IO (sequential access)\n");
282
283
  fprintf(stderr, "    -M             Switch to indicate that the I/O streams have missing values\n");
  fprintf(stderr, "    -m <missval>   Set the missing value of non NetCDF files (default: %g)\n", cdiInqMissval());
Uwe Schulzweida's avatar
Uwe Schulzweida committed
284
  fprintf(stderr, "    -O             Overwrite existing output file, if checked\n");
285
  fprintf(stderr, "    --operators    List of all operators\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
286
  fprintf(stderr, "    --pedantic     Warnings count as errors\n");
287
#ifdef _OPENMP
Uwe Schulzweida's avatar
Uwe Schulzweida committed
288
289
  fprintf(stderr, "    -P <nthreads>  Set number of OpenMP threads\n");
#endif
Uwe Schulzweida's avatar
Uwe Schulzweida committed
290
  fprintf(stderr, "    --percentile <method>\n");
291
  fprintf(stderr, "                   Percentile method: nrank, nist, rtype8, numpy, numpy_lower, numpy_higher, numpy_nearest\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
292
  fprintf(stderr, "    --precision <float_digits[,double_digits]>\n");
293
  fprintf(stderr, "                   Precision to use in displaying floating-point data (default: 7,15)\n");
294
  if (ITSME) fprintf(stderr, "    --pointsearchmethod [full/kdtree/nanoflann/spherepart/latbins]\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
295
  fprintf(stderr, "    --reduce_dim   Reduce NetCDF dimensions\n");
296
297
  fprintf(stderr, "    --no_remap_weights Switch off generation of remap weights\n");
  if (ITSME) fprintf(stderr, "    --remap_weights [0/1] Generate remap weights (default: 1)\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
298
  fprintf(stderr, "    -R, --regular  Convert GRIB1 data from global reduced to regular Gaussian grid (cgribex only)\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
299
  fprintf(stderr, "    -r             Generate a relative time axis\n");
300
301
  fprintf(stderr, "    -S             Create an extra output stream for the module TIMSTAT. This stream\n");
  fprintf(stderr, "                   contains the number of non missing values for each output period.\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
302
  fprintf(stderr, "    -s, --silent   Silent mode\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
303
  fprintf(stderr, "    --seed <seed>  Seed for a new sequence of pseudo-random numbers.\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
304
  fprintf(stderr, "    --sortname     Alphanumeric sorting of NetCDF parameter names\n");
305
  fprintf(stderr, "    -t <codetab>   Set GRIB1 default parameter code table name or file (cgribex only)\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
306
  fprintf(stderr, "                   Predefined tables: ");
307
308
  for (int id = 0; id < tableInqNumber(); id++)
    if ((name = tableInqNamePtr(id))) fprintf(stderr, " %s", name);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
309
310
  fprintf(stderr, "\n");

Uwe Schulzweida's avatar
Uwe Schulzweida committed
311
  fprintf(stderr, "    --timestat_date <srcdate>\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
312
  fprintf(stderr, "                   Target timestamp (temporal statistics): "
313
                  "first, middle, midhigh or last source timestep.\n");
314
  fprintf(stderr, "    -V, --version  Print the version number\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
315
  fprintf(stderr, "    -v, --verbose  Print extra details for some operators\n");
316
  fprintf(stderr, "    -w             Disable warning messages\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
317
  fprintf(stderr, "    --worker <num> Number of worker to decode/decompress GRIB records\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
318
  fprintf(stderr, "    -z szip        SZIP compression of GRIB1 records\n");
319
  fprintf(stderr, "       aec         AEC compression of GRIB2 records\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
320
  fprintf(stderr, "       jpeg        JPEG compression of GRIB2 records\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
321
  fprintf(stderr, "        zip[_1-9]  Deflate compression of NetCDF4 variables\n");
322
323
#ifdef HIRLAM_EXTENSIONS
  fprintf(stderr, "    --Dkext <debLev>   Setting debugLevel for extensions\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
324
325
  fprintf(stderr, "    --outputGribDataScanningMode <mode>   Setting grib scanning mode for data in output file <0, 64, 96>; "
                  "Default is 64\n");
326
#endif  // HIRLAM_EXTENSIONS
Uwe Schulzweida's avatar
Uwe Schulzweida committed
327
  reset_text_color(stderr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
328
329
330
  fprintf(stderr, "\n");

  fprintf(stderr, "  Operators:\n");
331
  fprintf(stderr, "    Use option --operators for a list of all operators.\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
332
333

  fprintf(stderr, "\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
334
  fprintf(stderr, "  CDO version %s, Copyright (C) 2003-2020 Uwe Schulzweida\n", VERSION);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
335
  fprintf(stderr, "  This is free software and comes with ABSOLUTELY NO WARRANTY\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
336
  fprintf(stderr, "  Report bugs to <https://mpimet.mpg.de/cdo>\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
337
338
}

339
static void
Uwe Schulzweida's avatar
Uwe Schulzweida committed
340
cdo_init_is_tty()
Uwe Schulzweida's avatar
Uwe Schulzweida committed
341
342
343
{
  struct stat statbuf;
  fstat(0, &statbuf);
344
  if (S_ISCHR(statbuf.st_mode)) stdin_is_tty = true;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
345
  fstat(1, &statbuf);
346
347
  if (S_ISCHR(statbuf.st_mode))
    {
348
      stdout_is_tty = true;
349
      progress::stdout_is_tty = true;
350
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
351
  fstat(2, &statbuf);
352
  if (S_ISCHR(statbuf.st_mode)) stderr_is_tty = true;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
353
354
}

355
static void
356
cdoPrintHelp(const char **help)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
357
{
358
  if (!help)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
359
    fprintf(stderr, "No help available for this operator!\n");
Uwe Schulzweida's avatar
Uwe Schulzweida committed
360
361
  else
    {
362
363
364
      size_t help_size = 0;
      while (help[help_size]) help_size++;
      for (size_t i = 0; i < help_size; i++)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
365
        {
366
          const bool lprint = !(help[i][0] == '\0' && help[i + 1][0] == ' ');
367
          if (lprint)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
368
            {
369
              if (colorEnabled())
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
370
                {
371
372
373
                  if (cstrIsEqual(help[i], "NAME") || cstrIsEqual(help[i], "SYNOPSIS") || cstrIsEqual(help[i], "DESCRIPTION")
                      || cstrIsEqual(help[i], "OPERATORS") || cstrIsEqual(help[i], "NAMELIST") || cstrIsEqual(help[i], "PARAMETER")
                      || cstrIsEqual(help[i], "ENVIRONMENT") || cstrIsEqual(help[i], "NOTE") || cstrIsEqual(help[i], "EXAMPLES"))
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
374
                    {
375
                      set_text_color(stdout, BRIGHT);
376
                      fprintf(stdout, "%s", help[i]);
377
                      reset_text_color(stdout);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
378
379
380
                      fprintf(stdout, "\n");
                    }
                  else
381
                    fprintf(stdout, "%s\n", help[i]);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
382
383
384
                }
              else
                {
385
                  fprintf(stdout, "%s\n", help[i]);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
386
387
388
                }
            }
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
389
390
391
    }
}

392
393
#undef IsBigendian
#define IsBigendian() (u_byteorder.c[sizeof(long) - 1])
Uwe Schulzweida's avatar
Uwe Schulzweida committed
394

395
396
static void
setDefaultDataType(const char *datatypestr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
397
{
Uwe Schulzweida's avatar
Uwe Schulzweida committed
398
  static const union
399
400
401
402
403
404
405
406
407
408
409
  {
    unsigned long l;
    unsigned char c[sizeof(long)];
  } u_byteorder = { 1 };
  enum
  {
    D_UINT,
    D_INT,
    D_FLT,
    D_CPX
  };
Uwe Schulzweida's avatar
Uwe Schulzweida committed
410
411
  int dtype = -1;

412
  const int datatype = tolower(*datatypestr);
413
414
415
416
417
418
419
  // clang-format off
  if      (datatype == 'i') { dtype = D_INT;  datatypestr++; }
  else if (datatype == 'u') { dtype = D_UINT; datatypestr++; }
  else if (datatype == 'f') { dtype = D_FLT;  datatypestr++; }
  else if (datatype == 'c') { dtype = D_CPX;  datatypestr++; }
  else if (datatype == 'p') {                 datatypestr++; }
  // clang-format on
Uwe Schulzweida's avatar
Uwe Schulzweida committed
420

421
  if (isdigit((int) *datatypestr))
422
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
423
      int nbits = atoi(datatypestr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
424
      datatypestr += 1;
425
      if (nbits >= 10) datatypestr += 1;
426

427
      if (dtype == -1)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
428
        {
429
          if (nbits > 0 && nbits < 32)
430
            CdoDefault::DataType = nbits;
431
          else if (nbits == 32)
432
            CdoDefault::DataType = (CdoDefault::FileType == CDI_FILETYPE_GRB) ? CDI_DATATYPE_PACK32 : CDI_DATATYPE_FLT32;
433
          else if (nbits == 64)
434
            CdoDefault::DataType = CDI_DATATYPE_FLT64;
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
435
436
437
          else
            {
              fprintf(stderr, "Unsupported number of bits %d!\n", nbits);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
438
              fprintf(stderr, "Use I8/I16/I32/F32/F64 for nc1/nc2/nc4/nc4c/nc5; U8/U16/U32 for nc4/nc4c/nc5; F32/F64 for "
439
                              "grb2/srv/ext/ieg; P1 - P24 for grb1/grb2.\n");
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
440
441
442
              exit(EXIT_FAILURE);
            }
        }
443
      else
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
444
        {
445
          // clang-format off
446
          if (dtype == D_INT)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
447
            {
448
449
450
              if      (nbits ==  8) CdoDefault::DataType = CDI_DATATYPE_INT8;
              else if (nbits == 16) CdoDefault::DataType = CDI_DATATYPE_INT16;
              else if (nbits == 32) CdoDefault::DataType = CDI_DATATYPE_INT32;
451
              else cdoAbort("Unsupported number of bits = %d for datatype INT!", nbits);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
452
            }
453
          else if (dtype == D_UINT)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
454
            {
455
456
457
              if      (nbits ==  8) CdoDefault::DataType = CDI_DATATYPE_UINT8;
              else if (nbits == 16) CdoDefault::DataType = CDI_DATATYPE_UINT16;
              else if (nbits == 32) CdoDefault::DataType = CDI_DATATYPE_UINT32;
458
              else cdoAbort("Unsupported number of bits = %d for datatype UINT!", nbits);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
459
            }
460
          else if (dtype == D_FLT)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
461
            {
462
463
              if      (nbits == 32) CdoDefault::DataType = CDI_DATATYPE_FLT32;
              else if (nbits == 64) CdoDefault::DataType = CDI_DATATYPE_FLT64;
464
              else cdoAbort("Unsupported number of bits = %d for datatype FLT!", nbits);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
465
            }
466
          else if (dtype == D_CPX)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
467
            {
468
469
              if      (nbits == 32) CdoDefault::DataType = CDI_DATATYPE_CPX32;
              else if (nbits == 64) CdoDefault::DataType = CDI_DATATYPE_CPX64;
470
              else cdoAbort("Unsupported number of bits = %d for datatype CPX!", nbits);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
471
            }
472
          // clang-format on
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
473
        }
474
475
    }

476
  if (*datatypestr != 0)
477
    {
478
      if (*datatypestr == 'l' || *datatypestr == 'L')
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
479
        {
480
          if (IsBigendian()) CdoDefault::Byteorder = CDI_LITTLEENDIAN;
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
481
482
          datatypestr++;
        }
483
      else if (*datatypestr == 'b' || *datatypestr == 'B')
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
484
        {
485
          if (!IsBigendian()) CdoDefault::Byteorder = CDI_BIGENDIAN;
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
486
487
          datatypestr++;
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
488
      else
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
489
        {
490
          cdoAbort("Unsupported character in number of bytes: >%s< !", datatypestr);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
491
        }
492
493
    }
}
494
/*
Uwe Schulzweida's avatar
Uwe Schulzweida committed
495
496
static
void setDefaultDataTypeByte(char *datatypestr)
497
{
498
499
  static union {unsigned long l; unsigned char c[sizeof(long)];} u_byteorder =
{1}; int datatype = -1;
500

Uwe Schulzweida's avatar
Uwe Schulzweida committed
501
502
503
504
505
  if ( isdigit((int) *datatypestr) )
    {
      datatype = atoi(datatypestr);
      datatypestr++;

506
507
508
509
510
      if      ( datatype == 1 ) CdoDefault::DataType = CDI_DATATYPE_PACK8;
      else if ( datatype == 2 ) CdoDefault::DataType = CDI_DATATYPE_PACK16;
      else if ( datatype == 3 ) CdoDefault::DataType = CDI_DATATYPE_PACK24;
      else if ( datatype == 4 ) CdoDefault::DataType = CDI_DATATYPE_FLT32;
      else if ( datatype == 8 ) CdoDefault::DataType = CDI_DATATYPE_FLT64;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
511
      else
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
512
513
        {
          fprintf(stderr, "Unsupported datatype %d!\n", datatype);
514
515
          fprintf(stderr, "Use 4/8 for filetype nc/srv/ext/ieg and 1/2/3 for
grb1/grb2.\n"); exit(EXIT_FAILURE);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
516
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
517
518
    }

Uwe Schulzweida's avatar
Uwe Schulzweida committed
519
  if ( *datatypestr != 0 )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
520
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
521
      if ( *datatypestr == 'l' || *datatypestr == 'L' )
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
522
        {
523
          if ( IsBigendian() ) CdoDefault::Byteorder = CDI_LITTLEENDIAN;
524
          datatypestr++;
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
525
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
526
      else if ( *datatypestr == 'b' || *datatypestr == 'B' )
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
527
        {
528
          if ( ! IsBigendian() ) CdoDefault::Byteorder = CDI_BIGENDIAN;
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
529
530
          datatypestr++;
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
531
      else
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
532
        {
533
          cdoAbort("Unsupported character in number of bytes: %s!", datatypestr);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
534
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
535
536
    }
}
537
*/
538
static void
539
setDefaultFileType(const char *filetypestr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
540
{
541
  if (filetypestr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
542
    {
543
      size_t len = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
544

545
      // clang-format off
546
547
548
549
550
551
552
553
554
555
556
557
      if      (cstr2IsEqualLen(filetypestr, "grb2", len)) CdoDefault::FileType = CDI_FILETYPE_GRB2;
      else if (cstr2IsEqualLen(filetypestr, "grb1", len)) CdoDefault::FileType = CDI_FILETYPE_GRB;
      else if (cstr2IsEqualLen(filetypestr, "grb",  len)) CdoDefault::FileType = CDI_FILETYPE_GRB;
      else if (cstr2IsEqualLen(filetypestr, "nc2",  len)) CdoDefault::FileType = CDI_FILETYPE_NC2;
      else if (cstr2IsEqualLen(filetypestr, "nc4c", len)) CdoDefault::FileType = CDI_FILETYPE_NC4C;
      else if (cstr2IsEqualLen(filetypestr, "nc4",  len)) CdoDefault::FileType = CDI_FILETYPE_NC4;
      else if (cstr2IsEqualLen(filetypestr, "nc5",  len)) CdoDefault::FileType = CDI_FILETYPE_NC5;
      else if (cstr2IsEqualLen(filetypestr, "nc1",  len)) CdoDefault::FileType = CDI_FILETYPE_NC;
      else if (cstr2IsEqualLen(filetypestr, "nc",   len)) CdoDefault::FileType = CDI_FILETYPE_NC2;
      else if (cstr2IsEqualLen(filetypestr, "srv",  len)) CdoDefault::FileType = CDI_FILETYPE_SRV;
      else if (cstr2IsEqualLen(filetypestr, "ext",  len)) CdoDefault::FileType = CDI_FILETYPE_EXT;
      else if (cstr2IsEqualLen(filetypestr, "ieg",  len)) CdoDefault::FileType = CDI_FILETYPE_IEG;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
558
      else
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
559
        {
560
561
          cdoWarning("Unsupported filetype %s!", filetypestr);
          cdoWarning("Available filetypes: grb1/grb2/nc1/nc2/nc4/nc4c/nc5/srv/ext/ieg");
562
          cdoAbort("Unsupported filetype %s!", filetypestr);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
563
        }
564
      // clang-format on
Uwe Schulzweida's avatar
Uwe Schulzweida committed
565

566
567
      const char *ftstr = filetypestr + len;

568
      if (CdoDefault::FileType != CDI_UNDEFID && *ftstr != 0)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
569
        {
570
          if (*ftstr == '_')
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
571
            {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
572
              setDefaultDataType(++ftstr);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
573
574
575
            }
          else
            {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
576
              fprintf(stderr, "Unexpected character >%c< in file type >%s<!\n", *ftstr, filetypestr);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
577
              fprintf(stderr, "Use format[_nbits] with:\n");
578
579
              fprintf(stderr, "    format = grb1, grb2, nc1, nc2, nc4, nc4c, nc5, srv, ext or ieg\n");
              fprintf(stderr, "    nbits  = 32/64 for grb2/nc1/nc2/nc4/nc4c/nc5/srv/ext/ieg; 1 - 24 for grb1/grb2\n");
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
580
581
582
              exit(EXIT_FAILURE);
            }
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
583
584
585
    }
}

586
#define NTESTS 11
587
#include <inttypes.h>
588
static int
Uwe Schulzweida's avatar
Uwe Schulzweida committed
589
getMemAlignment()
590
591
592
593
{
  int ma = -1;
  double *ptr[NTESTS];
  int64_t iptr;
594
595
  const size_t tsize[NTESTS] = { 1, 3, 5, 9, 17, 33, 69, 121, 251, 510, 1025 };
  const size_t ma_check[4] = { 8, 16, 32, 64 };
596
  int ma_result[4] = { 1, 1, 1, 1 };
597

598
  for (int i = 0; i < NTESTS; ++i)
599
    {
600
      ptr[i] = (double *) malloc(tsize[i]);
601
      iptr = (int64_t) ptr[i];
602
603
      for (int k = 0; k < 4; ++k)
        if (iptr % ma_check[k]) ma_result[k] = 0;
604
    }
605
  for (auto &p : ptr) free(p);
606

607
  for (int i = NTESTS - 1; i >= 0; i--)
608
    {
609
      ptr[i] = (double *) malloc(tsize[i] + 5);
610
      iptr = (int64_t) ptr[i];
611
612
      for (int k = 0; k < 4; ++k)
        if (iptr % ma_check[k]) ma_result[k] = 0;
613
    }
614
  for (auto &p : ptr) free(p);
615

616
617
  for (int k = 0; k < 4; ++k)
    if (ma_result[k]) ma = ma_check[k];
618

Uwe Schulzweida's avatar
Uwe Schulzweida committed
619
  return ma;
620
621
}

622
623
static void
defineCompress(const char *arg)
624
{
625
  const size_t len = strlen(arg);
626

627
  if (strncmp(arg, "szip", len) == 0)
628
    {
629
      Options::cdoCompType = CDI_COMPRESS_SZIP;
630
      Options::cdoCompLevel = 0;
631
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
632
  else if (strncmp(arg, "aec", len) == 0 || strncmp(arg, "ccsds", len) == 0)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
633
    {
634
635
      Options::cdoCompType = CDI_COMPRESS_AEC;
      Options::cdoCompLevel = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
636
    }
637
  else if (strncmp(arg, "jpeg", len) == 0)
638
    {
639
640
      Options::cdoCompType = CDI_COMPRESS_JPEG;
      Options::cdoCompLevel = 0;
641
    }
642
  else if (strncmp(arg, "zip", 3) == 0)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
643
    {
644
      Options::cdoCompType = CDI_COMPRESS_ZIP;
645
      Options::cdoCompLevel = (len == 5 && arg[3] == '_' && isdigit(arg[4])) ? atoi(&arg[4]) : 1;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
646
    }
647
  else
Uwe Schulzweida's avatar
Uwe Schulzweida committed
648
    {
649
      cdoAbort("Compression type '%s' unsupported!", arg);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
650
651
652
    }
}

653
static void
654
defineChunktype(const std::string &arg)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
655
{
Uwe Schulzweida's avatar
Uwe Schulzweida committed
656
  // clang-format off
657
658
659
660
  if      ("auto"  == arg) Options::cdoChunkType = CDI_CHUNK_AUTO;
  else if ("grid"  == arg) Options::cdoChunkType = CDI_CHUNK_GRID;
  else if ("lines" == arg) Options::cdoChunkType = CDI_CHUNK_LINES;
  else cdoAbort("Chunk type '%s' unsupported!", arg.c_str());
Uwe Schulzweida's avatar
Uwe Schulzweida committed
661
  // clang-format on
662
663
}

664
std::vector<std::string>
Uwe Schulzweida's avatar
Uwe Schulzweida committed
665
defineVarnames(const char *const arg)
666
667
{

668
669
  std::string strArgs = std::string(arg);
  std::vector<std::string> newVarnames;
670

671
672
673
  const char delim = ',';
  size_t previous = 0;
  size_t current = strArgs.find(delim);
674

675
676
677
678
679
  while (current != std::string::npos)
    {
      newVarnames.push_back(strArgs.substr(previous, current - previous));
      previous = current + 1;
      current = strArgs.find(delim, previous);
680
    }
681
682
  newVarnames.push_back(strArgs.substr(previous, current - previous));

683
  return newVarnames;
684
685
}

686
static void
Uwe Schulzweida's avatar
Uwe Schulzweida committed
687
get_env_vars()
Uwe Schulzweida's avatar
Uwe Schulzweida committed
688
{
689
690
  cdo::Username = getenv("LOGNAME");
  if (cdo::Username == nullptr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
691
    {
692
693
      cdo::Username = getenv("USER");
      if (cdo::Username == nullptr) cdo::Username = "unknown";
Uwe Schulzweida's avatar
Uwe Schulzweida committed
694
695
    }

Uwe Schulzweida's avatar
Uwe Schulzweida committed
696
  auto envstr = getenv("CDO_DOWNLOAD_PATH");
697
  if (envstr && *envstr)
698
    {
699
      cdo::DownloadPath = envstr;
700
701
702
      if (Options::cdoVerbose) fprintf(stderr, "CDO_DOWNLOAD_PATH = %s\n", cdo::DownloadPath);
    }

703
704
705
706
707
708
709
  envstr = getenv("CDO_ICON_GRIDS");
  if (envstr && *envstr)
    {
      cdo::IconGrids = envstr;
      if (Options::cdoVerbose) fprintf(stderr, "CDO_ICON_GRIDS = %s\n", cdo::IconGrids);
    }

Uwe Schulzweida's avatar
Uwe Schulzweida committed
710
  envstr = getenv("CDO_DISABLE_HISTORY");
711
  if (envstr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
712
    {
713
      if (parameter2bool(envstr) == true)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
714
        {
715
          Options::CDO_Reset_History = true;
716
          if (Options::cdoVerbose) fprintf(stderr, "CDO_DISABLE_HISTORY = %s\n", envstr);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
717
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
718
719
    }

Uwe Schulzweida's avatar
Uwe Schulzweida committed
720
  envstr = getenv("CDO_RESET_HISTORY");
721
  if (envstr)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
722
    {
723
      if (parameter2bool(envstr) == true)
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
724
        {
725
          Options::CDO_Reset_History = true;
726
          if (Options::cdoVerbose) fprintf(stderr, "CDO_RESET_HISTORY = %s\n", envstr);
Yvonne Kuestermann's avatar
Yvonne Kuestermann committed
727
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
728
729
    }

Uwe Schulzweida's avatar
Uwe Schulzweida committed
730
  envstr = getenv("CDO_HISTORY_INFO");
Oliver Heidmann's avatar