Split.cc 14.8 KB
Newer Older
Uwe Schulzweida's avatar
Uwe Schulzweida committed
1 2 3 4
/*
  This file is part of CDO. CDO is a collection of Operators to
  manipulate and analyse Climate model Data.

Uwe Schulzweida's avatar
Uwe Schulzweida committed
5
  Copyright (C) 2003-2020 Uwe Schulzweida, <uwe.schulzweida AT mpimet.mpg.de>
Uwe Schulzweida's avatar
Uwe Schulzweida committed
6 7 8 9 10 11 12 13 14 15 16 17
  See COPYING file for copying and redistribution conditions.

  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; version 2 of the License.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
*/

18 19 20 21
/*
   This module contains the following operators:

      Split      splitcode       Split codes
22 23
      Split      splitparam      Split parameters
      Split      splitname       Split variables
24 25 26
      Split      splitlevel      Split levels
      Split      splitgrid       Split grids
      Split      splitzaxis      Split zaxis
27
      Split      splittabnum     Split table numbers
28 29
*/

Ralf Müller's avatar
Ralf Müller committed
30
#include <cdi.h>
Oliver Heidmann's avatar
Oliver Heidmann committed
31

32
#include "process_int.h"
33
#include "cdo_history.h"
34
#include "util_files.h"
35
#include "cdo_zaxis.h"
36
#include "cdi_lockedIO.h"
Uwe Schulzweida's avatar
Uwe Schulzweida committed
37

38
static void
Uwe Schulzweida's avatar
Uwe Schulzweida committed
39
gen_filename(char *filename, bool swap_obase, const char *obase, const char *suffix)
40
{
41 42
  if (swap_obase) strcat(filename, obase);
  if (suffix[0]) strcat(filename, suffix);
43 44
}

45 46
void *
Split(void *process)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
47
{
48
  int nchars = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
49
  int varID;
50
  int levelID, levID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
51
  int varID2, levelID2;
52
  int itmp[999];
Uwe Schulzweida's avatar
Uwe Schulzweida committed
53 54
  double ftmp[999];
  int nsplit = 0;
55
  size_t nmiss;
56
  bool swap_obase = false;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
57
  const char *uuid_attribute = nullptr;
58 59
  std::vector<int> vlistIDs;
  std::vector<CdoStreamID> streamIDs;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
60

61
  cdoInitialize(process);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
62

Uwe Schulzweida's avatar
Uwe Schulzweida committed
63
  if (processSelf().m_ID != 0) cdoAbort("This operator can't be combined with other operators!");
64

65
  const auto lcopy = unchangedRecord();
66

Uwe Schulzweida's avatar
Uwe Schulzweida committed
67
  // clang-format off
68 69 70 71 72 73 74
  const auto SPLITCODE   = cdoOperatorAdd("splitcode",   0, 0, nullptr);
  const auto SPLITPARAM  = cdoOperatorAdd("splitparam",  0, 0, nullptr);
  const auto SPLITNAME   = cdoOperatorAdd("splitname",   0, 0, nullptr);
  const auto SPLITLEVEL  = cdoOperatorAdd("splitlevel",  0, 0, nullptr);
  const auto SPLITGRID   = cdoOperatorAdd("splitgrid",   0, 0, nullptr);
  const auto SPLITZAXIS  = cdoOperatorAdd("splitzaxis",  0, 0, nullptr);
  const auto SPLITTABNUM = cdoOperatorAdd("splittabnum", 0, 0, nullptr);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
75
  // clang-format on
Uwe Schulzweida's avatar
Uwe Schulzweida committed
76

77
  const auto operatorID = cdoOperatorID();
Uwe Schulzweida's avatar
Uwe Schulzweida committed
78

79
  for (int i = 0; i < operatorArgc(); ++i)
80
    {
81
      if (cdoOperatorArgv(i) == "swap")
82
        swap_obase = true;
83 84
      else if (cdoOperatorArgv(i).find("uuid=") == 0)
        uuid_attribute = &cdoOperatorArgv(i)[0] + 5;
85
      else
86
        cdoAbort("Unknown parameter: >%s<", cdoOperatorArgv(0).c_str());
87
    }
88

89 90
  const auto streamID1 = cdoOpenRead(0);
  const auto vlistID1 = cdoStreamInqVlist(streamID1);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
91

Uwe Schulzweida's avatar
Uwe Schulzweida committed
92 93 94
  VarList varList1;
  varListInit(varList1, vlistID1);

95
  const auto nvars = vlistNvars(vlistID1);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
96

97
  char filename[8192];
98
  if (!swap_obase)
99
    {
100
      strcpy(filename, cdoGetObase());
101 102
      nchars = strlen(filename);
    }
103

104 105
  auto refname = cdoGetStreamName(0);
  char filesuffix[32] = { 0 };
106
  cdoGenFileSuffix(filesuffix, sizeof(filesuffix), cdoInqFiletype(streamID1), vlistID1, refname);
107 108

  if (operatorID == SPLITCODE)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
109 110
    {
      nsplit = 0;
111 112
      for (varID = 0; varID < nvars; varID++)
        {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
113
          const auto code = varList1[varID].code;
114
          int index;
115
          for (index = 0; index < varID; index++)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
116
            if (code == varList1[index].code) break;
117 118 119 120 121 122 123 124

          if (index == varID)
            {
              itmp[nsplit] = code;
              nsplit++;
            }
        }

125 126
      vlistIDs.resize(nsplit);
      streamIDs.resize(nsplit);
127
      std::vector<int> codes(nsplit);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
128
      for (int index = 0; index < nsplit; ++index) codes[index] = itmp[index];
129 130 131 132 133 134

      for (int index = 0; index < nsplit; ++index)
        {
          vlistClearFlag(vlistID1);
          for (varID = 0; varID < nvars; varID++)
            {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
135
              const auto code = varList1[varID].code;
136 137
              if (codes[index] == code)
                {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
138
                  for (levID = 0; levID <  varList1[varID].nlevels; levID++)
139 140
                    {
                      vlistDefIndex(vlistID1, varID, levID, index);
141
                      vlistDefFlag(vlistID1, varID, levID, true);
142 143 144 145
                    }
                }
            }

146
          const auto vlistID2 = vlistCreate();
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
          cdoVlistCopyFlag(vlistID2, vlistID1);
          vlistIDs[index] = vlistID2;

          if (codes[index] > 9999)
            {
              sprintf(filename + nchars, "%05d", codes[index]);
              gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);
            }
          else if (codes[index] > 999)
            {
              sprintf(filename + nchars, "%04d", codes[index]);
              gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);
            }
          else
            {
              sprintf(filename + nchars, "%03d", codes[index]);
              gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);
            }

166
          streamIDs[index] = cdoOpenWrite(filename);
167
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
168
    }
169
  else if (operatorID == SPLITPARAM)
170 171 172
    {
      char paramstr[32];
      nsplit = 0;
173 174
      for (varID = 0; varID < nvars; varID++)
        {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
175
          const auto param = varList1[varID].param;
176
          int index;
177
          for (index = 0; index < varID; index++)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
178
            if (param == varList1[index].param) break;
179 180 181 182 183 184 185 186

          if (index == varID)
            {
              itmp[nsplit] = param;
              nsplit++;
            }
        }

187 188
      vlistIDs.resize(nsplit);
      streamIDs.resize(nsplit);
189
      std::vector<int> params(nsplit);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
190
      for (int index = 0; index < nsplit; ++index) params[index] = itmp[index];
191 192 193 194 195 196

      for (int index = 0; index < nsplit; ++index)
        {
          vlistClearFlag(vlistID1);
          for (varID = 0; varID < nvars; varID++)
            {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
197
              const auto param = varList1[varID].param;
198 199
              if (params[index] == param)
                {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
200
                  for (levID = 0; levID < varList1[varID].nlevels; levID++)
201 202
                    {
                      vlistDefIndex(vlistID1, varID, levID, index);
203
                      vlistDefFlag(vlistID1, varID, levID, true);
204 205 206 207
                    }
                }
            }

208
          const auto vlistID2 = vlistCreate();
209 210 211 212 213 214 215 216 217
          cdoVlistCopyFlag(vlistID2, vlistID1);
          vlistIDs[index] = vlistID2;

          cdiParamToString(params[index], paramstr, sizeof(paramstr));

          filename[nchars] = '\0';
          strcat(filename, paramstr);
          gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);

218
          streamIDs[index] = cdoOpenWrite(filename);
219
        }
220
    }
221
  else if (operatorID == SPLITTABNUM)
222 223
    {
      nsplit = 0;
224 225
      for (varID = 0; varID < nvars; varID++)
        {
226
          const auto tabnum = tableInqNum(vlistInqVarTable(vlistID1, varID));
227
          int index;
228 229 230 231 232 233 234 235 236 237
          for (index = 0; index < varID; index++)
            if (tabnum == tableInqNum(vlistInqVarTable(vlistID1, index))) break;

          if (index == varID)
            {
              itmp[nsplit] = tabnum;
              nsplit++;
            }
        }

238 239
      vlistIDs.resize(nsplit);
      streamIDs.resize(nsplit);
240
      std::vector<int> tabnums(nsplit);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
241
      for (int index = 0; index < nsplit; ++index) tabnums[index] = itmp[index];
242 243 244 245 246 247

      for (int index = 0; index < nsplit; ++index)
        {
          vlistClearFlag(vlistID1);
          for (varID = 0; varID < nvars; varID++)
            {
248
              const auto tabnum = tableInqNum(vlistInqVarTable(vlistID1, varID));
249 250
              if (tabnums[index] == tabnum)
                {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
251
                  for (levID = 0; levID < varList1[varID].nlevels; levID++)
252 253
                    {
                      vlistDefIndex(vlistID1, varID, levID, index);
254
                      vlistDefFlag(vlistID1, varID, levID, true);
255 256 257
                    }
                }
            }
258 259

          const auto vlistID2 = vlistCreate();
260 261 262 263 264 265
          cdoVlistCopyFlag(vlistID2, vlistID1);
          vlistIDs[index] = vlistID2;

          sprintf(filename + nchars, "%03d", tabnums[index]);
          gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);

266
          streamIDs[index] = cdoOpenWrite(filename);
267
        }
268
    }
269
  else if (operatorID == SPLITNAME)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
270 271 272
    {
      nsplit = nvars;

273 274
      vlistIDs.resize(nsplit);
      streamIDs.resize(nsplit);
275 276 277 278 279

      for (int index = 0; index < nsplit; index++)
        {
          vlistClearFlag(vlistID1);
          varID = index;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
280
          for (levID = 0; levID < varList1[varID].nlevels; levID++)
281 282
            {
              vlistDefIndex(vlistID1, varID, levID, index);
283
              vlistDefFlag(vlistID1, varID, levID, true);
284 285
            }

286
          const auto vlistID2 = vlistCreate();
287 288 289 290
          cdoVlistCopyFlag(vlistID2, vlistID1);
          vlistIDs[index] = vlistID2;

          filename[nchars] = '\0';
Uwe Schulzweida's avatar
Uwe Schulzweida committed
291
          strcat(filename, varList1[varID].name);
292 293
          gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);

294
          streamIDs[index] = cdoOpenWrite(filename);
295
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
296
    }
297
  else if (operatorID == SPLITLEVEL)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
298
    {
299
      const auto nzaxis = vlistNzaxis(vlistID1);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
300
      nsplit = 0;
301 302
      for (int index = 0; index < nzaxis; index++)
        {
303 304
          const auto zaxisID = vlistZaxis(vlistID1, index);
          const auto nlevs = zaxisInqSize(zaxisID);
305 306
          for (levID = 0; levID < nlevs; levID++)
            {
307
              const auto level = cdoZaxisInqLevel(zaxisID, levID);
308
              int i;
309 310 311 312 313 314
              for (i = 0; i < nsplit; i++)
                if (IS_EQUAL(level, ftmp[i])) break;
              if (i == nsplit) ftmp[nsplit++] = level;
            }
        }

315 316
      vlistIDs.resize(nsplit);
      streamIDs.resize(nsplit);
317
      Varray<double> levels(nsplit);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
318
      for (int index = 0; index < nsplit; ++index) levels[index] = ftmp[index];
319 320 321 322 323 324

      for (int index = 0; index < nsplit; ++index)
        {
          vlistClearFlag(vlistID1);
          for (varID = 0; varID < nvars; varID++)
            {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
325 326
              const auto zaxisID = varList1[varID].zaxisID;
              for (levID = 0; levID < varList1[varID].nlevels; levID++)
327
                {
328
                  const auto level = cdoZaxisInqLevel(zaxisID, levID);
329 330 331
                  if (IS_EQUAL(levels[index], level))
                    {
                      vlistDefIndex(vlistID1, varID, levID, index);
332
                      vlistDefFlag(vlistID1, varID, levID, true);
333 334 335
                    }
                }
            }
336 337

          const auto vlistID2 = vlistCreate();
338 339 340 341 342 343
          cdoVlistCopyFlag(vlistID2, vlistID1);
          vlistIDs[index] = vlistID2;

          sprintf(filename + nchars, "%06g", levels[index]);
          gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);

344
          streamIDs[index] = cdoOpenWrite(filename);
345
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
346
    }
347
  else if (operatorID == SPLITGRID)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
348
    {
349
      int gridID;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
350 351 352

      nsplit = vlistNgrids(vlistID1);

353 354
      vlistIDs.resize(nsplit);
      streamIDs.resize(nsplit);
355
      std::vector<int> gridIDs(nsplit);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
356
      for (int index = 0; index < nsplit; ++index) gridIDs[index] = vlistGrid(vlistID1, index);
357 358 359 360 361 362

      for (int index = 0; index < nsplit; ++index)
        {
          vlistClearFlag(vlistID1);
          for (varID = 0; varID < nvars; varID++)
            {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
363
              gridID = varList1[varID].gridID;
364 365
              if (gridIDs[index] == gridID)
                {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
366
                  for (levID = 0; levID < varList1[varID].nlevels; levID++)
367 368
                    {
                      vlistDefIndex(vlistID1, varID, levID, index);
369
                      vlistDefFlag(vlistID1, varID, levID, true);
370 371 372
                    }
                }
            }
373 374

          const auto vlistID2 = vlistCreate();
375 376 377
          cdoVlistCopyFlag(vlistID2, vlistID1);
          vlistIDs[index] = vlistID2;

Uwe Schulzweida's avatar
Uwe Schulzweida committed
378
          sprintf(filename + nchars, "%02d", vlistGridIndex(vlistID1, gridIDs[index]) + 1);
379 380
          gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);

381
          streamIDs[index] = cdoOpenWrite(filename);
382
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
383
    }
384
  else if (operatorID == SPLITZAXIS)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
385 386 387
    {
      nsplit = vlistNzaxis(vlistID1);

388 389
      vlistIDs.resize(nsplit);
      streamIDs.resize(nsplit);
390
      std::vector<int> zaxisIDs(nsplit);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
391
      for (int index = 0; index < nsplit; ++index) zaxisIDs[index] = vlistZaxis(vlistID1, index);
392 393 394 395 396 397

      for (int index = 0; index < nsplit; ++index)
        {
          vlistClearFlag(vlistID1);
          for (varID = 0; varID < nvars; varID++)
            {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
398
              const auto zaxisID = varList1[varID].zaxisID;
399 400
              if (zaxisIDs[index] == zaxisID)
                {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
401
                  for (levID = 0; levID < varList1[varID].nlevels; levID++)
402 403
                    {
                      vlistDefIndex(vlistID1, varID, levID, index);
404
                      vlistDefFlag(vlistID1, varID, levID, true);
405 406 407
                    }
                }
            }
408 409

          const auto vlistID2 = vlistCreate();
410 411 412
          cdoVlistCopyFlag(vlistID2, vlistID1);
          vlistIDs[index] = vlistID2;

Uwe Schulzweida's avatar
Uwe Schulzweida committed
413
          sprintf(filename + nchars, "%02d", vlistZaxisIndex(vlistID1, zaxisIDs[index]) + 1);
414 415
          gen_filename(filename, swap_obase, cdoGetObase(), filesuffix);

416
          streamIDs[index] = cdoOpenWrite(filename);
417
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
418 419 420 421 422 423
    }
  else
    {
      cdoAbort("not implemented!");
    }

424
  for (int index = 0; index < nsplit; index++)
425
    {
426
      if (uuid_attribute) cdo_def_tracking_id(vlistIDs[index], uuid_attribute);
427

428
      cdoDefVlist(streamIDs[index], vlistIDs[index]);
429 430
    }

431
  Varray<double> array;
432
  if (!lcopy)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
433
    {
434
      auto gridsizemax = vlistGridsizeMax(vlistID1);
435 436
      if (vlistNumber(vlistID1) != CDI_REAL) gridsizemax *= 2;
      array.resize(gridsizemax);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
437 438
    }

439
  int nrecs;
440
  int tsID = 0;
441
  while ((nrecs = cdoStreamInqTimestep(streamID1, tsID)))
Uwe Schulzweida's avatar
Uwe Schulzweida committed
442
    {
443
      for (int index = 0; index < nsplit; index++) cdoDefTimestep(streamIDs[index], tsID);
444 445 446

      for (int recID = 0; recID < nrecs; recID++)
        {
447
          cdoInqRecord(streamID1, &varID, &levelID);
448

Uwe Schulzweida's avatar
Uwe Schulzweida committed
449
          const auto index = vlistInqIndex(vlistID1, varID, levelID);
450
          const auto vlistID2 = vlistIDs[index];
451 452
          varID2 = vlistFindVar(vlistID2, varID);
          levelID2 = vlistFindLevel(vlistID2, varID, levelID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
453
          // printf("%d %d %d %d %d %d\n", index, vlistID2, varID, levelID, varID2, levelID2);
454

455
          cdoDefRecord(streamIDs[index], varID2, levelID2);
456 457
          if (lcopy)
            {
458
              cdoCopyRecord(streamIDs[index], streamID1);
459 460 461
            }
          else
            {
462 463
              cdoReadRecord(streamID1, array.data(), &nmiss);
              cdoWriteRecord(streamIDs[index], array.data(), nmiss);
464 465
            }
        }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
466 467 468 469

      tsID++;
    }

470
  cdoStreamClose(streamID1);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
471

Uwe Schulzweida's avatar
Uwe Schulzweida committed
472 473
  for (auto &streamID : streamIDs) cdoStreamClose(streamID);
  for (auto &vlistID : vlistIDs) vlistDestroy(vlistID);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
474 475 476

  cdoFinish();

Uwe Schulzweida's avatar
Uwe Schulzweida committed
477
  return nullptr;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
478
}