cmortable_parser.cc 8.75 KB
Newer Older
Uwe Schulzweida's avatar
Uwe Schulzweida committed
1
2
3
#include <errno.h>
#include "cdo_int.h"
#include "pmlist.h"
4
5
#include "json/jsmn.h"

Uwe Schulzweida's avatar
Uwe Schulzweida committed
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

static
char *readLineFromBuffer(char *buffer, size_t *buffersize, char *line, size_t len)
{
  int ichar;
  size_t ipos = 0;

  while ( *buffersize )
    {
      ichar = *buffer;
      (*buffersize)--;
      buffer++;
      if ( ichar == '\r' )
        {
          if ( *buffersize )
            {
              ichar = *buffer;
              if ( ichar == '\n' )
                {
                  (*buffersize)--;
                  buffer++;
                }
            }
          break;
        }
      if ( ichar == '\n' ) break;
      line[ipos++] = ichar;
      if ( ipos >= len )
        {
35
          fprintf(stderr, "readLineFromBuffer: end of line not found (maxlen = %zu)!\n", len);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
36
37
38
          break;
        }
    }
39

Uwe Schulzweida's avatar
Uwe Schulzweida committed
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
  line[ipos] = 0;

  if ( *buffersize == 0 && ipos == 0 ) buffer = NULL;

  return buffer;
}

static
char *skipSeparator(char *pline)
{
  while ( isspace((int) *pline) ) pline++;
  if ( *pline == '=' || *pline == ':' ) pline++;
  while ( isspace((int) *pline) ) pline++;

  return pline;
}

static
char *getElementName(char *pline, char *name)
{
  while ( isspace((int) *pline) ) pline++;
  size_t len = strlen(pline);
  size_t pos = 0;
  while ( pos < len && !isspace((int) *(pline+pos)) && *(pline+pos) != '=' && *(pline+pos) != ':' ) pos++;

  strncpy(name, pline, pos);
  name[pos] = 0;

  pline += pos;
  return pline;
}

static
Uwe Schulzweida's avatar
Uwe Schulzweida committed
73
char *getElementValue(char *pline)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
74
75
76
{
  while ( isspace((int) *pline) ) pline++;
  size_t len = strlen(pline);
77
78
79
80
  if ( *pline != '"' && *pline != '\'' )
    for ( size_t i = 1; i < len; ++i )
      if ( pline[i] == '!' ) { pline[i] = 0; len = i; break; }
  while ( isspace((int) *(pline+len-1)) && len ) { *(pline+len-1) = 0; len--; }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
81
82
83
84

  return pline;
}

85

Uwe Schulzweida's avatar
Uwe Schulzweida committed
86
void cmortablebuf_to_pmlist(list_t *pmlist, size_t buffersize, char *buffer)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
87
88
89
90
{
  char line[4096];
  char name[256];
  char *pline;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
91
92
  const char *listentry[] = {"axis_entry", "variable_entry"};
  int nentry = sizeof(listentry)/sizeof(listentry[0]);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
93
  int linenumber = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
94
  list_t *kvlist = NULL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
95
96
97
98
99
100
101
102
103

  while ( (buffer = readLineFromBuffer(buffer, &buffersize, line, sizeof(line))) )
    {
      linenumber++;
      pline = line;
      while ( isspace((int) *pline) ) pline++;
      if ( *pline == '#' || *pline == '!' || *pline == '\0' ) continue;
      //  len = (int) strlen(pline);

Uwe Schulzweida's avatar
Uwe Schulzweida committed
104
105
106
107
108
109
110
      int ientry = -1;
      for ( ientry = 0; ientry < nentry; ++ientry )
        if ( strncmp(pline, listentry[ientry], strlen(listentry[ientry])) == 0 ) break;
      
      if ( ientry < nentry )
	{
	  pline += strlen(listentry[ientry]);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
111

Uwe Schulzweida's avatar
Uwe Schulzweida committed
112
113
114
115
116
117
118
119
120
121
122
          kvlist = kvlist_new(listentry[ientry]);
          list_append(pmlist, &kvlist);

	  pline = skipSeparator(pline);
	  pline = getElementValue(pline);

	  if ( *pline ) kvlist_append(kvlist, "name", (const char **)&pline, 1);
	}
      else
	{
	  pline = getElementName(pline, name);
123
	  pline = skipSeparator(pline);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
124
125
126
127
128
129
130
131
132
	  pline = getElementValue(pline);

	  if ( kvlist == NULL )
            {
              kvlist = kvlist_new("Header");
              list_append(pmlist, &kvlist);
            }

	  if ( *pline ) kvlist_append(kvlist, name, (const char **)&pline, 1);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
133
	}
134
135
136
    }
}

137
// not used
138
139
140
141
int dump_json(const char *js, jsmntok_t *t, size_t count, int level)
{
  int i, j, k;
  if (count == 0)  return 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
142

143
144
145
146
  if (t->type == JSMN_PRIMITIVE)
    {
      printf("%.*s", t->end - t->start, js+t->start);
      return 1;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
147
    }
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
  else if (t->type == JSMN_STRING)
    {
      printf("'%.*s'", t->end - t->start, js+t->start);
      return 1;
    }
  else if (t->type == JSMN_OBJECT)
    {
      printf("\n");
      //  printf("Object: size %d\n", t->size);
      printf("Object: size %d count %d level %d\n", t->size, (int)count, level);
      j = 0;
      for (i = 0; i < t->size; i++)
        {
          for (k = 0; k < level; k++) printf("  ");
          j += dump_json(js, t+1+j, count-j, level+1);
          printf(": ");
          j += dump_json(js, t+1+j, count-j, level+1);
          printf("\n");
        }
      return j+1;
    }
  else if (t->type == JSMN_ARRAY)
    {
      j = 0;
      printf("\n");
      for (i = 0; i < t->size; i++)
        {
          for (k = 0; k < level-1; k++) printf("  ");
          printf("   - ");
          j += dump_json(js, t+1+j, count-j, level+1);
          printf("\n");
        }
      return j+1;
    }
  return 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
183
184
}

185
static
186
void kvlist_append_json(list_t *kvlist, const char *key, const char *js, jsmntok_t *t, int nvalues)
187
{
188
189
190
191
192
  keyValues_t *keyval = (keyValues_t *) malloc(sizeof(keyValues_t));
  keyval->key = strdup(key);
  keyval->nvalues = nvalues;
  keyval->values = (char **) malloc(nvalues*sizeof(char*));
  for ( int i = 0; i < nvalues; ++i )
193
    {
194
195
196
197
198
199
      size_t len = t[i].end - t[i].start;
      char *value = (char*) malloc((len+1)*sizeof(char));
      snprintf(value, len+1, "%.*s", (int)len, js+t[i].start);
      value[len] = 0;
      // printf("set %s: '%s'\n", key, value);
      keyval->values[i] = value;
200
    }
201
  list_append(kvlist, &keyval);
202
203
204
}

static
205
int json_to_pmlist(list_t *pmlist, const char *js, jsmntok_t *t, int count)
206
207
{
  bool debug = false;
208
  char name[4096];
209
  int i = 0;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
210
  int nobj = t[0].size;
211
  if ( t[0].type == JSMN_OBJECT )
Uwe Schulzweida's avatar
Uwe Schulzweida committed
212
    while ( nobj-- )
213
214
      {
        ++i;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
215
        int pmlname = i;
216
217
218
219
220
221
        if ( debug ) printf("  object: %.*s\n", t[i].end - t[i].start, js+t[i].start);
        ++i;
        if ( t[i].type == JSMN_OBJECT )
          {
            int ic = 0;
          NEXT:
222
223
            snprintf(name, sizeof(name), "%.*s", t[pmlname].end - t[pmlname].start, js+t[pmlname].start);
            name[sizeof(name)-1] = 0;
224
            // printf("new object: %s\n", name);
225
226
            list_t *kvlist = kvlist_new(name);
            list_append(pmlist, &kvlist);
227
228
229
230
231
232
233
                
            if ( t[i+2].type == JSMN_OBJECT )
              {
                if ( ic == 0 ) ic = t[i].size;
                else           ic--;
                
                ++i;
234
                kvlist_append_json(kvlist, "name", js, &t[i], 1);
235
236
237
238
                if ( debug ) printf("    name: '%.*s'\n", t[i].end - t[i].start, js+t[i].start);
                ++i;
              }
            int n = t[i].size;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
239
            while ( n-- )
240
241
              {
                ++i;
242
243
                snprintf(name, sizeof(name), "%.*s", t[i].end - t[i].start, js+t[i].start);
                name[sizeof(name)-1] = 0;
244
245
246
247
                if ( debug ) printf("    %.*s:", t[i].end - t[i].start, js+t[i].start);
                ++i;
                if ( t[i].type == JSMN_ARRAY )
                  {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
248
                    int nae = t[i].size;
249
                    kvlist_append_json(kvlist, name, js, &t[i+1], nae);
Uwe Schulzweida's avatar
Uwe Schulzweida committed
250
                    while ( nae-- )
251
252
253
254
255
256
257
                      {
                        ++i;
                        if ( debug ) printf(" '%.*s'", t[i].end - t[i].start, js+t[i].start);
                      }
                  }
                else
                  {
258
                    kvlist_append_json(kvlist, name, js, &t[i], 1);
259
260
261
262
263
264
265
266
267
268
                    if ( debug ) printf(" '%.*s'", t[i].end - t[i].start, js+t[i].start);
                  }
                if ( debug ) printf("\n");
              }
            if ( ic > 1 ) goto NEXT;
          }
      }

  if ( debug ) printf("Processed %d of %d tokens!\n", i, count-1);

269
270
271
272
  return 0;
}


Uwe Schulzweida's avatar
Uwe Schulzweida committed
273
void cmortablebuf_to_pmlist_json(list_t *pmlist, size_t buffersize, char *buffer, const char *filename)
274
275
{
  /* Prepare parser */
Uwe Schulzweida's avatar
Uwe Schulzweida committed
276
  jsmn_parser *p = jsmn_new();
277

Uwe Schulzweida's avatar
Uwe Schulzweida committed
278
279
  int status = jsmn_parse(p, buffer, buffersize);
  if ( status != 0 )
280
    {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
281
      switch (status)
282
        {
Uwe Schulzweida's avatar
Uwe Schulzweida committed
283
284
285
        case JSMN_ERROR_INVAL: fprintf(stderr, "JSON error: Invalid character in %s (line=%d character='%c')!\n", filename, p->lineno, buffer[p->pos]); break;
        case JSMN_ERROR_PART:  fprintf(stderr, "JSON error: End of string not found in %s (line=%d)!\n", filename, p->lineno); break;
        default:               fprintf(stderr, "JSON error in %s (line=%d)\n", filename, p->lineno); break;
286
287
288
        }
    }

Uwe Schulzweida's avatar
Uwe Schulzweida committed
289
290
  json_to_pmlist(pmlist, buffer, p->tokens, (int)p->toknext);
  jsmn_destroy(p);
291
292
293
}


294
list_t *cmortable_to_pmlist(FILE *fp, const char *name)
Uwe Schulzweida's avatar
Uwe Schulzweida committed
295
{
296
297
298
  listbuf_t *listbuf = listbuf_new();
  if ( listbuf_read(listbuf, fp, name) ) cdoAbort("Read error on CMOR table %s!", name);
  
299
  list_t *pmlist = NULL;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
300

301
  if ( listbuf->buffer[0] == '{' )
302
303
304
305
    {
      pmlist = list_new(sizeof(list_t *), free_kvlist, name);
      cmortablebuf_to_pmlist_json(pmlist, listbuf->size, listbuf->buffer, name);
    }
306
  else if ( strncmp(listbuf->buffer, "table_id:", 9) == 0 )
307
308
309
310
311
312
313
314
    {
      pmlist = list_new(sizeof(list_t *), free_kvlist, name);
      cmortablebuf_to_pmlist(pmlist, listbuf->size, listbuf->buffer);
    }
  else if ( listbuf->buffer[0] == '&' || listbuf->buffer[0] == '#' )
    {
      pmlist = namelistbuf_to_pmlist(listbuf);
    }
Uwe Schulzweida's avatar
Uwe Schulzweida committed
315
  else
316
    cdoAbort("Invalid CMOR table (file: %s)!", name);
317

318
319
320
  listbuf_destroy(listbuf);

  return pmlist;
Uwe Schulzweida's avatar
Uwe Schulzweida committed
321
}