Commit d97a2667 authored by Uwe Schulzweida's avatar Uwe Schulzweida
Browse files

Cleanup json parser.

parent 35fd263a
......@@ -284,35 +284,24 @@ int json_to_pmlist(list_t *pmlist, const char *js, jsmntok_t *t, int count)
}
void cmortablebuf_to_pmlist_json(list_t *pmlist, size_t buffersize, char *buffer)
void cmortablebuf_to_pmlist_json(list_t *pmlist, size_t buffersize, char *buffer, const char *filename)
{
char *js = buffer;
size_t jslen = buffersize;
/* Prepare parser */
jsmn_parser p;
jsmn_init(&p);
/* Allocate some tokens as a start */
size_t tokcount = 32;
jsmntok_t *tok = (jsmntok_t *) Malloc(sizeof(*tok) * tokcount);
jsmn_parser *p = jsmn_new();
int r;
AGAIN:
r = jsmn_parse(&p, js, jslen, tok, tokcount);
if ( r < 0 )
int status = jsmn_parse(p, buffer, buffersize);
if ( status != 0 )
{
if ( r == JSMN_ERROR_NOMEM )
switch (status)
{
tokcount = tokcount * 2;
tok = (jsmntok_t *) Realloc(tok, sizeof(*tok) * tokcount);
goto AGAIN;
case JSMN_ERROR_INVAL: fprintf(stderr, "JSON error: Invalid character in %s (line=%d character='%c')!\n", filename, p->lineno, buffer[p->pos]); break;
case JSMN_ERROR_PART: fprintf(stderr, "JSON error: End of string not found in %s (line=%d)!\n", filename, p->lineno); break;
default: fprintf(stderr, "JSON error in %s (line=%d)\n", filename, p->lineno); break;
}
}
json_to_pmlist(pmlist, js, tok, (int)p.toknext);
Free(tok);
json_to_pmlist(pmlist, buffer, p->tokens, (int)p->toknext);
jsmn_destroy(p);
}
......@@ -324,7 +313,7 @@ list_t *cmortable_to_pmlist(FILE *fp, const char *name)
list_t *pmlist = list_new(sizeof(list_t *), free_kvlist, name);
if ( listbuf->buffer[0] == '{' )
cmortablebuf_to_pmlist_json(pmlist, listbuf->size, listbuf->buffer);
cmortablebuf_to_pmlist_json(pmlist, listbuf->size, listbuf->buffer, name);
else if ( strncmp(listbuf->buffer, "table_id:", 9) == 0 )
cmortablebuf_to_pmlist(pmlist, listbuf->size, listbuf->buffer);
else
......
#include <stdio.h>
#include <stdlib.h>
#include "jsmn.h"
static
void jsmn_init(jsmn_parser *parser)
{
parser->tokens = NULL;
parser->num_tokens = 0;
parser->toksuper = -1;
parser->toknext = 0;
parser->pos = 0;
parser->lineno = 0;
}
jsmn_parser *jsmn_new(void)
{
jsmn_parser *parser = (jsmn_parser *) malloc(sizeof(jsmn_parser));
jsmn_init(parser);
return parser;
}
void jsmn_destroy(jsmn_parser *parser)
{
if ( parser )
{
if ( parser->tokens ) free(parser->tokens);
jsmn_init(parser);
free(parser);
}
}
/**
* Allocates a fresh unused token from the token pull.
*/
static
jsmntok_t *jsmn_alloc_token(jsmn_parser *parser, jsmntok_t *tokens, size_t num_tokens)
jsmntok_t *jsmn_alloc_token(jsmn_parser *parser)
{
if (parser->toknext >= num_tokens) return NULL;
const unsigned int TOK_MEM_INCR = 64;
jsmntok_t *tok = &tokens[parser->toknext++];
if ( parser->toknext >= parser->num_tokens )
{
parser->num_tokens += TOK_MEM_INCR;
parser->tokens = (jsmntok_t *) realloc(parser->tokens, sizeof(jsmntok_t) * parser->num_tokens);
if ( parser->tokens == NULL )
{
fprintf(stderr, "%s: Failed to allocated more memory!", __func__);
exit(-1);
}
}
jsmntok_t *tok = &parser->tokens[parser->toknext++];
tok->start = tok->end = -1;
tok->size = 0;
#ifdef JSMN_PARENT_LINKS
tok->parent = -1;
#endif
return tok;
}
......@@ -29,10 +74,9 @@ void jsmn_fill_token(jsmntok_t *token, jsmntype_t type, int start, int end)
token->size = 0;
}
/**
* Fills next available token with JSON primitive.
*/
static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *tokens, size_t num_tokens)
// Fills next available token with JSON primitive.
static
int jsmn_parse_primitive(jsmn_parser *parser, const char *js, size_t len)
{
jsmntok_t *token;
int start = parser->pos;
......@@ -62,29 +106,21 @@ static int jsmn_parse_primitive(jsmn_parser *parser, const char *js, size_t len,
#endif
found:
if (tokens == NULL)
{
parser->pos--;
return 0;
}
token = jsmn_alloc_token(parser, tokens, num_tokens);
if (token == NULL)
{
parser->pos = start;
return JSMN_ERROR_NOMEM;
}
token = jsmn_alloc_token(parser);
jsmn_fill_token(token, JSMN_PRIMITIVE, start, parser->pos);
#ifdef JSMN_PARENT_LINKS
token->parent = parser->toksuper;
#endif
parser->pos--;
return 0;
}
/**
* Fills next token with JSON string.
*/
static int jsmn_parse_string(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *tokens, size_t num_tokens)
static int jsmn_parse_string(jsmn_parser *parser, const char *js, size_t len)
{
jsmntok_t *token;
int start = parser->pos;
......@@ -99,14 +135,7 @@ static int jsmn_parse_string(jsmn_parser *parser, const char *js, size_t len, js
/* Quote: end of string */
if (c == '\"')
{
if (tokens == NULL) return 0;
token = jsmn_alloc_token(parser, tokens, num_tokens);
if (token == NULL)
{
parser->pos = start;
return JSMN_ERROR_NOMEM;
}
token = jsmn_alloc_token(parser);
jsmn_fill_token(token, JSMN_STRING, start+1, parser->pos);
#ifdef JSMN_PARENT_LINKS
token->parent = parser->toksuper;
......@@ -150,15 +179,14 @@ static int jsmn_parse_string(jsmn_parser *parser, const char *js, size_t len, js
return JSMN_ERROR_PART;
}
/**
* Parse JSON string and fill tokens.
*/
int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *tokens, unsigned int num_tokens)
// Parse JSON string and fill tokens.
int jsmn_parse(jsmn_parser *parser, const char *js, size_t len)
{
int r;
int i;
jsmntok_t *token;
int count = parser->toknext;
parser->lineno = 1;
for (; parser->pos < len && js[parser->pos] != '\0'; parser->pos++)
{
......@@ -168,13 +196,10 @@ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *token
switch (c)
{
case '{': case '[':
count++;
if (tokens == NULL) break;
token = jsmn_alloc_token(parser, tokens, num_tokens);
if (token == NULL) return JSMN_ERROR_NOMEM;
token = jsmn_alloc_token(parser);
if (parser->toksuper != -1)
{
tokens[parser->toksuper].size++;
parser->tokens[parser->toksuper].size++;
#ifdef JSMN_PARENT_LINKS
token->parent = parser->toksuper;
#endif
......@@ -184,11 +209,10 @@ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *token
parser->toksuper = parser->toknext - 1;
break;
case '}': case ']':
if (tokens == NULL) break;
type = (c == '}' ? JSMN_OBJECT : JSMN_ARRAY);
#ifdef JSMN_PARENT_LINKS
if (parser->toknext < 1) return JSMN_ERROR_INVAL;
token = &tokens[parser->toknext - 1];
token = &parser->tokens[parser->toknext - 1];
for (;;)
{
if (token->start != -1 && token->end == -1)
......@@ -203,12 +227,12 @@ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *token
if (token->type != type || parser->toksuper == -1) return JSMN_ERROR_INVAL;
break;
}
token = &tokens[token->parent];
token = &parser->tokens[token->parent];
}
#else
for (i = parser->toknext - 1; i >= 0; i--)
{
token = &tokens[i];
token = &parser->tokens[i];
if (token->start != -1 && token->end == -1)
{
if (token->type != type) return JSMN_ERROR_INVAL;
......@@ -221,7 +245,7 @@ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *token
if (i == -1) return JSMN_ERROR_INVAL;
for (; i >= 0; i--)
{
token = &tokens[i];
token = &parser->tokens[i];
if (token->start != -1 && token->end == -1) {
parser->toksuper = i;
break;
......@@ -230,30 +254,34 @@ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *token
#endif
break;
case '\"':
r = jsmn_parse_string(parser, js, len, tokens, num_tokens);
r = jsmn_parse_string(parser, js, len);
if (r < 0) return r;
count++;
if (parser->toksuper != -1 && tokens != NULL)
tokens[parser->toksuper].size++;
if ( parser->toksuper != -1 )
parser->tokens[parser->toksuper].size++;
break;
case '\r':
if ( parser->pos+1 < len && js[parser->pos+1] == '\n' ) parser->pos++;
case '\n':
parser->lineno++;
break;
case '\t' : case '\r' : case '\n' : case ' ':
case '\t': case ' ':
break;
case ':':
parser->toksuper = parser->toknext - 1;
break;
case ',':
if (tokens != NULL && parser->toksuper != -1 &&
tokens[parser->toksuper].type != JSMN_ARRAY &&
tokens[parser->toksuper].type != JSMN_OBJECT)
if (parser->toksuper != -1 &&
parser->tokens[parser->toksuper].type != JSMN_ARRAY &&
parser->tokens[parser->toksuper].type != JSMN_OBJECT)
{
#ifdef JSMN_PARENT_LINKS
parser->toksuper = tokens[parser->toksuper].parent;
parser->toksuper = parser->tokens[parser->toksuper].parent;
#else
for (i = parser->toknext - 1; i >= 0; i--)
{
if (tokens[i].type == JSMN_ARRAY || tokens[i].type == JSMN_OBJECT)
if (parser->tokens[i].type == JSMN_ARRAY || parser->tokens[i].type == JSMN_OBJECT)
{
if (tokens[i].start != -1 && tokens[i].end == -1)
if (parser->tokens[i].start != -1 && parser->tokens[i].end == -1)
{
parser->toksuper = i;
break;
......@@ -269,9 +297,9 @@ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *token
case '5': case '6': case '7' : case '8': case '9':
case 't': case 'f': case 'n' :
/* And they must not be keys of the object */
if (tokens != NULL && parser->toksuper != -1)
if (parser->toksuper != -1)
{
jsmntok_t *t = &tokens[parser->toksuper];
jsmntok_t *t = &parser->tokens[parser->toksuper];
if (t->type == JSMN_OBJECT || (t->type == JSMN_STRING && t->size != 0))
return JSMN_ERROR_INVAL;
}
......@@ -279,11 +307,10 @@ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *token
/* In non-strict mode every unquoted value is a primitive */
default:
#endif
r = jsmn_parse_primitive(parser, js, len, tokens, num_tokens);
r = jsmn_parse_primitive(parser, js, len);
if (r < 0) return r;
count++;
if (parser->toksuper != -1 && tokens != NULL)
tokens[parser->toksuper].size++;
if ( parser->toksuper != -1 )
parser->tokens[parser->toksuper].size++;
break;
#ifdef JSMN_STRICT
......@@ -294,26 +321,11 @@ int jsmn_parse(jsmn_parser *parser, const char *js, size_t len, jsmntok_t *token
}
}
if (tokens != NULL)
for (i = parser->toknext - 1; i >= 0; i--)
{
for (i = parser->toknext - 1; i >= 0; i--)
{
/* Unmatched opened object or array */
if (tokens[i].start != -1 && tokens[i].end == -1) return JSMN_ERROR_PART;
}
/* Unmatched opened object or array */
if (parser->tokens[i].start != -1 && parser->tokens[i].end == -1) return JSMN_ERROR_PART;
}
return count;
}
/**
* Creates a new parser based over a given buffer with an array of tokens
* available.
*/
void jsmn_init(jsmn_parser *parser)
{
parser->pos = 0;
parser->toknext = 0;
parser->toksuper = -1;
return 0;
}
......@@ -40,12 +40,12 @@ enum jsmnerr {
* end end position in JSON data string
*/
typedef struct {
jsmntype_t type;
int start;
int end;
int size;
jsmntype_t type;
int start;
int end;
int size;
#ifdef JSMN_PARENT_LINKS
int parent;
int parent;
#endif
} jsmntok_t;
......@@ -54,22 +54,23 @@ typedef struct {
* the string being parsed now and current position in that string
*/
typedef struct {
unsigned int pos; /* offset in the JSON string */
unsigned int toknext; /* next token to allocate */
int toksuper; /* superior token node, e.g parent object or array */
jsmntok_t *tokens;
unsigned int num_tokens;
unsigned int pos; /* offset in the JSON string */
unsigned int lineno;
unsigned int toknext; /* next token to allocate */
int toksuper; /* superior token node, e.g parent object or array */
} jsmn_parser;
/**
* Create JSON parser over an array of tokens
*/
void jsmn_init(jsmn_parser *parser);
/**
* Run JSON parser. It parses a JSON data string into and array of tokens, each describing
* a single JSON object.
*/
int jsmn_parse(jsmn_parser *parser, const char *js, size_t len,
jsmntok_t *tokens, unsigned int num_tokens);
int jsmn_parse(jsmn_parser *parser, const char *js, size_t len);
jsmn_parser *jsmn_new(void);
void jsmn_destroy(jsmn_parser *parser);
#ifdef __cplusplus
}
......
......@@ -2,29 +2,14 @@
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <sys/stat.h>
#include "jsmn.h"
/* Function realloc_it() is a wrapper function for standart realloc()
* with one difference - it frees old memory pointer in case of realloc
* failure. Thus, DO NOT use old data pointer in anyway after call to
* realloc_it(). If your code has some kind of fallback algorithm if
* memory can't be re-allocated - use standart realloc() instead.
*/
static inline void *realloc_it(void *ptrmem, size_t size) {
void *p = realloc(ptrmem, size);
if (!p) {
free (ptrmem);
fprintf(stderr, "realloc(): errno=%d\n", errno);
}
return p;
}
/*
* An example of reading JSON from stdin and printing its content to stdout.
* The output looks like YAML, but I'm not sure if it's really compatible.
*/
static
int dump(const char *js, jsmntok_t *t, size_t count, int indent)
{
......@@ -72,66 +57,50 @@ int dump(const char *js, jsmntok_t *t, size_t count, int indent)
return 0;
}
int main() {
int r;
int eof_expected = 0;
char *js = NULL;
size_t jslen = 0;
char buf[BUFSIZ];
jsmn_parser p;
jsmntok_t *tok;
size_t tokcount = 2;
int main()
{
FILE *fp = stdin;
int filedes = fileno(fp);
struct stat buf;
size_t filesize = 0;
if ( fstat(filedes, &buf) == 0 ) filesize = (size_t) buf.st_size;
if ( filesize == 0 )
{
fprintf(stderr, "Empty stream!\n");
return -1;
}
char *buffer = (char*) malloc(filesize);
size_t nitems = fread(buffer, 1, filesize, fp);
/* Prepare parser */
jsmn_init(&p);
if ( nitems != filesize )
{
free(buffer);
fprintf(stderr, "Read failed on stdin!\n");
return -1;
}
/* Allocate some tokens as a start */
tok = malloc(sizeof(*tok) * tokcount);
if (tok == NULL) {
fprintf(stderr, "malloc(): errno=%d\n", errno);
return 3;
}
/* Prepare parser */
jsmn_parser *p = jsmn_new();
for (;;) {
/* Read another chunk */
r = fread(buf, 1, sizeof(buf), stdin);
if (r < 0) {
fprintf(stderr, "fread(): %d, errno=%d\n", r, errno);
return 1;
}
if (r == 0) {
if (eof_expected != 0) {
return 0;
} else {
fprintf(stderr, "fread(): unexpected EOF\n");
return 2;
}
}
int status = jsmn_parse(p, buffer, filesize);
if ( status != 0 )
{
const char *filename = "stdin";
switch (status)
{
case JSMN_ERROR_INVAL: fprintf(stderr, "JSON error: Invalid character in %s (line=%d character='%c')!\n", filename, p->lineno, buffer[p->pos]); break;
case JSMN_ERROR_PART: fprintf(stderr, "JSON error: End of string not found in %s (line=%d)!\n", filename, p->lineno); break;
default: fprintf(stderr, "JSON error in %s (line=%d)\n", filename, p->lineno); break;
}
}
js = realloc_it(js, jslen + r + 1);
if (js == NULL) {
return 3;
}
strncpy(js + jslen, buf, r);
jslen = jslen + r;
dump(buffer, p->tokens, p->toknext, 0);
free(buffer);
again:
r = jsmn_parse(&p, js, jslen, tok, tokcount);
if (r < 0) {
if (r == JSMN_ERROR_NOMEM) {
tokcount = tokcount * 2;
tok = realloc_it(tok, sizeof(*tok) * tokcount);
if (tok == NULL) {
return 3;
}
goto again;
}
} else {
dump(js, tok, p.toknext, 0);
eof_expected = 1;
}
}
jsmn_destroy(p);
return EXIT_SUCCESS;
return EXIT_SUCCESS;
}
......@@ -12,6 +12,7 @@ void namelist_init(namelist_parser *parser)
parser->num_tokens = 0;
parser->toknext = 0;
parser->pos = 0;
parser->lineno = 0;
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment