Split the json_tokener_parse tests off from test1 into their own test and add several more cases to check various incremental parsing situations.

.gitignore

@@ -23,6 +23,7 @@ test2
 test4
 testSubDir
 test_parse_int64
+test_parse
 test_cast
 test_null
 Debug

Makefile.am

@@ -38,7 +38,7 @@ libjson_la_SOURCES = \
 	linkhash.c \
 	printbuf.c
 
-check_PROGRAMS = test1 test2 test4 test_parse_int64 test_null test_cast
+check_PROGRAMS = test1 test2 test4 test_parse_int64 test_null test_cast test_parse
 
 test1_SOURCES = test1.c
 test1_LDADD = $(lib_LTLIBRARIES)
@@ -58,7 +58,10 @@ test_null_LDADD = $(lib_LTLIBRARIES)
 test_cast_SOURCES = test_cast.c
 test_cast_LDADD = $(lib_LTLIBRARIES)
 
-TESTS = test1.test test2.test test4.test parse_int64.test test_null.test test_cast.test 
+test_parse_SOURCES = test_parse.c
+test_parse_LDADD = $(lib_LTLIBRARIES)
+
+TESTS = test1.test test2.test test4.test parse_int64.test test_null.test test_cast.test test_parse.test
 EXTRA_DIST += $(TESTS)
 testsubdir=testSubDir
 TESTS_ENVIRONMENT       = top_builddir=$(top_builddir)

test1.c

@@ -31,9 +31,7 @@ static int sort_fn (const void *j1, const void *j2)
 
 int main(int argc, char **argv)
 {
-  json_tokener *tok;
   json_object *my_string, *my_int, *my_object, *my_array;
-  json_object *new_obj;
   int i;
 
   MC_SET_DEBUG(1);
@@ -104,112 +102,6 @@ int main(int argc, char **argv)
   }
   printf("my_object.to_string()=%s\n", json_object_to_json_string(my_object));
 
-  new_obj = json_tokener_parse("\"\003\"");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("/* hello */\"foo\"");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("// hello\n\"foo\"");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("\"\\u0041\\u0042\\u0043\"");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("null");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("True");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("12");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("12.3");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("[\"\\n\"]");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("[\"\\nabc\\n\"]");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("[null]");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("[]");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("[false]");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("[\"abc\",null,\"def\",12]");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("{}");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("{ \"foo\": \"bar\" }");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("{ \"foo\": \"bar\", \"baz\": null, \"bool0\": true }");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("{ \"foo\": [null, \"foo\"] }");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  new_obj = json_tokener_parse("{ \"abc\": 12, \"foo\": \"bar\", \"bool0\": false, \"bool1\": true, \"arr\": [ 1, 2, 3, null, 5 ] }");
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);
-
-  enum json_tokener_error error = json_tokener_success;
-  new_obj = json_tokener_parse_verbose("{ foo }", &error);
-  assert (error == json_tokener_error_parse_object_key_name);
-  assert (new_obj == NULL);
-
-  new_obj = json_tokener_parse("{ foo }");
-  assert (new_obj == NULL);
-  
-  // if(is_error(new_obj)) printf("got error as expected\n");
-
-  new_obj = json_tokener_parse("foo");
-  assert (new_obj == NULL);
-  new_obj = json_tokener_parse_verbose("foo", &error);
-  assert (new_obj == NULL);
-  assert (error == json_tokener_error_parse_boolean);
-
-  new_obj = json_tokener_parse("{ \"foo");
-  if(is_error(new_obj)) printf("got error as expected\n");
-
-  /* test incremental parsing */
-  tok = json_tokener_new();
-  new_obj = json_tokener_parse_ex(tok, "{ \"foo", 6);
-  if(is_error(new_obj)) printf("got error as expected\n");
-  new_obj = json_tokener_parse_ex(tok, "\": {\"bar", 8);
-  if(is_error(new_obj)) printf("got error as expected\n");
-  new_obj = json_tokener_parse_ex(tok, "\":13}}", 6);
-  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
-  json_object_put(new_obj);  
-  json_tokener_free(tok);
-
   json_object_put(my_string);
   json_object_put(my_int);
   json_object_put(my_object);

test1.expected

@@ -33,26 +33,3 @@ my_object=
 	bool0: false
 	bool1: true
 my_object.to_string()={ "abc": 12, "foo": "bar", "bool0": false, "bool1": true }
-new_obj.to_string()="\u0003"
-new_obj.to_string()="foo"
-new_obj.to_string()="foo"
-new_obj.to_string()="ABC"
-new_obj.to_string()=null
-new_obj.to_string()=true
-new_obj.to_string()=12
-new_obj.to_string()=12.300000
-new_obj.to_string()=[ "\n" ]
-new_obj.to_string()=[ "\nabc\n" ]
-new_obj.to_string()=[ null ]
-new_obj.to_string()=[ ]
-new_obj.to_string()=[ false ]
-new_obj.to_string()=[ "abc", null, "def", 12 ]
-new_obj.to_string()={ }
-new_obj.to_string()={ "foo": "bar" }
-new_obj.to_string()={ "foo": "bar", "baz": null, "bool0": true }
-new_obj.to_string()={ "foo": [ null, "foo" ] }
-new_obj.to_string()={ "abc": 12, "foo": "bar", "bool0": false, "bool1": true, "arr": [ 1, 2, 3, null, 5 ] }
-got error as expected
-got error as expected
-got error as expected
-new_obj.to_string()={ "foo": { "bar": 13 } }

test_parse.c

@@ -0,0 +1,270 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+#include <assert.h>
+
+#include "json.h"
+#include "json_tokener.h"
+
+static void test_basic_parse(void);
+static void test_verbose_parse(void);
+static void test_incremental_parse(void);
+
+int main(int argc, char **argv)
+{
+	MC_SET_DEBUG(1);
+
+	test_basic_parse();
+	printf("==================================\n");
+	test_verbose_parse();
+	printf("==================================\n");
+	test_incremental_parse();
+	printf("==================================\n");
+}
+
+static void test_basic_parse()
+{
+  json_object *new_obj;
+
+  new_obj = json_tokener_parse("\"\003\"");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("/* hello */\"foo\"");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("// hello\n\"foo\"");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("\"\\u0041\\u0042\\u0043\"");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("null");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("True");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("12");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("12.3");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("[\"\\n\"]");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("[\"\\nabc\\n\"]");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("[null]");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("[]");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("[false]");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("[\"abc\",null,\"def\",12]");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("{}");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("{ \"foo\": \"bar\" }");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("{ \"foo\": \"bar\", \"baz\": null, \"bool0\": true }");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("{ \"foo\": [null, \"foo\"] }");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+
+  new_obj = json_tokener_parse("{ \"abc\": 12, \"foo\": \"bar\", \"bool0\": false, \"bool1\": true, \"arr\": [ 1, 2, 3, null, 5 ] }");
+  printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj));
+  json_object_put(new_obj);
+}
+
+static void test_verbose_parse()
+{
+	json_object *new_obj;
+	enum json_tokener_error error = json_tokener_success;
+
+	new_obj = json_tokener_parse_verbose("{ foo }", &error);
+	assert (error == json_tokener_error_parse_object_key_name);
+	assert (new_obj == NULL);
+
+	new_obj = json_tokener_parse("{ foo }");
+	assert (new_obj == NULL);
+
+	new_obj = json_tokener_parse("foo");
+	assert (new_obj == NULL);
+	new_obj = json_tokener_parse_verbose("foo", &error);
+	assert (new_obj == NULL);
+
+ 	/* b/c the string starts with 'f' parsing return a boolean error */
+	assert (error == json_tokener_error_parse_boolean);
+
+	printf("json_tokener_parse_versbose() OK\n");
+}
+
+struct incremental_step {
+	const char *string_to_parse;
+	int length;
+	int char_offset;
+	enum json_tokener_error expected_error;
+	int reset_tokener;
+} incremental_steps[] = {
+
+	/* Check that full json messages can be parsed, both w/ and w/o a reset */
+	{ "{ \"foo\": 123 }", -1, -1, json_tokener_success,  0 },
+	{ "{ \"foo\": 456 }", -1, -1, json_tokener_success,  1 },
+	{ "{ \"foo\": 789 }", -1, -1, json_tokener_success,  1 },
+
+	/*  Check a basic incremental parse */
+	{ "{ \"foo",          -1, -1, json_tokener_continue, 0 },
+	{ "\": {\"bar",       -1, -1, json_tokener_continue, 0 },
+	{ "\":13}}",          -1, -1, json_tokener_success,  1 },
+
+	/* Check that json_tokener_reset actually resets */
+	{ "{ \"foo",          -1, -1, json_tokener_continue, 1 },
+	{ ": \"bar\"}",       -1, 0, json_tokener_error_parse_unexpected, 1 },
+
+	/* Check incremental parsing with trailing characters */
+	{ "{ \"foo",          -1, -1, json_tokener_continue, 0 },
+	{ "\": {\"bar",       -1, -1, json_tokener_continue, 0 },
+	{ "\":13}}XXXX",      10, 6, json_tokener_success,  0 },
+	{ "XXXX",              4, 0, json_tokener_error_parse_unexpected, 1 },
+
+	/* Check that trailing characters can change w/o a reset */
+	{ "{\"x\": 123 }\"X\"", -1, 11, json_tokener_success, 0 },
+	{ "\"Y\"",            -1, -1, json_tokener_success, 1 },
+
+	/* To stop parsing a number we need to reach a non-digit, e.g. a \0 */
+	{ "1",                 1, 1, json_tokener_continue, 0 },
+	{ "2",                 2, 1, json_tokener_success, 0 },
+
+	/* Strings have a well defined end point, so we can stop at the quote */
+	{ "\"blue\"",         -1, -1, json_tokener_success, 0 },
+
+	{ "[1,2,3]",          -1, -1, json_tokener_success, 0 },
+
+	/* This behaviour doesn't entirely follow the json spec, but until we have
+	   a way to specify how strict to be we follow Postel's Law and be liberal
+	   in what we accept (up to a point). */
+	{ "[1,2,3,]",         -1, -1, json_tokener_success, 0 },
+	{ "[1,2,,3,]",        -1, 5, json_tokener_error_parse_unexpected, 0 },
+
+	{ NULL, json_tokener_success },
+};
+
+static void test_incremental_parse()
+{
+	json_object *new_obj;
+	enum json_tokener_error jerr;
+	json_tokener *tok;
+	const char *string_to_parse;
+	int ii;
+	int num_ok, num_error;
+
+	num_ok = 0;
+	num_error = 0;
+
+	printf("Starting incremental tests.\n");
+
+	string_to_parse = "{ \"foo"; /* } */
+	printf("json_tokener_parse(%s) ... ", string_to_parse);
+	new_obj = json_tokener_parse(string_to_parse);
+	if (new_obj == NULL) printf("got error as expected\n");
+
+	/* test incremental parsing in various forms */
+	tok = json_tokener_new();
+	for (ii = 0; incremental_steps[ii].string_to_parse != NULL; ii++)
+	{
+		int this_step_ok = 0;
+		struct incremental_step *step = &incremental_steps[ii];
+		int length = step->length;
+		int expected_char_offset = step->char_offset;
+		if (length == -1)
+			length = strlen(step->string_to_parse);
+		if (expected_char_offset == -1)
+			expected_char_offset = length;
+
+		printf("json_tokener_parse_ex(tok, %-12s, %3d) ... ",
+			step->string_to_parse, length);
+		new_obj = json_tokener_parse_ex(tok, step->string_to_parse, length);
+
+		jerr = json_tokener_get_error(tok);
+		if (step->expected_error != json_tokener_success)
+		{
+			if (new_obj != NULL)
+				printf("ERROR: invalid object returned: %s\n",
+					json_object_to_json_string(new_obj));
+			else if (jerr != step->expected_error)
+				printf("ERROR: got wrong error: %s\n",
+					json_tokener_error_desc(jerr));
+			else if (tok->char_offset != expected_char_offset)
+				printf("ERROR: wrong char_offset %d != expected %d\n",
+					tok->char_offset,
+					expected_char_offset);
+			else
+			{
+				printf("OK: got correct error: %s\n", json_tokener_error_desc(jerr));
+				this_step_ok = 1;
+			}
+		}
+		else
+		{
+			if (new_obj == NULL)
+				printf("ERROR: expected valid object, instead: %s\n",
+					json_tokener_error_desc(jerr));
+			else if (tok->char_offset != expected_char_offset)
+				printf("ERROR: wrong char_offset %d != expected %d\n",
+					tok->char_offset,
+					expected_char_offset);
+			else
+			{
+				printf("OK: got object of type [%s]: %s\n",
+					json_type_to_name(json_object_get_type(new_obj)),
+					json_object_to_json_string(new_obj));
+				this_step_ok = 1;
+			}
+		}
+
+		if (new_obj)
+			json_object_put(new_obj);
+
+		if (step->reset_tokener)
+			json_tokener_reset(tok);
+
+		if (this_step_ok)
+			num_ok++;
+		else
+			num_error++;
+	}
+
+	json_tokener_free(tok);
+
+	printf("End Incremental Tests OK=%d ERROR=%d\n", num_ok, num_error);
+
+	return;
+}

test_parse.expected

@@ -0,0 +1,46 @@
+new_obj.to_string()="\u0003"
+new_obj.to_string()="foo"
+new_obj.to_string()="foo"
+new_obj.to_string()="ABC"
+new_obj.to_string()=null
+new_obj.to_string()=true
+new_obj.to_string()=12
+new_obj.to_string()=12.300000
+new_obj.to_string()=[ "\n" ]
+new_obj.to_string()=[ "\nabc\n" ]
+new_obj.to_string()=[ null ]
+new_obj.to_string()=[ ]
+new_obj.to_string()=[ false ]
+new_obj.to_string()=[ "abc", null, "def", 12 ]
+new_obj.to_string()={ }
+new_obj.to_string()={ "foo": "bar" }
+new_obj.to_string()={ "foo": "bar", "baz": null, "bool0": true }
+new_obj.to_string()={ "foo": [ null, "foo" ] }
+new_obj.to_string()={ "abc": 12, "foo": "bar", "bool0": false, "bool1": true, "arr": [ 1, 2, 3, null, 5 ] }
+==================================
+json_tokener_parse_versbose() OK
+==================================
+Starting incremental tests.
+json_tokener_parse({ "foo) ... got error as expected
+json_tokener_parse_ex(tok, { "foo": 123 },  14) ... OK: got object of type [object]: { "foo": 123 }
+json_tokener_parse_ex(tok, { "foo": 456 },  14) ... OK: got object of type [object]: { "foo": 456 }
+json_tokener_parse_ex(tok, { "foo": 789 },  14) ... OK: got object of type [object]: { "foo": 789 }
+json_tokener_parse_ex(tok, { "foo      ,   6) ... OK: got correct error: continue
+json_tokener_parse_ex(tok, ": {"bar    ,   8) ... OK: got correct error: continue
+json_tokener_parse_ex(tok, ":13}}      ,   6) ... OK: got object of type [object]: { "foo": { "bar": 13 } }
+json_tokener_parse_ex(tok, { "foo      ,   6) ... OK: got correct error: continue
+json_tokener_parse_ex(tok, : "bar"}    ,   8) ... OK: got correct error: unexpected character
+json_tokener_parse_ex(tok, { "foo      ,   6) ... OK: got correct error: continue
+json_tokener_parse_ex(tok, ": {"bar    ,   8) ... OK: got correct error: continue
+json_tokener_parse_ex(tok, ":13}}XXXX  ,  10) ... OK: got object of type [object]: { "foo": { "bar": 13 } }
+json_tokener_parse_ex(tok, XXXX        ,   4) ... OK: got correct error: unexpected character
+json_tokener_parse_ex(tok, {"x": 123 }"X",  14) ... OK: got object of type [object]: { "x": 123 }
+json_tokener_parse_ex(tok, "Y"         ,   3) ... OK: got object of type [string]: "Y"
+json_tokener_parse_ex(tok, 1           ,   1) ... OK: got correct error: continue
+json_tokener_parse_ex(tok, 2           ,   2) ... OK: got object of type [int]: 12
+json_tokener_parse_ex(tok, "blue"      ,   6) ... OK: got object of type [string]: "blue"
+json_tokener_parse_ex(tok, [1,2,3]     ,   7) ... OK: got object of type [array]: [ 1, 2, 3 ]
+json_tokener_parse_ex(tok, [1,2,3,]    ,   8) ... OK: got object of type [array]: [ 1, 2, 3 ]
+json_tokener_parse_ex(tok, [1,2,,3,]   ,   9) ... OK: got correct error: unexpected character
+End Incremental Tests OK=20 ERROR=0
+==================================

test_parse.test

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Common definitions
+if test -z "$srcdir"; then
+    srcdir="${0%/*}"
+    test "$srcdir" = "$0" && srcdir=.
+    test -z "$srcdir" && srcdir=.
+fi
+. "$srcdir/test-defs.sh"
+
+run_output_test test_parse
+exit $?