ref type convertes, add spec about yajl opts, note about allow_partial_values

Author: uvlad7

CHANGELOG.md

@@ -7,10 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-### Added
-
-- Possibility to reuse config using `JsonScanner::Config`
-
 ### Fixed
 
 - Potential problems with garbage collection of the `result` array and other `VALUE`s
@@ -19,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Report `bytes_consumed` in `ParseError`s
+- Possibility to reuse config using `JsonScanner::Config`
+- Warn about `yajl`'s bug when `yajl_allow_partial_values` is set
 
 ## [0.2.0] - 2024-12-27
 

README.md

@@ -56,6 +56,38 @@ JsonScanner.scan('{"a": 1, "b": 2}', [[JsonScanner::ANY_KEY]], with_path: true)
 # => [[[["a"], [6, 7, :number]], [["b"], [14, 15, :number]]]]
 ```
 
+It supports multiple options
+
+```ruby
+JsonScanner.scan('[0, 42, 0]', [[(1..-1)]], with_path: true)
+# => [[[[1], [4, 6, :number]], [[2], [8, 9, :number]]]]
+JsonScanner.scan('[0, 42,', [[(1..-1)]], verbose_error: true)
+# JsonScanner::ParseError (parse error: premature EOF)
+#                                        [0, 42,
+#                      (right here) ------^
+JsonScanner.scan('[0, /* answer */ 42, 0]', [[(1..-1)]], allow_comments: true)
+# => [[[17, 19, :number], [21, 22, :number]]]
+JsonScanner.scan("\"\x81\x83\"", [[]], dont_validate_strings: true)
+# => [[[0, 4, :string]]]
+JsonScanner.scan("{\"\x81\x83\": 42}", [[JsonScanner::ANY_KEY]], dont_validate_strings: true, with_path: true)
+# => [[[["\x81\x83"], [7, 9, :number]]]]
+JsonScanner.scan('[0, 42, 0]garbage', [[(1..-1)]], allow_trailing_garbage: true)
+# => [[[4, 6, :number], [8, 9, :number]]]
+JsonScanner.scan('[0, 42, 0]  [0, 34]', [[(1..-1)]], allow_multiple_values: true)
+# => [[[4, 6, :number], [8, 9, :number], [16, 18, :number]]]
+JsonScanner.scan('[0, 42, 0,', [[(1..-1)]], allow_partial_values: true)
+# => [[[4, 6, :number], [8, 9, :number]]]
+
+# This is a bug of yajl that affects only numbers
+JsonScanner.scan('[0, 42, 0', [[(1..-1)]], allow_partial_values: true)
+# => [[[4, 6, :number], [-1, 0, :number]]]
+JsonScanner.scan('[0, 42, true', [[(1..-1)]], allow_partial_values: true)
+# => [[[4, 6, :number], [8, 12, :boolean]]]
+
+JsonScanner.scan('{"a": 1}', [[JsonScanner::ANY_KEY]], with_path: true, symbolize_path_keys: true)
+# => [[[[:a], [6, 7, :number]]]]
+```
+
 You can also create a config and reuse it
 
 ```ruby

ext/json_scanner/json_scanner.c

@@ -140,6 +140,7 @@ void scan_ctx_debug(scan_ctx *ctx)
       break;
     case PATH_INDEX:
       fprintf(stderr, "%ld", ctx->current_path[i].value.index);
+      break;
     }
     if (i < ctx->current_path_len - 1)
       fprintf(stderr, ", ");
@@ -193,7 +194,7 @@ VALUE scan_ctx_init(scan_ctx *ctx, VALUE path_ary, VALUE string_keys)
         break;
       case T_FIXNUM:
       case T_BIGNUM:
-        RB_NUM2LONG(entry);
+        NUM2LONG(entry);
         break;
       default:
       {
@@ -204,9 +205,9 @@ VALUE scan_ctx_init(scan_ctx *ctx, VALUE path_ary, VALUE string_keys)
           return rb_exc_new_cstr(rb_eArgError, "path elements must be strings, integers, or ranges");
         if (range_beg != any_key_sym || range_end != any_key_sym)
         {
-          if (RB_NUM2LONG(range_beg) < 0L)
+          if (NUM2LONG(range_beg) < 0L)
             return rb_exc_new_cstr(rb_eArgError, "range start must be positive");
-          end_val = RB_NUM2LONG(range_end);
+          end_val = NUM2LONG(range_end);
           if (end_val < -1L)
             return rb_exc_new_cstr(rb_eArgError, "range end must be positive or -1");
           if (end_val == -1L && open_ended)
@@ -273,8 +274,8 @@ VALUE scan_ctx_init(scan_ctx *ctx, VALUE path_ary, VALUE string_keys)
         else
         {
           paths[i].elems[j].type = MATCHER_INDEX_RANGE;
-          paths[i].elems[j].value.range.start = RB_NUM2LONG(range_beg);
-          paths[i].elems[j].value.range.end = RB_NUM2LONG(range_end);
+          paths[i].elems[j].value.range.start = NUM2LONG(range_beg);
+          paths[i].elems[j].value.range.end = NUM2LONG(range_end);
           // (value..-1) works as expected, (value...-1) is forbidden above
           if (paths[i].elems[j].value.range.end == -1L)
             paths[i].elems[j].value.range.end = LONG_MAX;
@@ -347,37 +348,38 @@ typedef enum
 } value_type;
 
 // noexcept
-VALUE create_point(scan_ctx *sctx, value_type type, size_t length, size_t curr_pos)
+VALUE create_point(scan_ctx *sctx, value_type type, size_t length)
 {
-  VALUE values[3];
-  VALUE point = rb_ary_new_capa(3);
+  VALUE values[3], point;
+  size_t curr_pos = yajl_get_bytes_consumed(sctx->handle);
+  point = rb_ary_new_capa(3);
   // noexcept
-  values[1] = RB_ULONG2NUM(curr_pos);
+  values[1] = ULL2NUM(curr_pos);
   switch (type)
   {
     // FIXME: size_t can be longer than ulong
   case null_value:
-    values[0] = RB_ULONG2NUM(curr_pos - length);
+    values[0] = LL2NUM(curr_pos - length);
     values[2] = null_sym;
     break;
   case boolean_value:
-    values[0] = RB_ULONG2NUM(curr_pos - length);
+    values[0] = LL2NUM(curr_pos - length);
     values[2] = boolean_sym;
     break;
   case number_value:
-    values[0] = RB_ULONG2NUM(curr_pos - length);
+    values[0] = LL2NUM(curr_pos - length);
     values[2] = number_sym;
     break;
   case string_value:
-    values[0] = RB_ULONG2NUM(curr_pos - length);
+    values[0] = LL2NUM(curr_pos - length);
     values[2] = string_sym;
     break;
   case object_value:
-    values[0] = RB_ULONG2NUM(sctx->starts[sctx->current_path_len]);
+    values[0] = LL2NUM(sctx->starts[sctx->current_path_len]);
     values[2] = object_sym;
     break;
   case array_value:
-    values[0] = RB_ULONG2NUM(sctx->starts[sctx->current_path_len]);
+    values[0] = LL2NUM(sctx->starts[sctx->current_path_len]);
     values[2] = array_sym;
     break;
   }
@@ -402,7 +404,7 @@ VALUE create_path(scan_ctx *sctx)
         entry = rb_str_new(sctx->current_path[i].value.key.val, sctx->current_path[i].value.key.len);
       break;
     case PATH_INDEX:
-      entry = RB_ULONG2NUM(sctx->current_path[i].value.index);
+      entry = LONG2NUM(sctx->current_path[i].value.index);
       break;
     default:
       entry = Qnil;
@@ -459,7 +461,7 @@ void save_point(scan_ctx *sctx, value_type type, size_t length)
     {
       if (point == Qundef)
       {
-        point = create_point(sctx, type, length, yajl_get_bytes_consumed(sctx->handle));
+        point = create_point(sctx, type, length);
         if (sctx->with_path)
         {
           path = create_path(sctx);
@@ -499,6 +501,7 @@ int scan_on_boolean(void *ctx, int bool_val)
 int scan_on_number(void *ctx, const char *val, size_t len)
 {
   scan_ctx *sctx = (scan_ctx *)ctx;
+  // yajl_get_bytes_consumed works incorrectly for numbers in yajl_allow_partial_values mode when partial value is a number
   if (sctx->current_path_len > sctx->max_path_len)
     return true;
   increment_arr_index(sctx);
@@ -793,9 +796,27 @@ VALUE scan(int argc, VALUE *argv, VALUE self)
   {
     char *str = (char *)yajl_get_error(handle, verbose_error, (unsigned char *)json_text, json_text_len);
     err_msg = rb_utf8_str_new_cstr(str);
-    bytes_consumed = RB_ULONG2NUM(yajl_get_bytes_consumed(handle));
+    bytes_consumed = ULL2NUM(yajl_get_bytes_consumed(handle));
     yajl_free_error(handle, (unsigned char *)str);
   }
+  // // Needed when yajl_allow_partial_values is set
+  // if (ctx->current_path_len > 0)
+  // {
+  //   if (ctx->current_path_len > ctx->max_path_len)
+  //     ctx->current_path_len = ctx->max_path_len;
+  //   for (int i = ctx->current_path_len - 1; i > 0; i--)
+  //   {
+  //     switch (ctx->current_path[i].type)
+  //     {
+  //     case PATH_KEY:
+  //       scan_on_end_object(ctx);
+  //       break;
+  //     case PATH_INDEX:
+  //       scan_on_end_array(ctx);
+  //       break;
+  //     }
+  //   }
+  // }
   // callback_err = ctx->rb_err;
   if (free_ctx)
   {

spec/json_scanner_spec.rb

@@ -22,7 +22,7 @@
     )
   end
 
-  it "supports symbols" do
+  it "supports 'symbolize_path_keys'" do
     expect(
       described_class.scan('{"a": {"b": 1}}', [[:a, "b"]], with_path: true),
     ).to eq([[[%w[a b], [12, 13, :number]]]])
@@ -118,6 +118,9 @@
     expect do
       described_class.scan "{1}", [], verbose_error: true
     end.to raise_error described_class::ParseError, /invalid object key(?=.*\(right here\))/m
+    expect do
+      described_class.scan("[0, 42,", [[(1..-1)]], verbose_error: true)
+    end.to raise_error described_class::ParseError, /parse error: premature EOF.*\[0, 42,.*\(right here\) ------\^/m
   end
 
   it "includes bytes consumed in the exception" do
@@ -229,6 +232,83 @@
     end.to raise_error(described_class::ParseError)
   end
 
+  context "yajl params" do
+    it "supports 'allow_comments'" do
+      params = ["[0, /* answer */ 42, 0]", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_comments: true)).to eq(
+        [[[17, 19, :number], [21, 22, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'dont_validate_strings'" do
+      params = ["\"\x81\x83\"", [[]]]
+      expect(described_class.scan(*params, dont_validate_strings: true)).to eq(
+        [[[0, 4, :string]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+      params = ["{\"\x81\x83\": 42}", [[JsonScanner::ANY_KEY]]]
+      expect(described_class.scan(*params, dont_validate_strings: true, with_path: true)).to eq(
+        [[[["\x81\x83".dup.force_encoding(Encoding::BINARY)], [7, 9, :number]]]],
+      )
+      expect do
+        described_class.scan(*params, with_path: true)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'allow_trailing_garbage'" do
+      params = ["[0, 42, 0]garbage", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_trailing_garbage: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+    end
+
+    it "supports 'allow_multiple_values'" do
+      params = ["[0, 42, 0]  [0, 34]", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_multiple_values: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number], [16, 18, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+      expect(described_class.scan("[0, 42, 0]  [0, 34]", [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :array], [12, 19, :array]]],
+      )
+      expect(described_class.scan('{"42": 34}  [0, 34]', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :object], [12, 19, :array]]],
+      )
+      expect(described_class.scan('[0, 42, 0]  {"42": 34}', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :array], [12, 22, :object]]],
+      )
+      expect(described_class.scan('{"42": 34}  {"0": 34}', [[]], allow_multiple_values: true)).to eq(
+        [[[0, 10, :object], [12, 21, :object]]],
+      )
+    end
+
+    it "supports 'allow_partial_values'" do
+      params = ["[0, 42, 0,", [[(1..-1)]]]
+      expect(described_class.scan(*params, allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [8, 9, :number]]],
+      )
+      expect do
+        described_class.scan(*params)
+      end.to raise_error(described_class::ParseError)
+      expect(described_class.scan("[0, 42, 0", [[(1..-1)]], allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [-1, 0, :number]]],
+      )
+      expect(described_class.scan("[0, 42, true", [[(1..-1)]], allow_partial_values: true)).to eq(
+        [[[4, 6, :number], [8, 12, :boolean]]],
+      )
+    end
+  end
+
   describe described_class::Config do
     it "saves state" do
       key = "abracadabra".dup