Merge pull request #5 from JuliaData/jq/missing

Fix #4 by appropriately converting nothing to missing for data tables

Author: quinnj

src/JSONTables.jl

@@ -23,8 +23,18 @@ Tables.istable(::Type{<:Table}) = true
 Tables.columnaccess(::Type{Table{true, T}}) where {T} = true
 Tables.columns(x::Table{true}) = x
 
+miss(x) = ifelse(x === nothing, missing, x)
+struct MissingVector{T} <: AbstractVector{T}
+    x::T
+end
+Base.IndexStyle(::Type{<:MissingVector}) = Base.IndexLinear()
+# Base.length(x::MissingVector) = length(x.x)
+Base.size(x::MissingVector) = size(x.x)
+@inline Base.getindex(x::MissingVector, i::Int) = miss(x.x[i])
+Base.copy(x::MissingVector) = map(y->y isa JSON3.Object || y isa JSON3.Array ? copy(y) : miss(y), x)
+
 Base.propertynames(x::Table{true}) = Tuple(keys(getfield(x, :source)))
-Base.getproperty(x::Table{true}, nm::Symbol) = getproperty(getfield(x, :source), nm)
+Base.getproperty(x::Table{true}, nm::Symbol) = MissingVector(getproperty(getfield(x, :source), nm))
 
 # row source
 Tables.rowaccess(::Type{Table{false, T}}) where {T} = true
@@ -35,8 +45,25 @@ Base.length(x::Table{false}) = length(x.source)
 Base.IteratorEltype(::Type{Table{false, T}}) where {T} = Base.HasEltype()
 Base.eltype(x::Table{false, JSON3.Array{T}}) where {T} = T
 
-Base.iterate(x::Table{false}) = iterate(x.source)
-Base.iterate(x::Table{false}, st) = iterate(x.source, st)
+struct MissingRow{T}
+    x::T
+end
+Base.propertynames(x::MissingRow) = propertynames(getfield(x, :x))
+Base.getproperty(x::MissingRow, nm::Symbol) = miss(getproperty(getfield(x, :x), nm))
+
+@inline function Base.iterate(x::Table{false})
+    st = iterate(x.source)
+    st === nothing && return nothing
+    val, state = st
+    return MissingRow(val), state
+end
+
+@inline function Base.iterate(x::Table{false}, st)
+    st = iterate(x.source, st)
+    st === nothing && return nothing
+    val, state = st
+    return MissingRow(val), state
+end
 
 # write
 struct ObjectTable{T}

test/runtests.jl

@@ -4,17 +4,17 @@ using Test, JSONTables, Tables, JSON3
 
 cjson = replace(replace("""{
 "a": [1,2,3],
-"b": [4.1, 5.2, 6.3],
-"c": ["7", "8", "9"]
+"b": [4.1, null, 6.3],
+"c": ["7", "8", null]
 }""", " "=>""), "\n"=>"")
 
 rjson = replace(replace("""[
 {"a": 1, "b": 4.1, "c": "7"},
-{"a": 2, "b": 5.2, "c": "8"},
-{"a": 3, "b": 6.3, "c": "9"}
+{"a": 2, "b": null, "c": "8"},
+{"a": 3, "b": 6.3, "c": null}
 ]""", " "=>""), "\n"=>"")
 
-ctable = (a=[1,2,3], b=[4.1, 5.2, 6.3], c=["7", "8", "9"])
+ctable = (a=[1,2,3], b=[4.1, missing, 6.3], c=["7", "8", missing])
 rtable = Tables.rowtable(ctable)
 
 cjtable = JSONTables.jsontable(cjson)
@@ -37,10 +37,10 @@ rjtable = JSONTables.jsontable(rjson)
 @test Base.IteratorEltype(typeof(rjtable)) == Base.HasEltype()
 @test eltype(rjtable) == Any
 
-@test Tables.columntable(cjtable) == ctable
-@test Tables.columntable(rjtable) == ctable
-@test Tables.rowtable(cjtable) == rtable
-@test Tables.rowtable(rjtable) == rtable
+@test isequal(Tables.columntable(cjtable), ctable)
+@test isequal(Tables.columntable(rjtable), ctable)
+@test isequal(Tables.rowtable(cjtable), rtable)
+@test isequal(Tables.rowtable(rjtable), rtable)
 
 @test JSONTables.objecttable(ctable) == cjson
 @test JSONTables.objecttable(rtable) == cjson