Merge pull request #224 from JuliaComputing/jb/0.7

WIP port to 0.7/1.0

Author: shashi

.travis.yml

@@ -4,13 +4,18 @@ os:
   - linux
   - osx
 julia:
-  - 0.6
+  - 0.7
+  - 1.0
+  - nightly
+matrix:
+  allow_failures:
+    - julia: nightly
 notifications:
   email: false
 # uncomment the following lines to override the default test script
-script:
-  - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-  - julia -e 'Pkg.add("TextParse"); Pkg.add("IndexedTables"); Pkg.checkout("TextParse"); Pkg.checkout("IndexedTables"); Pkg.clone(pwd()); Pkg.build("JuliaDB"); Pkg.test("JuliaDB"; coverage=true)'
+#script:
+#  - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
+#  - julia -e 'Pkg.add("TextParse"); Pkg.add("IndexedTables"); Pkg.checkout("TextParse"); Pkg.checkout("IndexedTables"); Pkg.clone(pwd()); Pkg.build("JuliaDB"); Pkg.test("JuliaDB"; coverage=true)'
 after_success:
   # push coverage results to Coveralls
   - julia -e 'cd(Pkg.dir("JuliaDB")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())'

REQUIRE

@@ -1,12 +1,14 @@
-julia 0.6 0.7-
-IndexedTables 0.7.3
-NamedTuples 4.0.0
-TextParse 0.3.0
-MemPool 0.0.6
-Dagger 0.5.2
-Glob 1.1.1
+julia 1.0
+IndexedTables 0.8.1
+TextParse 0.6.0
+MemPool 0.1.1
+Dagger 0.7.2
+Glob 1.2.0
 OnlineStats 0.17.0
-PooledArrays 0.2.1
+OnlineStatsBase
+PooledArrays 0.4.1
 WeakRefStrings
 StatsBase
 DataValues
+Nullables
+RecipesBase

appveyor.yml

@@ -1,9 +1,11 @@
 environment:
   matrix:
-  - JULIAVERSION: "julialang/bin/winnt/x86/0.5/julia-0.5-latest-win32.exe"
-  - JULIAVERSION: "julialang/bin/winnt/x64/0.5/julia-0.5-latest-win64.exe"
-  - JULIAVERSION: "julianightlies/bin/winnt/x86/julia-latest-win32.exe"
-  - JULIAVERSION: "julianightlies/bin/winnt/x64/julia-latest-win64.exe"
+  - julia_version: 0.7
+  - julia_version: 1.0
+
+platform:
+  - x86 # 32-bit
+  - x64 # 64-bit
 
 branches:
   only:
@@ -17,18 +19,11 @@ notifications:
     on_build_status_changed: false
 
 install:
-# Download most recent Julia Windows binary
-  - ps: (new-object net.webclient).DownloadFile(
-        $("http://s3.amazonaws.com/"+$env:JULIAVERSION),
-        "C:\projects\julia-binary.exe")
-# Run installer silently, output to C:\projects\julia
-  - C:\projects\julia-binary.exe /S /D=C:\projects\julia
+  - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1"))
 
 build_script:
-# Need to convert from shallow to complete for Pkg.clone to work
-  - IF EXIST .git\shallow (git fetch --unshallow)
-  - C:\projects\julia\bin\julia -e "versioninfo();
-      Pkg.clone(pwd(), \"JuliaDB\"); Pkg.build(\"JuliaDB\")"
+  - echo "%JL_BUILD_SCRIPT%"
+  - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%"
 
 test_script:
-  - C:\projects\julia\bin\julia -e "Pkg.test(\"JuliaDB\")"
+  - C:\julia\bin\julia -e "using Pkg; Pkg.test(\"JuliaDB\")"

docs/make.jl

@@ -3,7 +3,7 @@ using Documenter, JuliaDB, IndexedTables, Dagger, TextParse, Glob
 load_dir(x) = map(file -> joinpath("lib", x, file), readdir(joinpath(Base.source_dir(), "src", "lib", x)))
 
 makedocs(
-   modules = [JuliaDB, IndexedTables, NamedTuples],
+   modules = [JuliaDB, IndexedTables],
    clean = false,
    format = [:html],#, :latex],
    sitename = "JuliaDB",

src/JuliaDB.jl

@@ -1,17 +1,16 @@
 __precompile__()
 module JuliaDB
 
-using IndexedTables, Dagger, NamedTuples, OnlineStats
+using IndexedTables, Dagger, OnlineStats, Distributed, Serialization, Nullables, Printf, Statistics
 
-import Base: collect, select, join
+import Base: collect, join
 import IndexedTables: NextTable, table, NDSparse, ndsparse, Tup, groupjoin
 import TextParse: csvread
-import IndexedTables: Table
-import Dagger: compute, distribute, free!, gather, load, save
+import Dagger: compute, distribute, load, save
 using DataValues
 
 # re-export
-export IndexedTable, AbstractNDSparse, NDSparse, NextTable, Columns, colnames,
+export AbstractNDSparse, NDSparse, NextTable, Columns, colnames,
        table, ndsparse, compute, groupby, summarize, groupreduce, groupjoin,
        ColDict, insertafter!, insertbefore!, @cols, setcol, pushcol,
        popcol, insertcol, insertcolafter, insertcolbefore,

src/dcolumns.jl

@@ -19,7 +19,7 @@ function DColumns(arrays::Tup)
 
     i = findfirst(x->isa(x, ArrayOp), arrays)
     wrap = isa(arrays, Tuple) ? tuple :
-                                namedtuple(fieldnames(arrays)...)
+                                namedtuple(keys(arrays)...)∘tuple
     if i == 0
         error("""At least 1 array passed to
                  DColumns must be a DArray""")
@@ -53,7 +53,7 @@ function DColumns(arrays::Tup)
     T = isa(arrays, Tuple) ? Tuple{map(eltype, arrays)...} :
         wrap{map(eltype, arrays)...}
 
-    DArray(T, domain(darrays[1]), domainchunks(darrays[1]), cs, (i, x...)->vcat(x...))
+    DArray(T, domain(darrays[1]), domainchunks(darrays[1]), cs, dvcat)
 end
 
 function itable(keycols::DColumns, valuecols::DColumns)
@@ -69,7 +69,7 @@ function extractarray(t::Union{DNDSparse,DColumns}, accessor)
         lengths = length.(domain.(cs))
         dmnchunks = DomainBlocks((1,), (cumsum(lengths),))
         T = promote_eltype_chunktypes(cs)
-        DArray(T, ArrayDomain(1:sum(lengths)), dmnchunks, [cs...], (i, x...)->vcat(x...))
+        DArray(T, ArrayDomain(1:sum(lengths)), dmnchunks, [cs...], dvcat)
     end
 
     cs = map(delayed(accessor), t.chunks)
@@ -101,15 +101,15 @@ function columns(t::Union{DNDSparse, DColumns}, which::Tuple...)
         ls = length.(domain.(cs))
         d = ArrayDomain((1:sum(ls),))
         dchunks = DomainBlocks((1,), (cumsum(ls),))
-        DArray(eltype(T), d, dchunks, cs, (i, x...) -> vcat(x...))
+        DArray(eltype(T), d, dchunks, cs, dvcat)
     end
 end
 
 function _columns_as(t, which)
     stripas(w) = isa(w, As) ? w.src : w
     which_ = ntuple(i->as(stripas(which[i]), i), length(which))
     cs = columns(t, which_)
-    asvecs = find(isas, which)
+    asvecs = findall(isas, which)
     outvecs = Any[cs...]
     outvecs[asvecs] = map((w,x) -> w.f(x), [which[asvecs]...], outvecs[asvecs])
     tup = IndexedTables._output_tuple(which)

src/flatten.jl

@@ -9,8 +9,8 @@ function mapslices(f, x::DNDSparse, dims; name=nothing)
     # Note: the key doesn't need to be put in a tuple, this is
     # also bad for sortperm, but is required since DArrays aren't
     # parameterized by the container type Columns
-    vals = isempty(dims) ?  values(x) : (keys(x, (dims...)), values(x))
-    tmp = ndsparse((keys(x, (iterdims...)),), vals,
+    vals = isempty(dims) ?  values(x) : (keys(x, (dims...,)), values(x))
+    tmp = ndsparse((keys(x, (iterdims...,)),), vals,
                    allowoverlap=false, closed=true)
 
     cs = delayedmap(tmp.chunks) do c

src/indexing.jl

@@ -47,7 +47,7 @@ end
 
 # update a given domain to include a new key
 function update_domain(d::IndexSpace{<:NamedTuple}, key::Tuple)
-    knt = namedtuple(fieldnames(first(d))...)(key...)
+    knt = namedtuple(keys(first(d))...)(key)
     IndexSpace(Interval(min(first(d.interval), knt),
                         max(last(d.interval), knt)),
                Interval(map(min, first(d.interval), knt),
@@ -75,10 +75,6 @@ function insert_row!(x::DNDSparse{K,T}, idxs::Tuple, val) where {K,T}
     ds[i] = update_domain(ds[i], idxs)
     tmp = cs[i]
     cs[i] = compute(get_context(), delayed(x->(x[idxs...] = val; x))(cs[i]))
-    isa(cs[i].handle, DRef) && Dagger.addrefcount(cs[i].handle, 1)
-    isa(tmp.handle, DRef) && Dagger.addrefcount(tmp.handle, -1)
-    # TODO: tmp is not freed yet. Should ensure Dagger.free_chunks is called on it.
-
     x.domains = ds
     x.chunks = cs
 end

src/interval.jl

@@ -1,5 +1,3 @@
-using Base.Test
-
 """
 An interval type tailored specifically to store intervals of
 indices of an NDSparse object. Some of the operations on this
@@ -17,7 +15,7 @@ Base.last(int::Interval) = int.last
 Base.isempty(int::Interval) = first(int) > last(int)
 Base.isless(x::Interval, y::Interval) = x.last < y.first
 Base.in(x, int::Interval) = first(int) <= x <= last(int)
-Base.in(x::Range, int::Interval) = hasoverlap(Interval(first(x),last(x)), int)
+Base.in(x::AbstractRange, int::Interval) = hasoverlap(Interval(first(x),last(x)), int)
 Base.in(x::AbstractArray, int::Interval) = any(a in int for a in x)
 Base.in(x::Colon, int::Interval) = true
 Base.in(int::Interval, x::Union{AbstractArray, Colon}) = x in int

src/io.jl

@@ -1,6 +1,5 @@
 export loadfiles, ingest, ingest!, load, save, loadndsparse, loadtable
-import Base: serialize, deserialize
-import Dagger: refcount_chunks
+import Serialization: serialize, deserialize
 
 const JULIADB_DIR = ".juliadb"
 const JULIADB_FILECACHE = "csv_metadata"
@@ -22,13 +21,13 @@ end
 
 function offset_index!(x, o)
     l = length(x)
-    copy!(columns(x.index)[1], o:o+l-1)
+    copyto!(columns(x.index)[1], o:o+l-1)
     x
 end
 
 function offset_index!(x::DNDSparse, o=1)
     lengths = map(a->get(a.nrows), x.domains)
-    offs = [0, cumsum(lengths[1:end-1]);] .+ 1
+    offs = [0; cumsum(lengths[1:end-1])] .+ 1
     fromchunks(delayedmap(offset_index!, x.chunks, offs))
 end
 
@@ -222,33 +221,25 @@ function _makerelative!(t, dir::AbstractString)
     end
 end
 
-function serialize(io::AbstractSerializer, A::Union{DNextTable,DNDSparse})
-    @async refcount_chunks(A)
-    invoke(serialize, Tuple{AbstractSerializer,Any}, io, A)
-end
-
 deserialize(io::AbstractSerializer, DT::Type{DNDSparse{K,V}}) where {K,V} = _deser(io, DT)
 deserialize(io::AbstractSerializer, DT::Type{DNextTable{T,K}}) where {T,K} = _deser(io, DT)
 function _deser(io::AbstractSerializer, t)
-    nf = nfields(t)
+    nf = fieldcount(t)
     x = ccall(:jl_new_struct_uninit, Any, (Any,), t)
-    t.mutable && Base.Serializer.deserialize_cycle(io, x)
+    t.mutable && Serialization.deserialize_cycle(io, x)
     for i in 1:nf
         tag = Int32(read(io.io, UInt8)::UInt8)
-        if tag != Base.Serializer.UNDEFREF_TAG
-            ccall(:jl_set_nth_field, Void, (Any, Csize_t, Any), x, i-1, Base.Serializer.handle_deserialize(io, tag))
+        if tag != Serialization.UNDEFREF_TAG
+            ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), x, i-1, Serialization.handle_deserialize(io, tag))
         end
     end
-    finalizer(x, free!)
     return x
 end
 
-refcount_chunks(A::Union{DNextTable,DNDSparse}) = refcount_chunks(A.chunks)
-
 using WeakRefStrings
 
 function MemPool.mmwrite(io::AbstractSerializer, arr::StringArray)
-    Base.serialize_type(io, MemPool.MMSer{StringArray})
+    Serialization.serialize_type(io, MemPool.MMSer{StringArray})
     serialize(io, eltype(arr))
     MemPool.mmwrite(io, arr.buffer)
     MemPool.mmwrite(io, arr.offsets)