Merge pull request #98 from yuehhua/datasets

Add datasets

Author: yuehhua

Manifest.toml

@@ -26,9 +26,9 @@ version = "0.0.4"
 
 [[ArrayLayouts]]
 deps = ["FillArrays", "LinearAlgebra", "SparseArrays"]
-git-tree-sha1 = "e3e0a1e7dcbfdb1fc1061bfd889581a1d942cfcb"
+git-tree-sha1 = "bd09f450716f55c5a47b24de277a8825e2450729"
 uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a"
-version = "0.4.5"
+version = "0.4.7"
 
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -39,6 +39,24 @@ git-tree-sha1 = "ecdec412a9abc8db54c0efc5548c64dfce072058"
 uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
 version = "0.5.10"
 
+[[Blosc]]
+deps = ["Blosc_jll"]
+git-tree-sha1 = "84cf7d0f8fd46ca6f1b3e0305b4b4a37afe50fd6"
+uuid = "a74b3585-a348-5f62-a45c-50e91977d574"
+version = "0.7.0"
+
+[[Blosc_jll]]
+deps = ["Libdl", "Lz4_jll", "Pkg", "Zlib_jll", "Zstd_jll"]
+git-tree-sha1 = "aa9ef39b54a168c3df1b2911e7797e4feee50fbe"
+uuid = "0b7ba130-8d10-5ba8-a3d6-c5182647fed9"
+version = "1.14.3+1"
+
+[[BufferedStreams]]
+deps = ["Compat", "Test"]
+git-tree-sha1 = "5d55b9486590fdda5905c275bb21ce1f0754020f"
+uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
+version = "1.0.0"
+
 [[CEnum]]
 git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
 uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
@@ -86,6 +104,12 @@ git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7"
 uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
 version = "0.3.0"
 
+[[Compat]]
+deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
+git-tree-sha1 = "215f1c81cfd1c5416cd78740bff8ef59b24cd7c0"
+uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
+version = "3.15.0"
+
 [[CompilerSupportLibraries_jll]]
 deps = ["Libdl", "Pkg"]
 git-tree-sha1 = "7c4f882c41faa72118841185afc58a2eb00ef612"
@@ -152,15 +176,21 @@ uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 version = "0.8.3"
 
 [[ExprTools]]
-git-tree-sha1 = "6f0517056812fd6aa3af23d4b70d5325a2ae4e95"
+git-tree-sha1 = "7fce513fcda766962ff67c5596cb16c463dfd371"
 uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
-version = "0.1.1"
+version = "0.1.2"
+
+[[FileIO]]
+deps = ["Pkg"]
+git-tree-sha1 = "992b4aeb62f99b69fcf0cb2085094494cc05dfb3"
+uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+version = "1.4.3"
 
 [[FillArrays]]
 deps = ["LinearAlgebra", "Random", "SparseArrays"]
-git-tree-sha1 = "56dc5338eb79a05e5ea120f510ed77efe7e9784d"
+git-tree-sha1 = "b955c227b0d1413a1a97e2ca0635a5de019d7337"
 uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "0.9.5"
+version = "0.9.6"
 
 [[FixedPointNumbers]]
 deps = ["Statistics"]
@@ -214,6 +244,18 @@ git-tree-sha1 = "6e62e16c779458412951a71f4d535f05a1e0bb89"
 uuid = "3ebe565e-a4b5-49c6-aed2-300248c3a9c1"
 version = "0.1.1"
 
+[[HDF5]]
+deps = ["Blosc", "HDF5_jll", "Libdl", "Mmap", "Random"]
+git-tree-sha1 = "0713cbabdf855852dfab3ce6447c87145f3d9ea8"
+uuid = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"
+version = "0.13.6"
+
+[[HDF5_jll]]
+deps = ["Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "85bd2e586a10ae0eab856125bf5245e0d36384a7"
+uuid = "0234f1f7-429e-5d53-9886-15a909be8d59"
+version = "1.10.5+5"
+
 [[HTTP]]
 deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"]
 git-tree-sha1 = "2ac03263ce44be4222342bca1c51c36ce7566161"
@@ -242,16 +284,16 @@ deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [[JLD2]]
-deps = ["CodecZlib", "DataStructures", "MacroTools", "Mmap", "Pkg", "Printf", "Requires", "UUIDs"]
-git-tree-sha1 = "d2c0db66530ff444846d6e84bcf948a74ce31635"
+deps = ["CodecZlib", "DataStructures", "FileIO", "Mmap", "Pkg", "Printf", "UUIDs"]
+git-tree-sha1 = "9353b717ee4e27beab4e902c92a06bb5f160d2cf"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.2.0"
+version = "0.1.14"
 
 [[JSON]]
 deps = ["Dates", "Mmap", "Parsers", "Unicode"]
-git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
+git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-version = "0.21.0"
+version = "0.21.1"
 
 [[Juno]]
 deps = ["Base64", "Logging", "Media", "Profile"]
@@ -287,9 +329,21 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
 [[LoopVectorization]]
 deps = ["DocStringExtensions", "LinearAlgebra", "OffsetArrays", "SIMDPirates", "SLEEFPirates", "UnPack", "VectorizationBase"]
-git-tree-sha1 = "224c9768765c2a3b588fec71cff48b8eb1c80c48"
+git-tree-sha1 = "3242a8f411e19eda9adc49d0b877681975c11375"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
-version = "0.8.25"
+version = "0.8.26"
+
+[[Lz4_jll]]
+deps = ["Libdl", "Pkg"]
+git-tree-sha1 = "51b1db0732bbdcfabb60e36095cc3ed9c0016932"
+uuid = "5ced341a-0733-55b8-9ab6-a4889d929147"
+version = "1.9.2+2"
+
+[[MAT]]
+deps = ["BufferedStreams", "CodecZlib", "HDF5", "SparseArrays"]
+git-tree-sha1 = "7e36f6a52274ddb8515ec1f559306be3f412d6a6"
+uuid = "23992714-dd62-5051-b70f-ba57cb901cac"
+version = "0.8.1"
 
 [[MacroTools]]
 deps = ["Markdown", "Random"]
@@ -320,10 +374,10 @@ uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27"
 version = "0.5.0"
 
 [[MetaGraphs]]
-deps = ["JLD2", "LightGraphs", "Random"]
-git-tree-sha1 = "43ebbe06b22d213e4a8750424f9c7d1311bee2a6"
+deps = ["JLD2", "LightGraphs"]
+git-tree-sha1 = "c6a4c88304e1ecef6fc372f12d3b8e427e128c1a"
 uuid = "626554b9-1ddb-594c-aa3c-2596fe9399a5"
-version = "0.6.4"
+version = "0.6.3"
 
 [[Missings]]
 deps = ["DataAPI"]
@@ -415,9 +469,9 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
 [[SIMDPirates]]
 deps = ["VectorizationBase"]
-git-tree-sha1 = "26ccdd1466f3071e27e81b43216ea238b62c0c42"
+git-tree-sha1 = "450d163d3279a1d35e3aad3352a5167ef21b84a4"
 uuid = "21efa798-c60a-11e8-04d3-e1a92915a26a"
-version = "0.8.24"
+version = "0.8.25"
 
 [[SLEEFPirates]]
 deps = ["Libdl", "SIMDPirates", "VectorizationBase"]
@@ -515,9 +569,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [[VectorizationBase]]
 deps = ["CpuId", "Libdl", "LinearAlgebra"]
-git-tree-sha1 = "c2a34c8065076a867fc36522c1a3441156a63445"
+git-tree-sha1 = "03e2fbb479a1ea350398195b6fbf439bae0f8260"
 uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
-version = "0.12.32"
+version = "0.12.33"
 
 [[VersionParsing]]
 git-tree-sha1 = "80229be1f670524750d905f8fc8148e5a8c4537f"
@@ -536,11 +590,17 @@ git-tree-sha1 = "fdd89e5ab270ea0f2a0174bd9093e557d06d4bfa"
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
 version = "1.2.11+16"
 
+[[Zstd_jll]]
+deps = ["Libdl", "Pkg"]
+git-tree-sha1 = "4de91f4313d9e88162d461e282fe3066ab3a3c09"
+uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
+version = "1.4.5+1"
+
 [[Zygote]]
 deps = ["AbstractFFTs", "ArrayLayouts", "ChainRules", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "Future", "IRTools", "InteractiveUtils", "LinearAlgebra", "LoopVectorization", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"]
-git-tree-sha1 = "fa45127afd117fca6a8540f92d5e3799daf8339a"
+git-tree-sha1 = "b0a948a0a78e3e41515714fa1ef4f40a284ffa06"
 uuid = "e88e6eb3-aa80-5325-afca-941959d7151f"
-version = "0.5.5"
+version = "0.5.6"
 
 [[ZygoteRules]]
 deps = ["MacroTools"]

Project.toml

@@ -13,8 +13,10 @@ GraphSignals = "3ebe565e-a4b5-49c6-aed2-300248c3a9c1"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 IRTools = "7869d1d1-7146-5819-86e3-90919afe41df"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MAT = "23992714-dd62-5051-b70f-ba57cb901cac"
 MetaGraphs = "626554b9-1ddb-594c-aa3c-2596fe9399a5"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -36,9 +38,11 @@ Flux = "0.10 - 0.11"
 GraphSignals = "0.1"
 HTTP = "0.8"
 IRTools = "0.4"
-JLD2 = "0.2"
+JLD2 = "0.1 - 0.2"
+JSON = "0.21"
 LightGraphs = "1.3"
-MetaGraphs = "0.6"
+MAT = "0.8"
+MetaGraphs = "< 0.6.4"
 PyCall = "1.91"
 Requires = "1.0.0"
 ScatterNNlib = "0.1"

src/GeometricFlux.jl

@@ -19,8 +19,9 @@ using ZygoteRules
 import Flux: maxpool, meanpool
 
 export
-    # datasets
+    datasets,
     traindata,
+    validdata,
     testdata,
 
     # layers/gn

src/datasets/Datasets.jl

@@ -2,29 +2,40 @@ module Datasets
     using DataDeps: DataDep, register, @datadep_str
     using HTTP
     using JLD2
+    using JSON
+    using LightGraphs: SimpleDiGraph, add_edge!
+    using MAT
     using PyCall
-    using SparseArrays: SparseMatrixCSC
+    using SparseArrays: SparseMatrixCSC, sparse
 
     export
         Dataset,
         Planetoid,
         Cora,
+        PPI,
+        Reddit,
+        QM7b,
+        # Entities,
         dataset,
         traindata,
+        validdata,
         testdata
 
-    abstract type Dataset end
-
+    include("./dataset.jl")
     include("./planetoid.jl")
     include("./cora.jl")
     include("./ppi.jl")
     include("./reddit.jl")
-    # include("./qm7b.jl")
+    include("./qm7b.jl")
     # include("./entities.jl")
     include("./datautils.jl")
 
     function __init__()
         planetoid_init()
         cora_init()
+        ppi_init()
+        reddit_init()
+        qm7b_init()
+        # entities_init()
     end
 end

src/datasets/dataset.jl

@@ -0,0 +1,13 @@
+abstract type Dataset end
+
+# function dataset(::Dataset)
+#     throw()
+# end
+
+# function traindata(::Dataset)
+    
+# end
+
+# function testdata(::Dataset)
+    
+# end

src/datasets/datautils.jl

@@ -5,3 +5,8 @@ function download_file(url, path)
         end
     end
 end
+
+function unzip(zipfile::String)
+    f = replace(zipfile, ".zip"=>"")
+    run(`unzip $f`)
+end

src/datasets/ppi.jl

@@ -1,5 +1,5 @@
 ppi_init() = register(DataDep(
-    "Cora full datasets",
+    "PPI",
     """
     The protein-protein interaction networks from the `"Predicting
     Multicellular Function through Multi-layer Tissue Networks"
@@ -8,7 +8,63 @@ ppi_init() = register(DataDep(
     total) and gene ontology sets as labels (121 in total).
     """,
     "https://s3.us-east-2.amazonaws.com/dgl.ai/dataset/ppi.zip",
-    "";
-    fetch_method=http_download,
-    post_fetch_method=DataDeps.unpack,
+    "1f5b2b09ac0f897fa6aa1338c64ab75a5473674cbba89380120bede8cddb2a6a";
+    post_fetch_method=preprocess_ppi,
 ))
+
+function preprocess_ppi(local_path)
+    unzip(local_path)
+
+    for phase in ["train", "test", "valid"]
+        graph_file = @datadep_str "PPI/$(phase)_graph.json"
+        id_file = @datadep_str "PPI/$(phase)_graph_id.npy"
+        X_file = @datadep_str "PPI/$(phase)_feats.npy"
+        y_file = @datadep_str "PPI/$(phase)_labels.npy"
+
+        py"""
+        import numpy as np
+        ids = np.load($id_file)
+        X = np.load($X_file)
+        y = np.load($y_file)
+        """
+
+        X = Matrix{Float32}(py"X")
+        y = SparseMatrixCSC{Int32,Int64}(Array(py"y"))
+        ids = Array(py"ids")
+        graph = read_ppi_graph(graph_file)
+        
+        jld2file = replace(local_path, "ppi.zip"=>"ppi.$(phase).jld2")
+        @save jld2file graph X y ids
+    end
+end
+
+function read_ppi_graph(filename::String)
+    d = JSON.Parser.parsefile(filename)
+    g = SimpleDiGraph{Int32}(length(d["nodes"]))
+
+    for pair in d["links"]
+        add_edge!(g, pair["source"], pair["target"])
+    end
+    g
+end
+
+struct PPI <: Dataset
+end
+
+function traindata(::PPI)
+    file = datadep"PPI/ppi.train.jld2"
+    @load file graph X y ids
+    graph, X, y, ids
+end
+
+function validdata(::PPI)
+    file = datadep"PPI/ppi.valid.jld2"
+    @load file graph X y ids
+    graph, X, y, ids
+end
+
+function testdata(::PPI)
+    file = datadep"PPI/ppi.test.jld2"
+    @load file graph X y ids
+    graph, X, y, ids
+end