Merge pull request #184 from JuliaAI/dev

For a 0.11 release

Author: ablaom

.github/workflows/CI.yml

@@ -17,7 +17,6 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.0'
           - '1.6'
           - '1' # automatically expands to the latest stable 1.x release of Julia
         os:

Project.toml

@@ -2,7 +2,7 @@ name = "DecisionTree"
 uuid = "7806a523-6efd-50cb-b5f6-3fa6f1930dbb"
 license = "MIT"
 desc = "Julia implementation of Decision Tree (CART) and Random Forest algorithms"
-version = "0.10.13"
+version = "0.11.0"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
@@ -13,9 +13,9 @@ ScikitLearnBase = "6e75b9c4-186b-50bd-896f-2d2496a4843e"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [compat]
-AbstractTrees = "0.3"
+AbstractTrees = "0.3, 0.4"
 ScikitLearnBase = "0.5"
-julia = "1"
+julia = "1.6"
 
 [extras]
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"

README.md

@@ -19,7 +19,7 @@ Available via:
 * pre-pruning (max depth, min leaf size)
 * post-pruning (pessimistic pruning)
 * multi-threaded bagging (random forests)
-* adaptive boosting (decision stumps)
+* adaptive boosting (decision stumps), using [SAMME](https://www.intlpress.com/site/pub/pages/journals/items/sii/content/vols/0002/0003/a008/)
 * cross validation (n-fold)
 * support for ordered features (encoded as `Real`s or `String`s)
 
@@ -92,7 +92,7 @@ apply_tree(model, [5.9,3.0,5.1,1.9])
 # apply model to all the sames
 preds = apply_tree(model, features)
 # generate confusion matrix, along with accuracy and kappa scores
-confusion_matrix(labels, preds)
+DecisionTree.confusion_matrix(labels, preds)
 # get the probability of each label
 apply_tree_proba(model, [5.9,3.0,5.1,1.9], ["Iris-setosa", "Iris-versicolor", "Iris-virginica"])
 # run 3-fold cross validation of pruned tree,
@@ -312,6 +312,13 @@ Available models are: `AdaBoostStumpClassifier`,
 `RandomForestClassifier`, `RandomForestRegressor`.
 
 
+## Feature Importances
+
+The following methods provide measures of feature importance for all models:
+`impurity_importance`, `split_importance`, `permutation_importance`. Query the document
+strings for details.
+
+
 ## Saving Models
 Models can be saved to disk and loaded back with the use of the [JLD2.jl](https://github.com/JuliaIO/JLD2.jl) package.
 ```julia

src/DecisionTree.jl

@@ -1,5 +1,3 @@
-__precompile__()
-
 module DecisionTree
 
 import Base: length, show, convert, promote_rule, zero
@@ -9,11 +7,11 @@ using Random
 using Statistics
 import AbstractTrees
 
-export Leaf, Node, Ensemble, print_tree, depth, build_stump, build_tree,
+export Leaf, Node, Root, Ensemble, print_tree, depth, build_stump, build_tree,
        prune_tree, apply_tree, apply_tree_proba, nfoldCV_tree, build_forest,
        apply_forest, apply_forest_proba, nfoldCV_forest, build_adaboost_stumps,
        apply_adaboost_stumps, apply_adaboost_stumps_proba, nfoldCV_stumps,
-       majority_vote, ConfusionMatrix, confusion_matrix, mean_squared_error, R2, load_data
+       load_data, impurity_importance, split_importance, permutation_importance
 
 # ScikitLearn API
 export DecisionTreeClassifier, DecisionTreeRegressor, RandomForestClassifier,
@@ -42,17 +40,32 @@ end
 
 const LeafOrNode{S, T} = Union{Leaf{T}, Node{S, T}}
 
+struct Root{S, T}
+    node    :: LeafOrNode{S, T}
+    n_feat  :: Int
+    featim  :: Vector{Float64}   # impurity importance
+end
+
 struct Ensemble{S, T}
-    trees :: Vector{LeafOrNode{S, T}}
+    trees   :: Vector{LeafOrNode{S, T}}
+    n_feat  :: Int
+    featim  :: Vector{Float64}
 end
 
+
 is_leaf(l::Leaf) = true
 is_leaf(n::Node) = false
 
-zero(String) = ""
+zero(::Type{String}) = ""
 convert(::Type{Node{S, T}}, lf::Leaf{T}) where {S, T} = Node(0, zero(S), lf, Leaf(zero(T), [zero(T)]))
+convert(::Type{Root{S, T}}, node::LeafOrNode{S, T}) where {S, T} = Root{S, T}(node, 0, Float64[])
+convert(::Type{LeafOrNode{S, T}}, tree::Root{S, T}) where {S, T} = tree.node
 promote_rule(::Type{Node{S, T}}, ::Type{Leaf{T}}) where {S, T} = Node{S, T}
 promote_rule(::Type{Leaf{T}}, ::Type{Node{S, T}}) where {S, T} = Node{S, T}
+promote_rule(::Type{Root{S, T}}, ::Type{Leaf{T}}) where {S, T} = Root{S, T}
+promote_rule(::Type{Leaf{T}}, ::Type{Root{S, T}}) where {S, T} = Root{S, T}
+promote_rule(::Type{Root{S, T}}, ::Type{Node{S, T}}) where {S, T} = Root{S, T}
+promote_rule(::Type{Node{S, T}}, ::Type{Root{S, T}}) where {S, T} = Root{S, T}
 
 # make a Random Number Generator object
 mk_rng(rng::Random.AbstractRNG) = rng
@@ -75,10 +88,12 @@ include("abstract_trees.jl")
 
 length(leaf::Leaf) = 1
 length(tree::Node) = length(tree.left) + length(tree.right)
+length(tree::Root) = length(tree.node)
 length(ensemble::Ensemble) = length(ensemble.trees)
 
 depth(leaf::Leaf) = 0
 depth(tree::Node) = 1 + max(depth(tree.left), depth(tree.right))
+depth(tree::Root) = depth(tree.node)
 
 function print_tree(io::IO, leaf::Leaf, depth=-1, indent=0; feature_names=nothing)
     n_matches = count(leaf.values .== leaf.majority)
@@ -90,6 +105,13 @@ function print_tree(leaf::Leaf, depth=-1, indent=0; feature_names=nothing)
 end
 
 
+function print_tree(io::IO, tree::Root, depth=-1, indent=0; sigdigits=2, feature_names=nothing)
+    return print_tree(io, tree.node, depth, indent; sigdigits=sigdigits, feature_names=feature_names)
+end
+function print_tree(tree::Root, depth=-1, indent=0; sigdigits=2, feature_names=nothing)
+    return print_tree(stdout, tree, depth, indent; sigdigits=sigdigits, feature_names=feature_names)
+end
+
 """
     print_tree([io::IO,] tree::Node, depth=-1, indent=0; sigdigits=4, feature_names=nothing)
 
@@ -113,9 +135,9 @@ Feature 3 < -28.15 ?
    └─ 8 : 1227/3508
 ```
 
-To facilitate visualisation of trees using third party packages, a `DecisionTree.Leaf` object or
-`DecisionTree.Node` object can be wrapped to obtain a tree structure implementing the
-AbstractTrees.jl interface. See  [`wrap`](@ref)` for details.
+To facilitate visualisation of trees using third party packages, a `DecisionTree.Leaf` object,
+`DecisionTree.Node` object or  `DecisionTree.Root` object can be wrapped to obtain a tree structure implementing the 
+AbstractTrees.jl interface. See  [`wrap`](@ref)` for details. 
 """
 function print_tree(io::IO, tree::Node, depth=-1, indent=0; sigdigits=2, feature_names=nothing)
     if depth == indent
@@ -149,6 +171,12 @@ function show(io::IO, tree::Node)
     print(io,   "Depth:  $(depth(tree))")
 end
 
+function show(io::IO, tree::Root)
+    println(io, "Decision Tree")
+    println(io, "Leaves: $(length(tree))")
+    print(io,   "Depth:  $(depth(tree))")
+end
+
 function show(io::IO, ensemble::Ensemble)
     println(io, "Ensemble of Decision Trees")
     println(io, "Trees:      $(length(ensemble))")

src/abstract_trees.jl

@@ -66,6 +66,7 @@ In the first case `dc` gets just wrapped, no information is added. No. 2 adds fe
 as well as class labels. In the last two cases either of this information is added (Note the 
 trailing comma; it's needed to make it a tuple).
 """
+wrap(tree::DecisionTree.Root, info::NamedTuple = NamedTuple()) = wrap(tree.node, info)
 wrap(node::DecisionTree.Node, info::NamedTuple = NamedTuple()) = InfoNode(node, info)
 wrap(leaf::DecisionTree.Leaf, info::NamedTuple = NamedTuple()) = InfoLeaf(leaf, info)