diff --git a/README.md b/README.md index b3f92c6..9cb618b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ `sgtlearn` is a Python package for learning [Shape Generalized Trees (SGTs)](https://neurips.cc/virtual/2025/loc/san-diego/poster/115950). -- 🌳 **Shape Generalized Trees (SGTs):** A class of decision trees where each node applies a learnable, axis-aligned shape function to a feature for non-linear and interpretable splits. +- 🌳 **Shape Generalized Trees (SGTs):** A class of decision trees where each node applies a learnable, axis-aligned shape function to one or two logical features for non-linear and interpretable splits. - 👁 **Interpretability:** Each node's shape function can be visualized directly. - ⚡ **ShapeCART Algorithm:** An efficient induction method for learning SGTs from data. - 🔀 **Extensions:** @@ -12,10 +12,9 @@ - **SGTK:** Multi-way branching generalization. - **Shape²CART & ShapeCARTK:** Algorithms for learning S²GTs and SGTKs. + > [!NOTE] -> This codebase is an efficient, but working implementation of the algorithms in the paper "Empowering Decision Trees via Shape Function Branching". Please refer to the [ROADMAP](ROADMAP.md) for a detailed list of features that are currently implemented and those that are planned for future releases. For the canonical code base for the paper, please refer to https://github.com/optimal-uoft/Empowering-DTs-via-Shape-Functions. Features in the paper that are not yet implemented in this codebase include: -> * Bivariate shape functions (Shape$^2$CART) + Higher branching factors for bivariate splits (Shape$^2$SGT$_K$) -> * Visualization for bivariate splits (ex. contour plots) +> This codebase is an efficient implementation of the algorithms in "Empowering Decision Trees via Shape Function Branching." See the [ROADMAP](ROADMAP.md) for implementation status and the [canonical research code](https://github.com/optimal-uoft/Empowering-DTs-via-Shape-Functions) for the paper's original implementation. ## Installation diff --git a/ROADMAP.md b/ROADMAP.md index 6bfbf81..88eddd6 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -17,6 +17,12 @@ - [x] **NaN routing at predict**: if training saw missing at that split, follow the stored direction; otherwise route to the majority child. ## v0.3.0 -- [ ] multioutput support -- [ ] Shape$^2$CART -- [ ] Shape$^2$CART Random Forest Ensembling +- [x] multioutput support +- [x] Opt-in Shape$^2$CART for SGT estimators, including continuous/categorical + pairs, joint missing routing, and multiway branching ([tutorial](https://sgtlearn.readthedocs.io/en/latest/tutorials/bivariate-branching.html)) +- [x] Shape$^2$CART Random Forest Ensembling +- [x] Pair-aware TAO refinement ([#48](https://github.com/optimal-uoft/sgtlearn/issues/48)) +- [x] Shape$^2$CART routing heatmap visualization ([#28](https://github.com/optimal-uoft/sgtlearn/issues/28)) + +See the implementation specification in [#42](https://github.com/optimal-uoft/sgtlearn/issues/42) +and the umbrella issue [#27](https://github.com/optimal-uoft/sgtlearn/issues/27). diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bb50f40..7727d72 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -77,6 +77,10 @@ add_library(sgtlearn_core STATIC src/Splitters/categorical/CategoricalRegressionSplitter.h src/Splitters/categorical/CategoricalRegressionSplitter.cpp src/Discretizers/univariate/UnivariateClassificationDiscretizer.cpp + src/Discretizers/pair/PairClassificationDiscretizer.h + src/Discretizers/pair/PairClassificationDiscretizer.cpp + src/Discretizers/pair/PairRegressionDiscretizer.h + src/Discretizers/pair/PairRegressionDiscretizer.cpp src/Splitters/univariate/SquaredErrorSplitter.h src/Splitters/univariate/SquaredErrorSplitter.cpp src/Discretizers/univariate/UnivariateRegressionDiscretizer.cpp @@ -250,4 +254,4 @@ if (SGTLEARN_BUILD_TESTS) endif () -# endregion \ No newline at end of file +# endregion diff --git a/cpp/bindings/ShapeGeneralizedTrees.cpp b/cpp/bindings/ShapeGeneralizedTrees.cpp index 15ce024..52daccd 100644 --- a/cpp/bindings/ShapeGeneralizedTrees.cpp +++ b/cpp/bindings/ShapeGeneralizedTrees.cpp @@ -52,7 +52,8 @@ PYBIND11_MODULE(ShapeGeneralizedTrees, m) { size_t coordinate_descent_max_iters, size_t coordinate_descent_patience, bool coordinate_descent_smart_init, uint64_t random_state, - py::object max_features) { + py::object max_features, size_t pairwise_candidates, + double pairwise_penalty) { return ClassificationShapeGeneralizedTreePy( std::move(criterion), std::move(num_classes), num_partitions, outer_min_leaf_size, outer_min_gain_split, outer_max_depth, @@ -60,7 +61,8 @@ PYBIND11_MODULE(ShapeGeneralizedTrees, m) { inner_max_depth, inner_max_leaf_nodes, coordinate_descent_max_iters, coordinate_descent_patience, coordinate_descent_smart_init, random_state, - std::move(max_features)); + std::move(max_features), pairwise_candidates, + pairwise_penalty); }), py::arg("criterion") = "gini", py::arg("num_classes"), py::arg("num_partitions") = 2, @@ -76,7 +78,9 @@ PYBIND11_MODULE(ShapeGeneralizedTrees, m) { py::arg("coordinate_descent_patience") = 5, py::arg("coordinate_descent_smart_init") = true, py::arg("random_state") = 42, - py::arg("max_features") = py::none()) + py::arg("max_features") = py::none(), + py::arg("pairwise_candidates") = 0, + py::arg("pairwise_penalty") = 0.0) .def("fit", &ClassificationShapeGeneralizedTreePy::fit, py::arg("X"), py::arg("y"), py::arg("sample_weight") = py::none(), py::arg("features"), @@ -104,6 +108,8 @@ PYBIND11_MODULE(ShapeGeneralizedTrees, m) { &ClassificationShapeGeneralizedTreePy::classesPerOutput) .def_property_readonly( "is_fitted", &ClassificationShapeGeneralizedTreePy::isFitted) + .def_property_readonly( + "has_pair_nodes", &ClassificationShapeGeneralizedTreePy::hasPairNodes) .def_property_readonly( "feature_importance", &ClassificationShapeGeneralizedTreePy::featureImportance, @@ -122,14 +128,16 @@ PYBIND11_MODULE(ShapeGeneralizedTrees, m) { size_t coordinate_descent_max_iters, size_t coordinate_descent_patience, bool coordinate_descent_smart_init, uint64_t random_state, - py::object max_features) { + py::object max_features, size_t pairwise_candidates, + double pairwise_penalty) { return RegressionShapeGeneralizedTreePy( std::move(criterion), num_partitions, outer_min_leaf_size, outer_min_gain_split, outer_max_depth, outer_max_leaf_nodes, inner_min_leaf_size, inner_min_gain_split, inner_max_depth, inner_max_leaf_nodes, coordinate_descent_max_iters, coordinate_descent_patience, coordinate_descent_smart_init, - random_state, std::move(max_features)); + random_state, std::move(max_features), pairwise_candidates, + pairwise_penalty); }), py::arg("criterion") = "squared_error", py::arg("num_partitions") = 2, py::arg("outer_min_leaf_size") = 1, @@ -142,6 +150,8 @@ PYBIND11_MODULE(ShapeGeneralizedTrees, m) { py::arg("coordinate_descent_patience") = 5, py::arg("coordinate_descent_smart_init") = true, py::arg("random_state") = 42, py::arg("max_features") = py::none(), + py::arg("pairwise_candidates") = 0, + py::arg("pairwise_penalty") = 0.0, R"(Regression tree: inner bins are round-robin seeded. ``squared_error`` runs coordinate descent and keeps the map only if branch MSE improves clearly vs the seed; otherwise the snapshot is restored and the branch objective is rebuilt. @@ -164,6 +174,8 @@ is accepted for API parity with ClassificationShapeGeneralizedTree but ignored.) &RegressionShapeGeneralizedTreePy::nOutputs) .def_property_readonly("is_fitted", &RegressionShapeGeneralizedTreePy::isFitted) + .def_property_readonly( + "has_pair_nodes", &RegressionShapeGeneralizedTreePy::hasPairNodes) .def_property_readonly( "feature_importance", &RegressionShapeGeneralizedTreePy::featureImportance, diff --git a/cpp/bindings/TreeAlternatingOptimization.cpp b/cpp/bindings/TreeAlternatingOptimization.cpp index a3d15e9..40c7044 100644 --- a/cpp/bindings/TreeAlternatingOptimization.cpp +++ b/cpp/bindings/TreeAlternatingOptimization.cpp @@ -23,6 +23,7 @@ #include "algorithms/TAO/TreeAlternatingOptimization.h" #include +#include #include #include #include @@ -130,13 +131,17 @@ TaoRunContext TaoRunContext::make(py::object tree, const py::array &X, void TreeAlternatingOptimization(py::object tree, const py::array &X, const py::array &y, py::object sample_weight = py::none(), - size_t n_runs = 10, double lambda_ = 0.0) { + size_t n_runs = 10, double lambda_ = 0.0, + double tao_pair_scale = 1.1) { + if (!std::isfinite(tao_pair_scale) || tao_pair_scale < 0.0) + throw std::invalid_argument( + "tao_pair_scale must be finite and non-negative"); TaoRunContext ctx = TaoRunContext::make(tree, X, y, sample_weight); if (!ctx.isFitted()) throw std::logic_error("TreeAlternatingOptimization: model is not fitted"); py::gil_scoped_release release; - tao::optimize(ctx.adapter(), n_runs, lambda_); + tao::optimize(ctx.adapter(), n_runs, lambda_, tao_pair_scale); } } // namespace @@ -159,11 +164,13 @@ PYBIND11_MODULE(TreeAlternatingOptimization, m) { m.def("TreeAlternatingOptimization", &TreeAlternatingOptimization, py::arg("tree"), py::arg("X"), py::arg("y"), py::arg("sample_weight") = py::none(), py::arg("n_runs") = 10, - py::arg("lambda_") = 0.0, + py::arg("lambda_") = 0.0, py::arg("tao_pair_scale") = 1.1, "Refine a fitted ClassificationShapeGeneralizedTree or " "RegressionShapeGeneralizedTree in place. X is " "(n_samples, n_features) float32; y is 1-D class labels (uint) or " "float targets matching the tree type. Runs up to n_runs bottom-up " - "sweeps; lambda_ penalizes non-constant routing splits by " - "lambda_ * totalSampleWeight in weighted reward units."); + "sweeps. In weighted reward units, lambda_ penalizes single-feature " + "routers by lambda_ * nodeSampleCount and pair routers by " + "tao_pair_scale * lambda_ * nodeSampleCount; dummy routers are " + "unpenalized."); } diff --git a/cpp/bindings/_sgt_estimators.h b/cpp/bindings/_sgt_estimators.h index 1822747..d361cf6 100644 --- a/cpp/bindings/_sgt_estimators.h +++ b/cpp/bindings/_sgt_estimators.h @@ -17,6 +17,8 @@ #include "Discretizers/categorical/CategoricalClassificationDiscretizer.h" #include "Discretizers/categorical/CategoricalRegressionDiscretizer.h" +#include "Discretizers/pair/PairClassificationDiscretizer.h" +#include "Discretizers/pair/PairRegressionDiscretizer.h" #include "Domain/LearningCriterion.h" #include "Domain/FeatureInfo.h" #include "Discretizers/univariate/UnivariateDiscretizer.h" @@ -45,6 +47,39 @@ inline py::list routingFeaturesPy(const ShapeFunctionNode &n) { return feats; } +inline py::list logicalFeaturesPy(const ShapeFunctionNode &n) { + py::list feats; + for (size_t f : n.logicalFeatureIndices) + feats.append(f); + return feats; +} + +inline py::list pairAxesPy(const ShapeFunctionNode &n, + const std::array &axes) { + py::list out; + for (size_t axis = 0; axis < axes.size(); ++axis) { + py::dict item; + item["logical_feature"] = n.logicalFeatureIndices.at(axis); + item["kind"] = axes[axis].type == FeatureType::Categorical + ? "categorical" + : "continuous"; + py::list columns; + py::list categories; + for (size_t raw : axes[axis].indices) { + columns.append(raw); + if (axes[axis].type == FeatureType::Categorical) + categories.append(raw); + } + item["columns"] = columns; + item["categories"] = categories; + item["catchall"] = axes[axis].type == FeatureType::Categorical + ? py::cast("missing") + : py::none(); + out.append(item); + } + return out; +} + inline py::object primaryRoutingFeaturePy(const ShapeFunctionNode &n) { if (n.routingFeatures.empty()) return py::none(); @@ -258,7 +293,8 @@ class ClassificationShapeGeneralizedTreePy { double innerMinGainSplit, size_t innerMaxDepth, size_t innerMaxLeafNodes, size_t coordinateDescentMaxIters, size_t coordinateDescentPatience, bool coordinateDescentSmartInit, uint64_t random_state, - py::object max_features = py::none()) { + py::object max_features = py::none(), size_t pairwiseCandidates = 0, + double pairwisePenalty = 0.0) { criterionStr_ = criterion; const LearningCriterion crit = parseClassificationCriterion(criterion); const TreeBuildingParams outer{outerMinLeafSize, outerMinGainSplit, @@ -271,7 +307,8 @@ class ClassificationShapeGeneralizedTreePy { cd.smartInit = coordinateDescentSmartInit; impl_ = std::make_unique( crit, parseNumClassesPy(numClasses), numPartitions, outer, inner, cd, - random_state, parseMaxFeaturesPy(max_features)); + random_state, parseMaxFeaturesPy(max_features), pairwiseCandidates, + pairwisePenalty); } void fit(const py::array &X, const py::array &y, @@ -334,6 +371,7 @@ class ClassificationShapeGeneralizedTreePy { size_t numNodes() const { return impl_->numNodes(); } bool isFitted() const { return impl_->isFitted(); } size_t nOutputs() const { return impl_->nOutputs(); } + bool hasPairNodes() const { return impl_->hasPairNodes(); } py::list classesPerOutput() const { py::list out; @@ -401,6 +439,46 @@ class ClassificationShapeGeneralizedTreePy { } else { d["feature"] = primaryRoutingFeaturePy(n); d["features"] = routingFeaturesPy(n); + const auto *pairDisc = dynamic_cast( + n.innerDiscretizer.get()); + if (pairDisc) { + d["routing_kind"] = "pair"; + d["pair_features"] = logicalFeaturesPy(n); + d["pair_axes"] = pairAxesPy(n, pairDisc->axes()); + py::list innerTree; + py::list leafBins; + const auto &routingTree = pairDisc->routingTree(); + for (size_t nodeIndex = 0; nodeIndex < routingTree.size(); ++nodeIndex) { + const PairRoutingTreeNode &inner = routingTree[nodeIndex]; + py::dict innerNode; + innerNode["id"] = nodeIndex; + innerNode["is_leaf"] = inner.isLeaf; + if (inner.isLeaf) { + innerNode["bin"] = inner.bin; + leafBins.append(inner.bin); + } else { + innerNode["feature"] = inner.rawFeature; + innerNode["axis"] = inner.featurePosition; + innerNode["kind"] = inner.featureType == FeatureType::Categorical + ? "categorical" + : "continuous"; + innerNode["threshold"] = + inner.featureType == FeatureType::Continuous + ? py::cast(inner.threshold) + : py::none(); + innerNode["category"] = + inner.featureType == FeatureType::Categorical + ? py::cast(inner.rawFeature) + : py::none(); + innerNode["left"] = inner.left; + innerNode["right"] = inner.right; + innerNode["missing"] = inner.missing; + } + innerTree.append(innerNode); + } + d["pair_inner_tree"] = innerTree; + d["pair_leaf_bins"] = leafBins; + } const bool isCategorical = isCategoricalInnerDiscretizer(*n.innerDiscretizer); d["is_categorical"] = isCategorical; @@ -427,8 +505,9 @@ class ClassificationShapeGeneralizedTreePy { py::list bsc; for (size_t v : n.binSampleCounts) bsc.append(v); d["bin_sample_counts"] = bsc; - d["nan_prediction_partition"] = - n.binToPartition.empty() ? 0 : n.binToPartition.back(); + if (!pairDisc) + d["nan_prediction_partition"] = + n.binToPartition.empty() ? 0 : n.binToPartition.back(); py::list ch; if (i < childIdx.size()) { for (size_t c : childIdx[i]) ch.append(c); @@ -454,7 +533,8 @@ class RegressionShapeGeneralizedTreePy { size_t innerMinLeafSize, double innerMinGainSplit, size_t innerMaxDepth, size_t innerMaxLeafNodes, size_t coordinateDescentMaxIters, size_t coordinateDescentPatience, bool coordinateDescentSmartInit, - uint64_t random_state, py::object max_features = py::none()) { + uint64_t random_state, py::object max_features = py::none(), + size_t pairwiseCandidates = 0, double pairwisePenalty = 0.0) { criterionStr_ = criterion; const LearningCriterion crit = parseRegressionCriterion(criterion); const TreeBuildingParams outer{outerMinLeafSize, outerMinGainSplit, @@ -467,7 +547,7 @@ class RegressionShapeGeneralizedTreePy { cd.smartInit = coordinateDescentSmartInit; impl_ = std::make_unique( crit, numPartitions, outer, inner, cd, random_state, - parseMaxFeaturesPy(max_features)); + parseMaxFeaturesPy(max_features), pairwiseCandidates, pairwisePenalty); } void fit(const py::array &X, const py::array &y, @@ -507,6 +587,7 @@ class RegressionShapeGeneralizedTreePy { size_t numNodes() const { return impl_->numNodes(); } bool isFitted() const { return impl_->isFitted(); } size_t nOutputs() const { return impl_->nOutputs(); } + bool hasPairNodes() const { return impl_->hasPairNodes(); } py::array_t featureImportance() const { return colToNumpy(impl_->featureImportance()); @@ -570,6 +651,51 @@ class RegressionShapeGeneralizedTreePy { d["n_samples"] = total; d["feature"] = primaryRoutingFeaturePy(n); d["features"] = routingFeaturesPy(n); + const auto *pairDisc = dynamic_cast( + n.innerDiscretizer.get()); + const auto *taoPairDisc = + dynamic_cast( + n.innerDiscretizer.get()); + if (pairDisc || taoPairDisc) { + d["routing_kind"] = "pair"; + d["pair_features"] = logicalFeaturesPy(n); + d["pair_axes"] = pairAxesPy( + n, pairDisc ? pairDisc->axes() : taoPairDisc->axes()); + py::list innerTree; + py::list leafBins; + const auto &routingTree = + pairDisc ? pairDisc->routingTree() : taoPairDisc->routingTree(); + for (size_t nodeIndex = 0; nodeIndex < routingTree.size(); ++nodeIndex) { + const PairRoutingTreeNode &inner = routingTree[nodeIndex]; + py::dict innerNode; + innerNode["id"] = nodeIndex; + innerNode["is_leaf"] = inner.isLeaf; + if (inner.isLeaf) { + innerNode["bin"] = inner.bin; + leafBins.append(inner.bin); + } else { + innerNode["feature"] = inner.rawFeature; + innerNode["axis"] = inner.featurePosition; + innerNode["kind"] = inner.featureType == FeatureType::Categorical + ? "categorical" + : "continuous"; + innerNode["threshold"] = + inner.featureType == FeatureType::Continuous + ? py::cast(inner.threshold) + : py::none(); + innerNode["category"] = + inner.featureType == FeatureType::Categorical + ? py::cast(inner.rawFeature) + : py::none(); + innerNode["left"] = inner.left; + innerNode["right"] = inner.right; + innerNode["missing"] = inner.missing; + } + innerTree.append(innerNode); + } + d["pair_inner_tree"] = innerTree; + d["pair_leaf_bins"] = leafBins; + } const bool isCategorical = isCategoricalInnerDiscretizer(*n.innerDiscretizer); d["is_categorical"] = isCategorical; @@ -596,8 +722,9 @@ class RegressionShapeGeneralizedTreePy { py::list bsc; for (size_t v : n.binSampleCounts) bsc.append(v); d["bin_sample_counts"] = bsc; - d["nan_prediction_partition"] = - n.binToPartition.empty() ? 0 : n.binToPartition.back(); + if (!pairDisc && !taoPairDisc) + d["nan_prediction_partition"] = + n.binToPartition.empty() ? 0 : n.binToPartition.back(); py::list ch; if (i < childIdx.size()) { for (size_t c : childIdx[i]) ch.append(c); diff --git a/cpp/src/Discretizers/pair/PairClassificationDiscretizer.cpp b/cpp/src/Discretizers/pair/PairClassificationDiscretizer.cpp new file mode 100644 index 0000000..50268d8 --- /dev/null +++ b/cpp/src/Discretizers/pair/PairClassificationDiscretizer.cpp @@ -0,0 +1,351 @@ +#include "Discretizers/pair/PairClassificationDiscretizer.h" + +#include "Criterion.h" +#include "algorithms/missing_values.h" + +#include +#include +#include +#include +#include +#include + +PairClassificationDiscretizer::PairClassificationDiscretizer( + LearningCriterion criterion, FeatureInfo first, FeatureInfo second) + : criterion_(criterion), axes_{std::move(first), std::move(second)} { + axisOffsets_[1] = axes_[0].indices.n_elem; + axisOffsets_[2] = axisOffsets_[1] + axes_[1].indices.n_elem; + if (axisOffsets_[1] == 0 || axisOffsets_[2] == axisOffsets_[1]) + throw std::invalid_argument("pair CART logical features cannot be empty"); + for (const FeatureInfo &axis : axes_) + if (axis.type == FeatureType::Continuous && axis.indices.n_elem != 1) + throw std::invalid_argument( + "pair CART continuous logical features require one column"); + routingFeatures_ = arma::join_cols(axes_[0].indices, axes_[1].indices); +} + +bool PairClassificationDiscretizer::axisMissing( + size_t axis, const arma::fmat &X, size_t sample) const { + if (axes_[axis].type == FeatureType::Continuous) + return !missing_values::is_finite(X(axes_[axis].indices(0), sample)); + for (size_t raw : axes_[axis].indices) + if (X(raw, sample) >= 0.5F) + return false; + return true; +} + +bool PairClassificationDiscretizer::axisMissing( + size_t axis, const std::vector &values) const { + if (axes_[axis].type == FeatureType::Continuous) + return !missing_values::is_finite(values[axisOffsets_[axis]]); + for (size_t pos = axisOffsets_[axis]; pos < axisOffsets_[axis + 1]; ++pos) + if (values[pos] >= 0.5F) + return false; + return true; +} + +size_t PairClassificationDiscretizer::routingPosition( + size_t rawFeature) const { + const auto it = + std::find(routingFeatures_.begin(), routingFeatures_.end(), rawFeature); + if (it == routingFeatures_.end()) + throw std::runtime_error("pair CART routing feature not found"); + return static_cast(it - routingFeatures_.begin()); +} + +double PairClassificationDiscretizer::impurity( + const std::vector> &stats) const { + if (criterion_ == LearningCriterion::Gini) + return Criterion::gini(stats); + if (criterion_ == LearningCriterion::Entropy) + return Criterion::entropy(stats); + throw std::invalid_argument("pair CART requires a classification criterion"); +} + +PairClassificationDiscretizer::Split +PairClassificationDiscretizer::bestSplit( + const BuildNode &node, const arma::fmat &X, const arma::Mat &y, + const std::vector &classes, const arma::Row &weights, + size_t minLeafSize, double totalWeight) const { + Split best; + double bestChildScore = std::numeric_limits::infinity(); + + const auto makeStats = [&y, &classes, &weights]( + const std::vector &samples) { + std::vector> stats(classes.size()); + for (size_t o = 0; o < classes.size(); ++o) + stats[o].assign(classes[o], 0.0); + for (size_t sample : samples) + for (size_t o = 0; o < classes.size(); ++o) + stats[o][y(o, sample)] += weights(sample); + return stats; + }; + const auto sumWeight = [&weights](const std::vector &samples) { + double total = 0.0; + for (size_t sample : samples) + total += weights(sample); + return total; + }; + + for (size_t axis = 0; axis < 2; ++axis) { + std::vector valid; + std::vector missing; + for (size_t sample : node.samples) + (axisMissing(axis, X, sample) ? missing : valid).push_back(sample); + + const auto missingStats = makeStats(missing); + const double missingWeight = sumWeight(missing); + const auto consider = [&](size_t rawFeature, double threshold, + std::vector leftSamples, + std::vector rightSamples, + const std::vector> &leftStats, + const std::vector> &rightStats, + double leftWeight, double rightWeight) { + if (leftSamples.size() < minLeafSize || + rightSamples.size() < minLeafSize) + return; + const double childScore = + node.weight > 0.0 + ? (leftWeight * impurity(leftStats) + + rightWeight * impurity(rightStats) + + missingWeight * impurity(missingStats)) / + node.weight + : 0.0; + if (childScore >= + bestChildScore - std::numeric_limits::epsilon()) + return; + best.found = true; + bestChildScore = childScore; + best.featurePosition = axis; + best.rawFeature = rawFeature; + best.featureType = axes_[axis].type; + best.threshold = threshold; + best.left = std::move(leftSamples); + best.right = std::move(rightSamples); + best.missing = missing; + best.gain = totalWeight > 0.0 + ? (node.weight / totalWeight) * + (node.impurity - childScore) + : 0.0; + }; + + if (axes_[axis].type == FeatureType::Categorical) { + for (size_t rawFeature : axes_[axis].indices) { + std::vector left; + std::vector right; + for (size_t sample : valid) + (X(rawFeature, sample) >= 0.5F ? right : left).push_back(sample); + consider(rawFeature, 0.5, left, right, makeStats(left), + makeStats(right), sumWeight(left), sumWeight(right)); + } + continue; + } + + const size_t rawFeature = axes_[axis].indices(0); + std::vector order = std::move(valid); + std::stable_sort(order.begin(), order.end(), + [&X, rawFeature](size_t a, size_t b) { + return X(rawFeature, a) < X(rawFeature, b); + }); + + std::vector> left(classes.size()); + std::vector> right = makeStats(order); + for (size_t o = 0; o < classes.size(); ++o) + left[o].assign(classes[o], 0.0); + double leftWeight = 0.0; + const double validWeight = node.weight - missingWeight; + + for (size_t i = 1; i < order.size(); ++i) { + const size_t moved = order[i - 1]; + const double w = weights(moved); + leftWeight += w; + for (size_t o = 0; o < classes.size(); ++o) { + left[o][y(o, moved)] += w; + right[o][y(o, moved)] -= w; + } + if (i < minLeafSize) + continue; + if (order.size() - i < minLeafSize) + break; + + const float previous = X(rawFeature, order[i - 1]); + const float current = X(rawFeature, order[i]); + if (current <= previous + 1e-7F) + continue; + double threshold = static_cast(previous) / 2.0 + + static_cast(current) / 2.0; + if (!std::isfinite(threshold) || threshold == current) + threshold = previous; + consider( + rawFeature, threshold, + std::vector(order.begin(), order.begin() + i), + std::vector(order.begin() + i, order.end()), left, right, + leftWeight, validWeight - leftWeight); + } + } + return best; +} + +void PairClassificationDiscretizer::Train( + const arma::fmat &X, arma::uvec &features, const arma::Mat &y, + const std::vector &classes, size_t minLeafSize, + double minGainSplit, size_t maxDepth, size_t maxLeafNodes, + const arma::Row &sampleWeights) { + this->resetTrainedOutputs(); + tree_.clear(); + if (features.n_elem != routingFeatures_.n_elem || y.n_cols != X.n_cols || + classes.size() != y.n_rows) + throw std::invalid_argument("invalid pair CART training shapes"); + + arma::Row weights = sampleWeights; + if (weights.n_elem == 0) + weights.ones(X.n_cols); + if (weights.n_elem != X.n_cols) + throw std::invalid_argument("pair CART sample weight length mismatch"); + for (size_t rawFeature : routingFeatures_) + if (rawFeature >= X.n_rows) + throw std::invalid_argument("pair CART feature out of range"); + + const auto makeNode = [&y, &classes, &weights, this]( + std::vector samples, size_t depth) { + BuildNode node; + node.samples = std::move(samples); + node.depth = depth; + node.stats.resize(classes.size()); + for (size_t o = 0; o < classes.size(); ++o) + node.stats[o].assign(classes[o], 0.0); + for (size_t sample : node.samples) { + const double w = weights(sample); + node.weight += w; + for (size_t o = 0; o < classes.size(); ++o) + node.stats[o][y(o, sample)] += w; + } + node.impurity = impurity(node.stats); + return node; + }; + + std::vector all(X.n_cols); + std::iota(all.begin(), all.end(), 0); + std::vector nodes; + nodes.push_back(makeNode(std::move(all), 0)); + const double totalWeight = nodes.front().weight; + size_t finiteLeafCount = 1; + + const auto splitNode = [&](size_t index, Split split) { + const size_t depth = nodes[index].depth; + const bool missingGoesLeft = split.left.size() >= split.right.size(); + nodes[index].routing.isLeaf = false; + nodes[index].routing.featurePosition = split.featurePosition; + nodes[index].routing.rawFeature = split.rawFeature; + nodes[index].routing.featureType = split.featureType; + nodes[index].routing.threshold = split.threshold; + nodes[index].routing.left = nodes.size(); + nodes.push_back(makeNode(std::move(split.left), depth + 1)); + nodes[index].routing.right = nodes.size(); + nodes.push_back(makeNode(std::move(split.right), depth + 1)); + if (split.missing.empty()) { + nodes[index].routing.missing = missingGoesLeft + ? nodes[index].routing.left + : nodes[index].routing.right; + } else { + nodes[index].routing.missing = nodes.size(); + nodes.push_back(makeNode(std::move(split.missing), depth + 1)); + } + ++finiteLeafCount; + }; + + if (maxLeafNodes == 0) { + const auto grow = [&](auto &&self, size_t index) -> void { + if (maxDepth != 0 && nodes[index].depth >= maxDepth) + return; + Split split = bestSplit(nodes[index], X, y, classes, weights, + minLeafSize, totalWeight); + if (!split.found || + split.gain + std::numeric_limits::epsilon() < minGainSplit) + return; + splitNode(index, std::move(split)); + self(self, nodes[index].routing.left); + self(self, nodes[index].routing.right); + if (nodes[index].routing.missing != nodes[index].routing.left) + self(self, nodes[index].routing.missing); + }; + grow(grow, 0); + } else { + while (finiteLeafCount < maxLeafNodes) { + size_t bestIndex = nodes.size(); + Split best; + for (size_t i = 0; i < nodes.size(); ++i) { + if (!nodes[i].routing.isLeaf || + (maxDepth != 0 && nodes[i].depth >= maxDepth)) + continue; + Split candidate = bestSplit(nodes[i], X, y, classes, weights, + minLeafSize, totalWeight); + if (!candidate.found || + candidate.gain + std::numeric_limits::epsilon() < + minGainSplit) + continue; + if (bestIndex == nodes.size() || candidate.gain > best.gain) { + bestIndex = i; + best = std::move(candidate); + } + } + if (bestIndex == nodes.size()) + break; + splitNode(bestIndex, std::move(best)); + } + } + + for (BuildNode &node : nodes) { + if (!node.routing.isLeaf) + continue; + node.routing.bin = this->inSampleDiscretizations_.size(); + this->inSampleDiscretizations_.push_back(std::move(node.samples)); + this->leafStats_.push_back(std::move(node.stats)); + this->leafNumSamples_.push_back( + this->inSampleDiscretizations_.back().size()); + this->leafNodeWeights_.push_back(node.weight); + } + tree_.reserve(nodes.size()); + for (const BuildNode &node : nodes) + tree_.push_back(node.routing); + this->numLeaves_ = this->leafStats_.size(); + this->markTrained(); +} + +size_t PairClassificationDiscretizer::routeValues( + const std::vector &values) const { + if (values.size() != routingFeatures_.n_elem) + throw std::invalid_argument( + "pair router values do not match routing features"); + size_t index = 0; + while (!tree_[index].isLeaf) { + const PairRoutingTreeNode &node = tree_[index]; + if (axisMissing(node.featurePosition, values)) { + index = node.missing; + continue; + } + const float value = values[routingPosition(node.rawFeature)]; + index = node.featureType == FeatureType::Categorical + ? (value >= 0.5F ? node.right : node.left) + : (value <= node.threshold ? node.left : node.right); + } + return tree_[index].bin; +} + +size_t PairClassificationDiscretizer::routeToBin( + const std::vector &featureValues) const { + this->ensureTrained(); + return routeValues(featureValues); +} + +void PairClassificationDiscretizer::transform( + const arma::fmat &X, arma::Row &binLoc) const { + this->ensureTrained(); + binLoc.set_size(X.n_cols); + std::vector values(routingFeatures_.n_elem); + for (arma::uword i = 0; i < X.n_cols; ++i) { + for (size_t j = 0; j < routingFeatures_.n_elem; ++j) + values[j] = X(routingFeatures_(j), i); + binLoc(i) = routeValues(values); + } +} diff --git a/cpp/src/Discretizers/pair/PairClassificationDiscretizer.h b/cpp/src/Discretizers/pair/PairClassificationDiscretizer.h new file mode 100644 index 0000000..52c361a --- /dev/null +++ b/cpp/src/Discretizers/pair/PairClassificationDiscretizer.h @@ -0,0 +1,81 @@ +#pragma once + +#include "Discretizers/ClassificationDiscretizer.h" +#include "Domain/FeatureInfo.h" +#include "Domain/LearningCriterion.h" + +#include +#include +#include +#include + +struct PairRoutingTreeNode { + bool isLeaf = true; + size_t rawFeature = 0; + size_t featurePosition = 0; + FeatureType featureType = FeatureType::Continuous; + double threshold = 0.0; + size_t left = 0; + size_t right = 0; + size_t missing = 0; + size_t bin = 0; +}; + +/** Ordinary axis-aligned CART over exactly two logical features. */ +class PairClassificationDiscretizer final : public ClassificationDiscretizer { +public: + PairClassificationDiscretizer(LearningCriterion criterion, + FeatureInfo first, FeatureInfo second); + + void Train(const arma::fmat &X, arma::uvec &features, + const arma::Mat &y, + const std::vector &nClassesPerOutput, + size_t minLeafSize, double minGainSplit, size_t maxDepth, + size_t maxLeafNodes, + const arma::Row &sampleWeights = arma::Row()) override; + + void transform(const arma::fmat &X, arma::Row &binLoc) const override; + size_t routeToBin(const std::vector &featureValues) const override; + + const std::vector &routingTree() const { return tree_; } + const std::array &axes() const { return axes_; } + +private: + struct BuildNode { + PairRoutingTreeNode routing; + std::vector samples; + std::vector> stats; + double weight = 0.0; + double impurity = 0.0; + size_t depth = 0; + }; + + struct Split { + bool found = false; + size_t featurePosition = 0; + size_t rawFeature = 0; + FeatureType featureType = FeatureType::Continuous; + double threshold = 0.0; + double gain = 0.0; + std::vector left; + std::vector right; + std::vector missing; + }; + + double impurity(const std::vector> &stats) const; + Split bestSplit(const BuildNode &node, const arma::fmat &X, + const arma::Mat &y, + const std::vector &classes, + const arma::Row &weights, size_t minLeafSize, + double totalWeight) const; + size_t routeValues(const std::vector &values) const; + bool axisMissing(size_t axis, const arma::fmat &X, size_t sample) const; + bool axisMissing(size_t axis, const std::vector &values) const; + size_t routingPosition(size_t rawFeature) const; + + LearningCriterion criterion_; + std::array axes_; + arma::uvec routingFeatures_; + std::array axisOffsets_{}; + std::vector tree_; +}; diff --git a/cpp/src/Discretizers/pair/PairRegressionDiscretizer.cpp b/cpp/src/Discretizers/pair/PairRegressionDiscretizer.cpp new file mode 100644 index 0000000..0e05ba6 --- /dev/null +++ b/cpp/src/Discretizers/pair/PairRegressionDiscretizer.cpp @@ -0,0 +1,344 @@ +#include "Discretizers/pair/PairRegressionDiscretizer.h" + +#include "Criterion.h" +#include "algorithms/missing_values.h" + +#include +#include +#include +#include +#include +#include + +PairRegressionDiscretizer::PairRegressionDiscretizer( + LearningCriterion criterion, FeatureInfo first, FeatureInfo second) + : criterion_(criterion), axes_{std::move(first), std::move(second)} { + axisOffsets_[1] = axes_[0].indices.n_elem; + axisOffsets_[2] = axisOffsets_[1] + axes_[1].indices.n_elem; + if (axisOffsets_[1] == 0 || axisOffsets_[2] == axisOffsets_[1]) + throw std::invalid_argument("pair CART logical features cannot be empty"); + for (const FeatureInfo &axis : axes_) + if (axis.type == FeatureType::Continuous && axis.indices.n_elem != 1) + throw std::invalid_argument( + "pair CART continuous logical features require one column"); + routingFeatures_ = arma::join_cols(axes_[0].indices, axes_[1].indices); + if (criterion_ != LearningCriterion::SquaredError && + criterion_ != LearningCriterion::AbsoluteError) + throw std::invalid_argument("pair CART requires a regression criterion"); +} + +bool PairRegressionDiscretizer::axisMissing( + size_t axis, const arma::fmat &X, size_t sample) const { + if (axes_[axis].type == FeatureType::Continuous) + return !missing_values::is_finite(X(axes_[axis].indices(0), sample)); + for (size_t raw : axes_[axis].indices) + if (X(raw, sample) >= 0.5F) + return false; + return true; +} + +bool PairRegressionDiscretizer::axisMissing( + size_t axis, const std::vector &values) const { + if (axes_[axis].type == FeatureType::Continuous) + return !missing_values::is_finite(values[axisOffsets_[axis]]); + for (size_t pos = axisOffsets_[axis]; pos < axisOffsets_[axis + 1]; ++pos) + if (values[pos] >= 0.5F) + return false; + return true; +} + +size_t PairRegressionDiscretizer::routingPosition(size_t rawFeature) const { + const auto it = + std::find(routingFeatures_.begin(), routingFeatures_.end(), rawFeature); + if (it == routingFeatures_.end()) + throw std::runtime_error("pair CART routing feature not found"); + return static_cast(it - routingFeatures_.begin()); +} + +double PairRegressionDiscretizer::impurity( + const std::vector &samples, const arma::Mat &y, + const arma::Row &weights) const { + if (samples.empty()) + return 0.0; + if (criterion_ == LearningCriterion::SquaredError) { + std::vector> stats( + y.n_rows, std::vector(2, 0.0)); + double totalWeight = 0.0; + for (size_t sample : samples) { + const double w = weights(sample); + totalWeight += w; + for (arma::uword o = 0; o < y.n_rows; ++o) { + const double value = y(o, sample); + stats[o][0] += w * value; + stats[o][1] += w * value * value; + } + } + return Criterion::squaredError(stats, totalWeight); + } + + double total = 0.0; + for (arma::uword o = 0; o < y.n_rows; ++o) { + std::vector values; + std::vector sampleWeights; + values.reserve(samples.size()); + sampleWeights.reserve(samples.size()); + for (size_t sample : samples) { + values.push_back(y(o, sample)); + sampleWeights.push_back(weights(sample)); + } + total += Criterion::absoluteError(values, sampleWeights).mae; + } + return total; +} + +PairRegressionDiscretizer::Split PairRegressionDiscretizer::bestSplit( + const BuildNode &node, const arma::fmat &X, const arma::Mat &y, + const arma::Row &weights, size_t minLeafSize, + double totalWeight) const { + Split best; + double bestChildScore = std::numeric_limits::infinity(); + const auto sumWeight = [&weights](const std::vector &samples) { + double total = 0.0; + for (size_t sample : samples) + total += weights(sample); + return total; + }; + + for (size_t axis = 0; axis < 2; ++axis) { + std::vector valid; + std::vector missing; + for (size_t sample : node.samples) + (axisMissing(axis, X, sample) ? missing : valid).push_back(sample); + const double missingWeight = sumWeight(missing); + + const auto consider = [&](size_t rawFeature, double threshold, + std::vector left, + std::vector right) { + if (left.size() < minLeafSize || right.size() < minLeafSize) + return; + const double leftWeight = sumWeight(left); + const double rightWeight = sumWeight(right); + const double childScore = + node.weight > 0.0 + ? (leftWeight * impurity(left, y, weights) + + rightWeight * impurity(right, y, weights) + + missingWeight * impurity(missing, y, weights)) / + node.weight + : 0.0; + if (childScore >= + bestChildScore - std::numeric_limits::epsilon()) + return; + best.found = true; + bestChildScore = childScore; + best.featurePosition = axis; + best.rawFeature = rawFeature; + best.featureType = axes_[axis].type; + best.threshold = threshold; + best.left = std::move(left); + best.right = std::move(right); + best.missing = missing; + best.gain = totalWeight > 0.0 + ? (node.weight / totalWeight) * + (node.impurity - childScore) + : 0.0; + }; + + if (axes_[axis].type == FeatureType::Categorical) { + for (size_t rawFeature : axes_[axis].indices) { + std::vector left; + std::vector right; + for (size_t sample : valid) + (X(rawFeature, sample) >= 0.5F ? right : left).push_back(sample); + consider(rawFeature, 0.5, std::move(left), std::move(right)); + } + continue; + } + + const size_t rawFeature = axes_[axis].indices(0); + std::vector order = std::move(valid); + std::stable_sort(order.begin(), order.end(), + [&X, rawFeature](size_t a, size_t b) { + return X(rawFeature, a) < X(rawFeature, b); + }); + for (size_t i = minLeafSize; i + minLeafSize <= order.size(); ++i) { + const float previous = X(rawFeature, order[i - 1]); + const float current = X(rawFeature, order[i]); + if (current <= previous + 1e-7F) + continue; + double threshold = static_cast(previous) / 2.0 + + static_cast(current) / 2.0; + if (!std::isfinite(threshold) || threshold == current) + threshold = previous; + consider(rawFeature, threshold, + std::vector(order.begin(), order.begin() + i), + std::vector(order.begin() + i, order.end())); + } + } + return best; +} + +void PairRegressionDiscretizer::Train( + const arma::fmat &X, arma::uvec &features, const arma::Mat &y, + size_t minLeafSize, double minGainSplit, size_t maxDepth, + size_t maxLeafNodes, const arma::Row &sampleWeights) { + this->resetTrainedOutputs(); + tree_.clear(); + if (features.n_elem != routingFeatures_.n_elem || y.n_cols != X.n_cols) + throw std::invalid_argument("invalid pair CART training shapes"); + + arma::Row weights = sampleWeights; + if (weights.n_elem == 0) + weights.ones(X.n_cols); + if (weights.n_elem != X.n_cols) + throw std::invalid_argument("pair CART sample weight length mismatch"); + for (size_t rawFeature : routingFeatures_) + if (rawFeature >= X.n_rows) + throw std::invalid_argument("pair CART feature out of range"); + + const auto makeNode = [&y, &weights, this](std::vector samples, + size_t depth) { + BuildNode node; + node.samples = std::move(samples); + node.depth = depth; + node.stats.assign(y.n_rows, std::vector(2, 0.0)); + for (size_t sample : node.samples) { + const double w = weights(sample); + node.weight += w; + for (arma::uword o = 0; o < y.n_rows; ++o) { + const double value = y(o, sample); + node.stats[o][0] += w * value; + node.stats[o][1] += w * value * value; + } + } + node.impurity = impurity(node.samples, y, weights); + return node; + }; + + std::vector all(X.n_cols); + std::iota(all.begin(), all.end(), 0); + std::vector nodes; + nodes.push_back(makeNode(std::move(all), 0)); + const double totalWeight = nodes.front().weight; + size_t finiteLeafCount = 1; + + const auto splitNode = [&](size_t index, Split split) { + const size_t depth = nodes[index].depth; + const bool missingGoesLeft = split.left.size() >= split.right.size(); + nodes[index].routing.isLeaf = false; + nodes[index].routing.featurePosition = split.featurePosition; + nodes[index].routing.rawFeature = split.rawFeature; + nodes[index].routing.featureType = split.featureType; + nodes[index].routing.threshold = split.threshold; + nodes[index].routing.left = nodes.size(); + nodes.push_back(makeNode(std::move(split.left), depth + 1)); + nodes[index].routing.right = nodes.size(); + nodes.push_back(makeNode(std::move(split.right), depth + 1)); + if (split.missing.empty()) { + nodes[index].routing.missing = missingGoesLeft + ? nodes[index].routing.left + : nodes[index].routing.right; + } else { + nodes[index].routing.missing = nodes.size(); + nodes.push_back(makeNode(std::move(split.missing), depth + 1)); + } + ++finiteLeafCount; + }; + + if (maxLeafNodes == 0) { + const auto grow = [&](auto &&self, size_t index) -> void { + if (maxDepth != 0 && nodes[index].depth >= maxDepth) + return; + Split split = + bestSplit(nodes[index], X, y, weights, minLeafSize, totalWeight); + if (!split.found || + split.gain + std::numeric_limits::epsilon() < minGainSplit) + return; + splitNode(index, std::move(split)); + self(self, nodes[index].routing.left); + self(self, nodes[index].routing.right); + if (nodes[index].routing.missing != nodes[index].routing.left) + self(self, nodes[index].routing.missing); + }; + grow(grow, 0); + } else { + while (finiteLeafCount < maxLeafNodes) { + size_t bestIndex = nodes.size(); + Split best; + for (size_t i = 0; i < nodes.size(); ++i) { + if (!nodes[i].routing.isLeaf || + (maxDepth != 0 && nodes[i].depth >= maxDepth)) + continue; + Split candidate = + bestSplit(nodes[i], X, y, weights, minLeafSize, totalWeight); + if (!candidate.found || + candidate.gain + std::numeric_limits::epsilon() < + minGainSplit) + continue; + if (bestIndex == nodes.size() || candidate.gain > best.gain) { + bestIndex = i; + best = std::move(candidate); + } + } + if (bestIndex == nodes.size()) + break; + splitNode(bestIndex, std::move(best)); + } + } + + for (BuildNode &node : nodes) { + if (!node.routing.isLeaf) + continue; + node.routing.bin = this->inSampleDiscretizations_.size(); + this->inSampleDiscretizations_.push_back(std::move(node.samples)); + this->leafStats_.push_back( + criterion_ == LearningCriterion::SquaredError + ? std::move(node.stats) + : std::vector>{}); + this->leafNumSamples_.push_back( + this->inSampleDiscretizations_.back().size()); + this->leafNodeWeights_.push_back(node.weight); + } + tree_.reserve(nodes.size()); + for (const BuildNode &node : nodes) + tree_.push_back(node.routing); + this->numLeaves_ = this->leafStats_.size(); + this->markTrained(); +} + +size_t PairRegressionDiscretizer::routeValues( + const std::vector &values) const { + if (values.size() != routingFeatures_.n_elem) + throw std::invalid_argument( + "pair router values do not match routing features"); + size_t index = 0; + while (!tree_[index].isLeaf) { + const PairRoutingTreeNode &node = tree_[index]; + if (axisMissing(node.featurePosition, values)) { + index = node.missing; + continue; + } + const float value = values[routingPosition(node.rawFeature)]; + index = node.featureType == FeatureType::Categorical + ? (value >= 0.5F ? node.right : node.left) + : (value <= node.threshold ? node.left : node.right); + } + return tree_[index].bin; +} + +size_t PairRegressionDiscretizer::routeToBin( + const std::vector &featureValues) const { + this->ensureTrained(); + return routeValues(featureValues); +} + +void PairRegressionDiscretizer::transform(const arma::fmat &X, + arma::Row &binLoc) const { + this->ensureTrained(); + binLoc.set_size(X.n_cols); + std::vector values(routingFeatures_.n_elem); + for (arma::uword i = 0; i < X.n_cols; ++i) { + for (size_t j = 0; j < routingFeatures_.n_elem; ++j) + values[j] = X(routingFeatures_(j), i); + binLoc(i) = routeValues(values); + } +} diff --git a/cpp/src/Discretizers/pair/PairRegressionDiscretizer.h b/cpp/src/Discretizers/pair/PairRegressionDiscretizer.h new file mode 100644 index 0000000..0f3be9d --- /dev/null +++ b/cpp/src/Discretizers/pair/PairRegressionDiscretizer.h @@ -0,0 +1,68 @@ +#pragma once + +#include "Discretizers/RegressionDiscretizer.h" +#include "Discretizers/pair/PairClassificationDiscretizer.h" +#include "Domain/LearningCriterion.h" + +#include +#include +#include +#include + +/** Ordinary axis-aligned CART over exactly two logical features. */ +class PairRegressionDiscretizer final : public RegressionDiscretizer { +public: + PairRegressionDiscretizer(LearningCriterion criterion, FeatureInfo first, + FeatureInfo second); + + void Train(const arma::fmat &X, arma::uvec &features, + const arma::Mat &y, size_t minLeafSize, + double minGainSplit, size_t maxDepth, size_t maxLeafNodes, + const arma::Row &sampleWeights = arma::Row()) override; + + void transform(const arma::fmat &X, arma::Row &binLoc) const override; + size_t routeToBin(const std::vector &featureValues) const override; + + const std::vector &routingTree() const { return tree_; } + const std::array &axes() const { return axes_; } + +private: + struct BuildNode { + PairRoutingTreeNode routing; + std::vector samples; + std::vector> stats; + double weight = 0.0; + double impurity = 0.0; + size_t depth = 0; + }; + + struct Split { + bool found = false; + size_t featurePosition = 0; + size_t rawFeature = 0; + FeatureType featureType = FeatureType::Continuous; + double threshold = 0.0; + double gain = 0.0; + std::vector left; + std::vector right; + std::vector missing; + }; + + double impurity(const std::vector &samples, + const arma::Mat &y, + const arma::Row &weights) const; + Split bestSplit(const BuildNode &node, const arma::fmat &X, + const arma::Mat &y, + const arma::Row &weights, size_t minLeafSize, + double totalWeight) const; + size_t routeValues(const std::vector &values) const; + bool axisMissing(size_t axis, const arma::fmat &X, size_t sample) const; + bool axisMissing(size_t axis, const std::vector &values) const; + size_t routingPosition(size_t rawFeature) const; + + LearningCriterion criterion_; + std::array axes_; + arma::uvec routingFeatures_; + std::array axisOffsets_{}; + std::vector tree_; +}; diff --git a/cpp/src/Estimators/ClassificationShapeGeneralizedTree.cpp b/cpp/src/Estimators/ClassificationShapeGeneralizedTree.cpp index 2c24472..1f10e89 100644 --- a/cpp/src/Estimators/ClassificationShapeGeneralizedTree.cpp +++ b/cpp/src/Estimators/ClassificationShapeGeneralizedTree.cpp @@ -14,6 +14,7 @@ #include "Criterion.h" #include "Discretizers/ClassificationDiscretizer.h" +#include "Discretizers/pair/PairClassificationDiscretizer.h" #include "Discretizers/factories/DiscretizerFactories.h" #include "Estimators/ShapeFunctions/ShapeFunctionSplitSearch.h" @@ -47,13 +48,15 @@ ClassificationShapeGeneralizedTree::ClassificationShapeGeneralizedTree( LearningCriterion criterion, std::vector numClasses, size_t numPartitions, TreeBuildingParams outerParams, TreeBuildingParams innerParams, CoordinateDescentParams cdParams, - uint64_t random_state, FeatureBaggingPickFn featureBagging) + uint64_t random_state, FeatureBaggingPickFn featureBagging, + size_t pairwiseCandidates, double pairwisePenalty) : ShapeGeneralizedTree(criterion, numPartitions, outerParams, innerParams), numClasses_(std::move(numClasses)), cdParams_(cdParams), random_state_(random_state), rng_(), featureBagging_(featureBagging ? std::move(featureBagging) : FeatureBaggingPickFn(pickAllFeatureIndices)), + pairwiseCandidates_(pairwiseCandidates), pairwisePenalty_(pairwisePenalty), outerTreeBuilder_(outerParams_.minLeafSize, outerParams_.minGainSplit, outerParams_.maxDepth, outerParams_.maxLeafNodes) { if (criterion != LearningCriterion::Entropy && @@ -72,6 +75,14 @@ ClassificationShapeGeneralizedTree::ClassificationShapeGeneralizedTree( if (numPartitions < 2) throw std::invalid_argument( "ClassificationShapeGeneralizedTree: numPartitions must be >= 2"); + if (!std::isfinite(pairwisePenalty_) || pairwisePenalty_ < 0.0) + throw std::invalid_argument("pairwise_penalty must be finite and non-negative"); +} + +bool ClassificationShapeGeneralizedTree::hasPairNodes() const { + return std::any_of(nodes_.begin(), nodes_.end(), [](const ShapeFunctionNode &node) { + return !node.isLeaf && node.logicalFeatureIndices.size() == 2; + }); } void ClassificationShapeGeneralizedTree::resolveOutputLayout(size_t nOutputs) { @@ -214,6 +225,23 @@ void ClassificationShapeGeneralizedTree::fit( const size_t xSubCols = static_cast(Xsub.n_cols); ShapeBestBranchingState best{}; + const arma::Row wsub = + subSampleWeights(fitSampleWeights_, subIdx); + + struct UnivariateProxy { + size_t logicalIndex; + size_t numPartitions; + double childImpurity; + std::vector partitions; + }; + std::vector univariateProxies; + std::vector retainedPairCandidates; + + const auto addNoSplitProxy = [&univariateProxies, xSubCols, + parentImp](size_t logicalIdx) { + univariateProxies.push_back( + {logicalIdx, 1, parentImp, std::vector(xSubCols, 0)}); + }; const auto applyTaskFields = [](ShapeBestBranchingState &state, @@ -228,24 +256,39 @@ void ClassificationShapeGeneralizedTree::fit( const size_t logicalIdx = featureSubset[fi]; const FeatureInfo &feature = features_[logicalIdx]; - const arma::Row wsub = - subSampleWeights(fitSampleWeights_, subIdx); - auto disc = makeClassificationDiscretizer(criterion_, feature); trainClassificationDiscretizer( *disc, feature, Xsub, ysub, classesPerOutput_, innerParams_.minLeafSize, innerParams_.minGainSplit, innerParams_.maxDepth, innerParams_.maxLeafNodes, wsub); - if (disc->numLeaves() < 2) + if (disc->numLeaves() < 2) { + if (pairwiseCandidates_ > 0) + addNoSplitProxy(logicalIdx); continue; + } const ShapeBranchAssignmentSearchResult featureBest = searchShapeBranchAssignmentFromDiscretizer( *disc, criterion_, parentImp, numPartitions_, outerParams_, cdParams_, outerTreeBuilder_.eps, rng_, /*useKMeansSeed=*/true, classesPerOutput_, nOutputs_); - if (!featureBest.found) + if (!featureBest.found) { + if (pairwiseCandidates_ > 0) + addNoSplitProxy(logicalIdx); continue; + } + + if (pairwiseCandidates_ > 0) { + std::vector partitions(xSubCols, 0); + const auto &perBin = disc->inSampleDiscretizations(); + for (size_t bin = 0; bin < perBin.size(); ++bin) + for (size_t sample : perBin[bin]) + partitions[sample] = featureBest.assignments[bin]; + univariateProxies.push_back( + {logicalIdx, featureBest.chosenK, + parentImp - featureBest.impurityDecrease, + std::move(partitions)}); + } featureHasBetterShapeBranching( featureBest, best, logicalIdx, xSubCols, feature.indices, @@ -254,6 +297,93 @@ void ClassificationShapeGeneralizedTree::fit( outerTreeBuilder_.eps, applyTaskFields); } + if (pairwiseCandidates_ > 0 && univariateProxies.size() >= 2) { + std::sort(univariateProxies.begin(), univariateProxies.end(), + [](const UnivariateProxy &a, const UnivariateProxy &b) { + return a.logicalIndex < b.logicalIndex; + }); + struct PairProxy { + double score; + size_t first; + size_t second; + }; + std::vector retained; + const auto proxyLess = [](const PairProxy &a, const PairProxy &b) { + if (a.score != b.score) + return a.score < b.score; + if (a.first != b.first) + return a.first < b.first; + return a.second < b.second; + }; + const double totalWeight = arma::accu(wsub); + for (size_t i = 0; i + 1 < univariateProxies.size(); ++i) { + for (size_t j = i + 1; j < univariateProxies.size(); ++j) { + const size_t numCells = univariateProxies[i].numPartitions * + univariateProxies[j].numPartitions; + auto crossed = std::vector>>( + numCells, makeEmptyHistogram()); + std::vector crossedWeights(numCells, 0.0); + for (size_t sample = 0; sample < xSubCols; ++sample) { + const size_t cell = + univariateProxies[i].partitions[sample] * + univariateProxies[j].numPartitions + + univariateProxies[j].partitions[sample]; + const double w = wsub(sample); + crossedWeights[cell] += w; + for (size_t o = 0; o < nOutputs_; ++o) + crossed[cell][o][ysub(o, sample)] += w; + } + double crossedImpurity = 0.0; + for (size_t cell = 0; cell < crossed.size(); ++cell) + if (totalWeight > 0.0) + crossedImpurity += crossedWeights[cell] / totalWeight * + impurityForClassCounts(crossed[cell]); + retained.push_back( + {crossedImpurity - + std::min(univariateProxies[i].childImpurity, + univariateProxies[j].childImpurity), + univariateProxies[i].logicalIndex, + univariateProxies[j].logicalIndex}); + std::sort(retained.begin(), retained.end(), proxyLess); + if (retained.size() > pairwiseCandidates_) + retained.resize(pairwiseCandidates_); + } + } + + retainedPairCandidates.reserve(retained.size()); + for (const PairProxy &pair : retained) + retainedPairCandidates.push_back( + {{pair.first, pair.second}, + {features_[pair.first], features_[pair.second]}}); + + for (const PairProxy &pair : retained) { + const FeatureInfo &first = features_[pair.first]; + const FeatureInfo &second = features_[pair.second]; + arma::uvec rawFeatures = arma::join_cols(first.indices, second.indices); + auto pairDisc = std::make_unique( + criterion_, first, second); + pairDisc->Train( + Xsub, rawFeatures, ysub, classesPerOutput_, + innerParams_.minLeafSize, innerParams_.minGainSplit, + innerParams_.maxDepth, innerParams_.maxLeafNodes, wsub); + if (pairDisc->numLeaves() < 2) + continue; + ShapeBranchAssignmentSearchResult pairBest = + searchShapeBranchAssignmentFromDiscretizer( + *pairDisc, criterion_, parentImp, numPartitions_, + outerParams_, cdParams_, outerTreeBuilder_.eps, rng_, + /*useKMeansSeed=*/true, classesPerOutput_, nOutputs_, + nullptr, nullptr, 0, /*hasNanRoutingBin=*/false); + pairBest.bestFeatureScore += pairwisePenalty_; + if (featureHasBetterShapeBranching( + pairBest, best, pair.first, xSubCols, rawFeatures, + std::unique_ptr>>( + std::move(pairDisc)), + outerTreeBuilder_.eps, applyTaskFields)) + best.logicalFeatureIndices = {pair.first, pair.second}; + } + } + if (!std::isfinite(best.penalizedChildScore) || best.penalizedChildScore >= std::numeric_limits::infinity() || best.branching.impurityDecrease <= outerTreeBuilder_.eps) { @@ -263,6 +393,8 @@ void ClassificationShapeGeneralizedTree::fit( node.isLeaf = false; node.splitFeatureIndex = best.branching.featureIndex; + node.logicalFeatureIndices = best.logicalFeatureIndices; + node.retainedPairCandidates = std::move(retainedPairCandidates); node.routingFeatures.assign(best.routingColumnIndices.begin(), best.routingColumnIndices.end()); node.innerDiscretizer = best.winningDiscretizer; @@ -313,8 +445,16 @@ void ClassificationShapeGeneralizedTree::fit( nodes_[0], findBestSplit, makeChildren, [this](ShapeFunctionNode &parent, std::vector &children) { - sumOfNodeImportancesByFeature_(parent.splitFeatureIndex) += - parent.informationGain; + const auto &logical = parent.logicalFeatureIndices; + if (logical.size() == 2) { + sumOfNodeImportancesByFeature_(logical[0]) += + parent.informationGain / 2.0; + sumOfNodeImportancesByFeature_(logical[1]) += + parent.informationGain / 2.0; + } else { + sumOfNodeImportancesByFeature_(parent.splitFeatureIndex) += + parent.informationGain; + } totalNodeImportanceSum_ += parent.informationGain; const size_t pid = parent.nodeIndex; nodes_[pid] = std::move(parent); diff --git a/cpp/src/Estimators/ClassificationShapeGeneralizedTree.h b/cpp/src/Estimators/ClassificationShapeGeneralizedTree.h index dfe7950..57e8648 100644 --- a/cpp/src/Estimators/ClassificationShapeGeneralizedTree.h +++ b/cpp/src/Estimators/ClassificationShapeGeneralizedTree.h @@ -86,7 +86,8 @@ class ClassificationShapeGeneralizedTree : public ShapeGeneralizedTree { size_t numPartitions, TreeBuildingParams outerParams = {}, TreeBuildingParams innerParams = {}, CoordinateDescentParams cdParams = {}, uint64_t random_state = 42, - FeatureBaggingPickFn featureBagging = {}); + FeatureBaggingPickFn featureBagging = {}, size_t pairwiseCandidates = 0, + double pairwisePenalty = 0.0); /** Convenience overload: single-output / shared class count ``{numClasses}``. */ ClassificationShapeGeneralizedTree( @@ -94,11 +95,12 @@ class ClassificationShapeGeneralizedTree : public ShapeGeneralizedTree { TreeBuildingParams outerParams = {}, TreeBuildingParams innerParams = {}, CoordinateDescentParams cdParams = {}, uint64_t random_state = 42, - FeatureBaggingPickFn featureBagging = {}) + FeatureBaggingPickFn featureBagging = {}, size_t pairwiseCandidates = 0, + double pairwisePenalty = 0.0) : ClassificationShapeGeneralizedTree( criterion, std::vector{numClasses}, numPartitions, outerParams, innerParams, cdParams, random_state, - std::move(featureBagging)) {} + std::move(featureBagging), pairwiseCandidates, pairwisePenalty) {} ~ClassificationShapeGeneralizedTree() = default; @@ -151,6 +153,8 @@ class ClassificationShapeGeneralizedTree : public ShapeGeneralizedTree { /** Number of outputs the tree was fitted on (>= 1). */ size_t nOutputs() const { return nOutputs_; } + bool hasPairNodes() const; + /** Fit-resolved per-output class counts (empty before ``fit``). */ const std::vector &classesPerOutput() const { return classesPerOutput_; @@ -167,6 +171,8 @@ class ClassificationShapeGeneralizedTree : public ShapeGeneralizedTree { uint64_t random_state_; std::mt19937_64 rng_; FeatureBaggingPickFn featureBagging_; + size_t pairwiseCandidates_ = 0; + double pairwisePenalty_ = 0.0; std::vector features_; /** Outer routing expansion; `fit` passes split logic via buildTree callbacks. */ diff --git a/cpp/src/Estimators/RegressionShapeGeneralizedTree.cpp b/cpp/src/Estimators/RegressionShapeGeneralizedTree.cpp index 8d6e660..a2aad0c 100644 --- a/cpp/src/Estimators/RegressionShapeGeneralizedTree.cpp +++ b/cpp/src/Estimators/RegressionShapeGeneralizedTree.cpp @@ -13,6 +13,7 @@ #include "Estimators/RegressionShapeGeneralizedTree.h" #include "Criterion.h" +#include "Discretizers/pair/PairRegressionDiscretizer.h" #include "Discretizers/factories/DiscretizerFactories.h" #include "Discretizers/RegressionDiscretizer.h" #include "Estimators/ShapeFunctions/ShapeFunctionSplitSearch.h" @@ -65,19 +66,77 @@ PartitionMoments aggregatePartitionFromBins( return out; } +double crossedRegressionImpurity( + const std::vector &firstPartitions, size_t firstK, + const std::vector &secondPartitions, size_t secondK, + const arma::Mat &y, const arma::Row &weights, + LearningCriterion criterion, size_t nOutputs) { + const size_t numCells = firstK * secondK; + std::vector cellWeights(numCells, 0.0); + double totalWeight = 0.0; + for (size_t sample = 0; sample < firstPartitions.size(); ++sample) { + const size_t cell = firstPartitions[sample] * secondK + secondPartitions[sample]; + const double w = weights(sample); + cellWeights[cell] += w; + totalWeight += w; + } + if (totalWeight <= 0.0) + return 0.0; + + if (criterion == LearningCriterion::SquaredError) { + std::vector>> stats( + numCells, std::vector>(nOutputs, + std::vector(2, 0.0))); + for (size_t sample = 0; sample < firstPartitions.size(); ++sample) { + const size_t cell = firstPartitions[sample] * secondK + secondPartitions[sample]; + const double w = weights(sample); + for (size_t o = 0; o < nOutputs; ++o) { + const double value = y(o, sample); + stats[cell][o][0] += w * value; + stats[cell][o][1] += w * value * value; + } + } + double result = 0.0; + for (size_t cell = 0; cell < numCells; ++cell) + result += cellWeights[cell] / totalWeight * + Criterion::squaredError(stats[cell], cellWeights[cell]); + return result; + } + + std::vector>> cellYs( + numCells, std::vector>(nOutputs)); + std::vector> cellWs(numCells); + for (size_t sample = 0; sample < firstPartitions.size(); ++sample) { + const size_t cell = firstPartitions[sample] * secondK + secondPartitions[sample]; + cellWs[cell].push_back(weights(sample)); + for (size_t o = 0; o < nOutputs; ++o) + cellYs[cell][o].push_back(y(o, sample)); + } + double result = 0.0; + for (size_t cell = 0; cell < numCells; ++cell) { + double cellImpurity = 0.0; + for (size_t o = 0; o < nOutputs; ++o) + cellImpurity += Criterion::absoluteError(cellYs[cell][o], cellWs[cell]).mae; + result += cellWeights[cell] / totalWeight * cellImpurity; + } + return result; +} + } // namespace RegressionShapeGeneralizedTree::RegressionShapeGeneralizedTree( LearningCriterion criterion, size_t numPartitions, TreeBuildingParams outerParams, TreeBuildingParams innerParams, CoordinateDescentParams cdParams, uint64_t random_state, - FeatureBaggingPickFn featureBagging) + FeatureBaggingPickFn featureBagging, size_t pairwiseCandidates, + double pairwisePenalty) : ShapeGeneralizedTree(criterion, numPartitions, outerParams, innerParams), cdParams_(cdParams), random_state_(random_state), rng_(), featureBagging_(featureBagging ? std::move(featureBagging) : FeatureBaggingPickFn(pickAllFeatureIndices)), + pairwiseCandidates_(pairwiseCandidates), pairwisePenalty_(pairwisePenalty), outerTreeBuilder_(outerParams_.minLeafSize, outerParams_.minGainSplit, outerParams_.maxDepth, outerParams_.maxLeafNodes) { if (criterion != LearningCriterion::SquaredError && @@ -88,6 +147,14 @@ RegressionShapeGeneralizedTree::RegressionShapeGeneralizedTree( if (numPartitions < 2) throw std::invalid_argument( "RegressionShapeGeneralizedTree: numPartitions must be >= 2"); + if (!std::isfinite(pairwisePenalty_) || pairwisePenalty_ < 0.0) + throw std::invalid_argument("pairwise_penalty must be finite and non-negative"); +} + +bool RegressionShapeGeneralizedTree::hasPairNodes() const { + return std::any_of(nodes_.begin(), nodes_.end(), [](const ShapeFunctionNode &node) { + return !node.isLeaf && node.logicalFeatureIndices.size() == 2; + }); } @@ -279,6 +346,22 @@ void RegressionShapeGeneralizedTree::fit( const size_t xSubCols = static_cast(Xsub.n_cols); ShapeBestBranchingState best{}; + const arma::Row wsub = + subSampleWeights(fitSampleWeights_, subIdx); + struct UnivariateProxy { + size_t logicalIndex; + size_t numPartitions; + double childImpurity; + std::vector partitions; + }; + std::vector univariateProxies; + std::vector retainedPairCandidates; + + const auto addNoSplitProxy = [&univariateProxies, xSubCols, + parentImp](size_t logicalIdx) { + univariateProxies.push_back( + {logicalIdx, 1, parentImp, std::vector(xSubCols, 0)}); + }; const auto applyTaskFields = [this](ShapeBestBranchingState &state, @@ -295,16 +378,16 @@ void RegressionShapeGeneralizedTree::fit( const size_t logicalIdx = featureSubset[fi]; const FeatureInfo &feature = features_[logicalIdx]; - const arma::Row wsub = - subSampleWeights(fitSampleWeights_, subIdx); - auto disc = makeRegressionDiscretizer(criterion_, feature); trainRegressionDiscretizer( *disc, feature, Xsub, ysub, innerParams_.minLeafSize, innerParams_.minGainSplit, innerParams_.maxDepth, innerParams_.maxLeafNodes, wsub); - if (disc->numLeaves() < 2) + if (disc->numLeaves() < 2) { + if (pairwiseCandidates_ > 0) + addNoSplitProxy(logicalIdx); continue; + } const ShapeBranchAssignmentSearchResult featureBest = searchShapeBranchAssignmentFromDiscretizer( @@ -316,8 +399,23 @@ void RegressionShapeGeneralizedTree::fit( criterion_ == LearningCriterion::AbsoluteError ? &wsub : nullptr, xSubCols); - if (!featureBest.found) + if (!featureBest.found) { + if (pairwiseCandidates_ > 0) + addNoSplitProxy(logicalIdx); continue; + } + + if (pairwiseCandidates_ > 0) { + std::vector partitions(xSubCols, 0); + const auto &perBin = disc->inSampleDiscretizations(); + for (size_t bin = 0; bin < perBin.size(); ++bin) + for (size_t sample : perBin[bin]) + partitions[sample] = featureBest.assignments[bin]; + univariateProxies.push_back( + {logicalIdx, featureBest.chosenK, + parentImp - featureBest.impurityDecrease, + std::move(partitions)}); + } featureHasBetterShapeBranching( featureBest, best, logicalIdx, xSubCols, feature.indices, @@ -326,6 +424,77 @@ void RegressionShapeGeneralizedTree::fit( outerTreeBuilder_.eps, applyTaskFields); } + if (pairwiseCandidates_ > 0 && univariateProxies.size() >= 2) { + std::sort(univariateProxies.begin(), univariateProxies.end(), + [](const UnivariateProxy &a, const UnivariateProxy &b) { + return a.logicalIndex < b.logicalIndex; + }); + struct PairProxy { + double score; + size_t first; + size_t second; + }; + const auto proxyLess = [](const PairProxy &a, const PairProxy &b) { + if (a.score != b.score) + return a.score < b.score; + if (a.first != b.first) + return a.first < b.first; + return a.second < b.second; + }; + std::vector retained; + for (size_t i = 0; i + 1 < univariateProxies.size(); ++i) { + for (size_t j = i + 1; j < univariateProxies.size(); ++j) { + const double crossed = crossedRegressionImpurity( + univariateProxies[i].partitions, univariateProxies[i].numPartitions, + univariateProxies[j].partitions, univariateProxies[j].numPartitions, + ysub, wsub, criterion_, nOutputs_); + retained.push_back( + {crossed - std::min(univariateProxies[i].childImpurity, + univariateProxies[j].childImpurity), + univariateProxies[i].logicalIndex, + univariateProxies[j].logicalIndex}); + std::sort(retained.begin(), retained.end(), proxyLess); + if (retained.size() > pairwiseCandidates_) + retained.resize(pairwiseCandidates_); + } + } + + retainedPairCandidates.reserve(retained.size()); + for (const PairProxy &pair : retained) + retainedPairCandidates.push_back( + {{pair.first, pair.second}, + {features_[pair.first], features_[pair.second]}}); + + for (const PairProxy &pair : retained) { + const FeatureInfo &first = features_[pair.first]; + const FeatureInfo &second = features_[pair.second]; + arma::uvec rawFeatures = arma::join_cols(first.indices, second.indices); + auto pairDisc = std::make_unique( + criterion_, first, second); + pairDisc->Train( + Xsub, rawFeatures, ysub, innerParams_.minLeafSize, + innerParams_.minGainSplit, innerParams_.maxDepth, + innerParams_.maxLeafNodes, wsub); + if (pairDisc->numLeaves() < 2) + continue; + ShapeBranchAssignmentSearchResult pairBest = + searchShapeBranchAssignmentFromDiscretizer( + *pairDisc, criterion_, parentImp, numPartitions_, outerParams_, + cdParams_, outerTreeBuilder_.eps, rng_, + /*useKMeansSeed=*/false, /*classesPerOutput=*/{}, nOutputs_, + criterion_ == LearningCriterion::AbsoluteError ? &ysub : nullptr, + criterion_ == LearningCriterion::AbsoluteError ? &wsub : nullptr, + xSubCols, /*hasNanRoutingBin=*/false); + pairBest.bestFeatureScore += pairwisePenalty_; + if (featureHasBetterShapeBranching( + pairBest, best, pair.first, xSubCols, rawFeatures, + std::unique_ptr>>( + std::move(pairDisc)), + outerTreeBuilder_.eps, applyTaskFields)) + best.logicalFeatureIndices = {pair.first, pair.second}; + } + } + if (!std::isfinite(best.penalizedChildScore) || best.penalizedChildScore >= std::numeric_limits::infinity() || best.branching.impurityDecrease <= outerTreeBuilder_.eps) { @@ -335,6 +504,8 @@ void RegressionShapeGeneralizedTree::fit( node.isLeaf = false; node.splitFeatureIndex = best.branching.featureIndex; + node.logicalFeatureIndices = best.logicalFeatureIndices; + node.retainedPairCandidates = std::move(retainedPairCandidates); node.routingFeatures.assign(best.routingColumnIndices.begin(), best.routingColumnIndices.end()); node.innerDiscretizer = best.winningDiscretizer; @@ -423,8 +594,15 @@ void RegressionShapeGeneralizedTree::fit( nodes_[0], findBestSplit, makeChildren, [this](ShapeFunctionNode &parent, std::vector &children) { - sumOfNodeImportancesByFeature_(parent.splitFeatureIndex) += - parent.informationGain; + if (parent.logicalFeatureIndices.size() == 2) { + sumOfNodeImportancesByFeature_(parent.logicalFeatureIndices[0]) += + parent.informationGain / 2.0; + sumOfNodeImportancesByFeature_(parent.logicalFeatureIndices[1]) += + parent.informationGain / 2.0; + } else { + sumOfNodeImportancesByFeature_(parent.splitFeatureIndex) += + parent.informationGain; + } totalNodeImportanceSum_ += parent.informationGain; const size_t pid = parent.nodeIndex; nodes_[pid] = parent; @@ -508,5 +686,3 @@ RegressionShapeGeneralizedTree::predict(const arma::fmat &X) const { } return yhat; } - - diff --git a/cpp/src/Estimators/RegressionShapeGeneralizedTree.h b/cpp/src/Estimators/RegressionShapeGeneralizedTree.h index 28c8369..3ecf886 100644 --- a/cpp/src/Estimators/RegressionShapeGeneralizedTree.h +++ b/cpp/src/Estimators/RegressionShapeGeneralizedTree.h @@ -80,7 +80,8 @@ class RegressionShapeGeneralizedTree : public ShapeGeneralizedTree { TreeBuildingParams outerParams = {}, TreeBuildingParams innerParams = {}, CoordinateDescentParams cdParams = {}, uint64_t random_state = 42, - FeatureBaggingPickFn featureBagging = {}); + FeatureBaggingPickFn featureBagging = {}, size_t pairwiseCandidates = 0, + double pairwisePenalty = 0.0); ~RegressionShapeGeneralizedTree() = default; @@ -114,6 +115,8 @@ class RegressionShapeGeneralizedTree : public ShapeGeneralizedTree { /** Number of outputs the tree was fitted on (>= 1). */ size_t nOutputs() const { return nOutputs_; } + bool hasPairNodes() const; + /** * Per outer-tree node index: for squared error, concatenated * ``[Σw·y0, Σw·y0², Σw·y1, Σw·y1², ...]`` (length ``2 * nOutputs``) at leaves @@ -142,6 +145,8 @@ class RegressionShapeGeneralizedTree : public ShapeGeneralizedTree { uint64_t random_state_; std::mt19937_64 rng_; FeatureBaggingPickFn featureBagging_; + size_t pairwiseCandidates_ = 0; + double pairwisePenalty_ = 0.0; std::vector features_; /** Outer routing expansion; `fit` passes split logic via buildTree callbacks. */ diff --git a/cpp/src/Estimators/ShapeFunctions/ShapeFunctionNode.h b/cpp/src/Estimators/ShapeFunctions/ShapeFunctionNode.h index bfef499..e7ec081 100644 --- a/cpp/src/Estimators/ShapeFunctions/ShapeFunctionNode.h +++ b/cpp/src/Estimators/ShapeFunctions/ShapeFunctionNode.h @@ -8,8 +8,10 @@ */ #include "Discretizers/InnerDiscretizerBase.h" +#include "Domain/FeatureInfo.h" #include +#include #include #include #include @@ -18,6 +20,11 @@ class ShapeGeneralizedTree; +struct RetainedPairCandidate { + std::array logicalFeatureIndices{}; + std::array features; +}; + /** * Outer-tree node: routing rule when internal, plus training sample indices * during fit. @@ -48,6 +55,12 @@ class ShapeFunctionNode { */ size_t splitFeatureIndex = 0; + /** Logical feature indices used by this router (one normally, two for a pair). */ + std::vector logicalFeatureIndices; + + /** Top-P pairs from initial screening; TAO may refit only these pairs. */ + std::vector retainedPairCandidates; + /** Row indices into X used for routing; undefined if isLeaf. */ std::vector routingFeatures; /** Maps each inner discretizer bin (including NaN) to a child partition. */ diff --git a/cpp/src/Estimators/ShapeFunctions/ShapeFunctionSplitSearch.cpp b/cpp/src/Estimators/ShapeFunctions/ShapeFunctionSplitSearch.cpp index d17dc37..3b25837 100644 --- a/cpp/src/Estimators/ShapeFunctions/ShapeFunctionSplitSearch.cpp +++ b/cpp/src/Estimators/ShapeFunctions/ShapeFunctionSplitSearch.cpp @@ -28,14 +28,16 @@ void refineShapeBranchAssignmentNested( std::vector>> &stats, std::vector &leafWeights, const std::vector &leafSampleCounts, - const std::vector &classesPerOutput, size_t nOutputs) { + const std::vector &classesPerOutput, size_t nOutputs, + bool hasNanRoutingBin) { if (k >= numRoutingBins || criterion == LearningCriterion::AbsoluteError) return; const std::vector snapshot = branchObj->assignments; const double objBeforeCd = branchObj->objective(); - coordinateDescent(k, *branchObj, rng, cdParams.maxIters, cdParams.patience); + coordinateDescent(k, *branchObj, rng, cdParams.maxIters, cdParams.patience, + hasNanRoutingBin); const double objAfterCd = branchObj->objective(); if (std::isfinite(objAfterCd) && objAfterCd <= objBeforeCd + kShapeFunctionCdImprovementEps) @@ -53,13 +55,14 @@ void refineShapeBranchAssignmentAbsoluteError( std::vector>> &maeLeafYs, std::vector> &maeLeafWs, std::vector &leafWeights, - const std::vector &leafSampleCounts) { + const std::vector &leafSampleCounts, bool hasNanRoutingBin) { if (k >= numRoutingBins || !mae_branch_config::coordinateDescentEnabled()) return; const std::vector snapshot = branchObj->assignments; const double objBeforeCd = branchObj->objective(); - coordinateDescent(k, *branchObj, rng, cdParams.maxIters, cdParams.patience); + coordinateDescent(k, *branchObj, rng, cdParams.maxIters, cdParams.patience, + hasNanRoutingBin); const double objAfterCd = branchObj->objective(); if (std::isfinite(objAfterCd) && objAfterCd <= objBeforeCd + kShapeFunctionCdImprovementEps) @@ -117,7 +120,7 @@ ShapeBranchAssignmentSearchResult searchShapeBranchAssignmentFromDiscretizer( std::mt19937_64 &rng, bool useKMeansSeed, const std::vector &classesPerOutput, size_t nOutputs, const arma::Mat *ysub, const arma::Row *wsub, - size_t xSubCols) { + size_t xSubCols, bool hasNanRoutingBin) { auto &stats = disc.leafStats(); auto &sizes = disc.leafNumSamples(); auto &leafWeights = disc.leafNodeWeights(); @@ -176,14 +179,14 @@ ShapeBranchAssignmentSearchResult searchShapeBranchAssignmentFromDiscretizer( maeLeafYs, maeLeafWs); refineShapeBranchAssignmentAbsoluteError( branchObj, k, numRoutingBins, cdParams, rng, maeLeafYsStorage, - maeLeafWsStorage, leafWeights, sizes); + maeLeafWsStorage, leafWeights, sizes, hasNanRoutingBin); } else { branchObj = makeBranchAssignment(criterion, trialAssignments, k, stats, leafWeights, sizes, classesPerOutput, nOutputs); refineShapeBranchAssignmentNested( branchObj, k, numRoutingBins, criterion, cdParams, rng, stats, - leafWeights, sizes, classesPerOutput, nOutputs); + leafWeights, sizes, classesPerOutput, nOutputs, hasNanRoutingBin); } if (!branchObj->partitionCountsMeetMinLeaf(outerParams.minLeafSize)) @@ -199,7 +202,10 @@ ShapeBranchAssignmentSearchResult searchShapeBranchAssignmentFromDiscretizer( if (score < result.bestFeatureScore - scoreEpsilon) { result.bestFeatureScore = score; result.chosenK = k; - result.assignments = trialAssignments; + // Coordinate descent updates the live branch object, not the seed vector. + // Persist that refined routing so the objective and eventual child buckets + // describe the same split. + result.assignments = branchObj->assignments; result.partitionSampleCounts = branchObj->partitionSampleCounts(); if (const auto *leafAgg = dynamic_castleafNodeWeights().end()); applyTaskFields(best, search, disc->leafStats()); best.routingColumnIndices = routingColumnIndices; + best.logicalFeatureIndices = {featureIndex}; best.winningDiscretizer = std::shared_ptr(std::move(disc)); return true; diff --git a/cpp/src/Estimators/ShapeFunctions/ShapeFunctionSplitSearch.h b/cpp/src/Estimators/ShapeFunctions/ShapeFunctionSplitSearch.h index 5838371..25d5a16 100644 --- a/cpp/src/Estimators/ShapeFunctions/ShapeFunctionSplitSearch.h +++ b/cpp/src/Estimators/ShapeFunctions/ShapeFunctionSplitSearch.h @@ -46,6 +46,7 @@ struct ShapeBestBranchingState { std::shared_ptr winningDiscretizer; /** Column indices into ``X`` used for routing at inference. */ arma::uvec routingColumnIndices; + std::vector logicalFeatureIndices; }; void markShapeFunctionNodeAsLeaf(ShapeFunctionNode &node); @@ -87,7 +88,7 @@ ShapeBranchAssignmentSearchResult searchShapeBranchAssignmentFromDiscretizer( std::mt19937_64 &rng, bool useKMeansSeed = false, const std::vector &classesPerOutput = {}, size_t nOutputs = 1, const arma::Mat *ysub = nullptr, const arma::Row *wsub = nullptr, - size_t xSubCols = 0); + size_t xSubCols = 0, bool hasNanRoutingBin = true); std::vector> routeSamplesToPartitions(const ShapeFunctionNode &parent, const arma::fmat &X); diff --git a/cpp/src/algorithms/CoordinateDescent.h b/cpp/src/algorithms/CoordinateDescent.h index 6c03a49..7eac238 100644 --- a/cpp/src/algorithms/CoordinateDescent.h +++ b/cpp/src/algorithms/CoordinateDescent.h @@ -30,22 +30,22 @@ inline double coordinateDescent(size_t numPartitions, BranchAssignment &assignmentObjective, std::mt19937_64 &rng, size_t maxIters = 10, - size_t patience = 5) { + size_t patience = 5, + bool hasNanRoutingBin = true) { size_t numBins = assignmentObjective.assignments.size(); if (numBins <= 1) return assignmentObjective.objective(); - const size_t nanBinIndex = numBins - 1; + const size_t finiteBinCount = hasNanRoutingBin ? numBins - 1 : numBins; - // 1. Omit the NaN bin from the objective state during standard coordinate descent - assignmentObjective.removeLeaf(nanBinIndex); + if (hasNanRoutingBin) + assignmentObjective.removeLeaf(numBins - 1); size_t consecutiveTrialsWithoutImprovement = 0; for (size_t i = 0; i < maxIters; ++i) { bool improved = false; - // 2. Shuffle and optimize ONLY the finite numeric bins (0 to numBins - 2) - std::vector permutation(nanBinIndex); + std::vector permutation(finiteBinCount); std::iota(permutation.begin(), permutation.end(), size_t{0}); std::shuffle(permutation.begin(), permutation.end(), rng); @@ -84,7 +84,11 @@ inline double coordinateDescent(size_t numPartitions, } - // 3. Factor the NaN bin back in by greedily finding its optimal partition + if (!hasNanRoutingBin) + return assignmentObjective.objective(); + + // Factor the NaN bin back in by greedily finding its optimal partition. + const size_t nanBinIndex = numBins - 1; size_t bestNanPartition = missing_values::partition_with_max_count_min_index_tie(assignmentObjective.partitionSampleCounts()); // Fallback assignmentObjective.addLeaf( nanBinIndex, @@ -110,4 +114,4 @@ inline double coordinateDescent(size_t numPartitions, assignmentObjective.addLeaf(nanBinIndex, bestNanPartition); return assignmentObjective.objective(); -} \ No newline at end of file +} diff --git a/cpp/src/algorithms/TAO/ClassificationTaoAdapter.cpp b/cpp/src/algorithms/TAO/ClassificationTaoAdapter.cpp index 92ca451..2384df6 100644 --- a/cpp/src/algorithms/TAO/ClassificationTaoAdapter.cpp +++ b/cpp/src/algorithms/TAO/ClassificationTaoAdapter.cpp @@ -178,4 +178,33 @@ void ClassificationTaoAdapter::recomputeLeafStats( classesPerOutput_, nOutputs_); } +void ClassificationTaoAdapter::refreshNodeBinMetadata( + ShapeFunctionNode &node, const std::vector &samples) { + if (!node.innerDiscretizer) + throw std::runtime_error( + "ClassificationTaoAdapter: accepted router has no discretizer"); + arma::Row bins; + node.innerDiscretizer->transform(X_, bins); + const size_t nBins = node.binToPartition.size(); + node.binSampleCounts.assign(nBins, 0); + node.splitBinWeights.assign(nBins, 0.0); + node.splitClassCounts.assign(nBins, std::vector>(nOutputs_)); + for (size_t b = 0; b < nBins; ++b) + for (size_t o = 0; o < nOutputs_; ++o) + node.splitClassCounts[b][o].assign(classesPerOutput_[o], 0.0); + for (arma::uword col : samples) { + const size_t bin = bins(col); + if (bin >= nBins) + throw std::runtime_error( + "ClassificationTaoAdapter: accepted router bin out of range"); + ++node.binSampleCounts[bin]; + const double weight = static_cast(w_(col)); + node.splitBinWeights[bin] += weight; + for (size_t o = 0; o < nOutputs_; ++o) + node.splitClassCounts[bin][o][y_(static_cast(o), col)] += + weight; + } + node.splitLeafStats.clear(); +} + } // namespace tao diff --git a/cpp/src/algorithms/TAO/ClassificationTaoAdapter.h b/cpp/src/algorithms/TAO/ClassificationTaoAdapter.h index b2fe80a..9443fb5 100644 --- a/cpp/src/algorithms/TAO/ClassificationTaoAdapter.h +++ b/cpp/src/algorithms/TAO/ClassificationTaoAdapter.h @@ -51,6 +51,10 @@ class ClassificationTaoAdapter final : public ShapeGeneralizedTaoAdapter { void recomputeLeafStats( const std::vector> &nodeSamples) override; + void refreshNodeBinMetadata( + ShapeFunctionNode &node, + const std::vector &samples) override; + private: /** * Per-child correctness rewards for one sample. diff --git a/cpp/src/algorithms/TAO/RegressionTaoAdapter.cpp b/cpp/src/algorithms/TAO/RegressionTaoAdapter.cpp index 2ce16b1..5e1e9be 100644 --- a/cpp/src/algorithms/TAO/RegressionTaoAdapter.cpp +++ b/cpp/src/algorithms/TAO/RegressionTaoAdapter.cpp @@ -156,4 +156,39 @@ void RegressionTaoAdapter::recomputeLeafStats( } } +void RegressionTaoAdapter::refreshNodeBinMetadata( + ShapeFunctionNode &node, const std::vector &samples) { + if (!node.innerDiscretizer) + throw std::runtime_error( + "RegressionTaoAdapter: accepted router has no discretizer"); + arma::Row bins; + node.innerDiscretizer->transform(X_, bins); + const size_t nBins = node.binToPartition.size(); + node.binSampleCounts.assign(nBins, 0); + node.splitBinWeights.assign(nBins, 0.0); + node.splitClassCounts.clear(); + if (squared_) + node.splitLeafStats.assign( + nBins, std::vector>( + y_.n_rows, std::vector(2, 0.0))); + else + node.splitLeafStats.clear(); + for (arma::uword col : samples) { + const size_t bin = bins(col); + if (bin >= nBins) + throw std::runtime_error( + "RegressionTaoAdapter: accepted router bin out of range"); + ++node.binSampleCounts[bin]; + const double weight = static_cast(w_(col)); + node.splitBinWeights[bin] += weight; + if (!squared_) + continue; + for (arma::uword o = 0; o < y_.n_rows; ++o) { + const double value = static_cast(y_(o, col)); + node.splitLeafStats[bin][o][0] += weight * value; + node.splitLeafStats[bin][o][1] += weight * value * value; + } + } +} + } // namespace tao diff --git a/cpp/src/algorithms/TAO/RegressionTaoAdapter.h b/cpp/src/algorithms/TAO/RegressionTaoAdapter.h index 666c8c5..8e3b1cd 100644 --- a/cpp/src/algorithms/TAO/RegressionTaoAdapter.h +++ b/cpp/src/algorithms/TAO/RegressionTaoAdapter.h @@ -52,6 +52,10 @@ class RegressionTaoAdapter final : public ShapeGeneralizedTaoAdapter { void recomputeLeafStats( const std::vector> &nodeSamples) override; + void refreshNodeBinMetadata( + ShapeFunctionNode &node, + const std::vector &samples) override; + private: /** * Per-child negative loss rewards for one sample under the current leaf diff --git a/cpp/src/algorithms/TAO/TaoAdapter.h b/cpp/src/algorithms/TAO/TaoAdapter.h index 92f37ec..2c2bccd 100644 --- a/cpp/src/algorithms/TAO/TaoAdapter.h +++ b/cpp/src/algorithms/TAO/TaoAdapter.h @@ -123,6 +123,12 @@ class TaoAdapter { */ virtual void recomputeLeafStats( const std::vector> &nodeSamples) = 0; + + /** Refresh exported per-bin metadata for a router accepted by TAO. */ + virtual void refreshNodeBinMetadata( + ShapeFunctionNode &node, + const std::vector &samples) = 0; + }; } // namespace tao diff --git a/cpp/src/algorithms/TAO/TaoObjective.cpp b/cpp/src/algorithms/TAO/TaoObjective.cpp index 9b3d2fa..550b48c 100644 --- a/cpp/src/algorithms/TAO/TaoObjective.cpp +++ b/cpp/src/algorithms/TAO/TaoObjective.cpp @@ -5,9 +5,6 @@ #include #include "algorithms/TAO/TaoObjective.h" -#include "Discretizers/univariate/UnivariateDiscretizer.h" -#include "algorithms/missing_values.h" - #include #include #include @@ -17,9 +14,9 @@ namespace tao { TaoObjective::TaoObjective(const NodeCareSet &care, const arma::fmat &X, - double lambda, double totalSampleWeight) + double lambda, double nodeSampleCount) : care_(care), X_(X), lambda_(lambda), - totalSampleWeight_(totalSampleWeight), nCare_(care.size()), + nodeSampleCount_(nodeSampleCount), nCare_(care.size()), totalCareWeight_(0.0) { if (care_.careWeights.empty()) { totalCareWeight_ = static_cast(nCare_); @@ -42,53 +39,44 @@ size_t TaoObjective::argMax(const std::vector &counts) { counts.begin(), std::max_element(counts.begin(), counts.end()))); } -size_t TaoObjective::routeValue(float value, const std::vector &thresholds, - const std::vector &binToPartition, - size_t nanPartition) { - if (binToPartition.empty()) - return nanPartition; - if (!missing_values::is_finite(value)) - return nanPartition; - const auto it = - std::lower_bound(thresholds.begin(), thresholds.end(), value); - size_t bin = static_cast(it - thresholds.begin()); - if (bin >= binToPartition.size()) - bin = binToPartition.size() - 1; - return binToPartition[bin]; -} - double TaoObjective::meanReward(double rewardSum) const { return rewardSum / totalCareWeight_; } -double TaoObjective::penalizedScore(double rewardSum) const { - if (lambda_ == 0.0 || totalCareWeight_ <= 0.0) +double TaoObjective::penalizedScore(double rewardSum, + double complexityScale) const { + if (lambda_ == 0.0 || complexityScale == 0.0 || totalCareWeight_ <= 0.0) return meanReward(rewardSum); return meanReward(rewardSum) - - lambda_ * totalSampleWeight_ / totalCareWeight_; + complexityScale * lambda_ * nodeSampleCount_ / totalCareWeight_; } -double TaoObjective::rewardSumForPartition( - size_t feature, const std::vector &thresholds, +double TaoObjective::rewardSumForDiscretizer( + const ClassificationDiscretizer &disc, const std::vector &binToPartition) const { + arma::Row bins; + disc.transform(X_, bins); double rewardSum = 0.0; for (size_t i = 0; i < nCare_; ++i) { - const float v = X_(feature, care_.careCols[i]); - const size_t child = - routeValue(v, thresholds, binToPartition, care_.dummyChild); + const size_t bin = bins(care_.careCols[i]); + if (bin >= binToPartition.size()) + throw std::runtime_error( + "TaoObjective::rewardSumForDiscretizer: bin out of range"); + const size_t child = binToPartition[bin]; rewardSum += careWeight(i) * care_.careRewards[i][child]; } return rewardSum; } -double TaoObjective::scoreCurrent(const ShapeFunctionNode &node) const { +double TaoObjective::scoreCurrent(const ShapeFunctionNode &node, + double complexityScale) const { double rewardSum = 0.0; for (size_t i = 0; i < nCare_; ++i) { const size_t child = node.routeSampleToPartition( X_, static_cast(care_.careCols[i])); rewardSum += careWeight(i) * care_.careRewards[i][child]; } - return penalizedScore(rewardSum); + return penalizedScore(rewardSum, complexityScale); } double TaoObjective::scoreDummy() const { @@ -98,31 +86,19 @@ double TaoObjective::scoreDummy() const { return meanReward(rewardSum); } -double TaoObjective::scoreRouting(size_t feature, - const std::vector &thresholds, - const std::vector &binToPartition) - const { - return penalizedScore( - rewardSumForPartition(feature, thresholds, binToPartition)); -} - double TaoObjective::scoreDiscretizer( - size_t feature, ClassificationDiscretizer &disc, - std::vector &thresholdsOut, - std::vector &binToPartitionOut) const { + ClassificationDiscretizer &disc, + std::vector &binToPartitionOut, + double complexityScale) const { if (disc.numLeaves() < 1) return -std::numeric_limits::infinity(); - const std::vector &thr = numericInnerThresholds(disc); const std::vector>> &leafStats = disc.leafStats(); if (leafStats.empty()) throw std::runtime_error( "TaoObjective::scoreDiscretizer: discretizer has no leaf stats"); - thresholdsOut.resize(thr.size()); - for (size_t b = 0; b < thr.size(); ++b) - thresholdsOut[b] = static_cast(thr[b]); binToPartitionOut.resize(leafStats.size()); // TAO trains the inner discretizer on scalar care-set pseudo-labels (child @@ -140,7 +116,8 @@ double TaoObjective::scoreDiscretizer( binToPartitionOut[b] = argMax(leafStats[b][0]); } - return scoreRouting(feature, thresholdsOut, binToPartitionOut); + return penalizedScore( + rewardSumForDiscretizer(disc, binToPartitionOut), complexityScale); } } // namespace tao diff --git a/cpp/src/algorithms/TAO/TaoObjective.h b/cpp/src/algorithms/TAO/TaoObjective.h index f57c6bf..1011009 100644 --- a/cpp/src/algorithms/TAO/TaoObjective.h +++ b/cpp/src/algorithms/TAO/TaoObjective.h @@ -7,7 +7,7 @@ * Encapsulates scoring of candidate routing rules at one internal node. Rewards * come from the task-specific ``NodeCareSet``; this class only sums per-sample * rewards under a routing map (current node rule, dummy constant rule, or a - * single-feature rule induced by a trained classification discretizer). + * rule induced by a trained classification discretizer). */ #include "Discretizers/ClassificationDiscretizer.h" @@ -22,15 +22,13 @@ namespace tao { /** * Mean care-set reward under a routing rule, with optional split penalty. * - * Non-dummy candidates subtract ``lambda * totalSampleWeight / careWeight`` from - * the mean reward (equivalently ``lambda * totalSampleWeight`` from the weighted - * reward sum). ``scoreDummy`` is unpenalized. ``totalSampleWeight`` is the sum - * of training sample weights (or the sample count when weights are uniform). + * Candidates subtract ``complexityScale * lambda * nodeSampleCount`` from the + * weighted reward sum. ``scoreDummy`` uses scale zero. */ class TaoObjective { public: TaoObjective(const NodeCareSet &care, const arma::fmat &X, double lambda, - double totalSampleWeight); + double nodeSampleCount); /** Number of care samples at this node. */ size_t nCare() const { return nCare_; } @@ -41,7 +39,8 @@ class TaoObjective { const NodeCareSet &careSet() const { return care_; } /** Mean care reward under the node's current routing rule, minus split penalty. */ - double scoreCurrent(const ShapeFunctionNode &node) const; + double scoreCurrent(const ShapeFunctionNode &node, + double complexityScale) const; /** Mean care reward when every care sample routes to ``dummyChild``. */ double scoreDummy() const; @@ -50,37 +49,29 @@ class TaoObjective { * Extract routing from a trained discretizer and score it on the care set. * * Bins are mapped to child partitions by argmax over discretizer leaf stats. - * Writes the induced thresholds and bin-to-partition map to the out-params. + * Writes the induced bin-to-partition map to the out-param. * * @returns Penalized mean reward, or ``-infinity`` when ``disc`` has no bins. */ - double scoreDiscretizer(size_t feature, ClassificationDiscretizer &disc, - std::vector &thresholdsOut, - std::vector &binToPartitionOut) const; - - /** Mean care reward for an explicit single-feature routing rule, minus penalty. */ - double scoreRouting(size_t feature, const std::vector &thresholds, - const std::vector &binToPartition) const; + double scoreDiscretizer(ClassificationDiscretizer &disc, + std::vector &binToPartitionOut, + double complexityScale) const; private: static size_t argMax(const std::vector &counts); - static size_t routeValue(float value, const std::vector &thresholds, - const std::vector &binToPartition, - size_t nanPartition); - - double rewardSumForPartition(size_t feature, - const std::vector &thresholds, - const std::vector &binToPartition) const; + double rewardSumForDiscretizer( + const ClassificationDiscretizer &disc, + const std::vector &binToPartition) const; double meanReward(double rewardSum) const; - double penalizedScore(double rewardSum) const; + double penalizedScore(double rewardSum, double complexityScale) const; double careWeight(size_t i) const; const NodeCareSet &care_; const arma::fmat &X_; double lambda_; - double totalSampleWeight_; + double nodeSampleCount_; size_t nCare_; double totalCareWeight_; }; diff --git a/cpp/src/algorithms/TAO/TreeAlternatingOptimization.cpp b/cpp/src/algorithms/TAO/TreeAlternatingOptimization.cpp index a19ccdb..d742725 100644 --- a/cpp/src/algorithms/TAO/TreeAlternatingOptimization.cpp +++ b/cpp/src/algorithms/TAO/TreeAlternatingOptimization.cpp @@ -8,6 +8,7 @@ #include "algorithms/TAO/TreeAlternatingOptimization.h" #include "Discretizers/ClassificationDiscretizer.h" +#include "Discretizers/pair/PairClassificationDiscretizer.h" #include "Discretizers/factories/DiscretizerFactories.h" #include "Discretizers/InnerDiscretizerBase.h" #include "algorithms/TAO/TaoObjective.h" @@ -53,24 +54,12 @@ computeNodeSamples(const std::vector &nodes, return nodeSamples; } -double totalSampleWeight(const arma::Row &sampleWeights, - arma::uword numSamples) { - if (sampleWeights.is_empty()) - return static_cast(numSamples); - double sum = 0.0; - for (arma::uword i = 0; i < sampleWeights.n_elem; ++i) - sum += static_cast(sampleWeights(i)); - if (sum <= 0.0) - return static_cast(numSamples); - return sum; -} - } // namespace bool optimizeNodeInPlace( TaoAdapter &adapter, const std::vector> &nodeSamples, size_t nodeIdx, - double lambda) { + double lambda, double taoPairScale) { auto &nodes = adapter.nodes(); auto &childIndices = adapter.childIndices(); const arma::fmat &X = adapter.X(); @@ -94,10 +83,14 @@ bool optimizeNodeInPlace( if (care.empty()) return false; - const double nTotal = - totalSampleWeight(adapter.sampleWeights(), X.n_cols); - TaoObjective objective(care, X, lambda, nTotal); - const double currScore = objective.scoreCurrent(node); + TaoObjective objective(care, X, lambda, + static_cast(samples.size())); + const double currentScale = node.logicalFeatureIndices.empty() + ? 0.0 + : (node.logicalFeatureIndices.size() == 2 + ? taoPairScale + : 1.0); + const double currScore = objective.scoreCurrent(node, currentScale); const double dummyScore = objective.scoreDummy(); double bestSingleScore = -std::numeric_limits::infinity(); @@ -115,10 +108,9 @@ bool optimizeNodeInPlace( innerParams.minGainSplit, innerParams.maxDepth, innerParams.maxLeafNodes, care.wexp); - std::vector thresholds; std::vector binToPartition; - const double score = - objective.scoreDiscretizer(f, *disc, thresholds, binToPartition); + const double score = objective.scoreDiscretizer( + *disc, binToPartition, /*complexityScale=*/1.0); if (score > bestSingleScore) { bestSingleScore = score; bestFeature = f; @@ -129,8 +121,40 @@ bool optimizeNodeInPlace( } } - if (dummyScore >= currScore && dummyScore >= bestSingleScore) { + double bestPairScore = -std::numeric_limits::infinity(); + bool havePair = false; + std::array bestPairLogical{}; + std::vector bestPairRoutingFeatures; + std::vector bestPairBinToPartition; + std::shared_ptr bestPairDiscretizer; + for (const RetainedPairCandidate &pair : node.retainedPairCandidates) { + arma::uvec rawFeatures = arma::join_cols(pair.features[0].indices, + pair.features[1].indices); + auto disc = std::make_unique( + routerCriterion, pair.features[0], pair.features[1]); + disc->Train(care.Xexp, rawFeatures, care.yexp, std::vector{k}, + innerParams.minLeafSize, innerParams.minGainSplit, + innerParams.maxDepth, innerParams.maxLeafNodes, care.wexp); + std::vector binToPartition; + const double score = objective.scoreDiscretizer( + *disc, binToPartition, taoPairScale); + if (!havePair || score > bestPairScore || + (score == bestPairScore && + pair.logicalFeatureIndices < bestPairLogical)) { + bestPairScore = score; + bestPairLogical = pair.logicalFeatureIndices; + bestPairRoutingFeatures.assign(rawFeatures.begin(), rawFeatures.end()); + bestPairBinToPartition = std::move(binToPartition); + bestPairDiscretizer = + std::shared_ptr(std::move(disc)); + havePair = true; + } + } + + if (dummyScore >= currScore && dummyScore >= bestSingleScore && + dummyScore >= bestPairScore) { node.isLeaf = false; + node.logicalFeatureIndices.clear(); node.routingFeatures = {0}; featOne(0) = 0; auto disc = makeClassificationDiscretizer(routerCriterion, @@ -141,21 +165,40 @@ bool optimizeNodeInPlace( std::shared_ptr(std::move(disc)); node.binToPartition = {objective.dummyChild(), objective.dummyChild()}; node.numPartitions = k; + node.informationGain = 0.0; + adapter.refreshNodeBinMetadata(node, samples); return true; } if (haveSingle && bestSingleScore > currScore && - bestSingleScore > dummyScore) { + bestSingleScore > dummyScore && bestSingleScore >= bestPairScore) { node.isLeaf = false; + node.splitFeatureIndex = bestFeature; + node.logicalFeatureIndices = {bestFeature}; node.routingFeatures = {bestFeature}; node.innerDiscretizer = std::move(bestDiscretizer); node.binToPartition = std::move(bestBinToPartition); node.numPartitions = k; + adapter.refreshNodeBinMetadata(node, samples); + return true; + } + if (havePair && bestPairScore > currScore && + bestPairScore > dummyScore && bestPairScore > bestSingleScore) { + node.isLeaf = false; + node.splitFeatureIndex = bestPairLogical[0]; + node.logicalFeatureIndices.assign(bestPairLogical.begin(), + bestPairLogical.end()); + node.routingFeatures = std::move(bestPairRoutingFeatures); + node.innerDiscretizer = std::move(bestPairDiscretizer); + node.binToPartition = std::move(bestPairBinToPartition); + node.numPartitions = k; + adapter.refreshNodeBinMetadata(node, samples); return true; } return false; } -void optimize(TaoAdapter &adapter, size_t nRuns, double lambda) { +void optimize(TaoAdapter &adapter, size_t nRuns, double lambda, + double taoPairScale) { auto &nodes = adapter.nodes(); auto &childIndices = adapter.childIndices(); const size_t rootIndex = adapter.rootIndex(); @@ -182,7 +225,8 @@ void optimize(TaoAdapter &adapter, size_t nRuns, double lambda) { stack.emplace_back(child, false); continue; } - if (optimizeNodeInPlace(adapter, nodeSamples, nodeIdx, lambda)) { + if (optimizeNodeInPlace(adapter, nodeSamples, nodeIdx, lambda, + taoPairScale)) { changed = true; nodeSamples = computeNodeSamples(nodes, childIndices, rootIndex, X); diff --git a/cpp/src/algorithms/TAO/TreeAlternatingOptimization.h b/cpp/src/algorithms/TAO/TreeAlternatingOptimization.h index aecf86d..b5844c8 100644 --- a/cpp/src/algorithms/TAO/TreeAlternatingOptimization.h +++ b/cpp/src/algorithms/TAO/TreeAlternatingOptimization.h @@ -13,11 +13,14 @@ * 1. Build the care set via ``TaoAdapter::buildCareSet``. * 2. Score the current rule and a constant (dummy) rule that sends all care * samples to ``dummyChild``. - * 3. For each candidate feature, train a classification discretizer over child- - * partition pseudolabels and score the induced routing rule. - * 4. Accept the best non-worsening rule (current, dummy, or single-feature split). - * ``lambda`` penalizes non-dummy splits by ``lambda * totalSampleWeight`` in - * weighted reward units (cost-complexity style; see ``TaoObjective``). + * 3. For each candidate feature or feature pair, train a classification + * discretizer over child-partition pseudolabels and score the induced routing + * rule. + * 4. Accept the best non-worsening rule (current, dummy, single-feature, or pair + * split). In weighted reward units, ``lambda`` penalizes single-feature splits + * by ``lambda * nodeSampleCount`` and pair splits by + * ``taoPairScale * lambda * nodeSampleCount``. Dummy rules are unpenalized + * (cost-complexity style; see ``TaoObjective``). * * **Outer loop** (``optimize``): * @@ -47,14 +50,16 @@ namespace tao { * @param adapter Task adapter (care set, discretizer params, leaf refresh). * @param nodeSamples Current sample partition per node index. * @param nodeIdx Internal node to optimize. - * @param lambda Per-sample complexity rate; non-dummy scores pay - * ``lambda * totalSampleWeight`` in weighted reward units. + * @param lambda Per-sample complexity rate; single-feature scores pay + * ``lambda * nodeSampleCount`` in weighted reward units. + * @param taoPairScale Multiplier applied to the complexity penalty for pair + * routers. * @returns ``true`` if the node's routing rule was updated. */ bool optimizeNodeInPlace( TaoAdapter &adapter, const std::vector> &nodeSamples, size_t nodeIdx, - double lambda); + double lambda, double taoPairScale); /** * Run TAO refinement on a fitted tree. @@ -62,7 +67,10 @@ bool optimizeNodeInPlace( * @param adapter Concrete adapter bound to the tree and training data. * @param nRuns Maximum number of bottom-up sweeps (early-stops on no change). * @param lambda Cost-complexity rate passed to ``optimizeNodeInPlace``. + * @param taoPairScale Multiplier applied to the complexity penalty for pair + * routers. */ -void optimize(TaoAdapter &adapter, size_t nRuns = 10, double lambda = 0.0); +void optimize(TaoAdapter &adapter, size_t nRuns = 10, double lambda = 0.0, + double taoPairScale = 1.1); } // namespace tao diff --git a/docs/api/ensemble.rst b/docs/api/ensemble.rst index 25d7204..855aa13 100644 --- a/docs/api/ensemble.rst +++ b/docs/api/ensemble.rst @@ -5,7 +5,8 @@ Ensembles Bootstrap-aggregated random forests over Shape Generalized Trees. -Both forest estimators accept ``tao_n_runs`` and ``tao_lambda``; these are +Both forest estimators accept ``tao_n_runs``, ``tao_lambda``, and +``tao_pair_scale``; these are forwarded to each base tree and TAO runs on that tree's bootstrap sample (or the full training set when ``bootstrap=False``) at the end of each tree's ``fit``. See :doc:`tao` for post-hoc refinement on the full ``(X, y)``. @@ -14,11 +15,19 @@ Multi-output ``y`` is supported the same way as for single trees (see :doc:`estimators`): one joint forest over all outputs, with sklearn-shaped ``predict`` / ``predict_proba`` returns. +Forests accept the Shape²CART options ``pairwise_candidates`` and +``pairwise_penalty`` and forward them to every base estimator. Pair candidates +are restricted to each node's ``max_features`` logical-feature subset. See +:doc:`estimators` for candidate-count semantics, categorical and joint missing +routing, multiway support, and the feature-importance warning for pair nodes. +See :doc:`../tutorials/bivariate-branching` for a worked classifier example. + :attr:`~sgtlearn.ensemble.RandomSGForestClassifier.mean_feature_importances_` and :attr:`~sgtlearn.ensemble.RandomSGForestClassifier.std_feature_importance_` (and the regressor counterparts) summarize per-tree :attr:`~sgtlearn.SGTClassifier.feature_importances_` across the forest, aligned with the shared :attr:`processed_features_`. +They are unavailable if any base tree has undergone positive-run TAO refinement. RandomSGForestClassifier ------------------------ diff --git a/docs/api/estimators.rst b/docs/api/estimators.rst index 11b6f29..b2b1448 100644 --- a/docs/api/estimators.rst +++ b/docs/api/estimators.rst @@ -9,14 +9,14 @@ Both estimators accept ``tao_n_runs`` and ``tao_lambda``; TAO runs automatically at the end of :meth:`~sklearn.base.BaseEstimator.fit` when ``tao_n_runs > 0``. See :doc:`tao` for behaviour and post-hoc :func:`~sgtlearn.tao.TAO_refine`. -After fitting, :attr:`~sgtlearn.base.BaseShapeCART.feature_importances_` gives -normalized importances over logical features, aligned with +After fitting with TAO disabled, :attr:`~sgtlearn.base.BaseShapeCART.feature_importances_` +gives normalized impurity importances over logical features, aligned with :attr:`~sgtlearn.base.BaseShapeCART.processed_features_`. Forests expose :attr:`~sgtlearn.ensemble.RandomSGForestClassifier.mean_feature_importances_` and :attr:`~sgtlearn.ensemble.RandomSGForestClassifier.std_feature_importance_` -instead (see :doc:`ensemble`). Prefer built-in importances when TAO is off -(``tao_n_runs=0``); see :doc:`../tutorials/feature-importance` for permutation -importance with categoricals. +instead (see :doc:`ensemble`). After any positive-run TAO refinement these +attributes are unavailable; see :doc:`../tutorials/feature-importance` for +permutation importance. Multi-output targets -------------------- @@ -71,3 +71,42 @@ pre-resolve it once with :func:`configure_feature_dict` and pass the result as .. autoclass:: sgtlearn._features.ProcessedFeatures :members: + +Bivariate branching (Shape²CART) +--------------------------------- + +See :doc:`../tutorials/bivariate-branching` for a worked S²GT classification +example and tuning guidance. + +All four estimators — :class:`SGTClassifier`, :class:`SGTRegressor`, +:class:`~sgtlearn.RandomSGForestClassifier`, and +:class:`~sgtlearn.RandomSGForestRegressor` — support opt-in bivariate +Shape²CART nodes. Set ``pairwise_candidates`` to a positive value to enable +pair screening; its default is ``0`` and therefore preserves the existing +axis-aligned behaviour exactly. + +``pairwise_candidates`` may be an integer (an absolute number of candidate +pairs) or a float (the fraction of logical features used to determine that +number, rounded up). Candidate pairs are formed only from the logical feature +subset selected for the node by ``max_features``. ``pairwise_penalty`` +(default ``0``) is applied only while selecting between univariate and +bivariate candidates; raw gain and minimum-leaf checks remain unchanged. + +A retained pair is fit with an ordinary axis-aligned CART over the two logical +features. Continuous and grouped categorical features are supported. Missing +values are routed jointly per feature, so a finite interval on one axis and a +missing value on the other is a distinct bin (as are the converse and both +missing); a missing branch may continue splitting on the other feature. +Multiway outer branching uses the same inner pair tree. + +Without TAO, pair gains are divided equally between the two logical features. Accessing +``feature_importances_`` after a fit containing pair nodes emits a warning, +because this attribution is intentionally a symmetric convention rather than +a unique or fully trustworthy decomposition. + +Pair-aware TAO is available through ``tao_pair_scale`` (default ``1.1``): it +is finite and non-negative, affects only the TAO complexity penalty, and TAO +only reconsiders pairs retained during initial screening. After TAO, +``feature_importances_`` is unavailable. The existing ``plot_tree`` API renders +exact Shape²CART routing heatmaps with marginal histograms, partition-changing +threshold labels, categorical labels, and missing margins when present in ``X``. diff --git a/docs/api/plotting.rst b/docs/api/plotting.rst index d2b0cea..bc62d25 100644 --- a/docs/api/plotting.rst +++ b/docs/api/plotting.rst @@ -10,6 +10,16 @@ plot_tree .. autofunction:: plot_tree +When a fitted estimator contains Shape²CART nodes, the same ``plot_tree`` API +renders the exact pair routing heatmap. Continuous/categorical combinations +use the corresponding rectangle or category-matrix layout, with independent +missing-value margins (and a both-missing corner) and one color for each of the +``K`` outer partitions. Passing ``X`` adds top/right marginal histograms and +shows missing margins only when the corresponding node data contain missing +values. Continuous axes label only thresholds where the final partition +changes, formatted with ``precision``; categorical axes retain category labels. +See :doc:`../tutorials/bivariate-branching` for a worked example. + export_graphviz --------------- diff --git a/docs/api/tao.rst b/docs/api/tao.rst index f2a1c5d..76742c4 100644 --- a/docs/api/tao.rst +++ b/docs/api/tao.rst @@ -24,7 +24,7 @@ package root as ``sgtlearn.tao``): TAO parameters on estimators ---------------------------- -All four public tree estimators accept the same two constructor / ``fit``-time +All four public tree estimators accept the same three constructor / ``fit``-time TAO knobs: ``tao_n_runs`` : int, default=10 @@ -37,8 +37,20 @@ TAO knobs: ``tao_lambda * n_samples`` in weighted reward units to be accepted. With the default ``0.0``, weighted training accuracy / loss does not decrease. -These map directly to the keyword arguments of :func:`~sgtlearn.tao.TAO_refine` -(``n_runs`` and ``lambda_``). +``tao_pair_scale`` : float, default=1.1 + Finite, non-negative multiplier for the TAO complexity penalty of a + retained bivariate candidate. It applies only to that TAO penalty and + never reuses ``pairwise_penalty``. TAO only reconsiders pairs retained by + the node's initial pair screening; it does not search new pairs. See + :doc:`../tutorials/bivariate-branching` for tuning guidance. + +.. warning:: + + Impurity-based feature importances are unavailable after any TAO call with + ``n_runs > 0``. Use held-out permutation importance instead. + +These map directly to ``n_runs``, ``lambda_``, and ``tao_pair_scale`` on +:func:`~sgtlearn.tao.TAO_refine`. Supported estimators ~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/index.rst b/docs/index.rst index fac471c..63d5985 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,8 +3,8 @@ sgtlearn ``sgtlearn`` is a Python package for learning **Shape Generalized Trees (SGTs)** — a class of decision trees where each internal node applies a learnable, -axis-aligned *shape function* to a feature, producing non-linear yet -interpretable splits. +axis-aligned *shape function* to one or two logical features, producing +non-linear yet interpretable splits. It implements the algorithms from the NeurIPS 2025 paper `Empowering Decision Trees via Shape Function Branching @@ -15,13 +15,14 @@ Highlights ---------- - 🌳 **Shape Generalized Trees (SGTs):** each node applies a learnable, - axis-aligned shape function to a feature for non-linear, interpretable splits. + axis-aligned shape function for non-linear, interpretable splits. - 👁 **Interpretability:** every node's shape function can be visualized directly with :func:`~sgtlearn.plot_tree`. - ⚡ **ShapeCART algorithm:** an efficient native (C++/pybind11) induction method for learning SGTs from data. -- 🔀 **Extensions:** multi-way branching (:math:`\mathrm{SGT}_K`) and bootstrap - ensembling via :class:`~sgtlearn.RandomSGForestClassifier` / +- 🔀 **Extensions:** bivariate branching (:math:`\mathrm{S}^2\mathrm{GT}`), + multi-way branching (:math:`\mathrm{SGT}_K`), and bootstrap ensembling via + :class:`~sgtlearn.RandomSGForestClassifier` / :class:`~sgtlearn.RandomSGForestRegressor`. .. note:: @@ -44,6 +45,7 @@ Highlights :caption: Tutorials tutorials/shape-functions + tutorials/bivariate-branching tutorials/categorical-features tutorials/feature-importance tutorials/sgt-k diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 3dae079..f75b0f5 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -24,8 +24,9 @@ Train and visualize an SGT classifier on the built-in "Plus Sign" dataset: Inner vs. outer depth --------------------- -Every SGT node's split is itself a small *inner* tree (the shape function) that -carves one feature into bins; the *outer* tree routes samples through those bins. +By default, every SGT node's split is itself a small *inner* tree (the shape +function) that carves one feature into bins; the *outer* tree routes samples +through those bins. ``num_partitions`` sets the outer branching factor (the :math:`\mathrm{SGT}_K` arity), and ``inner_max_depth`` controls how rich each shape function may be: @@ -145,3 +146,26 @@ Columns you do not mention stay as individual continuous features. See :doc:`tutorials/categorical-features` tutorial for a worked example with plots. For permutation importance on raw categoricals, see :doc:`tutorials/feature-importance`. + +Bivariate branching +------------------- + +To let a node consider interactions between two logical features, enable +Shape²CART explicitly. The inner model is a small, ordinary axis-aligned CART +over the pair; ``pairwise_candidates=0`` (the default) leaves the usual +univariate training path unchanged: + +.. code-block:: python + + model = SGTClassifier( + max_depth=3, + pairwise_candidates=8, # or a fraction such as 0.5 + pairwise_penalty=0.0, + random_state=42, + ).fit(X_train, y_train) + +The same options are available on ``SGTRegressor`` and both random-forest +estimators. Candidate pairs are screened within the logical-feature subset +selected by ``max_features``. See the :doc:`tutorials/bivariate-branching` +tutorial for a univariate comparison, routing heatmaps, tuning guidance, +categorical and missing-value behaviour, and pair-aware TAO. diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 85c98db..fbac26f 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -1,10 +1,8 @@ Release Roadmap =============== -Which features are implemented today and which are planned. Features from the -paper not yet in this codebase include bivariate shape functions -(:math:`\mathrm{Shape}^2\mathrm{CART}`), higher branching factors for bivariate -splits, and contour-plot visualization for bivariate splits. +Which features are implemented today and which are planned. Shape²CART, +pair-aware TAO, and dedicated routing heatmaps are available in v0.3.0. v0.1.0 ------ @@ -28,6 +26,16 @@ v0.2.0 v0.3.0 ------ -- ⬜ Multioutput support -- ⬜ :math:`\mathrm{Shape}^2\mathrm{CART}` -- ⬜ :math:`\mathrm{Shape}^2\mathrm{CART}` random forest ensembling +- ✅ Multioutput support +- ✅ Opt-in :math:`\mathrm{Shape}^2\mathrm{CART}` for SGT classifiers and + regressors, including continuous/categorical pairs, joint missing routing, + and multiway outer branching (see :doc:`tutorials/bivariate-branching`) +- ✅ :math:`\mathrm{Shape}^2\mathrm{CART}` random forest ensembling +- ✅ Pair-aware TAO refinement (see + `issue #48 `_) +- ✅ Shape²CART routing heatmap visualization (see + `issue #28 `_) + +The bivariate work is specified in `issue #42 +`_ and tracked under the +umbrella `issue #27 `_. diff --git a/docs/tutorials/bivariate-branching.ipynb b/docs/tutorials/bivariate-branching.ipynb new file mode 100644 index 0000000..17da014 --- /dev/null +++ b/docs/tutorials/bivariate-branching.ipynb @@ -0,0 +1,293 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "s2gt-title", + "metadata": {}, + "source": [ + "# S$^2$GT — Bivariate branching\n", + "\n", + "A standard Shape Generalized Tree (SGT) routes each node with a shape function over one logical feature. **S$^2$GT** lets a node instead learn a shape function over a pair of logical features, so one interpretable node can represent interactions that require several univariate nodes. In `sgtlearn`, the opt-in **Shape²CART** training path learns these bivariate nodes.\n", + "\n", + "> **When to use this.** Enable bivariate branching when validation data suggests that interactions matter and a univariate SGT needs extra depth to express them. It increases node-fitting cost, so start with a small candidate budget and compare against the default univariate model." + ] + }, + { + "cell_type": "markdown", + "id": "mechanics", + "metadata": {}, + "source": [ + "## What changes inside a node\n", + "\n", + "The outer SGT is unchanged. The difference is the small inner model used to route one outer node:\n", + "\n", + "1. Fit the best univariate shape function among the node's `max_features` subset.\n", + "2. Screen unordered pairs from that same subset and fully fit the best `pairwise_candidates` pairs.\n", + "3. For each retained pair, fit an ordinary **axis-aligned CART over two logical features**. These are not oblique splits.\n", + "4. Compare the best pair with the best univariate candidate after applying `pairwise_penalty` to pair selection.\n", + "5. Map the winning inner leaves to the node's `num_partitions` outer children.\n", + "\n", + "A model with bivariate branching enabled may still choose a univariate node whenever it scores better. `pairwise_candidates=0` disables pair search and exactly preserves the default path." + ] + }, + { + "cell_type": "markdown", + "id": "example-heading", + "metadata": {}, + "source": [ + "## A three-way interaction in one node\n", + "\n", + "This target has three nested regions: corners, an edge-cross, and a center square. Neither feature alone identifies all three classes, but their joint location does. We fit both classifiers at outer depth 1 so the comparison isolates the routing rule rather than added tree depth." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "imports-data", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-02T19:30:20.275175Z", + "iopub.status.busy": "2026-09-02T19:30:20.274999Z", + "iopub.status.idle": "2026-09-02T19:30:21.081310Z", + "shell.execute_reply": "2026-09-02T19:30:21.080824Z" + } + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from matplotlib.colors import ListedColormap\n", + "\n", + "from sgtlearn import SGTClassifier, plot_tree\n", + "\n", + "\n", + "def three_way_regions(X):\n", + " inside_x = np.abs(X[:, 0]) < 0.85\n", + " inside_y = np.abs(X[:, 1]) < 0.85\n", + " return np.where(inside_x & inside_y, 2, np.where(inside_x | inside_y, 1, 0))\n", + "\n", + "\n", + "rng = np.random.default_rng(0)\n", + "X_train = rng.uniform(-2.5, 2.5, size=(2400, 2))\n", + "y_train = three_way_regions(X_train)\n", + "\n", + "axis = np.linspace(-2.5, 2.5, 201)\n", + "x1, x2 = np.meshgrid(axis, axis)\n", + "X_grid = np.column_stack([x1.ravel(), x2.ravel()])\n", + "y_grid = three_way_regions(X_grid)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fit-models", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-02T19:30:21.082564Z", + "iopub.status.busy": "2026-09-02T19:30:21.082449Z", + "iopub.status.idle": "2026-09-02T19:30:21.141044Z", + "shell.execute_reply": "2026-09-02T19:30:21.140592Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Univariate SGT grid accuracy: 0.660\n", + "Bivariate S²GT grid accuracy: 1.000\n" + ] + } + ], + "source": [ + "common = dict(\n", + " max_depth=1,\n", + " inner_max_depth=4,\n", + " inner_max_leaf_nodes=9,\n", + " num_partitions=3,\n", + " tao_n_runs=0, # compare the greedy routing rules directly\n", + " random_state=0,\n", + ")\n", + "\n", + "univariate = SGTClassifier(pairwise_candidates=0, **common).fit(X_train, y_train)\n", + "bivariate = SGTClassifier(pairwise_candidates=1, **common).fit(X_train, y_train)\n", + "\n", + "print(f\"Univariate SGT grid accuracy: {univariate.score(X_grid, y_grid):.3f}\")\n", + "print(f\"Bivariate S²GT grid accuracy: {bivariate.score(X_grid, y_grid):.3f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "compare-regions", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-02T19:30:21.143826Z", + "iopub.status.busy": "2026-09-02T19:30:21.143726Z", + "iopub.status.idle": "2026-09-02T19:30:21.270489Z", + "shell.execute_reply": "2026-09-02T19:30:21.270002Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABI8AAAF9CAYAAACNl0i3AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlcelbwAAAAlwSFlzAAAPYQAAD2EBqD+naQAALfVJREFUeJzt3Qm43tOdB/CTJoSExB5CKtbS2oqotcRS+xZTY6ul1DB2Ritm1DJMqpNS7TxF1aStmYTWEoLYSlVViV1tDVFiC0U2kUhy7zy//zzvfd57c8/Neu+7fT7P8z7Jfbf/+b83Ob/3//2fc/7dmpubmxMAAAAAtOML7d0JAAAAAMIjAAAAADpk5BEAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gENeJ73/teuuyyyyrdDICa8uyzz6Z/+Id/SK+88kpDbh+A6uubK719WBQ9FulVkDF79ux0+OGHL9Dnc8wxx6T999+/aj7L0aNHp//5n/9Jv/71r1OvXr1Stfnd736XVllllUo3A2CJu//++9O1116bLrzwwrTpppvO8/g111yTHnjggTRixIi0/PLLL9R7v//+++mWW25JZ5555hJscddsf9y4cenyyy9Pw4YNSxtssMESbVdzc3O644470p/+9Kf07rvvpjXWWCNtueWW6aCDDkrLLLNMu6957rnn0l133ZVef/319Pnnn6f+/fun9dZbLx188MFp1VVXTf/7v/+bbrvttvlue9lll0033HDDEt0foLGNHz8+DR06tNV9ffr0SWuvvXY68sgj0/rrr9/qMbVh0WvDjBkz0tFHH138fbnllks77LBDOu6441KPHvNGC+pGfenWHP9CYAlpampKt956a6v7fvKTn6RHHnmk+KJY/oV0s802SxtuuGHVfPY/+MEPiqLzySefpBVWWCFVmwcffDAtvfTSaccdd6x0UwCWqAiHTj755CJE2n333ed5/IQTTkjXX399+vDDDxc6RJ80aVJRgwYPHpxWXnnlJdjqzt9+nNSIYOaxxx5L22677RJr08cff5z22muv4mDrxBNPTF/5yleKg4QI8KZNm1ZsrzysmjJlSvE7iAOKQw89tKhDccDwt7/9Lf3mN79JL730Uho+fHjabbfdWp1Fj4Oz0047rThRVDrQCHGAEQciAEvKn//857TddtsVfU3p5HR8p49A+9577y3qzHe+852W56sNi14bYrDA7bff3tLPX3rppcXn/sMf/lDdqHNGHrFEfeELXyiGYLb98hvii2J82WTR7Lrrrj46gIXUr1+/eepSI22/Pf/xH/9RjGp69NFH0/bbb99yfwQ9xx9/fBEWlcydOzcdeOCB6YUXXkiPP/542mKLLVq91/nnn59+9rOfFUHSJptsUtxKXnvtteLPOFFUbZ8BUJ9i9Gp5fxN92le/+tV03nnntQqPKt03V3r7i1MbllpqqVZtf+edd4ppeCXqRv0SHtHlZs6cmY466qiWn3v27Jm++MUvFmczo3MvmTBhQvrud7+b/uVf/iWtvvrq6ec//3lx30UXXZQ22mijNHny5OIswl/+8pdiSOVJJ51UnJWOs5+RfK+77rot7xUD7MaMGZPuu+++IlUfMGBA0YbS9Ih47xhuX5pOF51i6GiqQJxtjduoUaOKTjb+HsM4f/nLX7ac7YjRVs8880zRicaQz29/+9vFENpy8XhMlfvoo4/S1ltvXST9MVorOuKrrrqq1ZpH8dp//dd/bfX6N954o3h9nCWI6XY77bRTOuyww1r2odTpDxw4sPjzF7/4RXF2ZqWVViraEyPAyk2dOrVo9/PPP5/mzJlTnHU49thji+cDVIPoc6+88sqiv58+fXrRr0UfGqNz4uAgRmmWxBfaOCsat6gdEYJcfPHF6fTTT09f//rXW71vaYRNjPY54ogjikAkalDo1q1bMXo2akI8Vj4Forw90YdGHXj77beL2hLvUb79sCDvG9PDSmdxI6Ap9cGnnnpq2mWXXVq2HaOa4gxwnB2OqWOHHHLIPPvVVnwmMXWs/OAg9O7dO40cObKoZSVRGx9++OFiX9oGR6X2n3LKKa0CJ4BqOrEd/euLL75YfK8tTa1SGxavNpQfh0TNu+mmm1ruUzfqlwWz6XIRakS4UbrFUP633nqrCE5++9vftjwvQp5YJyKGmkagEwn9WmutVRwgxBDJCGNiGGUETjGfORLxe+65p3hNvLY8rIohmBEWxXS0b3zjG8V7xOujIwyDBg1Km2++efH3+OJdaltHUwximH5sK864/td//VfRjk8//bR47KmnnioOEqIzjS/b0QnHekrx9zigKInAKbYdZ2fjy/7f//73IkT74x//WOx32zWP4iChXBxcfPnLXy6+2MfrI4Q744wziikFcUBVMnbs2OLgJj6D+Ox23nnnIrT62te+VoRvJfHlP34PcSAWoVK8Zzw/nvfBBx8s0u8bYEmbOHFi0f/GNKo4ybDxxhsXo1vOPffcol60t65F9K8h+uY//OEPRdjTVpwMuPnmm9OXvvSl4ucVV1yxpR784z/+Y9G3xjoQMbomRuG0bU/UsAiFoi1xYiTqT9vtL+j7RhviZEDYY489Wp6/zjrrtLxPnDSJqWKzZs0qaltsM54b4VhH4uTKZ599lp588sl2D7TKRwnHAUHcF9vuSN++fTt8HKAS3nvvveL7c6zLU74mj9qweLUhxEn9mGoeS3+Un7RQN+pYrHkEnenII4+MdbWap02b1uHzTjjhhOY111yz5edx48YVr1t//fWbp0+f3nL/Z5991nz00Uc39+rVq/mdd95puf/TTz9t/tKXvlS8Jl5bcu655zb36NGj+emnn261ve9973vFe3z44YfFz8OGDSte+8knnyzQfl144YXF86Pd5W2bNWtW88CBA5u33nrr5tmzZ7c8NnPmzOb11luveciQIcXPU6dObe7bt2/z/vvv3+p977nnnuZlllmm2JdyW221VfOee+7Z8nN8niuttFLz17/+9eY5c+a03B/72a1bt+Yzzzyz5b7Y7rLLLtv80EMPtdw3Y8aM4vXf+ta3Wu676aabin167bXXWm178uTJxfMBOsPVV19d9D33339/u48ff/zxxeOl/nrUqFHFz9Enzp07t+V5P/rRj4r+75VXXmm5b+zYscVzH3nkkZb7zjnnnOalllqqedKkSa22M2jQoObNN998vu0dPHhw884779zyc6k9u+yyS0u/H+2KetDe9hf0fW+77bbitY899tg8z73++uuLx37729+2un/EiBHFZ/DUU09ltzNhwoTmVVddtag1xxxzTPGaZ555prmpqWme56699trNX/ziF5sXxfjx44s2xucN0Jmin4z+ZtNNN20+5JBDitvuu+/e3Lt37+ZDDz20+f3332/1fLVh8WrDE088UdSHW265ZZ7H1I36ZdoaFRFTumLUTwyPjNEusdB2pNcxVStGDZVPkYqzsjFcsnzkUizKHdMK4kovJTFlK557ySWXtJquFlfniTOx5VPiQkxNiKvYxKicb33rW4u8LzHiqSSmHsT7xZSEyy67rNUZjjgjHCN/YipcnCWOhWFj3//5n/+51fvtueeeac0115zvduP18VnFVLTu3bu33B/7GaO54gx6+Zn1OJtQPtUhhqXGWZjyOcqlBc1jccGzzjqr5X2dUQaqUUypjbOhJXEGNPr9uLpLafRQrt/+0Y9+VEz5LU0fi1GYsdZDTBsuF6NFoz+N0aYxojPeP+4rH0lUEqOeSv1+tKt8+lxbC/O+7YkFxGMUUts1M6KexYikGO0UI2zbE6+Lha2jPsZV7GI6dEx7W2211YqRXHFluFL/H9MU2pu2fOONNxajtEpipG6MBgaopPgeXFowO/rW6O/imGObbbZJ55xzToevVRsWrDbEDI44pogZD/HZxi2WEPnpT39afI7qRv0SHtHl4hK/EVpERxRhSkxFiy/YMeUsplLFehHlX1TjMsDlosOKYlC+plFHz40v4rHNGHJfurhg/BnznkNMmVscbbf56quvFn/GlLWYUhHbKm23dHnjGCobAVNobz/ivvm1q/T69tZkiikTcXW2GHoaIVFuO7E+Rqx/VLLvvvsWa27E1I8Iv2IIakyJiPsW9gpHAAsq1s1ZlOe17deiTwvRx3YkprnFdOL//u//bgmPIoyJkD8u6VwS030j0I8QZsiQIcX6exEOxXS0OOExv3qQs7Dv256oNaVp4KFUa+IWwdX8akjU2TiQKh1M/fWvf03/9m//VnweUTvi7yH6/vYCrVgzsBSUXXDBBcWVeACqbcHsOMkQ/Vj0bdHvxxXZctSGBasNccL+V7/6VavPrnxKm7pRv4RHdLn//M//LMKfSLZjDaKS8hEw5ZZffvlWP5fO5EYH1lbbhdxKi0bnrvQSoUisGbQ42ravtM0Y7VS+NkW56FQXZj/a09HrY+2lOMgqXzS7vTPg8ZwY9VUSZxNikbu42kKccYg54hdeeGH6/ve/X6wR0nZxbYAloTS6tHyttnJxUiG0XW+hbb9WCpfK+7WcGH0aFw2ISw9vtdVWxbp0EeSUn7yI/i/Opsaac+X9afmIm47qQc7Cvm974nXxHu3VthiFGxeGWBhRJ6P///3vf1+MyCqFR3GgFSFbhFrlYV1cTCFu4cc//rHwCKha8Z08Rv7HhXM6Co+C2jD/2hAnpju6Upy6Ub+ER3S5mJoWX2rLg6Pw0EMPLdDr43Vx5bBYlLqtp59+utXPMd0qFkeN6V3zuxxm6Qzqghx0dCSu9lM6yOlom6Wr1sR+lBbrDnHm+eWXX245g55TmoYXi6uWthnirHNMvSg/K7ywSguQxy3OXsfnHUNY21tgFmBxlYLp6MMPOuigeR6PUamlRaiXlLg4QVxgIEYclaaLlU9DLtWrqCHlAU+MHo2RQ4tjQd+3o7oU/X6MHD3ggAM6nB7XnhiZuuuuu85zf7QnziiXh3gnn3xy0f9fccUVxcUhAGrNwoyMVBsWrDZ0RN2oX662RpeL0COmb5Vf5SuuWPbhhx8u8HvEpYojbIq1eUqeeOKJ4gxyW7EGUlzFJq4EUP4FfPbs2UWCHldhCKWztG+++WZaHHEGO77Mx2idtgFXzBuOA5UQU/diykK0q9SGUnsXZI2hGHob87djJFd8niXxBT9GdZ199tkL3fZYR6ntZxijmCKQMm0N6MzwKNZqu+aaa1qNQo2+Jy5XH1ekXJQ+bX6jnWLKV1z1MkKRCMnbBipRryKgmTRpUkt7zj///JbpwItqQd+3o7oUa1GU1r0rH4Ea73XnnXcW6z7lxGcaJzfKr/4Z4kqbsa3yAC+uwBlXb4s6HfUpTnCUmzx5cnEDqEbRP8ZaPDGd98ADD5zv89WGBasNHVE36pfwiC4X6+lE6BGXf481H2KETJxtPf300xf4PWIx51gQ9Jvf/GbxPnE544suuqi4P5QvIB3PiZAohtXHwcFee+1VBDexyFsEUKXpErG4XpzZ3meffYriEl+sx48fv0j7GAvHxYLeEfBEQBRrCcV+RjtLay3F9IqYohBnjGOOdQypjefEiKP4s3wfcmLh8JhGECOX4qAnXhfDSWPaWdvLVS+ICK1iQbyYbhefU6x3NGjQoOIsTCySB9BZIsSJvjn69Oh3oi9ef/3106WXXlpMN/inf/qnJb7NGGkUZ6RjWm5MYWu7plKE+xGcx/Tmvffeu1iEO0YDxQmCxbGg7xt9e9SGqHf77bdfUZdi6kDpy3lcoCHqWIwWjcXCI4CLdQSvu+66eUb3tp2WEaOfoq+PzzxqVKyfFxdwiM9h+PDhrZ4faxrF4t5R2/r161fUm6iTMTUhps5F24cOHbpYnwnAkhDf+aOvjFt8l41+Lr7PxyL/pVH/86M2LFht6Ii6UZ+6xSXXKt0I6ltMoYq0OsKUUiAS/+zi7HKMxIlOKUKbWE8hpizEAUMMjYwrssV6EPHlNHf1sTfeeCO9+OKLxZfXGPETV8mJaQhxfwRF5WKkUWwzRvnEYt0R2LQd4RPPiRFMH3zwQZo7d27xBbm9q8yEuEJO3GKNjPKr/ZSLfYjpFnHWI9pTOkAoF9uJbcYZ5Chqsa8RrkWo9Mc//rHVNIO4b8cdd5xnOzHSKApjnLWOg662+xWLkceBRPn0thAjo+J3ULoqRUlMVYupc7G9mFoRny9AV4jFrl944YVi7bcIWCKAbzsiJ86KxsidCFbK+7u4kuWYMWOKvjSCpxCje2L9tghW4opgbY0ePboI9XOPR12I2hQXYIiAPkYDRfuiz43+v6P2dLT9BXnfUr2M502cOLFoZ/TxERaVPx4jeeMiCtHPR589v2nP5W2L7UUbotZFWNWnT58OXxMjXWMkWLQ/FvqOsCr+bE+MXI2AK2pf7CNAZ4nv0fFduVzUjjhZ3HaacFAblmxt6Ii6UT+ER9SVuHrb3XffXXR2C3r1nmoTC8PGl/EYOVS65CUAAABUimlr1KwYflo+cC5G19x0003pxBNPrJngKEZCxaihkjiTG5fCjDPubRduBQAAgEpwtTVqVowwivV5Ys2I0jSHI444olj7qFbEcNpYkymmvcWw/9iHGOYfl0Ve0HnZAAAA0JlMW6OmRWgUax7FJY7jaj25tZGqWYyeivWFYs2nmE8ci6DGmk8AAABQDYRHAAAAAGRZ8wgAAACALOERAAAAAFkNtWB2U1NTevfdd9Pyyy9fM1fjAhpHrH81bdq01L9//2IRdaqH+gFUM/WjeqkfQL3Uj4YKjyI4GjBgQKWbAdChiRMnprXWWsunVEXUD6AWqB/VR/0A6qV+NFR4FCOOwsP/cFJabqmelW5Ow+q369OVbgLtmPTglj6XCps+e1ba+eZrWvoqqof6AVQz9aN6qR/VwfFHdXL8UVv1o6HCo9JUtQiOlltaeFQpfXo11D+7mvGp/xNVw7Ta6qN+ALVA/ag+6kd1cPxRnRx/1Fb9sKgGAAAAAFnCIwAAAACyhEcAAAAAZAmPAAAAAMgSHgEAAACQJTwCAAAAIEt4BAAAAECW8AgAAACALOERAAAAAFnCIwAAAACyhEcAAAAAZAmPAAAAAMgSHgEAAACQJTwCAAAAIEt4BAAAAECW8AgAAACALOERAAAAAFnCIwAAAACyhEcAAAAAZAmPAAAAAMgSHgEAAACQ1SPVmFdffTVNmDAhDRgwIG2yySaVbg4ANUL9AED9AKjz8OjJJ59MJ598cpo2bVpaZ5110jPPPJPWXXfdNHr06LTaaqtVunkAVCn1AwD1A6BBpq1NmTIlXXPNNemVV15JY8eOTa+99loRJJ111lmVbhoAVUz9AED9AGiQkUe77bZbq5+XW265tPfee6c777yzYm0CoPqpHwCoHwANEh611dzcnH7/+9+nr3zlK9nnzJo1q7iVTJ06tYtaB0C1Uj8AUD8A6nTaWluXXXZZeuGFF9L3v//97HOGDRuW+vbt23KLRbYBaGzqBwDqB0ADhEfXXntt+vd///d04403pk033TT7vKFDhxZrXZRuEydO7NJ2AlBd1A8A1A+ABpi2dt1116XTTz+9CI4OPPDADp/bs2fP4gYA6gcAjj8AGmDk0fXXX59OPfXUNGrUqHTwwQdXujkA1Aj1AwD1A6ABRh7ddttt6Tvf+U468sgjU1NTU7r55puL+5deeul0wAEHVLp5AFQp9QMA9QOgQcKjGTNmpCFDhqTPPvusmLJW0rt3b+ERAOoHAI4/ABo9PIoRR3EDAPUDAMcfAF2nptY8AgAAAKBrCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyOqRfwgAKNdv16dTn15KJ1Bdps6Yk9KoSrcCgHrmGzAALKBJD26ZPl26p88LqCrTP5+VUnqk0s0AoI6ZtgYAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAA6iM8euedd9JFF12UNt5443TAAQdUujkA1Aj1AwD1A6ABwqNZs2al7bffPjU1NaXNN988vfvuu5VuEgA1QP0AQP0AaJDwqGfPnmnChAnpkksuSauvvnqlmwNAjVA/AFA/ABokPArdu3evdBMAqEHqBwDqB8Ci65HqfKpC3EqmTp1a0fYAUBvUDwDUD4AaHXm0sIYNG5b69u3bchswYEClmwRADVA/AFA/ABokPBo6dGiaMmVKy23ixImVbhIANUD9AED9AGiQaWuxSGrcAED9AMDxB8CiqeuRRwAAAAA00MijPffcM7366qvp448/LhYzHThwYHH/Sy+9lHr16lXp5gFQpdQPANQPgAYJj0aMGJE+//zzee5fdtllK9IeAGqD+gGA+gHQIOFR//79K90EAGqQ+gGA+gGw6Kx5BAAAAECW8AgAAACA+pi2tqT02/Xp1KdXQ+56VTjihX+qdBNox8hvXOtzqbCpM+akNKrSrQAAljTHH5Xl+KM6Of6oreMPCQoALCBf/oFq5OQDAJ3NtDUAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsnqkhTBz5sx09dVXp/Hjx6dtttkmHXnkkWmppZZqefz2229Pn3zySTr22GMX5m0BqHPqBwDqB0CDjDzae++908UXX5yefPLJdOKJJ6addtopffjhhy2PR6j0l7/8pTPaCUANUz8AUD8AGiA8+sMf/pBeffXV4vbEE0+kV155Jc2ZMycNHjw4ffDBB6mrTJgwIY0YMSKNGjUqffTRR122XQAWjfoBgPoB0CDh0csvv5z233//1K9fv+LnddddNz388MPFzxEgTZo0KXW26667Lm2yySbF9Lhrrrkmrb/++umxxx7r9O0CsOjUDwDUD4AGCY9WXnnl9PHHH7e6r3fv3unOO+9M/fv3T7vuumunjkB6991302mnnZauvPLKNHr06CK4ijDruOOO67RtArD41A8A1A+ABgmPdtlllzRu3Lj02Weftbp/2WWXTWPGjElrrbVWGj58eOosMdqoR48e6Zhjjmm575RTTimm0T377LOdtl0AFo/6AYD6AdAg4dEqq6xSjPz505/+NM9jyyyzTBHuHHXUUWnttddOneGll15KAwcOLLZVsvHGG7c81p5Zs2alqVOntroB0LXUDwDUD4Da1mNhnnzOOedkH4tQ59e//nXqLNOmTUsrrLBCq/v69OmTunfvng2Fhg0bVlwdDoDKUj8AUD8AGmDkUcmtt95aXG2tPffee2+64oorUmeI6XERIJX79NNP09y5c1OvXr3afc3QoUPTlClTWm4TJ07slLYBMH/qBwCLQv0AqMHwKKaC7bjjjunSSy8tgpswc+bMdOaZZ6b99tsvrbbaap3RzrTBBhukt956q2WbYcKECS2Ptadnz57F6KTyGwCVoX4AoH4ANEh4dPjhh6fbbrst/fSnPy0WQR07dmwaNGhQuuOOO4oroMW6R50hgqmYnnbXXXe13HfDDTek1Vdfvdg+ANVN/QBA/QBogDWPSvbdd9/03HPPpa233jrts88+aaeddkqPPvpop47s2XDDDdO5555bXG3tpJNOSh9//HEaMWJEGjVqVHEVNgCqn/oBgPoB0AAjj8KkSZPScccdV6w5dPTRR6fHHnssXX755Wn27NmpM/3gBz9IN954Y2pqakr9+vVL48aNS4ccckinbhOAJUf9AED9AKg9Cz1k589//nM64IAD0sYbb5yef/75NGDAgHTssccWIdJ9992XRo4cmV2DaEnYc889ixsAtUX9AED9AGiQkUePP/54sTj2Qw89VARHYfDgwUWQtM4666Rrr722M9oJQI1TPwBQPwAaZOTRKaec0u4aQyuuuGL6zW9+kyZOnLik2gZAHVE/AFA/ABpk5NH8FqcujUYCAPUDgMXl+AOgRhfMBgAAAKAxCI8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjAAAAALKERwAAAABkCY8AAAAAyOqRGtCkB7dMny7ds9LNaFgjv3FtpZtAO967b5DPpcKmfz4rpfRIpZtBB9QPoBqpH9VP/agsxx/VyfFHbdUPI48AAAAAyBIeAQAAAFAf4dFzzz2XTjrppLTiiiumHXbYodLNAaBGqB8AqB8ADRAezZo1Kx1zzDFps802S0OGDCl+BgD1AwDHHwCdq2YWzO7Zs2d69tlni7+feeaZlW4OADVC/QBA/QBokJFHAAAAAHS9mhl5tChialv59LapU6dWtD0A1Ab1AwD1A6AKRh599NFHaZlllunwduihhy7WNoYNG5b69u3bchswYMASaz8AlaF+AKB+ADTIyKOVV145TZ48ucPndO/efbG2MXTo0HT22We3GnkkQAKobeoHAOoHQANNW4vRRZ29SGrcAKgv6gcA6gdA17FgNgAAAAD1sWD2VlttlV588cU0Z86c1NTU1HLmOda/6N27d6WbB0CVUj8AUD8AGiQ8euyxx4rQqKunLwBQ29QPANQPgAYJj5ZeeulKNwGAGqR+AKB+ACw6ax4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGT1SDXk008/Tffcc0+aMGFCGjBgQNp///1T7969K90sAKqc+gGA+gHQACOPxo4dmzbZZJM0atSo9MEHH6Srrroqrb/++unll1+udNMAqGLqBwDqB0CDjDxac80105NPPplWXnnl4ufm5ua0yy67pHPPPTfdeeedlW4eAFVK/QBA/QBokJFHm222WUtwFLp165a22Wab9MYbb1S0XQBUN/UDAPUDoEFGHrU1a9asNHr06LTzzjt3+Jy4lUydOrWLWgdAtVI/AFA/AGokPJoxY0b64Q9/2OFzNtpoo3TYYYe1+9iJJ55YhEGXXHJJ9vXDhg1LF1988WK3FYDqoX4AoH4AdK2ambZW7owzzkhjxoxJ9957b+rfv3/2eUOHDk1TpkxpuU2cOLFL2wlAdVE/AFA/AGpo5FGvXr3SRRddtNCvO+uss9INN9yQHnjggbTFFlt0+NyePXsWNwDqh/oBgPoB0LVqauTROeeck371q1+l+++/P2255ZaVbg4ANUL9AED9AGiABbOvvvrqdMUVV6QDDzywmLIWt7DMMsuk8847r9LNA6BKqR8AqB8ADRIebbDBBunCCy+sdDMAqDHqBwDqB0CDhEe77757cQMA9QMAxx8AXaem1jwCAAAAoGsJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgKweqYE0NzcXf06fPavSTWloU2fMqXQTaMf0z/2/qLRS31Tqq6ge6gdQzdSP6qV+VAfHH9XJ8Udt1Y9uzQ10lPL222+nAQMGVLoZAB2aOHFiWmuttXxKVUT9AGqB+lF91A+gXupHQ4VHTU1N6d13303LL7986tatW5dtd+rUqUVoFb+QPn36pHrWSPvaaPvbSPtaqf2N7njatGmpf//+6QtfMKu4mqgfnU8fU7/8bjuf+lG91I/Op4+pX3631VU/GmraWnwYlTybHwegjXDQ3Wj72mj720j7Won97du3b5dtiwWnfnQdfUz98rvtXOpHdVI/uo4+pn753VZH/XBqGwAAAIAs4REAAAAAWcKjLtCzZ8904YUXFn/Wu0ba10bb30ba10bcX6pTI/07bKR9bbT9baR9bcT9pTo10r/DRtrXRtvfRtrXWtjfhlowGwAAAICFY+QRAAAAAFnCIwAAAACyhEcAAAAAZAmPKuD+++9P3/72t9Nuu+2Wjj/++DRu3LhUzx555JF05JFHpm233Ta98sorqR5Mnjw5nXfeeWnw4MHp4IMPTmPGjEn1KpZFi3+z3/zmN4vf4XvvvZfq1dy5c9PIkSPTEUcckfbYY490+umnp9dff73SzYIW6kftUz/qk/pBtVM/ap/6UZ/m1tDxh/Coi11++eVp+PDhaaeddkrnn39+Wm211YoD8rvvvjvVozPPPDMNHTo0bbzxxunxxx9P06dPT7Vuzpw5affdd08PP/xwOuuss9L222+fhgwZkm688cZUj44++uji3+2GG25Y/A5nzZqV6tUJJ5xQ/F/cb7/90ne/+9308ccfpy222CK9/PLLlW4aqB/qR81RP9QPqoPjD8cftUb92KI6jz/iamt0nWnTps1z38EHH9y8xx571OWvYfLkycWfL7/8clzVr3ncuHHNtW7kyJHN3bt3b37vvfda7jvjjDOaBw4c2FzPv8NHHnmk+B2+8cYbzY3y/7Opqal5ww03bD7rrLMq1iYoUT/Uj1qjfqgfVAf1Q/2oNerHhlV5/GHkURdbbrnl2r3v888/T/Wob9++qd787ne/S4MGDUqrr756y30HHnhg+tvf/la1QwwXRz3+Dhf0/2e3bt1S79696/b/J7VF/ah96kf9Uj+oZupH7VM/6tdyNXT8ITyqsL/+9a/p5ptvTgcddFClm8ICevPNN1P//v1b3Vf6OR6jfowdOzY988wzRTgI1Ub9qD3qR+NQP6hm6kftUT8ax9gqPv7oUekG1LpYPDgWTO7IXnvtlS666KJ57v/www/TAQcckHbcccd02mmnpVoQ6zQ9+OCDHT4n1oxZaaWVUr2aPXt26tmzZ6v7ll122ZbHqA8vvvhisdB7LFoXi9fBkqZ+zEv9oB6oH3Q29WNe6gf14MUqP/4QHi2mCEl+/OMfd/icVVdddZ77Pvroo2LR5Zj6NHr06NS9e/dUC4477rgi8OrI8ssvn+r9dx4LKbf9fYaVV165Qq1iSYqrAsbVECMYnt//b1hU6se81A9qnfpBV1A/5qV+UOteqYHjD+HRYooRKHG1tIURQUP8w1hhhRXSXXfdlXr16pVqxQYbbFDcGtmWW26ZfvKTn6Smpqb0hS/8/8zPuApZ/FuIq8pR21599dU0ePDgtPfee6frrruumHcMnUH9aDzqR31TP+gq6kfjUT/q26s1cvxhzaMu9sknnxRD0GIR4hheGYthUVuOOuqoNHXq1PSzn/2s+Hny5MnpqquuSocddpjfZ40bP358S8d9/fXXt4SDUA3Uj9qnftQv9YNqpn7UPvWjfo2voeOPbnHJtUo3opGcffbZ6corr0ybbrppqxFHq622WrrjjjtSvYnFwIcPH55mzpyZnnvuubTJJpsUAUvM4zziiCNSLe/XCSecUExTmzRpUvra176Wbr311rq8MtkvfvGL4jZt2rT00ksvpS222KI443XBBRekfffdN9WTb3zjG+mBBx5IW2+9dauOe7vttiv+30IlqR/qR61RP9QPqoP6oX7UGvUjVeXxh/Coi8Xl3N9///157o+D8a9+9aup3kSw8sYbb8xz/9prr53WWGONVMs+++yzYohhTD8cOHBgqldvv/12cWtrvfXWa3c9r1oW4ViMKmsrfscbbbRRRdoEJerH/1M/aof6oX5QHdSP/6d+1A71I1Xl8YfwCAAAAICs6p1QBwAAAEDFCY8AAAAAyBIeAQAAAJAlPAIAAAAgS3gEAAAAQJbwCAAAAIAs4REAAAAAWcIjWEImT56cbrnllvToo4/6TAFQPwDoVI4/6Eo9unRrUIemTJmSzjnnnHT33XenuXPnpp122intsMMOlW4WAFVO/QBA/aBWGHkE83H77benp59+utV948aNS2PGjCn+PmPGjLTtttum8ePHp+22287nCYD6AcAic/xBNRIewXy8+eababfddktvvfVW8fPrr79e/Pzee+8VP6+xxhrphBNOSL179/ZZAqB+ALBYHH9Qjbo1Nzc3V7oRUM3iv8g+++xTjDC677770s4775xWX331NHr06Hmee9BBB6UePXqkm2++uSJtBaB6qB8AqB/UC2sewXx069Yt/fKXv0ybbbZZGjRoUPr73/+e7rzzTp8bAOoHAEuc4w+qkWlrsAD69euXTj311PTCCy+kCy64IK2yyio+NwDUDwA6heMPqo3wCBbA22+/na688sr05S9/OV1xxRVp+vTpPjcA1A8AOoXjD6qN8Ajmo6mpKR199NFpm222SU888USxptEZZ5zhcwNA/QBgiXP8QTWy5hHMx/Dhw4vpas8//3xxRbWRI0embbfdNu27775pyJAhxXNuueWWNHv27PTOO++k7t27pxtvvDH17NkzHXzwwT5fgAalfgCgflAvXG0NOjB16tR02mmnpcMPPzzttddeLfePGDEiPfXUU+mqq64qwqJjjz02zZw5s9Vr+/Tpk37+85/7fAEakPoBgPpBPREeAQAAAJBlzSMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkCU8AgAAACBLeAQAAABAlvAIAAAAgCzhEQAAAABZwiMAAAAAsoRHAAAAAGQJjwAAAADIEh4BAAAAkHL+Dwt10PJTV6mtAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "cmap = ListedColormap([\"#d95f76\", \"#f2a541\", \"#4c78a8\"])\n", + "predictions = [y_grid, univariate.predict(X_grid), bivariate.predict(X_grid)]\n", + "titles = [\"Target regions\", \"Univariate SGT\", \"Bivariate S²GT\"]\n", + "\n", + "fig, axes = plt.subplots(1, 3, figsize=(12, 3.7), constrained_layout=True)\n", + "for ax, values, title in zip(axes, predictions, titles):\n", + " ax.contourf(x1, x2, values.reshape(x1.shape), levels=[-0.5, 0.5, 1.5, 2.5], cmap=cmap)\n", + " ax.set(title=title, xlabel=\"x1\", ylabel=\"x2\", aspect=\"equal\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "comparison-explanation", + "metadata": {}, + "source": [ + "The univariate root can carve repeated intervals along only one axis, so it cannot distinguish the center from both arms of the cross. The bivariate root combines thresholds from both axes and routes all three regions directly." + ] + }, + { + "cell_type": "markdown", + "id": "heatmap-heading", + "metadata": {}, + "source": [ + "## Reading a bivariate node\n", + "\n", + "`plot_tree` renders an S$^2$GT node as its exact two-dimensional routing heatmap. Colors identify outer branches. Numeric ticks mark thresholds where the final branch changes; redundant inner-CART thresholds are omitted. Passing `X` adds marginal histograms, and missing-value margins appear only for axes with missing observations." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "plot-tree", + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-02T19:30:21.272063Z", + "iopub.status.busy": "2026-09-02T19:30:21.271941Z", + "iopub.status.idle": "2026-09-02T19:30:21.339614Z", + "shell.execute_reply": "2026-09-02T19:30:21.339060Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA/8AAAHiCAYAAAC+1Xw6AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlcelbwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAdNZJREFUeJzt/Qd83Nld7/+/p0gz0jT1Lqu4SLbXZb3VW0MKIYTAhUAuJOGXG/4kl9ASknBJ4NIv5YYQciFw4VIvNxAIISSkAOm7WW8v9q7XllzVbKuXmdFo+vwf54w0krzyrousMno9Hw8/bJ8Zab47Wlv+fD/v8zmOXC6XEwAAAAAAKFrO9b4AAAAAAABwc1H8AwAAAABQ5Cj+AQAAAAAochT/AAAAAAAUOYp/AAAAAACKHMU/AAAAAABFjuIfAAAAAIAiR/EPAAAAAECRo/gHAAAAAKDIUfwDAAAAAFDkKP4BAAAAAChyFP8AAAAAABQ5in8AAAAAAIocxT8AAAAAAEWO4h8AAAAAgCJH8Q8AAAAAQJGj+AcAAAAAoMhR/AMAAAAAUOQo/gEAAAAAKHIU/wAAAAAAFDmKfwAAAAAAihzFPwAAAAAARY7iHwAAAACAIkfxDwAAAABAkaP4BwAAAACgyFH8AwAAAABQ5Cj+AQAAAAAochT/AAAAAAAUOYp/AAAAAACKHMU/AAAAAABFjuIfAAAAAIAiR/EPAAAAAECRo/gHAAAAAKDIUfwDAAAAAFDkKP4BAAAAAChyFP8AAAAAABQ593pfAAAAN8unPvWpl6z9yI/8CG84AADYcij+AQDa6jcEruRm3Si40ZsSV/pvWOlz3KzXutrXBwAAG4Mjl8vl1vsiAAC4EddSoN6oKxW4a3kNxfDfwI0CAADWFp1/AACuwWYq8ov5vwEAAFwbBv4BAAAAAFDkKP4BAAAAAChyFP8AAAAAABQ5in8AAAAAAIocxT8AAAAAAEWO4h8AAAAAgCLnyOVyufW+CAAAAAAAcPO4eXMBAJtZNpvVxYsXFQgE5HA41vtycBVM3yESiaipqUlOJyFEAADWAsU/AGBTM4V/a2vrel/GpuJ2u+VyuZRIJF7xuT6fT5lMRvF4/CWPmcLd4/HYz2Nuwqzk5Z4zODiolpaWZWuf+tSntNp+5Ed+ZNU/JwAAmw3FPwBgUzMdf+PkFx5WwOfXRpDtatalmZcWy+upo8ylx/79G/r8I1/TRHhaJS6Xsrmc/vOr36h799224sd8/ttf1b98+6va075D/+2t71722L8e+bq+cOTr8nnLNRuP6Q13PagfePD1V/WcyGxUu9/0QOFrBwAAbj6KfwDAprYQ9TeFf9C/MYrJbDCoSLb0hj7H1MS47ZqHKqs0G40onU4rVFF53Z8vWO7ShfERvfct71RHY77b/tWnHtGffe7vtaO1XTtb2pc9/8T503r0xed0e/c+JVPJZe/t48ef078+8nX96o/9rPZ27FRP/1n96l9+XNtb2nT/gTte8TkHtnfb57BNAwCAtUPxDwDABvQv//A3SsTjis/NaXpqQnOxWTU2t+rd7/uwfPOFuLkpMDM99bKfp6q6Vt6yMvvrt7/+Py177HV33KdP/sfn9OL508uK/0hsVv/rn/5G7/2h/6KvPPWILf6X+spT39ahXXttUW90t23XbV379JUnv10o/l/uOQvF/1og8g8AQB7FPwAAG9Tpk8f1ng/8d+3cfYst/j/2mx/Wt77yJb3xB37YPn786NP6xr9/4WU/x5vf+k7t2rNvxcfGZ6Y0G59TTWh5ouCP//n/6YGDd2pPx05b/F/u7IUBfe99r1221t3WqX/42hev6TkAAGDtUPwDALBB7d530Bb+Rlm5T7v3H9KFgb7C43fd9x32x/XIZLP6k89+Uo1Vtbprz4HC+pce/aadCfDBt77ripP6o3MxBcp9y9bN7+PJhFLplNwu9ys8J621QNcfAIBFFP8AAGxQwYqqZb/3lHqUiM8Vfn+tsf+lBfz//pdPqu/SkH7zXT+nEneJXR+eHLPbAEzc/8LYsF2LxecUTybVP3xBjdV1Ki0pkdPhUDqTWfY5Fwp6p9Nl9/K/3HNcHO8HAMCao/gHAGCTup7Yvyn8//Rzf69neo7rN37859Rc27Bsr39dZbU+9bXFz2lSAOaIvo/9w1/qv73t3fb5ZpvAVGRm2euY31cFQ4XC/uWeYwYZroWVjg0kDQAA2Koo/gEA2KSuNfa/UPg/eeKYfv3H36fW+sZlj5uhf//rfb+ybO3jn/5rTUdm9Gv/v/cV1m7p7NKzp17U277z+wprz/Qet+vX8hwAALB2KP4BANgi/vKLn9a3nntcP/OD77A3AkyU3wj6/KoMhK7683z/g9+pD37id/R/Pv8p3bf/dj12/DkNjFzUT7/5/7um56wFOv0AAORR/AMArtqRI0f0pS99ye7pftOb3qS77777ZZ9vzqb/l3/5Fx09etR+zKFDh/R93/d9crlched84AMfUCKRWPZx5jmve93rtvRXprKqRuU+/7K1QKhCVbV11/05TbHfUFWrf/rGl5etP3jrXfqBB1+/4seY+H7p/EyABU019fof7/6A/vlb/66//NI/qb6yWr/5rvdrW33TVT0nHI1oMxsbG9Nf//Vf6y/+4i908eJF9ff3q7q6el2v6d/+7d/0iU98Qs8995xCoZC++7u/W7/yK79ifw0AgOHImVv/AAC8go9//OP6xV/8Rb3nPe9RJpPRn/7pn9q1n/iJn7hi4f+a17xGAwMD+rEf+zG7b9wUS93d3fqP//iPwr5vv9+vt771rdq/f3/hY++55x57o+BqhMNhW+AMfeNZBf2BDfF1zO5u0YXpuDaS7eUuJc+OaCMwxX/Lqw9pZmZGwWDwFffpb7TO/1ve8ha1tbWpvb1dP/3TP21vBtTU1Gi9PP/88/rQhz6kn/mZn9HBgwc1ODho/8w1Njbqq1/96rpdFwBgY1mbiTsAgE1tfHxcH/7wh22x//u///v259/+7d/Wz//8z9vieyWmA/nwww/rM5/5jH75l39Zv/qrv2oLu6997Ws6ceLEsueaLqUpohZ+XG3hD7ySa72Z8JWvfEWVlZV6+umnC2v/5//8HzU1Ndmi2vj0pz+t3/u939P27duv+vOahIBJtExMTCxbTyaT9ubXM888c93XZG6cffnLX9Yb3vAGW/Dfeeed+p3f+R37Z210dPSa/vsBAMWL4h8A8IpM8ZFKpfTDP/zDhbUf/dEfVTQa1de//vUVP6a2ttbG+013d8H09LRKS0tfEpE2Nwg++MEP2tjy0NAQXxGsm+/8zu+0/5+/7W1v0+zsrHp6evRzP/dztphubW297s9r/p8321te+9rXanJyslD4/+AP/qDt3G/btm1Vr8nclDNbbbxe73VfMwCguFD8AwBe0enTp23xsjSibYp7E9k3j63ERKL/7u/+Tu9973v19re/3XY3zbYB0zU13ckFgUBA5eXl9vN/8Ytf1O7du/Xv//7vV7wWU0CZwmbpD2A1Y/8f+9jH7I2rn/qpn7L/337P93yP3vGOd9zQm+zxePS5z33Obg8w8yxMR95sHzhz5oy9gWb+PK3WNZk/E7/+67+uH/iBH3jJtgoAwNbFwD8AKBJmhMulS5dsYW06fkuZTqPpwHd0dFzX556bm7NF+uVMYRGLxa54Pd/61rfs4zt37rR7/h977DG7ZuLPC5588slC99JsLXj3u9+td77znbpw4cKK58GbbqcpbC7naquVK7AxCh2XI6Pm0OJQw40gl3PJ1X79wwJXkyuysbvRZWVl+vu//3vdfvvt9s/TN77xjVX5vKYL//nPf15vfOMb7ZaB5uZm++ehvr5+1a7J3Bx785vfbP/s/Nmf/dmqXDcAoDhQ/ANAETBDvUw30OzNN53F//k//6cd+LXA7Ac23fRPfvKT1/X5TYffRPYvNzU1dcXO4mc/+1m7L/ncuXN2OJphYstdXV02xmz2JxuXx5ZNSuDP//zPdfbsWXvT4HLmBsH73//+ZV1O8zkGEln5S7PaCHaVJeWZXDkRsV6i1Qd1IbEx3p/oTbqO1Rzu98ILL9gbVubmlYnaV1RUrMrnNdteqqqqCjfUTFG/WtdkCn/T7TdDNs1NhfU+gQAAsLEQ+weATc4UAaaoNnuCzRR9EwX+r//1v+qXfumXVu019u7da9MDS4eHmePNTAGzZ8+eFT/m1KlTtvhYKPyNXbt22RsJvb29V3ythWP/zGkBV4pPmxsOS38Aq6mvr88OnjQzKA4cOGD/TK3G4UjmlAzzZ9Xs8TdDL03h/13f9V2KRCI3fE1mfoDp+JttON/85jeXba0BAMCg+AeATe6pp55SS0uL/uiP/sh21D/60Y/aLr/5vTn+azWYPcqmy/i///f/Lqz98R//serq6vSqV72qsGaGkC0cLbZv3z57BNrSKeZHjhyxQwLNY4YpgsxWhQVmqOD/+l//y84LMAkB4Fon+6/041oLdDPM8oEHHtBP/uRP6m//9m/tyRVmz/2NMB17U7Cbif2mODc3wkwix+zjNzcAzJ+L670m8+fmh37oh2zhbzr+5hQAAAAuR+wfADY5s5f/8knhr3nNa+yE/te//vX297fccssNvYbp1v/FX/yFjeSbPfqmGPn2t79th/ctnSZubg6Y/cvmZoEZSGYKlgcffFBvetObbJfyC1/4gn78x3/cXt9Ct9I818T2zceZmwPmef/wD/+w4n5/4GYzR1iaItrcmDLMjTWzd978v28m9Zuuu0nVmJtU5s+BYW5WGWZbzX/6T/9pxc9rjuRb6Mqbz7nw5+rf/u3f7AyMo0eP6r777ruuazJbcv71X//VbikwNxWWMq93xx13rOI7BADYrBy51cixAQDWjYkDv/rVr7Z76y/3xBNP2BsApjgxNwCud8//AnMMn5lMbgYKmqL98mjxn/7pn9ozxg8dOlRYMwWL+WE+5uDBg3YLweUx/4cffth+bnOd9957ry1irpbZ8x8KhfTEiX75N8jAv10VaTkmNuCe/+n8lor1Fo2EddeeNnvj6vJtG9faqV/tWQBmP73pxpvtJZdvrykpKbH/b5qbVubH5UyM33zsanulazKPmS04KzEnaXAjDQBgUPwDQBEwXfb3vOc9dor4StsCzHYA89iNFv8bEcX/1dlqxf9qDv8DAKAYEPsHgCLw13/911fs/JnI77PPPnvFI/kAAABQ/NhQCQBFoLa21saRF/YgX87E6VeKKQPF6mZsHwAAYDOj+AeAIvG1r33NTt43MwCWGh4e1hve8IYbnlYOAACAzYviHwCKhDkuzAy+M0P1/u7v/s6uff7zn7fH6pktAb/1W791w69hjhQz0/7NHIF0On1VH2P2dZvnmyPOzP78y5mjycyNi6U/+vv7b/hages56g8AgGLFnn8AKBI1NTX64he/qE984hP2OD3T6X/hhRf0a7/2a/rQhz50wxO/TdH//d///fZoP3NmudliYG4u3HrrrVf8mI985CP69V//dXV3d9uPMceVmZsQ733ve5cNK9y+fbvdurDAXH9bW9sNXS8AAAAWUfwDQJG555571NTUZM8NN2d+/9AP/dANF/5mXsBb3vIWffd3f7f+/M//3K697W1v03/+z/9ZJ0+eXPF4szNnzugXfuEXbArhrW99q10zH/vud79bb37zmwtnnRvmBsGVzkcHbtRC958TAAAAWxmxfwAoErlcTh/96Ed1+PBhveY1r7HReRP5N535hYL9en3zm9+0n+8Xf/EXC2vm16aTf+TIkRU/Zmpqyv68NBlw++2325+np6eXPXdwcFCPPPKIhoaGbug6AQAAsDI6/wBQJD796U/rd3/3d/WP//iPhS66Wfubv/kb/ezP/qwdBHi9+/5NiqCiokIdHR2Ftb1798rj8djHHnjggRWPGHz7299uO/0m5m9i/7//+79vf3/LLbcse665LhPzP3HihL1B8Ld/+7dqbW1d8VoSiYT9sWClOQIAAABYjuIfAIqE2Vf//PPP28j/Uv/lv/wXW5x/+ctfvu7Pbbr4VVVVL1k3awsd/pWYuP9P//RP2+LeFP/xeNxuH1jqT//0T+0WAofDobGxMbu1wPz+4YcfXvFz/s7v/I7dJnC5hvISBX0l2gjSDqcclTu10b7hN22Q9yecWfvrIPIPANjqKP4BoEgcOHDgio91dnbaIvxqmMn8ZkK/YTr7999/v0pLS+2JAZeLxWL2sZU8++yzdpjfZz7zGTsocGHvtTmVwNyk2L17t10z6YAFZujfr/zKr+h7v/d7NTIyovr6+pd83g9/+MN6//vfv6zzb1ICrnPDcvlntREkdrfoQjirjWR7eVrZsyPaCFzRyHpfAgAAWw7FPwBgmf/3//6fjd8vdPZN8d/e3q7x8XEbtzc3BIxIJGILb/PYSr7yla+osrKyUPgvdF/f9a532ccWiv+VTi0wLl26tGLxb15/4RqAK6HTDwDAchT/AIBl/vAP//Al74gZIGiO9vvSl76kH/iBH7Brn/vc5+yU/+/4ju9YNhjQpAzM/n1TuJubA2ZbgLkJYJhuvkkLLBT15vFgMLjstcz2hLKyMu3cubFi8wAAAJsZxT8A4BWZYv5nfuZn9BM/8ROamJiwNwJM/P4DH/iAGhoaCs97wxveoF/7tV/Thz70IXuT4Dd+4zfsHv73ve999jQCM/Bvx44ddjuAYRIAZs+/OY7Q3BD41re+pT/5kz/RRz7yEfl8Pr4yAAAAq4TiHwBwVT72sY/ZKf2m+2+G8/3BH/yB3vGOdyx7zqtf/erCNoBQKKRnnnlGn/jEJ/RP//RP9mPMKQQ/9VM/Jb/fb5/zgz/4g2pubtYnP/lJe8yf+dhHH320cCQgAAAAVocjZ1oxAABsUmbrgLnRMPSNZxX0B7QRZM3Av+m4NpLt5S4lN8jAv3A0opZXH7KDJS/f9mGGQq429v8DACA5eRMAAAAAAChuFP8AAAAAABQ59vwDAICiRNwfAIBFdP4BAAAAAChyFP8AAAAAABQ5in8AAAAAAIocxT8AAChK5tjAm3F0IAAAmxED/wAAm1oul7M/R2aj2iiy4bCikbg2knDapWQ0oo1g4Wu18LUDAAA3nyPHd14AwCY2NDSk1tbW9b4MXIfBwUG1tLTw3gEAsAYo/gEAm1o2m9XFixcVCATkcDjW+3LU29urO++8U08++aS6urrW5DXNffxE/7giT59RLpmWo9St6jfdIuWkb5yOqu/caX3g/3uTvvTVh3XfnQfW5Jpe6XojkYiamprkdLIDEQCAtUDsHwCwqZnicSN1j/1+f+HnYDB4018vG08q/Php5frH5Hd5pDKPSuvc8p37kqn9VVv5PRop99nn9s+kVT+e0i2tFfKUuLSeQqHQur4+AABbDcU/AACbVLx/TOHHTykXTxXWPG218u/MKTeUk8lB3F4Z1nDAW3j80vScJqIJ7WutUGNl+TpdOQAAWGsU/wAAbDK22//EaSX6xgprDo9bwbt3ydtep+zoSWXm1z2unPa25Lvs7vmIfTKd1TPnJ9U0PWdTAKXu9U0BAACAm4/iHwCAVVRTU6O2tjb7880QHxhT5LFTyi7t9m+rUeDuXXKVla74MbW1tfaa3nBXly7NuTUykz+J4OLUnCYiCe3bVqmGirKbcr0AAGBjYOAfAACbQDaRUuTJ04qfGy2smcF+gbt2yttRt2zYoe38n3/Y/trV8YCcdbuXDdszRf/xwSmlMotH7TVXlWtvi0kBMIAPAIBiROcfAIANLjE4rrDp9s8lC2ulLdUKHt4lV7nnmj6XuUlgCv3qgEfP909pNJxPAVyYjGk8Etf+bZWqD5ECAACg2FD8AwCwQWWTptt/RvGzI8u7/XfukLez/oaONvSWuHTH9mpb9B8fmlY6k1MildVTZyfUMp8CKCEFAABA0aD4BwBgA0oMTSj8aO/ybn9zlYKHu+TyXVu3/0rMzYOWap+qA149PzClsfkUwJBNASS0f1uF6kgBAABQFCj+AQDYQLLJtCJPn1H89HBhzVHiUuCOHfLuaLihbv+VlJW6dOf2ag1OxHTCpACyOcVTGT15dkKt1eXaY1IALmYBAACwmfGdHACAq5TJZPTzP//zN+39Slyc1MS/PrWs8C9tqlT1992hsp2NKxb+2WxWv/3bv33Dr20+97Yanx7cU6+awGKywNwQeOjESCEVAAAANieKfwAArsIv/MIvqLS0VH/wB3+g6enpVX3Psqm0wo/1avqrzys7m1js9h/epYrX7pfL513x4/74j/9YJSUl+u///b9rdHTxFIAbUVbq1l07arRvW4VczvzNBpMCeOLMuN0akM5kV+V1AADA2qL4BwDgZXzhC1+Q3+/XRz7yEb3mNa9RKpVSRUXFqr1niUtTmvj8U5o7damwVtpYoervvUPlu5pW7PYfO3ZMO3fu1M/+7M8qEAjYtX/4h39YtWsyr9lW49eDu+vtqQALBsZn9dDJEXsqAAAA2FwcOXPgLwAAWDFS73K57I9z585p27Ztq9rtjz5zTnO9Fxe/Kbud8t++XWVXKPoXlJeX28d///d/X88++6xNJPz93/+9Tp8+rerqamVHTypz/mH7XFfHA3LW7b7u6zT/TOgfn9XJCzPKZBf/ydBW69PuppDczAIAAGBToPMPAMCVvkk6nXr9619v9/o/8sgj+v7v/357I8AU3uaHeex6JIenNfmvTy8r/Esa5rv9Xc2vONTv537u5xSLxfTEE0+op6dH//f//l9NTU2prq5OP/7jP76qX09zLe21fj2wu15V/tLCev/YrB4+OaKJSH6bAgAA2Njo/AMAMO+hhx7SJz7xCXV3d+s3f/M3C51vs6/e3AAwXvva12r37t3667/+a0WjUXtD4LOf/exVvYe5VEaR585p7uSFxUW3U4FDnSrrXrnoNzcd/vAP/9DG/P/H//gfhefU1NRocnLS/t4U/ObXn/nMZ+xjXTs6dOxvf97evLjRzv+y68/l1DeWTwFklwQHzc2B7qYgKQAAADYwin8AwJY3MTGhPXv2LBua197ervPnz9tf/9Zv/ZYdqvfoo4/q8OHDhULY4/EonU7b7QGvJDkyrfCRHmWW7JcvqQspeG+X3MHylzzfdPLNTYaRkRFb4JvXa21t1cDAQGEWwfd+7/fqox/9qD7wgQ/YdfN8kwgwRf/ckT+2H7eaxf+C2Xhax/onNTmbLKyVe9w62FapKv/ijAAAALBxEPsHAGx5LS0tdoK/GaRniuyOjg719fUV3pdf+qVfsvvrFwp/wxTWDz74oH3+mTNnrvge5tIZRZ46o6l/P7pY+Luc8t+xXZXfdXDFwn/hmkw3/+mnn7Y3F0znf3BwsPD4m970Jm3fvl233nqrPerv0KFDNoVgmOefPL84QHC1+bxuHd5Vqz0tIc0fCKBYIq1HT43pxND0stkAAABgY6D4BwBsaX/1V3+leDyuP/mTP9H+/fvt2h133GH39i/1/ve//yUfe+lSvsA2NwtWkhyd0cQXnlbsxFBhraQ2qOo33S7fntYr7u03w/tMB/9jH/uYbrvtNrt211132Y7+UqbI/6Ef+iE9+eST9scnP/lJNTc328eaa1fvRIKVmGvvrAvYWQCVvsVZAOdGo3YWwFSUWQAAAGwkFP8AgC2tqanJ/mwKbeM3fuM39OlPf9oWt6FQyP5+Jb/6q7+qF198UXffffdLbhTkMhlFnj6rqX9/TpnwXH7R6ZD/tk5VftetcodW7vYvaGxstD//0R/9kf3ZdPZNYb9wTb/yK79i181cAnOtn/vc59TZ2al//Md/tDckKkJBTUZiWgt+b4nu2VWr3c2LKYDZRFpHSAEAALChsOcfALDl7du3T8ePHy+8D21tbTZG/8UvflGpVEp/9md/pne/+9325w9/+MOanZ1VMpnUq1/9an39619f9v6lxsKaMXv7ZxaLb3dNQKH7uuUO+a76vT5w4ICef/75wu/NMYMmBbBwTebGwE//9E/rW9/6lv15fHxcY2Nj+tEf/VH91Ud+YdWO+rsWkbmUnQUwHUsV1vxetw60VS1LBwAAgLVH8Q8AwPwAP7fbrTe+8Y3613/9V/uemGF7DQ0NdrK+KazNcL/3ve99qq+v13/7b//NDvxbkMtkFT3Wp9jxAWlhy7vp9h/sUPneFjkui+xfDRPrNycNfOd3fqf+7d/+za6ZIr+2tlaVlZV2JoBhTgMwiYAf/uEftteUHT25LsW/veZcTudGIjp1KaylW/931Ae0szEo10I8AAAArCli/wCAomUKdFMU+/1+/eAP/uDLPtcc8WeK7Z/8yZ8srJki39wQSCTy+9fNr83zfvmXf3lZ4Z8aD2vii08r9sJi4e+u9qv6e26Xb9+2ZYX/L/7iL6qiosJe08KAviv5y7/8S3tN73rXuwpr5kaEuSFgkgcLfvZnf1bveMc7ll3TenE6HNrRENR93fUKlZcU1s+MRPRIz4imY4vXDQAA1g7FPwCgKJnY/u/93u/Zzn1paan++Z//WdXV1Vd8/sGDB+3PP/MzP7NsX7/p9r/97W9f8WNst/+585r88rPKTM/H/J0O+W7tUNV3H5K7cnnM/84779Tv/M7v2JsKplA3e/VNB/9KRwUuDCD84Ac/WFgzMwhM7N90+TeyYFmJ7u2qU1djUAtzDSPxtI70jKr34oyynAgAAMCaIvYPACg6PT099sz797znPXaK/8LReGa/vNnfv3Qv/eXD/8zAPK/Xa39vTgF4zWteo6997WsveW5qIqLwkR6lp2YLa+4qv4L3dqukyv+S5589e1Y7duzQj/3Yj9mOvmE6/+YGwJ49e+zwwCsd+XfhwgV7s8BM+5+bm7NHDJq9/leynrH/lYRjSR3tn1J4bnEWQKCsRAfbKhUqZxYAAABrgc4/AKDofOYzn7E/f8/3fE9h7Qtf+IKNzL/wwgv60pe+tOLHmSL7da97ncrLy22h/uijj76k8M9ls4oe7dPkl55dLPwdDvkOtqvqjYdWLPyNf/mXf7E/v+ENb1i2VldXpxMnTuizn/3sih83ODio7/qu75LP57MT/R955JGXLfw3omB5qe7rrtMukwJYMhzwkZ7R+dkAS4YDAACAm4LiHwBQdO6//37785//+Z8vW18omt/5zncW1r785S/bo/o+9KEP2aP0vvKVr2hiYsLeJDh8+PCyj09NRm3RP3usz0wItGsm2m+Kfv+B9pcd6nfffffZn//mb/5m2frDD+c79OY0gQXmGsw1meGC5prMsD9zTeZEgnvvvVebkZkFYIp/cxMg4M3PAjDvoCn+zU2ApakAAACw+oj9AwCKkimeTUze7I9fqrGxUcPDw3a6vzEzM2MH8JlYvYn5r8R0+2ePD+aL/oW96g7Jt69Nvv1tcricV31Nppg3cwSWam5u1sWLF+26eU4kElEwGHzJYL+rtdFi/5cz+/1PD4d1ZjhSOBjBzAXY1RDU9oaAvVEAAABWF51/AEBR+r7v+z5bTJs99ku9/vWvtz+fOnXK/mxOAzBzAU6ePLni5zHRfjPQb/a584XC31VRrqo33ib/rR1XXfgb5sSBTCajt73tbcvWzfGCxsIsgkAgoD/7sz+74hyAzc7pdKirKWQHAvq9brtm7sX0XgrrSO+o3RIAAABWF51/AEDRMlP+Tef/m9/8pl71qlcVCnAz+d90+V/uaDzT7Y+9OKTo0fPLuv3lt2zLR/yvoegvfM5czg4TNN38r371q3rta19r19/61rfqU5/6lO34myMAb9RG7/wvlcnmbPT/7EiksOY0KYDGoDrrSQEAALBaKP4BAEXnd3/3d/XqV7/aFv4Le+3f8pa32ML6r/7qr7Rr1y719vZe8ePT07OaMZP8xxcLUleoXCEzyb82eEPX9sQTT+juu++2v37zm99stxyY6f/bt2/XmTNntBo2U/G/YGo2oWN9U4omFrdEVJSX6mB7pfzzMwIAAMD1o/gHABSNI0eO2An/09PTeu9736uPf/zjevzxx+2EfbNm9tubY/JMEmAluWxOsRODij53Wbd/T6v8t5puv2tVrvPpp5+2pwosXJO5QbEw+G+rFv8LKYDeizM6NxpdlgIwWwQ66/z2vQIAANeH4h8AUBRGR0dVX19vj/P7+te/rv3791/Tx6dnYgof6VFqLFxYcwXLFLy3W6V1IW0mm7X4XzAZTehY/5Rml6QAKn2lOtBGCgAAgOtF8Q8AKBpmYN61Fv1mH37s5JCiz543refCevmelvxAP/fqdPvX0mYv/o1MNquei2GdX5YCcKi7OaiOWlIAAABcK4p/AMCWlQ6bbn+vUqMzhTVXwJvv9tdXaLMqhuJ/wYRNAUwqlsgU1qr8pTqwrUq++ZMCAADAK+O7JgBgyzHd/rmeC4o8c25Zt79sd7MCt3bKUbL5uv3Fqtrv0QPd9TYF0DeWTwFMRpN6uGdE3U0htdf6mAUAAMBVoPgHAGwp6chcfm//yJJuv3++29+webv9xcztcuqW1go1VHj1fP+UYsmMHQ744tC0hqfn7CyAcg//pAEA4OXwnRIAsHW6/b0XFX3mrHLpJd3+rib5b+uUs4RviRtdTcCrB3bX6+SFGfWPzxa2BTx0ckR7mkPaVkMKAACAK+FfOgCAopeJxhV+tEfJS9OFNafPY7v9nsbKdb02XHsKYN+2SjVWlOnYwJTm5lMALwxO69L0nPZvIwUAAMBKKP4BAMXd7T99SdGnTLd/cWBc2a5G+W/bLmcp3wY3q5rgYgpgYD4FMB5J6OGTI9rdEtK2alIAAAAsxb96AABFKTNruv29Sl6cWt7tP9wlT3PVul4bVkeJy2k7/TYF0D+leCqjtEkBDORnAZjHyrjBAwCARfEPACi6bn/8zLAiT51RLrXY7ffubFDg9h10+4tQbdCrB/fU68TQtAYnYnZtLJzQQydGtLe1Qi1V5ZwIAADY8ij+AQBFIzObUPixXiUvTBbWnGWlCt7TJU9L9bpeG25+CuBAW5UaKsr0/MCUEqmsTQGYRMClqTk7J6CslCMcAQBbF8U/AKA4uv3nRhR54vTybv/2egXu2CGnp2Rdrw9rpz5Upgd3e2wKYGgynwIYDcf18Mlh7W2pUDMpAADAFkXxDwDY1DIx0+0/peTQxPJu/+Fd8rTWrOu1YX2Uup062F5lZwHYFEA6q1Qmp6MmBTCdTwF4S0gBAAC2Fud6XwAAANc9yf/ciCY+/9Sywt/bWa/q77ujqAr/kydP6r3vfa/27dunP/qjP1rvy9k06ivK9OCeBjVXlhfWRmbidhbAhcmY/X8IAICtguIfALDpZOaSmvnWiwp/+6RyybRdc3pLFPqOvQrdv7uoYv7/8R//oTe/+c3q6OhQJBLRyMjIel/SpksB3NpRpds7q+2vjVQmq+f6JvXMuQkllmwTAQCgmFH8AwA2lfj5UU18/kklBsYLa572Otvt926r1WaSTCb1hje8Qb/5m79ZWJubm9NrX/taffSjH7W/f/DBB3XixAm9733vk9frXcer3dzMIMBX7alXU2VZYW14Jq5vnRjRxan8bAAAAIoZe/4BAJtCNp5U+PHTSvSPFdYcnhIF794pb3udNqPS0lL9wi/8gl73utfp/vvv16te9Sq9//3v18DAgN7znvfY51Dwr+L77XbpUEe1GipiOj44raSdBZDVs+cn7YkAt7RWyMMsAABAkaL4BwBsePH+MYUfP6VcPFVY87TVKHjXLjvcbzMzBf8HP/hB/eiP/qhNAPzlX/6ljhw5Ip/Pt96XVrSaKstV7ffohcFpDU/P2TUzCHAimtC+1ko1LkkHAABQLIj9AwA2rGw8pemHTtj9/QuFv8PjVuiBPQo9uHfTF/4LfuM3fkMNDQ165zvfqV/7tV/THXfcsd6XVPRMh/+2jiodaq9SiSv/zyGTBHjm/ISePT+hZJpZAACA4kLnHwCwIcUHxhV5rNfeAFhgJvgHDu+Uq8yjYmKmzqfT+cGF9fX16305W4bD4VBTVbmqAh69MDBlTwIwLk7NaSKSsEcCmlkBAAAUAzr/AIANJZtIaebbJzXzzeOFwt9R6lbw/t12mn+xFf7GL/3SL2lqakof+chH7GC/M2fOrPclbSneEpc9DeCgTQE47FoindXT5ybsqQAmEQAAwGZH5x8AsGEkBscVfuyUsnPJwlppS7WCh3fJVV58Rb/xjW98Qx//+Mf1zW9+U/fdd5/d7//2t79djzzyiNxuvk2vZQqgpapcNX6Pnh+Y0mg4nwK4MBnTeCSu/dsqVR8iBQAA2LwcOZM1BABgHWWTKUWePKP42cUz7B0lLgXu2ilvZ70tzIqR6fbv27dP73jHO/Rbv/Vbdm18fFz79+/Xu971Lv36r/+6RkZG9JrXvMY+dvr0aYVCIdXV1enQoUP627/92xU/b3b0pDLnH7a/dnU8IGfd7jX8r9r8zD+NTNF/fGha6cziP5PMzYG9LRUqcROcBABsPhT/AIB1lRiaUNjs7Y8t6fY3Vyl4uEsuX3F2+xdMT09raGhI3d3dy7r8w8PD9jGzbmYB9PT0vORjzWkAHR0dK35eiv/VMZfM2BTA2HwKYGGLgEkB1IW8q/QqAACsDYp/AMC6yCbTijx9RvHTw8u7/XfskHdHQ9F2+9cCxf/qpgAGJ2I6YVIA2cUUQGu1T3taQoWTAgAA2OjYTAgAWHOJi5MKP9qr7GyisFbaWKngPV1y+emoYuMwN6G21fhUG/ToWP+UxiP5/2cHJ2ZtIuBAW6Vqg/w/CwDY+Cj+AQBrJptKK/r0Wc2dulRYc7hd8t+xXWU7G+n2Y8MqK3Xrrh01GpiY1YmhGWWyOcVTGT1xZtzeHNjTHJKbFAAAYAOj+AcArInkpSnNHOlZ3u1vqFDwXtPtZ4o6NkcKoK3Gr9qAV8cGpjQxnwIYGF9MAdQESAEAADYmin8AwM3v9j9zTnO9FwtrDrdT/tu2q6yriW4/Np1yj1t376hR//isTl7IpwDMcMDHT4+rrdan3U2kAAAAGw/FPwDgpkkOTyt8pEeZ6OK09JL6kIL3dssdoNuPzZ0CaK/12/3+x/onNRnNn1bRPzarsRmTAqhSdaC4T6sAAGwuFP8AgFWXS2UUee6c5k5eWFx0ORW4rVNl3c10+1E0fB63Du+sVd9YPgWQzeUUS2b02OkxddT61d0clMvJiQAAgPVH8Q8AWFXJEdPt71UmMldYK6kz3f4uuYPlvNsoyhRAR51fdUGvjvZPamo2nwI4PxbVSDiug22VqvKTAgAArC+KfwDAqsilM4o+d16xE0OLiy6n/Ic6VN7dIofTwTuNoubzunXPrlqdH42q56JJAUixRFqPnhpTZ51fXU0hufhzAABYJxT/AIAblhybUfiRHmXCS7r9tcH83v4Q3X5srRRAZ31AdSGTApjS9HwK4NxoVCMzcR1sr1SljxQAAGDtUfwDAK5bLpNR9GifYi8OSrn5RadD/ls7VL6nlW4/tiy/t0T37qq1RX/vfApgNpHWkd4xba/3a1cjKQAAwNqi+AcAXJfUWFgzZpL/TGzxm0pNQCHT7a/w8a5iyzMpgO0mBTA/C2AmlrLvydmR+RRAW5UqfKVb/n0CAKwNin8AwDXJZbKKHutT7PjA8m7/wXaV7zXdfiabA0sFykp0b1edzo1EdOpS2KYAonGTAhi1Nwd2NpoTAZiJAQC4uSj+AQBXLTURsXv709Ozi99Iqv35bn+ln3cSuAKnw6EdDUHVhcp0bD4FYO6dnRmJaGRmTgfbqxQqJwUAALh5KP4BAFfV7Z99vl+zL/Qv6/b79rfJt28b3X7gKgXnUwBnhyM6NRxWLidF4mk90jOqHQ0B7WwIykkKAABwE1D8AwBeVmpyvts/taTbX+W3k/xLquj2A9eTAjBR//r5EwHCc/kUwOnhyPwsgEoFSQEAAFYZxT8AYEW5bFazLwxo9pjp9s+3+x3z3f79dPuBG2UK/Pu66nR6OKwzwxF7A8DcCPh2z6i9OWCSAOZGAQAAq4HiHwDwEqmpaL7bPxld/IZR6ct3+6sDvGPAKjER/66mkBoqynS0b0qReD4FYAYDjkzP6UB7ld0qAADAjaL4BwAs7/YfH9TssT7ZkeSGQ/LtM93+NjlcTPIHbgYz7O/+7uUpgBmbAhjRrsagPRWAFAAA4EZQ/AMALLOnf+ZIj9ITkcI74gqVK3Rft0pqgrxLwBqlAOpDZTraP2mPAzQ7bnovhjU8PaeDbVX22EAAAK4HxT8AbHGm2x97cUjRo+eXdfvL926T/2A73X5gjVX4TAqg3kb/z47kb8aZowHzKYCQttf75WAWAADgGlH8A8AWlp6ZtXv7U+OXdfvN3v5auv3AenE5HdrdbGYBeHWsb0rRRNrem+u5OKPhGZMCqJTfSwoAAHD1KP4BYAvKZXOKnRxS9Nlzi91+mW5/a77b73at6/UByKv0eXT/7nr1XpzRudH8AM7p2aQePjlitwh01pECAABcHYp/ANhi0jMxhY/0KDUWLqy5gmV2kn9pXWhdrw3AyimAPS0V9kSAY/1Tmp1PAZy8MGNnARwgBQAAuAoU/wCwReRyOc2dvKCI6fZnsoX18j0t8t/aQbcf2OCq/B49sLtOPRfDOj+fApiyKYBRdTcH1VFLCgAAcGUU/wCwBaTDc/lu/+hMYc0V8Oa7/fUV63ptAK6ey+nU3kIKYFKxREbZXE4nhhZSAFXyefjnHQDgpfjuAADF3u3vme/2pxe7/WXdzQoc6pSjhL39wGZUbVIA3fU2BdA3lk8BTEbzswDMoMC2Gh8nAgAAlqH4B4AilYnMaebRXqWGpwtrTr9XoXu7VNpQua7XBuDGuV1O3dJqUgBeOwtgLplRJpvT8cFpXZrKzwIoJwUAAJhH8Q8AxdjtP3VR0afPKre029/VJP9tnXKW8Fc/UExqAl49uLveDgDsH5+1axPRhB46OaI9zSFtIwUAAKD4B4DikonGFX60V8lLU4U1p89j9/Z7Gun2A8WcAti3rVKNZhbAwGIK4AWTApg/EaCslBt/ALCV8V0AAIql23/6Ur7bn8oU1st2Ncp/23Y5+Uc/sCXUBL16YD4FMDCfAhiPJPTQiRF7XGBrdTmzAABgi6L4B4BNLjM73+2/uKTbX+5R8J4ueZqr1vXaAKy9EpdT+7dV2hMBnu+fUjyVUTqb0/MDU7o0HbOPkQIAgK2H4h8ANnG3P35mWJGnzizr9nt3NChwxw66/cAWVxf06sE99ToxNK3BiZhdGwvnZwGY4wJbqkgBAMBWQvEPAJtQJpbId/svTBbWnGWl+W5/S/W6XhuAjZUCONBWlU8BDEwpkcoqncnZ0wHMiQAmBeAt5chPANgKKP4BYLN1+8+NKPLkGeWS6cK6d3t9vtvvKVnX6wOwMdWHyvTgbo9NAQxN5lMAo+G4Hjo5bFMAzaQAAKDoUfwDwCaRmUso8tgpJQYnlnf7D++Sp7VmXa8NwMZX6nbqYHs+BfCCSQGks0plcjpqUgDTc/a0AG8JKQAAKFYU/wCwGbr950cVefK0cokl3f7OOgXu3Em3H8A1McV/lb9UxwendXFqzq6NzMQ1eWJEt7RWqKmyjBMBAKAIUfwDwAaWnUsq/PgpJQbGC2tOb4kCd++St612Xa8NwOZV6nbpUEe1GivnbAogaVMAWT3XN6lL02Xa11ohDykAACgqFP8AsEHF+0YVftx0+1OFNU97rYJ37ZTTW7qu1wagODRWlKn6shTA8PScJqOJ+RRA+XpfIgBglVD8A8AGk42bbv9pJfrHCmsOT4mCd++Ut71uXa8NQPGmABoqYvYmgEkBmB/Pnp+0JwLs21ZhnwMA2Nwo/gFgA4n3j9mYfy6+pNvfVqPgXbvscD8AuFlMl7/a79ELg9O2+2+YQYAT0YQdBmhSAgCAzYviHwA2gGw8ZQf6mcF+Cxwet434e9rrGL4FYE2Yff63dVTZLQAmBWDmAJgUwDPnJuwgQLMVgBQAAGxOFP8AsM7iA+OKPNZrbwAs8LRWK3B4l1xlnnW9NgBbj8PhUHNVuaoDHjsM0JwEYJgbAhORfArAnBgAANhcKP4BYJ1kE6bbf0bxcyOFNUepW4E7d8jbWU+3H8C68pa4dHtntS5MxvTikEkB5JRIZ/X0uQl7c2Bvi0kBOPkqAcAmQfEPAOsgMTSh8KO99ii/BaUt1Qqabn853X4AGycF0FLtU03Aq+cHpjQazqcAzA2B8Uhc+7dVqj5ECgAANgOKfwBYQ9lkSpGnzip+Zriw5ihxKXDnTnm30+0HsDF5S126Y3u1huZTAGmTAkhl9dTZCbVWl2tPS4VKXKQAAGAjo/gHgDWSuDDf7Y8t6fY3V+W7/T4vXwcAGz4F0GpTAB6bAhgLJ+z64ETM/np/W6XqgvxdBgAbFcU/ANxk2WRa0afPau70peXd/jt2yLujgb39ADaVslK37txeY4v+EyYFkM0pnsroyTPj2lbt0+6WECkAANiAKP4B4CZKXJzMd/tn8x0yo7SxUsF7uuTy0yEDsHlTANtqfKoNenSsf0rjkfzfcQMTsxqbnwVQSwoAADYUin8AuAmyKdPtP6e5UxcLaw63S/7bt6tsVyPdfgBFkwK4a0eNLfpPDM0ok81pLpnRE2fG1Vbj0+7mkNzMAgCADYHiHwBWWfLSlGZMtz+an4ptlDRUKHSv6fYzFRtA8aUA2mr8qg14bQpgIppPAfSPz9rTAQ60VdrTAgAA64viHwBWs9v/zDnN9S52++V2KnDbdpV1NdHtB1DUyj1u3b2zxhb9Jy8spgAePz2u9lqfuptIAQDAeqL4B4BVkByeVvhIjzJLu/31IQXv7ZY7QLcfwNZJAbTX+u1+/2P9k5qM5k836Rub1ehMXAfaq1Tt96z3ZQLAlkTxDwA3IJfOKPrsOcVOXlhcdJluf6fKupvp9gPYknwetw7vrFXfWFQnL4SVzeUUS2b02KkxddT51d0UlMvpXO/LBIAtheIfAK5TcnQm3+0PzxXWSuqC+W5/sJz3FYC2egqgoy4wnwKY0tRsPgVwfjSaTwG0VaqKFAAArBmKfwC4nm7/c+cVOzG0uOhyyn9rh8p3t8jhdPCeAsA8v7dE9+yqtUV/z8UZZXPSbCKtR0+NqbPOr66mkFz8vQkANx3FPwBcg+TYjMKPXNbtr53v9ofo9gPAlVIAnfUB1YW8Oto/pen5FMC50ahGZuI62F6pSh+zAADgZqL4B4CrkMtkFD3ap9iLg1JuftHpyHf797TS7QeAq0wB3Lur1hb9vUtSAEd6x7S9PqBdjWYWAOkpALgZKP4B4BWkxsOaMd3+mdjiX57VAYXu65a7wsf7BwDXmAIwhX5d0KQAJjUTS9n1syMRjczM6WBblSp8pbynALDKKP4B4Apymaxmj/Vp9vjA8m7/wXaV7zXdfiZVA8D1CpSV6N6uOp0biaj3Uli5nBSNmxTAqLY3BLSzgRQAAKwmin8AWEFqImL39qenZxf/wqzy57v9lX7eMwBYBU6HQzsagqoLlelY36Rm5lL2XuuZ4YhGpvOzAELlpAAAYDVQ/APA5d3+F/o1+3z/sm6/b3+bfPu20e0HgJsgaFIA3XU6OxzRqeF8CiAST+mRnlHtmE8BOJkFAAA3hOIfAOalJqMKP3JS6akl3f5Kn4L37VZJFd1+ALjZKYCdjUHVz58IEJ5PAZw2KQBzIkBbpYKkAADgulH8A9jyclnT7R/Q7DHT7Z9v9ztMt3+bfPva5HCxtx8A1oop8O/rqtPp4bCN/5u/lc2NgG/3jNqbAyYJYG4UAACuDcU/gC0tPRW1k/zTk9Hl3f57u1VSHVjXawOArcpE/LuaQmqoKNPRvim7BcDcBDh1KayR6TkdaK+yWwUAAFeP4h/Alu32x14cVPRon+xB04ZD8t2yTb4D7XT7AWADMMP+7uvOpwDMPADzt7UZCvhIz4h2NQbVWU8KAACuFsU/gC3HTPC33f6JSGHNFSq3k/xLaoLrem0AgOVcToe6TQogVKaj/ZP2OEBzz7bnYliXpud0sK3KHhsIAHh5FP8AtoxcNqfYiUFFnzu/rNtfvrdV/oOm2+9a70sEAFxBha9U93fX2+j/2ZH8zduZmJkFMGK3CHTW+eVgFgAAXBHFP4AtIT0TU/hIj1Jj4cKaK1imkJnkX0u3HwA2Swpgd7OZBeC1swBmE/kUwMkLM/MpgEr5vaQAAGAlFP8Air/bf3Io3+3PZAvrhW6/m24/AGw2lT6PHthdr96LMzo3mh/YOj2b1MMnR+wWgQ5SAADwEhT/AIpWOjzf7R9d3u03k/xL60Lrem0AgBtPAexpqbAnAhzrX0wBnJhPARwgBQAAy1D8Ayg6uVxOcycvKPLsueXd/t0t8h/qoNsPAEWkym9SAHV2AOD5+RTAlE0BjGp3c1DttcwCAACD4h9AUUmH5xR+tEepkZnCmsvvzXf7GyrW9doAADeHy+nUXpMCCJkUwKRiyYyyuZxeHFpIAVTJ5+GfvQC2Nv4WBFA83f7ei4o8c1ZKL3b7y7qbFTjUKUcJe/sBoNhVB/KzAHouzqhvbNauTUbzswDMoMC2Gh8nAgDYsij+AWx6meicZo70KjU8XVhz+r0K3dOl0sbKdb02AMDacrucuqW1sjALYC6ZUSab0/HBaV2ays8CKCcFAGALovgHsLm7/acuKfr0WeXSmcJ62a4m+W/vlLOEv+IAYKuqCXj14O56ewxg/3g+BTARTeihkyPa0xzSNlIAALYY/mUMYFPKROMKP9qr5KWpwprT51Hwni55mqrW9doAABsnBbBvWz4F8PzAYgrgBZMCmD8RoKyUfw4D2Br42w7Apuv2x88MK/LUGeVSS7r9Oxvlv327nPwjDgBwmdqg184CODk0o4GJfApgPJLQQydG7HGBrdXlzAIAUPQo/gFsGpnZuMKPnVLywmRhzVlequA93fI00+0HAFxZicup/W2Vaqgs0/P9U4qnMkpnczYRcGk6pv3bSAEAKG4U/wA2R7f/7LAiTy7v9nt3NChwh+n2l6zr9QEANo+6oFcP7qnXiaFpDU7E7NpYOD8LwBwX2FJFCgBAcaL4B7ChZWIJhR/rVXJoSbe/zHT7u+RpqV7XawMAbN4UwIG2qsIsgEQqq3QmZ08HMCcCmBSAt5QjYgEUF4p/ABu3239uJN/tT6YL697t9QrcsUNOD91+AMCNqQ+V6cHdHr04NK0Lk/kUwGg4rodODtsUQDMpAABFhOIfwIaTmUso8tgpJQYnCmtOb4kCh7vk3VazrtcGACgupW6nbm2vUmNFmV4wKYB0VqlMTkdNCmB6zp4W4C0hBQBg86P4B7Cxuv3nRxV58rRyiSXd/o46Be7caW8AAABwM5gtAFX+Uh0fnNbFqTm7NjIT1+SJEd3SWqGmyjJOBACwqVH8A9gQsnNJhR8/pcTAeGHN4S1R8O5d8rbVruu1AQC2hlK3S4c6qtVYMacXBqeUtCmArJ7rm9Sl6TLta62QhxQAgE2K4h/Auov3jSr8uOn2pwprnvZaBe8y3f7Sdb02AMDW01i5mAIw0X9jeHpOk9HEfAqgfL0vEQCuGcU/gHWTjScVfuK0En1jhTWHx53v9rfX8ZUBAKwb0+G/rbNaF6di9iaASQGYH8+en7Q3AsxNAJMUAIDNguIfwLqI948p8vgpZeNLuv3bahS4e5dcZXT7AQAbg+nyV/s9emFw2hb9hpkJMB5J2GGAZlAgAGwGFP8A1pQp9s1APzPYb1m3/66d8rTXMUwJALAxUwAdVbboNykAMwfApACeOTeh5spy7bUpAOd6XyYAvCyKfwBrJj4wnu/2zyULa57WagUOm26/h68EAGDDcjgcaq4qV3XAY48ENCcBGBemYhqPxG0KwJwYAAAbFcU/gJsumzDd/jOKnxsprDlK3QrcuUPeznq6/QCATcNb4tLtndW6MBnTi0MmBZBTIp3V0+cm1FJVrj0tpAAAbEwU/wBuqsTQhMKP9i7r9pe2VCl4uEuucrr9AIDNmQJoqfapJuDV8wNTGg3nUwBDkzGNReLav61S9SFSAAA2Fop/ADdFNplW5Kkzip8ZLqw5Slz5bv/2Brr9AIBNz1vq0h3bq23R/+LgtNLZnBKprJ46O6HW6nwKoMTFLAAAGwPFP4BVl7gwme/2xxKFtdKmSgXv6ZLL5+UdBwAUVQqg1aYAPDYFMBbOf+8bnIjZXx9oq1RtkO99ANYfxT+AVe32R58+q7nTl5Z1+/23b1fZzka6/QCAolVW6tad22ts0X9iKJ8CiKcyeuLMuLZV+7S7JUQKAMC6ovgHsCoSF+e7/bNLuv2N891+Px0PAMDWSAFsq/GpNujRsf4pjUfy3xMHJmbtLIAD2ypVQwoAwDqh+AdwQ7KptKLPnNNc78XCmsPtlP/2HSrbRbcfALA1UwB37ajRwPisTlyYUSab01wyo8fPjKutxqfdzSG5mQUAYI1R/AO4bslLU5ox3f5ofsqxUdJQoZDp9geYcgwA2NopgLZav93vb1IAE9F8CqB/fFZj4bj2t1Xa0wIAYK1Q/AO4ZrlURpFnz2mu58KSv02cCty2XWVdTeztBwBgXrnHrbt31tii/+R8CiBmUgCnx9Ve61d3U5AUAIA1QfEP4JokR6YVfqRHmaXd/rqQgvd2yx2k2w8AwEopgPZCCmBSk9GkXe8bi2o0PKcDbVWq9nt44wDcVBT/AK5KLp1R9Nnzip0cWlx0mW5/p8q6m+n2AwDwCnwetw7vrLVF/8kLYWVzOcUSGT12akwddfkUgMvp5H0EcFNQ/AN4RcnRGYWP9CgTniusldQGFbzPdPvLeQcBALiGFEBHXaAwC2BqNp8COD8a1ehMXAfaKlVFCgDATUDxD+Dlu/3PnVfsxJJuv9Mh/6FOle9ukcPp4N0DAOA6+L0lumdXrS36ey7OKJuTZhNpPXpqTJ11fnU1heTi+yyAVUTxD2BFqbGwZh45ubzbXxPId/tDPt41AABWIQXQWR9QXciro31Tmo7lUwDnFlIA7ZWq9DELAMDqoPgHsEwuk1H0aJ9iLw5KuSXd/ls7VL6nlW4/AAA3IQVwb1etzo5EdepSPgUQTaR1pHdM2+sD2tVoZgGQtgNwYyj+ARSkxk23v0eZmdjiXxLVAYVMt7+Cbj8AADczBbCjIaB6kwLon9RMLGXXz45ENDqTPxGgwlfKFwDAdaP4B6BcJqvZY32aPT6wvNt/oF3lt5huP5OHAQBYC4EykwKos0X/qUth5XJSJG5SAKPa3hDQzgZSAACuD8U/sMWlJiJ2kn96araw5q7y2739JZX+db02AAC2IqfDYYv8+lCZjvVNamYuZe/NnxmOaGQ6roPtlQqVkwIAcG0o/oGt3O1/oV+zz5tu/3y73+GQ70CbfPu20e0HAGCdBU0KoLvOFv2nTQpAJgWQ0iM9o9rZGLTbBMyNAgC4GhT/wBaUmozmu/2T0cKau9KX7/ZXBdb12gAAwCJT3JuBf2YWwLH+KYXnUwBmS8Dw9JwOtlUqSAoAwFWg+Ae2kFzWdPsHNHusf0m3X/LtN93+Njlc7O0HAGAjMjH/+7rqdHo4bJMA5ru4uRHw7d5R7WoI2nkApAAAvByKf2CLSE9F7ST/Zd3+ivlufzXdfgAANjqn06GuppAaKsp0tG/SDgI09/J7TQpg/kQAs1UAAFZC8Q9sgW5/7MVBRY/2yR4cvNDtv2WbfAfa6fYDALAZUwDd9XYOwJmRiF0zRwM+0jNitwh01pMCAPBSFP9AEUtPz+a7/RP5fxgYrlC5QqbbXxNc12sDAADXz+V0qLt5PgXQP6loPG3v8fdczM8CMCkAc2wgACyg+AeKUC6bU+zEoKLPnV/W7S/f2yr/QdPtd633JQIAgFVQ4SvV/d31dgDg2fkUwHQspW/3jNgtAp11fjk4EQAAxT9QfNIzMTvJPzUWLqy5gmV2b39pbWhdrw0AANycFMBumwLw6mjflGYT+RTAyQsz8ymASvm9pACArY7OP1BM3f6TQ/lufyZbWC/f0yL/rR1yuOn2AwBQzCp9Hj2wu169F2d0bjQ/4HdqNqmHT46ouymkDlIAwJZG8Q8UgXR4vts/uqTbH5jv9tfR7QcAYCulAPa0VMzPAphSbD4FcKKQAqiSz0sJAGxF/MkHNrFcLqe5nguKPHNuebd/d7P8hzrp9gMAsEVV+T16cHedei6EdX4snwKYnE3qoZMjdotAe62PWQDAFkPxD2xS6chcvts/MlNYc/m9Ct7brdKGinW9NgAAsP5cTqf2tuZTAMf6JxVLZpTN5fTi0LQuTcfyKQAP5QCwVfCnHdiM3f7ei4o+c1a59GK3v6y7WQHT7S9hbz8AAFhUHcjPAui5OKO+sVm7NhnNzwIwKYC2GlIAwFZA8Q9sIpmo6fb3Kjk8XVhz+r0K3dOl0sbKdb02AACwcbldTt3SWjmfApjSXDKjTDan44MmBTCnA9sqVU4KAChqFP/AZun2n7qk6NOm258prJftapL/9k45S/ijDAAAXllNwKsHd9fbYwD7x/MpgIlIws4C2NMS0rZqUgBAsaJiADa4TDSu8KO9Sl6aKqw5fR4F7+mSp6lqXa8NAABszhTAvm2LKYB4Kp8CeGFgWpemzIkAlSorpUwAig1/qoEN3O2PnxlW5KkzyqWWdPt3Nsp/+3Y5+aYMAABuQG3Qqwf31OvE0IwGJ/IpgHGTAjhhUgAVaq0u50QAoIhQ/AMbUGY2rvBjp5S8MFlYc5aX5rv9zdXrem0AAKB4lLicttPfWFGm5wfyKYB0Nmd/bWYB7N9mUgAMEwaKAcU/sNG6/WdHFHny9LJuv3d7gwJ3mm5/ybpeHwAAKE51ofwsAHMM4NBkzK6NheN66OSw9rZUqKWKFACw2VH8AxtEJpbId/uHJgprzrL5bn8L3X4AAHBzlbidOthepcbKfAogkcoqncnZuQBmFsD+tkp5OVIY2LQo/oGN0O0/P6rIE6eVS6YL697OegXu3CGnh24/AABYO/WhMj2422NTABfmUwCjJgVwYlh7WyvVXFnGLABgE6L4B9ZRZi6hyGOnlBhc0u33lihwuEvebTV8bQAAwLoodTt1q0kBVJTpBZMCSGeVyuR0tG9Sl6a8dhaAhxQAsKlQ/APr1O1P9I0qbLr9iSXd/o46Be7caW8AAAAArDdzHGCVv1THB6d1cWrOro3MxPWtEyPa11phtwg4HI71vkwAV4HiH1hj2bmkwk+cUqJ/vLDm8JYoePcuedtq+XoAAIANpdTt0qGOajVWxPTC4LSSNgWQ1bN9k2qcLtMtrRWkAIBNgOIfWENx0+1/3HT7U4U1T1utgnebbn8pXwsAALBhNVaWq8rvsSkAcwygYX6eiCbmUwDl632JAF4GxT+wBrJx0+0/rUTfWGHN4XHnu/3tdXwNAADApmD2+d/WWa2LUzG9MDBtEwAmCfDM+Uk1Tc/ZFIBJCgDYeCj+gZss3j+myOOnlI0v6fZvq1Hg7l1yldHtBwAAm09TZbmq/R47DHB4Jm7XzEyA8UjCDgM0swIAbCwU/8BNkk2kFHnytOLnRgtrjlK3AnfttIP9GI4DAACKIwUwp+ODU/Y0AJMCePrchJory7XXpgCc632ZAOZR/AM3QWJwXOHHTtnhfgtKW6oVPLxLrnIP7zkAACgKppnRXFWu6kA+BWBOAjAuTMU0Holrf1ul6kOkAICNgOIfWO1u/1NnFD87srzbf+cOeTvr6fYDAICi5C1x6fbOal2YjOnFITMLIKdEOqunzk6opapce1sqVEIKAFhXFP/AKkkMTSj8aO/ybn9zlYKHu+Ty0e0HAADFnwJoqfapOuC1KYDRcD4FMDQZ01gkrgPbKlVHCgBYNxT/wA3KJtP5bv+Z4cKao8SlwB075N3RQLcfAABsKWWlLt2xvdoW/S8OTiudzSmRyurJsxNqrS7XHpMCcDELAFhrFP/ADUhcmMx3+2OJwlppU6WC95huv5f3FgAAbNkUQGu1TzUBj54fmNJYOP9vpcGJmP31gbZK1Qb5txKwlij+gevs9kefPqu505eWdfv9t29X2c5Guv0AAAA2BeDWndtrbNF/YiifAoinMnrizLi21fi0pzkkNykAYE1Q/APXKHFpSuEjPcrOLun2N1YoeE+3XH7uYAMAAFyeAjCF/kIKYDyS/zfUwPisxsL5WQA1pACAm47iH7hK2VRa0WfOaa73YmHN4Xbmu/27muj2AwAAvIxyj1t37aixRf+JCzPKZHOaS2b0+JlxtdX4tJsUAHBTUfwDVyE5bLr9vcpE81NrjZKGCoXM3v4AZ9cCAABcbQqgrdZv9/sf65/SRDSfAuhfSAG0Vak6wClJwM1A8Q+8jFwqo8iz5zTXc2HJnxqnAoc6VdbdTLcfAADgOlMAd++sUd/YrHou5lMAsWRGj50eU3utX91NQWYBAKuM4h+4guTItN3bn4ks6fbXhRS8t0vuYDnvGwAAwA2mADrq/KqzKYBJTc4m7XrfWFSj4TkdbKtSlZ8UALBaKP6By+TSGUWfO6/YiaHFRZdT/kMdKt/dQrcfAABgFfm8bh3eVavzY1H1XAgrm8splsjo0VNj9uZAd1NILqeD9xy4QRT/wBLJ0Zl8tz88V1grqQ0qeG+33CG6/QAAADcrBdBZF5hPAUxpaj4FcH40qtGZuA62VaqSFABwQyj+gYVu/9E+xU4MSrn5t8TpkP9QZ77bz91mAACAm87vLdE9u2p1bjSq3oszyuak2URaR06NqbPOry5SAMB1o/jHlpcaC2vGdPtnYot/MGoCCt1nuv2+Lf/+AAAArHUKYHt9QPUhr472TWo6lrLr5+ZTAAfaq1TpK+WLAlwjin9sWblMNt/tf3Fgebf/YIfK95puv3OdrxAAAGCLpwC66nRuJKJTl8wsAClqUgC9o9pRH9DOxiCzAIBrQPGPLSk1Pt/tn17S7a8OKGT29lfS7QcAANgInA6HdjQEVR8q09H+Sc3MpwDOjEQ0MjNnUwAV5aQAgKtB8Y8txXT7Z5/v1+wL/cu7/QfaVX5LK91+AACADShQVqJ7u+p0dj4FkMtJkXhaR3pGtaMhoJ0NQTmZ0QS8LIp/bBmpiYid5J+emi2suav8dpJ/SZV/Xa8NAAAAr5wCMEV+fdCro/1TCs+lbC/n9HBEw/MnAoRIAQBXRPGPopfLmm7/gO3429vEhsMh34E2+fZto9sPAACwiQTLS3Vfd53ODEd02qQAJEXmUnqkZ9TOATBJAHOjAMByFP8oaqnJaL7bPxktrJk9/cH7TLc/sK7XBgAAgOtjivtdjWYWQD4FYIp/cxPAbAkYnp6zKQBzkwDAIop/FG+3/4X5br8ZDWs4JN++Nvn2t8nhYpI/AADAZmdi/vd31en0cNgmAcy/+sx2gG/3jmpXQ1DbSQEABRT/KDrpqaid5J+eWOz2uyrKFbpvt0qq6fYDAAAUEzPor6sppPqKMh3rm7SDAM1Oz16TApgxKYAqOzAQ2Ooo/lFU3f7Yi4OKHu1b1u0vv2WbneZPtx8AAKB4mSP/7uuut3MAzFGAhjka8Ns9I3aLQGc9swCwtVH8oyikp2fz3f7x/F/0hitUrpCZ5F8bXNdrAwAAwNpwOR3qbl5MAUQTadsT6rloZgHEdbC9Un4vKQBsTRT/2NRy2ZxiJwYVfe788m7/3lb5D5puv2u9LxEAAABrrNJXqvt31+vUpRmdHclvBZ2OJfXwyRG7RaCzzi8HJwJgi6H4x6aVnonZSf6psXBhzRUsU/DebpXWhdb12gAAALD+KYDdzRVqCJXZEwFm51MAJy/M2BMBDrSRAsDWQvGPzdntPzmU7/ZnsoX18j0t8t/aIYebbj8AAADyKv0ePbC7Xr0XZ3RuNJ8CmJrNpwDMFoGOWlIA2Boo/rGppMOm29+r1OhMYc0VMN3+LpXWV6zrtQEAAGDjpgD2tFSooSKfAojNpwBODM2nALZVyeelNEJx4/9wbAq5XE5zPRcUeebcsm5/2e5mBW7tlKOEbj8AAABeXpVJAXTX2QGAfWP5FMBkNKmHe0bU3RRSe62PWQAoWhT/2PDSkbn83v6RJd1+vze/t7+Bbj8AAACuntvl1C2tFWo0JwL0TyqWzCiTzenFoWldmo7pQFuVfB7KJBQf/q/Gxu72915U9JmzyqWXdPu7muS/rVPOEv73BQAAwPWpDuRnAZgBgP3js4spgJMj2t0cUlsNKQAUF6onbEiZaFzhR3uUvDRdWHP6PAqZbn9j5bpeGwAAAIonBbBvW6UaK00KYEpz8ymA44PTdhbA/m2VKicFgCJB8Y+N1+0/fUnRp0y3P1NYL9vVKP/t2+n2AwAAYNXVBLx6cHe9TlyY0cB8CmA8ksinAFpC2lZNCgCbH8U/NozMbNxO8k9emlrW7Q8e7pKnuWpdrw0AAADFnwIwnf78LIApxVMZpbM5vTAwreGpOe1vq1RZKeUTNi/+78WG6PbHzwwr8tQZ5VJLuv0757v9/CULAACANVIb9OrBPfU6MTStwYmYXRuLJPTQiRHtba1QS1U5JwJgU6L4x7rKzCYUfqxXyQuThTVneWm+299Sva7XBgAAgK2pxOW0U/9tCmBgSolU1qYATCLg0tScnRNQVspR09hcKP6xft3+syOKPHl6Wbffu71BgTtNt7+ErwwAAADWVV2oTK/a7bHHAA5N5lMAo+G4Hj45rL0tFWomBYBNhOIfay4TM93+U0oOTRTWnGWm279LntYaviIAAADYMErcTh1sr7InAjzfP6VEOqtUJqejJgUwnU8BeEtIAWDjo/jH2nb7z48q8sRp5ZLpwrq3s16BO3fI6aHbDwAAgI2pPlSmB/fkUwAX5lMAIzNxTZ4Y1i2tlWqqLGMWADY0in+sicxcUpHHTikxOF5Yc3pLFDi8S95ttXwVAAAAsOGVup261aQAKsr0/MCUkvMpgOf6JnVpymtTAB5SANigKP5x07v9ib4xhZ84pVxisdvvaa9T8K4dcnpL+QoAAABgU2moKFOVv1THB6d1cWrOrg3PxDVxYkT7tlWoqbJ8vS8ReAmKf9w02XhS4cdPK9E/VlhzeEsUvHuXvG10+wEAALB5lbpdOtRRrYaKmL0JkE8BZPXseZMCmNMtrRWkALChUPzjpoj3jyn8+Cnl4qnCmqetVsG7d9LtBwAAQNEwXf5qv0cvDE5reDqfAjCDACeiCe1rrVAjKQBsEBT/WFXZeErhJ04r0TdaWHN43Pluf3sd7zYAAACKjtnnf1tHle34m5sAJgFgkgDPnJ9U03Q+BWCSAsB6ovjHqokPjNmhfuYGwAJzdJ8Z6ucqY28/AAAAipfD4VBTVbmqAx47DNCcBGCYmQATkYQdBmhmBQDrheIfNyybSCny5GnFzy3p9pe6Fbhrp7wddRx5AgAAgC2VAri9s9oW/ccHp+xpAIl0Vk+fm1BzVbn2tpgUgHO9LxNbEMU/bog5ui9suv1zycJaaUu1gqbbX+7h3QUAAMCWTAE0L6QA+qc0Gs6nAC5MxjQeiWv/tkrVh0gBYG1R/OO6ZJOm239G8bMjhTVHiSvf7e+sp9sPAACALc9b4tId26tt0X98aFppkwJIZfXU2Qm1zKcASkgBYI1Q/OOaJYYmFH6sV9nYkm5/c5WCh7vk8tHtBwAAAJamAFqqfaoOeO0sgLH5FMCQTQEkbAqgLuTlDcNNR/GPq5ZNphV5+ozip4eXd/vv2CHvjga6/QAAAMAVlJW6dOf2ag1OxHTCpACyOcVTGT15dlyt1eXaY1IALmYB4Oah+MdVSVycVPjRXmVnE4W10qZKBe8x3X7uVAIAAABXkwLYVuNTbdCjY/1TtvNvmBsCY+GEDrRVqjbIv61xc1D842VlU2lFnz6ruVOXCmsOt0v+O7arbGcj3X4AAADgGpWVunXXjhoNTMzqxNCMMvMpgCfOjNubA3uaQ3KTAsAqo/jHFSUuTSl8pGd5t7+hQsF7u+Xyc0cSAAAAuJEUQFuNX7UBr44NTGliPgUwMD5r5wKYFEBNgH9zY/VQ/GPlbv8z5zTXe7Gw5nA75b9tu8q6muj2AwAAAKuk3OPW3Ttq1D8+q5MX8imAuWRGj58eV1utT7ubSAFgdVD8Y5nk8LTt9mei+SmkRklDhUJmb3+As0gBAACAm5ECaK/12/3+x/onNRnNn6rVPzarsRmTAqhSdYBTtXBjKP5h5VIZRZ47p7mTF5b83+FU4FCnyrqb6fYDAAAAN5nP49bhnbXqG8unALK5nGLJjB47PaaOWr+6m4NyOTkRANeH4h9Kjphuf68ykbnCu1FSF1Lw3i65g+W8QwAAAMAapgA66vyqW0gBzOZTAOfHohoJx3WwrVJVflIAuHYU/1tYLp1R9Lnzip0YWlx0OeU/1KHy7hY5nI71vDwAAABgy/J53Tq8q9YW/T02BSDFEmk9empMnXV+dTWF5OLf67gGFP9bVHJ0Jr+3P7yk218btJP83SG6/QAAAMBGSAF01gXmUwBTmppPAZwbjWpkJq6D7ZWq9JECwNWh+N9ichnT7e9T7MSglJtfdDrkv7VD5Xta6fYDAAAAG4zfW6J7dtXaor/3Yj4FMJtI60jvmLbX+7WrkRQAXhnF/xaSGgtrxnT7Z2KFNXdNQCHT7a/wreu1AQAAAHj5FMD2+oUUwKSmYym7fnZkPgXQVqUKXylvIa6I4n8LyGWyih7rU+z4wPJu/8F2le813X4mhgIAAACbQaCsRPd01encSESnLoVtCiAaT+uR3lHtqA9oZ6M5EYDZXXgpiv8il5qIaOaRk8pML+n2V/sVune33JV0+wEAAIDNxulwaEdDUHWhMpsCmJlPAZwZiWhkZk4H2qtUUU4KAMtR/Bdxt3/2+X7NvtC/rNvvO9Au3y10+wEAAIDNLlhWonu76nR2OKJTw2HlclIkntaRnlHtaAhoZ0NQTlIAmEfxX4RSkxGFH+lRemq2sOau8ttJ/iVV/nW9NgAAAACrmwIwUf/6kFdH+6cUnkvZ3t/pYZMCiOtAW6VCpABA8V9cclnT7R+wHX97289wOOTb3ybf/m3s7QcAAACKVLC8VPd11+nMcESnL4XtDQBzI+CRnlF7c8AkAcyNAmxddP6LRGoyqvCRHqUno4U1s6ffdvurA+t6bQAAAABuPlPc71pIAfRNKRLPpwDMYMDh6TkdbK+yWwWwNVH8F0O3//igZo/1yY76NBySb5/p9rfJ4WKSPwAAALCVmJj//d11Oj0ctkmAhRTAt3tG7M0Bc2QgKYCth+J/EzN7+mdMt38iUlhzhcoVuq9bJTXBdb02AAAAAOvHDPrragqpPlSmo/2T9jhAszO49+J8CqCtyh4biK2D4n+TdvtjLw4pevT8sm5/+S3b5D/QTrcfAAAAgFXhMymAehv9PzuSbxqaowEXUgCdpAC2DIr/TSY9M2sn+afGL+v2m739tXT7AQAAACzncjq0uzmkhgqvjvVNKZpI2x5ij00BxHWwvVJ+LymAYkfxv0nksjnFTg4p+uy5xW6/pPK9rfLfarr9rnW9PgAAAAAbW6XPo/t316v34ozOjeYHhU/Hknr45IjdItBZ55eDEwGKFsX/JpCeidlJ/qmxcGHNFSyzk/xL60Lrem0AAAAANlcKYE9LhRoqynSsf0qz8ymAkxdm7CyAA22kAIoVxf8GlsstdPvPS5lsYb18T4v8t3bI4abbDwAAAODaVfk9emB3nY3+n59PAUzNmhTAqLqbg+qoJQVQbCj+N6h02HT7e5UanSmsuQLefLe/vmJdrw0AAADA5udyOrW3kAKYVCyRUTaX04mhhRRAlXweSsZiwVdyA3b753ouKGL29qcXu/1lu5sVuLVTjhK6/QAAAABWT7VJAXTX2xRA31g+BTAZzc8CMIMC22p8zAIoAhT/G0gmMqeZR3uVGp4urLn8ptvfpdKGynW9NgAAAADFy+1y6pZWkwLw2lkAc8mMMtmcjg9O69JUfhZAOSmATY3if6N0+3svKvrMWeWWdvu7muS/rVPOEr5MAAAAAG6+moBXD+6utwMA+8dn7dpENKGHTo5oT3NI20gBbFpUlessE40r/GiPkpcWu/1On8fu7fc00u0HAAAAsPYpgH3bKtVoZgEMLKYAXjApgOk57d9GCmAzovhfz27/6UuKPn1WuVSmsF62q1H+27bLWcqXBgAAAMD6qQl69cB8CmBgPgUwHknYWQDmuMDW6nJmAWwiVJjrIDNruv29Sl6cKqw5yz0K3tMlT3PVelwSAAAAALxEictpO/3mRIDn+6cUT2WUzub0/MCULk3H7GNlNC43BYr/Ne72x88MK/LUmWXdfu+OBgXu2EG3HwAAAMCGVBf06sE99ToxNK3BiZhdGwsn9NCJEe1trVBLFSmAjY7if41kZhMKP9ar5IXJwpqzrDTf7W+pXqvLAAAAAIDrTgEcaKvKpwAGppRIZW0KwJwOYE4EMCkAbylHk29UFP9r0e0/N6LIk2eUS6YL697t9fluv6fkZl8CAAAAAKya+lCZHtztsSmAocl8CmA0HNdDJ4e1t6VCzaQANiSK/5soEzPd/lNKDk0s7/Yf3iVPa83NfGkAAAAAuGlK3U4dbK+yJwLYFEA6q1Qmp6MmBTA9Z08L8JaQAthIKP5vVrf//KgiT5xe3u3vrFPgzp10+wEAAAAUhfqKMj3o9+jFwWldmMqnAEZm4po8MaJbWivUVFnGiQAbBMX/KsvMJRV5/JQSA+OFNae3RIHDu+TdVrvaLwcAAAAA654CuLWjSg2VZXphYEpJmwLI6rm+SV2aLtO+1gp5SAGsO4r/VRTvG1X48VPKJRa7/Z72OgXv2iGnt3Q1XwoAAAAANhSzBaDaX6rjg9O6ODVn14an5zQZTcynAMrX+xK3NIr/VZCNJxV+/LQS/WOFNYenRMG7d8rbXrcaLwEAAAAAG16p26VDHdVqqIjZmwAmBWB+PHt+0p4IYG4CkAJYHxT/NyjeP5bv9sdThTVPW42Cd+2yw/0AAAAAYKsxXf5qv0cvDE7b7r9hBgFORBPa11qpxsqy9b7ELYfi/zpl4ylFnjxtB/stcHjctuj3tNcy1AIAAADAlmY6/Ld1VNktACYFYOYAmBTAM+cn1DRdZlMAJimAtUHxfx3iA+OKPNZrbwAs8LRW26F+rjLPan59AAAAAGDTcjgcaq4qV3XAY4cBmpMADHNDYCKSsEcCNlSQAlgLFP/XIJsw3f4zip8bKaw5St0K3LVT3o46uv0AAAAAsAJviUu3d1brwmRMLw6ZFEBOiXRWT5+bsDcH9raYFICT9+4movi/SonBcYUfO6XsXLKwVtpSraDp9pfT7QcAAACAV0oBtFT7VBPw6vmBKY2G8ykAc0NgPBLX/m2Vqg+RArhZKP5fQTZpuv1nFT87XFhzlLjy3f7Oerr9AAAAAHANvKUu3bG9WkPzKYC0SQGksnrq7IRaq8u1p7lCJaQAVh3F/8tIXJhQ+NFeZWNLuv3NVQoe7pLLR7cfAAAAAK43BdBqUwAemwIYCyfs+uBEzP56f1ul6oJe3txVRPG/gmwyrcjTZxQ/fVm3/44d8u5ooNsPAAAAAKugrNStO7fX2KL/hEkBZHOKpzJ68sy4vTmwpyWkEhezAFYDxf9lEhcn893+2fydJ6O0sVLBe7rk8nPnCQAAAABWOwWwrcan2qBHx/qnNB5ZSAHMaiwc14G2StWSArhhFP/zsqm0ok+f1dypS4U3x+F2yX/HdpXtbKTbDwAAAAA3OQVw144aDUzM6sTQjDLzKYAnzoyrrcan3c0huUkBXDeKf0nJS1OaOdKzvNvfUKHgvabbz7RJAAAAAFirFEBbjV+1Aa9NAUxE8zVa//isPR3ApADMaQG4dlu6+Lfd/mfOaa734uKi26nAbdtV1tVEtx8AAAAA1kG5x627d9bYov/khXwKYC6Z0eOnx9Ve61N3EymAa7Vli//k8LTCR3qUiebPljRK6kMK3tstd4BuPwAAAACsdwqgvdZv9/sf65/UZDR/Clvf2KxGZ0wKoErVAU5hu1pbrvjPpTOKPntOsZMXFhddptvfqbLuZrr9AAAAALCB+DxuHd5Zq76xqE5eCCubyymWzOix02PqqPWruzkol5MTAV7Jlir+k6MzCj/So0xkrrBWUhfMd/uD5et6bQAAAACAK6cAOuoC8ymAKU3N5lMA58eihVkAVX5SANrqxb/t9j93XrETQ4uLLqf8t3aofHeLHE7Hel4eAAAAAOAq+L0lumdXrc6PRtVzcUbZnDSbSOvRU2PqrPOrqykkF/Xdihy5XC6nIpYcm+/2h5d0+2vnu/0huv0AgOKRSyeUi44qNz2g7Mhxu+asv0WOim1y+OvkcNMRAQAUj2g8paP9U5qeTwEsbBE42F6pSh/f87ZM8Z/LZBQ92qfYi4PSwn+h05Hv9u9ppdsPACgquUxK6WP/KKVmV35CiU/uA/9ZDlfJWl8aAAA3jSlnz41G1TufAliwvT6gXY1mFgAp76KO/afGw5ox3f6ZWGHNXR1Q6L5uuSt863ptAADcHDkptfh97yXsY0V5vx8AsMVnAZhCvy7o1dH+Sc3EUnb97EhEIzNzOthWpQpf6Xpf5oawaTv/5rLj50flcLvk3VaTX8tkNXusT7PHB5Z3+w+2q3yv6fYzARIAULzSZ76h3MTpFR9z1OyUe/ur1/yaAABYK+YUgHMjEfVeCmuhyjV9f3NzYOeSFMDw9Jwy2ZyaKsu21Glvm7b4N8P7Ik+dsb+2+/crfXZvf3p6Me7orvYrZB/zr+OVAgCwNnJz00o//+kVOvwOufe/RY6yCr4UAICiF55L6VjfpGbm8ikAI+A1swCq8o/1T9m1PS0hddYFtFVsyuI/G09q/LNPKJfK5BfMHZylGzycDvn2t8m3bxvdfgCAtnr3n64/AGArpgDODkd0angxBXB56Vjicug79jao1O3SVrApc/BmkF+h8DeWFP7uKr+q3nib/AfaKfwBAFuOq/nQS9eaXroGAEAxczocNup/f1edgmWLw26X9oxTmZxOXQprq9h0xb+J9c+durjiY6Ut1ar67kMqqSLmDwDYmky03+GvX/y9v564PwBgywqWl+q+rjrVBVc++q9/bFaRJdsDitmmK/7Dj/VecVhxamRa2cTW+MIBAHAlzvZ7JYfT/rC/BgBgC0tmspqMJld8zJSWLwzkZwAUu0111F/szCWlRq8cyzBbAZLD0yrrXOx4AACw1Th9tXLc8eP211tpijEAACuZiCSUXpr3v8zkbFKDE7NqrS7uY+E3Vec/NTx9xcccbqdKm6vkaa5a02sCAGAjMkU/hT8AAFJt0Gt/LBz1d6UbBMVuU037N5H+6W8et1MaSluq5A6Wy+X32h8OTwn/yAEAAAAArCiXyymVySqWyCiWTCsaT2ssHJcJyd3eWV30U/83VfEPAAAAAACKPPYPAAAAAACuHcU/AAAAAABFjuIfAAAAAIAiR/EPAAAAAECRo/gHAAAAAKDIUfwDAAAAAFDkKP4BAAAAAChyFP8AAAAAABQ5in8AAAAAAIocxT8AAAAAAEWO4h8AAAAAgCJH8Q8AAAAAQJFzr/cFAAAAAADwSiajCc0m0nK7nGqsKOMNu0YU/wAAbAHZ8dPKRS5JJeVytdy+3pcDAMA1G5yY1eBETD6Pm+L/OhD7BwBgCzCFf3b0pLITZ9f7UgAAwGbo/GfjKcUHxpSZiclR6pa3vVbukK/weCaWUKJ/TOnInBxOp0pqAvJsq7G/XhA91qdsLKGS2pBcoTKlLk0rl83Kf7Bj2WPuKp+SFyaVTaRVUheSd1vN8muZSyreN6qMeS1PiUqqA/K0VC97zsu9Vi6dUdxc6/SsHA6H3Pbjq+Rwua7v3QQA4BrkUnHlps4pNzctuT1yVnXKUVa55PG5fLGemJFcXjn8tXJWbFv2OTIXnpGSs3L46+Uor1Z2ZlBKJ+TwN8hZ1T7/nGeVjQznPyAVU+b8w/aXjppdcgYaruu15K1QLnxBymXkarmDrzsA4Oq//+VymomlbIS/1O1UoKxE3pKX1mDJdEbjkYR9TkV56RU/39LnhcpLFZlLrbg9IHeVr3sl4VhSUVObupyq8pfKNV/jDk/PKZXJ2kSC+ZwzsaRSmVzhtc3rTsdSiiXScjocqvCVqKz0paV4NJ5SNJ6WQ1Kw/KXPeaXHX8k1PTtxcVIz33pRuVRG3s56ub0lijx+Wr79bSptrFRicFwzD59QLif59rbaojr24qDcVX5VfucBOT0l9vPYgn06psTAuFyBMnnaa+Xyepc/NjhhP87cPIidHJJeHFTmtk75bsn/QyRxYVIzD70oORwq39Oi9NSsZo/2qbShQqFX3yJniftlXyuXyWry355TejKq8t3NcpR77E2LyNNnVfWGW+Uq91zTGwkAwLXIzgwpc/qrUiYpR/VOOdxlyvQ9ImfTITlDzcpOD+YfdzjlbNynXGxS2QtPKxtskmvXd8nhyn9PtQX73JQ01S+Hr8b+yA6/IOWOKdd6l1xNB+XwVsjh9ihnPsDpksrzN9Md7vz33mt/rT45PEE5zM2KkiBfeADAVZuNp/XM+QmF51KqC3ptIT6XTKuzLqCupqBtyhoTkYSeOjuudDan2qBHyVRW2RU+37LnBTxKpLP2+525AbB0e8DsVb7uitecSOvZ8xP2xoHf67aft+dCRgfaqxQsK1HvxRlF4mkFvG5lc1LIFuYu+9qmYH/m3IR9vCbg0VwyYz9fa7VP+7ZV2JsBxtG+SQ1NxlTpK5WnxKXwUFI+b4kOtVepxO18xcdXtfjPptL5wj6VUdmuRgUPd9n18lu2KRdP5R9/pEe5dFa+Wzvk399m73BMfO5JW2BHnjmr0D3dyz+pw6HK1x+Uw7XCxTqkilffYhMD5uPNzYD4+VFb/NvX+nb+WoL3dqlsR2P+tT77hJLD04odH5T/1o6Xfa3UeNh+Xrmd8t+2vbCemY3L4abzDwC4eXKZlDJnvm4Lf2fdbrk6HsivNx2U0vH842e/Yb75ytX5Kjlru+z3ufSxTykXvqjspaMrdNsdcu16vRwOpy3ec1N9tlg3xb+zulO58JBypvvv8shVv2f5tVzPa+1+kxzmRgIAAFf7/S+Xs4W66Z53NQa1szGodCarb7w4rDMjEVswt9X6lc3l9PzAlC3o22pMkVxpu/tfPz6fYpu39HmmmD7QVqlEKmM/3/W87kqyCx87X7zfuaPGFuym0x9PZpY9N5bM6FV7GuznW/zYCVvsL/x3mI/75ovDdn6Bed6uxqC9UWEKe5fToXt21RZuRIxMz9nP8UqPX62rLv5TIzPKJdL21972usK6eWFHWant+ueS+cdN933hsdL6Cs2F52zxfrmS2uDKhb95rCZY2Crg9OYjHtlE6iXXYhIAqbGw/XXO3GYxsY/hKUkdL/9aC79OZzX+L0/YbQElVX67RcHlYw4iAODmMUW1KfINR9X2wrr9Zl5Spuz0QOFx8+tcdGT+A/M9j9zMRall+ed0+Ots4W/Nd/QXPsfLXosZAnjNr1VP4Q8AuGaT0Xxs3n5/mR/gZ5gIfjKd1aXpOVuEm9i8KZiNhvnOfanbZaP2Y+FE4fMtfV5jZf55pit++fOu9nVXMmU+Np7/2I5af6FTb6L/JWXLa9lqv6dQ+C89ncBYSCCYj6sNenVhMmavwxT/6Wz+e24mm9PT5yZUHfAoVFaq2pDXvl5svs6+0uNX6+o7//F84W2Y/fUveXz+P8o+vmTfxMKvc/OF+1JLn/eyjzmufC3uCp+c3vz1mG0CxsL2gpd7rZJKv7w7GxQ/PazsbEKJ2VEl+kYVfe68Kl+3325jAADgplhSlDtK5gv1pVJzi4+XV0nu/D8YnJfF9ZeZj+bbxx2OfMT/atzgawEAcLVMzH6B6WabPfCG2c9vfiwUziYav8AU6As8lyW0lz7P87LPu7rXfaVrfrnnXX6tL/nvWFKPLlzrwuOmkDfbB8yWhJGZuP1hmHkEh3fWvuLjPq97dYt/l29xD7wZoKf5Qnulx802gMJz53/tXMU99EtfyyQEPM1V1/V5zDYEsz0hORpWeiKiWO9FczvFbi+g+AcA3DSli99Dc4moHdR3pccdvjo5K1rX5Fpu+msBALa0pXvTO+r8qvJ7XrGINpH+wq8z2et63tW+7kqWfmzyss/7SkqXJM+XXlM6kyukAAyn06H7uurs4MDJ2aRNNEzNJhVPZXR+LKpbWite8fFVPeqvpD5kB+YZsd4Ldt+EkU2m7WA/+7h/fmjf4Lj92UzTT14yEXypbHt+mvBqWPpac6cuFq7FMIP/kmMzr/g5zEkBiUtTcvnLVNZZr8AdO1RaF3zFRAIAADfKYSbse/Lfc7KjJwrfx3LphHJzU1d83D7HDOOLzEfzr4VrfkpyNn1V13JDrwUAwApM0e125WPdJmq/lIm0L3TkzcT+kvnnTUXz8X3z/Wkqllz2MUufNz2bf8zsgTeF8fW87kpMlH/hNUzxvcBcj5kv8HKqAh65nY7C1oMFk7P5/6b6kLdwY8DMAmiqKreF/L1ddXbrwsLrvNLjV+uqO/9m/70ZwDf90ItKDk1q8ovPyBUss8fsBe/pto+bKfvmNIC5kxeUjcaVicRtkW1OBvAdaLvqi7q6a9lnr8VM8Z/68rP2xkR6Jmb+6xW4e9crfxKnw55EEHnitNyVPimTs8MCXQGvPT0AAICbxQzKc3d9l9Knv6rc9IDSx//ZTuQ3x+y5Ol6Vf3zX6/OPT/Up8+LnJG8wfySgcnK133ftr1nRJl16XkpGlT71H3KUlMnZcrscJeWr/loAAKzEdLoPtlXpub5JnR+N5k+J87jtRPzRcFy7m0Mq97jt83Y1hvTi0LROD0cKxf3lda55XldTSMcHp3X6Ujh/g8A87zpfdyXu+Y99tm9SfWOzMtvz/WVujc3E1VkfUO3LNI7N6x6wrzuh08NmTl3ORvej8ycDmNddSAI8enpM1f5Se1RgNptTOJYyJataqstf8fGr5chdy62C+TsLZlJ+ZmZOTvOFaagoHKtnH8/mlBqdsTcFzFA9c1SfO7j8gszxe2Z4n1m/PF6/0mPJkWmbLnCUuG2Xftm1jIWVCcfshH5XqNzu5b/a1zIy0TmlJqI2peDyeVVSF5Jj/u4MAAA3k/k+louOSvFpye2RI9gkx0KHvvD4iBSfkZxuOcoqXrJFwEz0N4kBhzdkjwi0a+FL+QSBq0TOmp2Lny8etq+Xy+S7D87q7fYIwBt5LQAArpXZ635pKmaH8JnOuNmz3lhR/pI982PhuO22myK6Jpg/Js8c7WeG+i0UzoXnzcypxJl/Xv/YrO3wmyP37u+uv+bXXYmJ2F+citkjA831NFWWKVie/57dc3HGTv43CYNtNb6XfKxJFlycmrOD+8yAvoryEjVWltvp/QtMQW/+W2fmkrbYN/MFmirLCzclXunxm1L8AwAAAACwEZiC3lviLBx/Z2L85ig9U6x31vm1p+Xq9sNvBZxpBwAAAADYlKZnEzozElFDqEwul8N22E3hX+Er1c6G/Ewb5NH5BwAAAABsWub4PhOJNykAM53fDMOrC3oLaQDkUfwDAAAAAFDkrvqoPwAAAAAAsDlR/AMAAAAAUOQo/gEAAAAAKHIU/wAAAAAAFDmKfwAAAAAAihzFPwAAAAAARY7iHwAAAACAIkfxDwAAAABAkaP4BwAAAACgyFH8AwAAAABQ5Cj+AQAAAAAochT/AAAAAAAUOYp/AAAAAACKHMU/AAAAAABFjuIfAAAAAIAiR/EPAAAAAECRo/gHAAAAAKDIUfwDAAAAAFDkKP4BAAAAAChyFP8AAAAAABQ5in8AAAAAAIocxT8AAAAAAEWO4h8AAAAAgCJH8Q8AAAAAQJGj+AcAAAAAoMhR/AMAAAAAUOQo/gEAAAAAKHIU/wAAAAAAFDmKfwAAAAAAihzFPwAAAAAARY7iHwAAAACAIkfxDwAAAABAkaP4BwAAAACgyFH8AwAAAABQ5Cj+AQAAAAAochT/AAAAAAAUOYp/AAAAAACKHMU/AAAAAABFjuIfAAAAAIAiR/EPAAAAAECRo/gHAAAAAKDIUfwDAAAAAFDkKP4BAAAAAChyFP8AAAAAABQ5in8AAAAAAIocxT8AAAAAAEWO4h8AAAAAgCJH8Q8AAAAAgIrb/x/rw5LP2R2vkQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(figsize=(12, 6))\n", + "plot_tree(\n", + " bivariate,\n", + " X=X_train,\n", + " feature_names=[\"x1\", \"x2\"],\n", + " class_names=[\"corners\", \"edge cross\", \"center\"],\n", + " precision=2,\n", + " node_aspect_ratio=1.2,\n", + " ax=ax,\n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "tuning", + "metadata": {}, + "source": [ + "## Tuning the pair search\n", + "\n", + "Tune these controls on validation data and measure fit time as well as predictive performance:\n", + "\n", + "| Parameter | Effect | Practical starting point |\n", + "| --- | --- | --- |\n", + "| `pairwise_candidates` | Enables pair search and limits how many screened pairs are fully fitted at each node. An integer is an absolute count; a float resolves to `ceil(value * n_logical_features)`. | Start with a small integer such as `1`–`5`; increase only if validation performance justifies the cost. |\n", + "| `pairwise_penalty` | Discourages selecting a pair over the best univariate candidate. It affects selection only, not raw-gain or minimum-leaf checks. | Start at `0`, then increase if pair nodes proliferate without improving validation performance. |\n", + "| `tao_pair_scale` | Multiplies `tao_lambda` for retained pair routers during TAO; the default is `1.1`. It does not reuse `pairwise_penalty`. | Keep `1.1` initially; tune it only when using regularized pair-aware TAO. |\n", + "\n", + "`max_features` still chooses the logical-feature subset available at each node, so it also limits which pairs can be formed. `num_partitions` is independent: it sets the number of outgoing outer branches for either a univariate or bivariate router." + ] + }, + { + "cell_type": "markdown", + "id": "data-types", + "metadata": {}, + "source": [ + "## Categorical and missing values\n", + "\n", + "Pairs may be continuous×continuous, continuous×categorical, or categorical×categorical. One-hot columns grouped with `feature_dict` remain one logical categorical feature during pair screening.\n", + "\n", + "Missing values are routed per feature rather than through one shared OR bin. For example, “`x1` is in this interval while `x2` is `NaN`” and “`x1` is `NaN` while `x2` is in this interval” are distinct inner bins, and a missing branch may continue splitting on the other feature. See [Categorical features](categorical-features.ipynb) for logical feature grouping." + ] + }, + { + "cell_type": "markdown", + "id": "importance-warning", + "metadata": {}, + "source": [ + "## Feature importance and TAO\n", + "\n", + "> **Warning**\n", + ">\n", + "> With `tao_n_runs=0`, a pair node's impurity gain is split equally between its two logical features for bookkeeping. That symmetric split is not a unique or fully trustworthy attribution of either feature's contribution. After any positive-run TAO refinement, impurity `feature_importances_` are unavailable for both univariate and bivariate models. Use held-out [permutation importance](feature-importance.ipynb) when reliable attribution matters.\n", + "\n", + "TAO can revisit only the pairs retained during initial Shape²CART screening. Set `tao_n_runs=0` to inspect the original greedy tree, or leave TAO enabled and evaluate the refined model without impurity importances." + ] + }, + { + "cell_type": "markdown", + "id": "summary", + "metadata": {}, + "source": [ + "## Takeaways\n", + "\n", + "- Leave `pairwise_candidates=0` for the default univariate SGT.\n", + "- Enable a small candidate budget when feature interactions may reduce outer-tree depth.\n", + "- Use `pairwise_penalty` to prefer simpler univariate routing when gains are close.\n", + "- Read pair nodes with `plot_tree`; the heatmap is the exact learned router, not a surrogate.\n", + "- Validate accuracy and runtime together.\n", + "\n", + "S$^2$GT and Shape²CART extend the algorithms introduced in [Empowering Decision Trees via Shape Function Branching](https://neurips.cc/virtual/2025/loc/san-diego/poster/115950). See the [estimator API](../api/estimators.rst), [plotting API](../api/plotting.rst), and [TAO API](../api/tao.rst) for the complete parameter contracts." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/tutorials/categorical-features.ipynb b/docs/tutorials/categorical-features.ipynb index 122df4f..659cc0e 100644 --- a/docs/tutorials/categorical-features.ipynb +++ b/docs/tutorials/categorical-features.ipynb @@ -7,7 +7,7 @@ "source": [ "# Categorical features with `feature_dict`\n", "\n", - "SGT splits are numeric: a node carves one feature with a shape function. To\n", + "By default, an SGT node carves one feature with a shape function. To\n", "model a **categorical** variable, one-hot encode it and group the columns into a\n", "single *logical feature* with `feature_dict`. A multi-column group is routed\n", "through the one-hot inner discretizer, so a node branches on category buckets —\n", @@ -223,7 +223,7 @@ "source": [ "See [Shape functions](shape-functions.ipynb) for what a node's inner tree does,\n", "and [SG forests](forests.ipynb) for ensembling. The full resolution rules live\n", - "in `sgtlearn.configure_feature_dict`." + "in `sgtlearn.configure_feature_dict`. Grouped categoricals can also participate in continuous×categorical and categorical×categorical [S²GT pair nodes](bivariate-branching.ipynb)." ] } ], diff --git a/docs/tutorials/feature-importance.ipynb b/docs/tutorials/feature-importance.ipynb index 5ce2158..ef3cb0d 100644 --- a/docs/tutorials/feature-importance.ipynb +++ b/docs/tutorials/feature-importance.ipynb @@ -22,9 +22,9 @@ "source": [ "## 1. Built-in impurity importances\n", "\n", - "After `fit`, each tree exposes normalized importances over **logical** features — the same order as `processed_features_`. Forests expose the mean and standard deviation across base trees.\n", + "After a fit with TAO disabled, each tree exposes normalized importances over **logical** features — the same order as `processed_features_`. Forests expose the mean and standard deviation across base trees.\n", "\n", - "> **When to use this.** Prefer built-in importances when the tree was grown **without TAO** (`tao_n_runs=0`). Impurity importances are accumulated from the ShapeCART splits during growth. TAO later rewrites routing rules without recomputing those importances, so after TAO they can drift from how the refined model actually uses features. If you ran TAO (the default), prefer permutation importance (part 2) instead." + "> **When to use this.** Built-in importances are available only when the tree was grown **without TAO** (`tao_n_runs=0`). After any positive-run TAO refinement, accessing them raises `AttributeError`; use permutation importance (part 2) instead. For an unrefined [S²GT](bivariate-branching.ipynb), each pair node's gain is split equally between its two logical features for bookkeeping. That symmetric split is not a unique or fully trustworthy attribution." ] }, { @@ -290,4 +290,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/docs/tutorials/forests.ipynb b/docs/tutorials/forests.ipynb index 668b308..c8c349f 100644 --- a/docs/tutorials/forests.ipynb +++ b/docs/tutorials/forests.ipynb @@ -69,7 +69,8 @@ "- `n_estimators` — number of trees. More = better, with diminishing returns.\n", "- `bootstrap` — resample rows per tree (True) for diversity; `max_samples` sets the resample size.\n", "- `max_features` — features considered per split (`\"sqrt\"`, `\"log2\"`, int, float). Lower = more diverse trees.\n", - "- `n_jobs` — parallel tree fitting (`-1` = all cores)." + "- `n_jobs` — parallel tree fitting (`-1` = all cores).\n", + "- `pairwise_candidates` / `pairwise_penalty` — opt into [S²GT bivariate branching](bivariate-branching.ipynb); pairs are screened within each node's `max_features` subset." ] }, { @@ -178,7 +179,7 @@ "id": "5dc4e12d", "metadata": {}, "source": [ - "For a single interpretable model instead, see the [quickstart](../quickstart.rst) and [structure vs. accuracy](structure-and-accuracy.ipynb). TAO also refines every tree in a forest — see [TAO › Forests](tao.ipynb)." + "For a single interpretable model instead, see the [quickstart](../quickstart.rst) and [structure vs. accuracy](structure-and-accuracy.ipynb). The same pair-search controls are available on forests; see [S²GT bivariate branching](bivariate-branching.ipynb). TAO also refines every tree in a forest — see [TAO › Forests](tao.ipynb)." ] } ], diff --git a/docs/tutorials/inspecting-trees.ipynb b/docs/tutorials/inspecting-trees.ipynb index 6b4a044..f22c83c 100644 --- a/docs/tutorials/inspecting-trees.ipynb +++ b/docs/tutorials/inspecting-trees.ipynb @@ -7,7 +7,7 @@ "source": [ "# Inspecting a Fitted SGT\n", "\n", - "A fitted SGT is fully introspectable: `tree_export()` returns a plain dict describing every node — its routing feature, bin thresholds, which child each bin goes to, and the leaf statistics. This notebook walks that structure and pairs it with `plot_tree`." + "A fitted SGT is fully introspectable: `tree_export()` returns a plain dict describing every node — its routing features, bins, child assignments, and leaf statistics. This notebook walks the default univariate structure and pairs it with `plot_tree`." ] }, { @@ -115,7 +115,7 @@ "id": "742b7d03", "metadata": {}, "source": [ - "## Reading a node\n", + "## Reading a univariate node\n", "\n", "- `feature` — the column this node routes on (None at a leaf).\n", "- `thresholds` — the inner-tree bin edges along that feature.\n", @@ -124,7 +124,9 @@ "- `nan_prediction_partition` — where non-finite values route.\n", "- Leaves carry `class_counts` (classifier) or `value` (regressor) instead of a feature.\n", "\n", - "The routing rule is exactly: `bin = searchsorted(thresholds, value, side=\"right\")`, then go to `children[bin_to_partition[bin]]`." + "The univariate routing rule is exactly: `bin = searchsorted(thresholds, value, side=\"right\")`, then go to `children[bin_to_partition[bin]]`.\n", + "\n", + "Bivariate exports set `routing_kind=\"pair\"` and add `pair_features`, `pair_axes`, `pair_inner_tree`, and `pair_leaf_bins`; missing routing is stored on inner-tree edges rather than in one node-level missing partition. See [S²GT bivariate branching](bivariate-branching.ipynb) for the fitted heatmap." ] }, { diff --git a/docs/tutorials/regression.ipynb b/docs/tutorials/regression.ipynb index ca74baa..571ebd8 100644 --- a/docs/tutorials/regression.ipynb +++ b/docs/tutorials/regression.ipynb @@ -127,7 +127,7 @@ "id": "991582ee", "metadata": {}, "source": [ - "See [structure vs. accuracy](structure-and-accuracy.ipynb) (the same depth knobs apply) and [TAO](tao.ipynb) (refines regressors too)." + "See [structure vs. accuracy](structure-and-accuracy.ipynb) (the same depth knobs apply), [S²GT bivariate branching](bivariate-branching.ipynb) (the same pair-search controls apply), and [TAO](tao.ipynb) (refines regressors too)." ] } ], diff --git a/docs/tutorials/sgt-k.ipynb b/docs/tutorials/sgt-k.ipynb index 7627bb9..460c5c8 100644 --- a/docs/tutorials/sgt-k.ipynb +++ b/docs/tutorials/sgt-k.ipynb @@ -143,7 +143,7 @@ "source": [ "The binary tree only matches `num_partitions=4`'s depth-1 accuracy at `max_depth=2` — a deeper tree with more nodes. So K is a lever for **compactness**: set it near the number of natural regions in a feature. Too-large K on simple structure just adds empty branches, and K only pays off when the inner shape function ([`inner_max_depth`](shape-functions.ipynb)) can produce enough bins to feed it.\n", "\n", - "See [Structure vs. accuracy](structure-and-accuracy.ipynb) for choosing K on real data with a train/test split." + "`num_partitions` also sets the outgoing branches for a bivariate router; see the three-way [S²GT example](bivariate-branching.ipynb). See [Structure vs. accuracy](structure-and-accuracy.ipynb) for choosing K on real data with a train/test split." ] } ], diff --git a/docs/tutorials/shape-functions.ipynb b/docs/tutorials/shape-functions.ipynb index a15235a..198c4ed 100644 --- a/docs/tutorials/shape-functions.ipynb +++ b/docs/tutorials/shape-functions.ipynb @@ -7,7 +7,7 @@ "source": [ "# Shape Functions — what an SGT node actually does\n", "\n", - "A classic decision-tree node asks one yes/no question (`x < t`). A **Shape Generalized Tree** node is richer: it runs a small *inner tree* over one feature, carving that feature into several bins, then routes each bin to a child. That inner tree **is** the node's shape function. This notebook shows how the shape-function knobs change what a single node can express." + "A classic decision-tree node asks one yes/no question (`x < t`). By default, a **Shape Generalized Tree** is richer: each node learns a potential non-linear function over one feature that is used to route samples to different branches. In ShapeCART, we represent this function by running a small *inner tree* over one feature, carving that feature into several bins, then routes each bin to a child. This notebook shows how the univariate shape-function knobs change what a single node can express." ] }, { @@ -109,7 +109,7 @@ "id": "632f2ad0", "metadata": {}, "source": [ - "More bins let one node express a more detailed shape, up to the point where the pattern is captured. See [SGT_K multi-way branching](sgt-k.ipynb) for the complementary *outer* branching knob, and [Structure vs. accuracy](structure-and-accuracy.ipynb) for how these interact with generalization." + "More bins let one node express a more detailed shape, up to the point where the pattern is captured. [S²GT bivariate branching](bivariate-branching.ipynb) extends the inner tree to two logical features. See [SGT_K multi-way branching](sgt-k.ipynb) for the complementary *outer* branching knob, and [Structure vs. accuracy](structure-and-accuracy.ipynb) for how these interact with generalization." ] } ], diff --git a/docs/tutorials/structure-and-accuracy.ipynb b/docs/tutorials/structure-and-accuracy.ipynb index 8186854..c5f5b13 100644 --- a/docs/tutorials/structure-and-accuracy.ipynb +++ b/docs/tutorials/structure-and-accuracy.ipynb @@ -182,6 +182,7 @@ "\n", "- `max_depth` and `inner_max_depth` add capacity — raise them until the **test** score stops improving.\n", "- `min_samples_leaf` removes capacity — raise it to fight overfitting (train drops; test moves non-monotonically here, so pick the value that maximizes the test score rather than assuming larger is always better).\n", + "- `pairwise_candidates` expands interaction search and `pairwise_penalty` discourages pair selection; see [S²GT bivariate branching](bivariate-branching.ipynb).\n", "- Once structure is chosen, [TAO](tao.ipynb) refines the rules inside that fixed structure." ] } diff --git a/docs/tutorials/tao.ipynb b/docs/tutorials/tao.ipynb index d6fb6af..65e568e 100644 --- a/docs/tutorials/tao.ipynb +++ b/docs/tutorials/tao.ipynb @@ -234,6 +234,8 @@ "- **Leave it on** (`tao_n_runs=10`) for the default: every model is refined once, no extra code.\n", "- **Fit with TAO off, then `tao.TAO_refine`** when you want to compare before/after, sweep `lambda_` without refitting, or refine a forest on the full training set. Safe to call repeatedly — each call refines further in place.\n", "\n", + "Pair-aware TAO reconsiders only pairs retained during initial screening and scales their complexity cost with `tao_pair_scale` (default `1.1`); see [S²GT bivariate branching](bivariate-branching.ipynb). After any positive-run TAO refinement, impurity feature importances are unavailable.\n", + "\n", "See the [TAO API reference](../api/tao.rst) for the full parameter list." ] } diff --git a/sgtlearn/_export.py b/sgtlearn/_export.py index 66d59cd..f97d83d 100644 --- a/sgtlearn/_export.py +++ b/sgtlearn/_export.py @@ -10,10 +10,10 @@ from __future__ import annotations from collections.abc import Sequence -from typing import Any +from typing import Any, cast import numpy as np -from matplotlib.patches import FancyArrowPatch +from matplotlib.patches import FancyArrowPatch, Rectangle __all__ = ["export_graphviz", "export_text", "plot_tree"] @@ -160,6 +160,43 @@ def _finite_routing_bins( return bin_to_partition +def _is_pair_node(node: dict) -> bool: + return node.get("routing_kind") == "pair" + + +def _pair_axis_missing(row: np.ndarray, axis: dict) -> bool: + columns = [int(c) for c in axis["columns"]] + if axis["kind"] == "continuous": + return not np.isfinite(row[columns[0]]) + return _active_onehot_column(row, columns) is None + + +def _route_pair_bin(node: dict, row: np.ndarray) -> int: + """Replay one exported pair router exactly.""" + inner = {int(n["id"]): n for n in node["pair_inner_tree"]} + axes = node["pair_axes"] + current = 0 + while not inner[current]["is_leaf"]: + split = inner[current] + if _pair_axis_missing(row, axes[int(split["axis"])]): + current = int(split["missing"]) + elif split["kind"] == "categorical": + current = int( + split["right"] if row[int(split["feature"])] >= 0.5 else split["left"] + ) + else: + current = int( + split["left"] + if row[int(split["feature"])] <= float(split["threshold"]) + else split["right"] + ) + return int(inner[current]["bin"]) + + +def _route_pair_partition(node: dict, row: np.ndarray) -> int: + return int(node["bin_to_partition"][_route_pair_bin(node, row)]) + + def _merge_routing_regions( thresholds: list[float], bin_to_partition: list[int], @@ -231,8 +268,15 @@ def _route_samples(tree: dict, X) -> dict[int, Any]: reach.setdefault(cid, np.empty(0, dtype=np.int64)) queue.append(cid) continue - feature = node["feature"] - if _is_categorical_node(node): + feature = node.get("feature") + if _is_pair_node(node): + part_idx = np.fromiter( + (_route_pair_partition(node, X_arr[int(row_i)]) for row_i in rows), + dtype=np.int64, + count=rows.size, + ) + children = list(node["children"]) + elif _is_categorical_node(node): feature_cols = [int(c) for c in node["features"]] bin_categories = _bin_categories_for_node(node) b2p = np.asarray(list(node["bin_to_partition"]), dtype=np.int64) @@ -659,6 +703,279 @@ def _draw_internal_panel( return [inset, *extra] +def _pair_axis_label( + estimator: Any, axis: dict, feat_names: list[str], *, prefer_logical_name: bool = True +) -> str: + processed = getattr(estimator, "processed_features_", None) + logical = int(axis["logical_feature"]) + if prefer_logical_name and processed is not None and processed.logical_names and logical < len(processed.logical_names): + return str(processed.logical_names[logical]) + return _category_label_for_columns(axis["columns"], feat_names) + + +def _pair_axis_cells( + node: dict, + axis_index: int, + axis: dict, + values: np.ndarray | None, + *, + include_missing: bool, +) -> list[tuple[float, float, object]]: + if axis["kind"] == "categorical": + categories = [int(c) for c in axis["categories"]] + categorical_cells: list[tuple[float, float, object]] = [ + (float(i), float(i + 1), category) + for i, category in enumerate(categories) + ] + if include_missing: + categorical_cells.append( + ( + float(len(categories)) + 0.08, + float(len(categories)) + 0.38, + None, + ) + ) + return categorical_cells + thresholds = sorted({ + float(split["threshold"]) + for split in node["pair_inner_tree"] + if not split["is_leaf"] + and int(split["axis"]) == axis_index + and split["kind"] == "continuous" + }) + finite = values[np.isfinite(values)] if values is not None else np.array([]) + if finite.size: + lo, hi = float(finite.min()), float(finite.max()) + elif thresholds: + delta = max((thresholds[-1] - thresholds[0]) / max(len(thresholds) - 1, 1), 1.0) + lo, hi = thresholds[0] - delta, thresholds[-1] + delta + else: + lo, hi = 0.0, 1.0 + if hi <= lo: + hi = lo + 1.0 + edges = [lo, *[t for t in thresholds if lo < t < hi], hi] + continuous_cells: list[tuple[float, float, object]] = [ + (edges[i], edges[i + 1], (edges[i] + edges[i + 1]) / 2) + for i in range(len(edges) - 1) + ] + if include_missing: + span = hi - lo + continuous_cells.append((hi + span * 0.04, hi + span * 0.12, None)) + return continuous_cells + + +def _pair_cell_row(node: dict, x_axis: dict, x_value, y_axis: dict, y_value) -> np.ndarray: + width = max([int(c) for c in node["features"]] + [0]) + 1 + row = np.zeros(width, dtype=np.float64) + for axis, value in ((x_axis, x_value), (y_axis, y_value)): + columns = [int(c) for c in axis["columns"]] + if value is None: + if axis["kind"] == "continuous": + row[columns[0]] = np.nan + elif axis["kind"] == "continuous": + row[columns[0]] = float(value) + else: + row[int(value)] = 1.0 + return row + + +def _pair_switch_boundaries( + cells: list[tuple[float, float, object]], + partitions: np.ndarray, + *, + axis: int, +) -> list[float]: + """Return finite cell boundaries where the outer partition changes.""" + boundaries = [] + for index in range(1, len(cells)): + if cells[index - 1][2] is None or cells[index][2] is None: + continue + before = np.take(partitions, index - 1, axis=axis) + after = np.take(partitions, index, axis=axis) + if np.any(before != after): + boundaries.append(cells[index][0]) + return boundaries + + +def _draw_internal_panel_pair( + host_ax, + center: tuple[float, float], + size: tuple[float, float], + node: dict, + palette, + estimator: Any, + feat_names: list[str], + X_rows: np.ndarray | None, + fontsize: int | None, + label: str, + prefer_logical_name: bool, + n_hist_bins: int, + precision: int, +) -> list: + """Draw exported pair-routing cells, including the two missing margins.""" + cx, cy = center + w, h = size + left, bottom = cx - w / 2, cy - h / 2 + with_histograms = X_rows is not None and len(X_rows) > 0 + heatmap_width = w * 0.78 if with_histograms else w + heatmap_height = h * 0.78 if with_histograms else h + inset = host_ax.inset_axes( + [left, bottom, heatmap_width, heatmap_height], + transform=host_ax.transAxes, + ) + inset.set_label("pair-heatmap") + x_axis, y_axis = node["pair_axes"] + x_values = X_rows[:, int(x_axis["columns"][0])] if X_rows is not None and x_axis["kind"] == "continuous" else None + y_values = X_rows[:, int(y_axis["columns"][0])] if X_rows is not None and y_axis["kind"] == "continuous" else None + x_has_missing = X_rows is None or any( + _pair_axis_missing(row, x_axis) for row in X_rows + ) + y_has_missing = X_rows is None or any( + _pair_axis_missing(row, y_axis) for row in X_rows + ) + x_cells = _pair_axis_cells( + node, 0, x_axis, x_values, include_missing=x_has_missing + ) + y_cells = _pair_axis_cells( + node, 1, y_axis, y_values, include_missing=y_has_missing + ) + counts: dict[tuple[int, int], int] = {} + if X_rows is not None: + for row in X_rows: + for xi, (x0, x1, xv) in enumerate(x_cells): + if _pair_axis_missing(row, x_axis) != (xv is None): + continue + if xv is not None and x_axis["kind"] == "continuous" and not (x0 <= row[int(x_axis["columns"][0])] <= x1): + continue + if xv is not None and x_axis["kind"] == "categorical" and _active_onehot_column(row, x_axis["columns"]) != xv: + continue + for yi, (y0, y1, yv) in enumerate(y_cells): + if _pair_axis_missing(row, y_axis) != (yv is None): + continue + if yv is not None and y_axis["kind"] == "continuous" and not (y0 <= row[int(y_axis["columns"][0])] <= y1): + continue + if yv is not None and y_axis["kind"] == "categorical" and _active_onehot_column(row, y_axis["columns"]) != yv: + continue + counts[(xi, yi)] = counts.get((xi, yi), 0) + 1 + break + break + partitions = np.empty((len(x_cells), len(y_cells)), dtype=np.intp) + for xi, (x0, x1, xv) in enumerate(x_cells): + for yi, (y0, y1, yv) in enumerate(y_cells): + part = _route_pair_partition(node, _pair_cell_row(node, x_axis, xv, y_axis, yv)) + partitions[xi, yi] = part + inset.add_patch(Rectangle((x0, y0), x1 - x0, y1 - y0, facecolor=palette[part], alpha=0.55, edgecolor="white", linewidth=0.5)) + inset.set_xlim(x_cells[0][0], x_cells[-1][1]) + inset.set_ylim(y_cells[0][0], y_cells[-1][1]) + histogram_axes = [] + if with_histograms: + x_hist = host_ax.inset_axes( + [left, bottom + h * 0.82, heatmap_width, h * 0.18], + transform=host_ax.transAxes, + ) + y_hist = host_ax.inset_axes( + [left + w * 0.82, bottom, w * 0.18, heatmap_height], + transform=host_ax.transAxes, + ) + x_hist.set_label("pair-x-histogram") + y_hist.set_label("pair-y-histogram") + x_counts = [ + sum(counts.get((xi, yi), 0) for yi in range(len(y_cells))) + for xi in range(len(x_cells)) + ] + y_counts = [ + sum(counts.get((xi, yi), 0) for xi in range(len(x_cells))) + for yi in range(len(y_cells)) + ] + if x_axis["kind"] == "continuous": + assert x_values is not None + finite_cells = [cell for cell in x_cells if cell[2] is not None] + finite_values = x_values[np.isfinite(x_values)] + x_hist.hist( + finite_values, + bins=n_hist_bins, + range=(finite_cells[0][0], finite_cells[-1][1]), + color="#777777", + alpha=0.65, + ) + if x_has_missing: + start, end, _ = x_cells[-1] + x_hist.bar((start + end) / 2, x_counts[-1], width=end - start, color="#777777", alpha=0.65) + else: + x_hist.bar( + [(start + end) / 2 for start, end, _ in x_cells], + x_counts, + width=[end - start for start, end, _ in x_cells], + color="#777777", + alpha=0.65, + ) + if y_axis["kind"] == "continuous": + assert y_values is not None + finite_cells = [cell for cell in y_cells if cell[2] is not None] + finite_values = y_values[np.isfinite(y_values)] + y_hist.hist( + finite_values, + bins=n_hist_bins, + range=(finite_cells[0][0], finite_cells[-1][1]), + orientation="horizontal", + color="#777777", + alpha=0.65, + ) + if y_has_missing: + start, end, _ = y_cells[-1] + y_hist.barh((start + end) / 2, y_counts[-1], height=end - start, color="#777777", alpha=0.65) + else: + y_hist.barh( + [(start + end) / 2 for start, end, _ in y_cells], + y_counts, + height=[end - start for start, end, _ in y_cells], + color="#777777", + alpha=0.65, + ) + x_hist.set_xlim(x_cells[0][0], x_cells[-1][1]) + y_hist.set_ylim(y_cells[0][0], y_cells[-1][1]) + x_hist.set_axis_off() + y_hist.set_axis_off() + histogram_axes = [x_hist, y_hist] + inset.set_xlabel(_pair_axis_label(estimator, x_axis, feat_names, prefer_logical_name=prefer_logical_name), fontsize=(fontsize - 1) if isinstance(fontsize, int) else None) + inset.set_ylabel(_pair_axis_label(estimator, y_axis, feat_names, prefer_logical_name=prefer_logical_name), fontsize=(fontsize - 1) if isinstance(fontsize, int) else None) + if x_axis["kind"] == "categorical": + inset.set_xticks([(a + b) / 2 for a, b, _ in x_cells]) + inset.set_xticklabels([_column_label(cast(int, v), feat_names) if v is not None else "NaN" for _, _, v in x_cells], rotation=30, ha="right", fontsize=(fontsize - 2) if isinstance(fontsize, int) else None) + else: + x_ticks = _pair_switch_boundaries(x_cells, partitions, axis=0) + x_labels = [f"{value:.{precision}f}" for value in x_ticks] + if x_cells[-1][2] is None: + start, end, _ = x_cells[-1] + x_ticks.append((start + end) / 2) + x_labels.append("NaN") + inset.set_xticks(x_ticks) + inset.set_xticklabels( + x_labels, + rotation=30, + ha="right", + fontsize=(fontsize - 2) if isinstance(fontsize, int) else None, + ) + if y_axis["kind"] == "categorical": + inset.set_yticks([(a + b) / 2 for a, b, _ in y_cells]) + inset.set_yticklabels([_column_label(cast(int, v), feat_names) if v is not None else "NaN" for _, _, v in y_cells], fontsize=(fontsize - 2) if isinstance(fontsize, int) else None) + else: + y_ticks = _pair_switch_boundaries(y_cells, partitions, axis=1) + y_labels = [f"{value:.{precision}f}" for value in y_ticks] + if y_cells[-1][2] is None: + start, end, _ = y_cells[-1] + y_ticks.append((start + end) / 2) + y_labels.append("NaN") + inset.set_yticks(y_ticks) + inset.set_yticklabels( + y_labels, + fontsize=(fontsize - 2) if isinstance(fontsize, int) else None, + ) + if label != "none": + inset.text(1.0, 1.0, f"n={node['n_samples']}", transform=inset.transAxes, ha="right", va="top", fontsize=(fontsize - 2) if isinstance(fontsize, int) else None, color="#444444") + return [inset, *histogram_axes] + + def plot_tree( estimator: Any, *, @@ -676,7 +993,50 @@ def plot_tree( node_aspect_ratio: float = 2.5, n_hist_bins: int = 20, ) -> list[Any]: - """Render a fitted SGT estimator with matplotlib (see module docstring).""" + """Render a fitted SGT estimator with matplotlib. + + Univariate nodes show their one-dimensional shape function. Bivariate + nodes show the exact two-dimensional routing heatmap; when ``X`` is + supplied, top/right marginal histograms and observed missing-value margins + are included. Continuous axes label only thresholds where the final outer + partition changes. + + Parameters + ---------- + estimator : SGTClassifier or SGTRegressor + Fitted estimator to render. + X : array-like of shape (n_samples, n_features), optional + Data used for node sample counts and bivariate marginal histograms. + max_depth : int, optional + Maximum outer-tree depth to display. + feature_names : list of str, optional + Display names for input columns. + class_names : list of str, bool, or None, default=None + Class labels shown on classifier leaves. + label : str, default="feature" + Controls field-name labels in node text. + impurity : bool, default=False + Whether to display node impurity. + proportion : bool, default=False + Whether to display sample proportions instead of counts. + precision : int, default=2 + Decimal precision for displayed values and routing thresholds. + cmap : colormap or color sequence, optional + Colors for classes and routing partitions. + ax : matplotlib.axes.Axes, optional + Axes on which to draw. + fontsize : int, optional + Node text size. + node_aspect_ratio : float, default=2.5 + Width-to-height ratio of node boxes. + n_hist_bins : int, default=20 + Number of bins in each bivariate marginal histogram. + + Returns + ------- + list + Matplotlib artists created by the tree exporter. + """ if not isinstance(estimator, (SGTClassifier, SGTRegressor)): raise TypeError( "plot_tree expects an SGTClassifier or SGTRegressor; got " @@ -734,10 +1094,11 @@ def plot_tree( resolved_class_names = list(class_names) # type: ignore[arg-type] n_features = estimator.n_features_in_ or 0 + stored_feature_names = estimator.feature_names_in_ if feature_names is not None: feat_names = list(feature_names) - elif getattr(estimator, "feature_names_in_", None) is not None: - feat_names = [str(n) for n in estimator.feature_names_in_] + elif stored_feature_names is not None: + feat_names = [str(n) for n in stored_feature_names] else: feat_names = [f"X[{i}]" for i in range(n_features)] @@ -836,7 +1197,23 @@ def _size_for(nid: int) -> tuple[float, float]: if node_rows is not None and feat_idx is not None: feat_vals = node_rows[:, feat_idx] - if _is_categorical_node(node): + if _is_pair_node(node): + panel_artists = _draw_internal_panel_pair( + host_ax=ax, + center=pos, + size=(internal_w, internal_h), + node=node, + palette=palette, + estimator=estimator, + feat_names=feat_names, + X_rows=node_rows, + fontsize=fontsize, + label=label, + prefer_logical_name=feature_names is None, + n_hist_bins=n_hist_bins, + precision=precision, + ) + elif _is_categorical_node(node): panel_artists = _draw_internal_panel_categorical( host_ax=ax, center=pos, @@ -864,11 +1241,15 @@ def _size_for(nid: int) -> tuple[float, float]: ) artists.extend(panel_artists) - feat_name = _logical_feature_label( - estimator, - node, - feat_names, - prefer_logical_name=feature_names is None, + feat_name = ( + " × ".join(_pair_axis_label(estimator, axis, feat_names, prefer_logical_name=feature_names is None) for axis in node["pair_axes"]) + if _is_pair_node(node) + else _logical_feature_label( + estimator, + node, + feat_names, + prefer_logical_name=feature_names is None, + ) ) feat_text = ax.text( pos[0] + internal_w * 0.55, diff --git a/sgtlearn/_features.py b/sgtlearn/_features.py index 543654a..795cfcb 100644 --- a/sgtlearn/_features.py +++ b/sgtlearn/_features.py @@ -19,7 +19,7 @@ class ProcessedFeatures: features List of ``{"type": "continuous"|"categorical", "indices": [...]}`` dicts in trainer order. Index ``i`` aligns with - ``estimator.feature_importances_[i]`` after :meth:`~sklearn.base.BaseEstimator.fit`. + ``estimator.feature_importances_[i]`` after a fit without TAO. logical_names Parallel names for ``features``. When ``feature_dict`` is supplied, these are the stringified keys; omitted columns are filled as diff --git a/sgtlearn/base.py b/sgtlearn/base.py index c3d1639..acd067a 100644 --- a/sgtlearn/base.py +++ b/sgtlearn/base.py @@ -2,7 +2,10 @@ from __future__ import annotations +import warnings from collections.abc import Mapping, Sequence +from math import ceil, isfinite +from numbers import Integral, Real from typing import Any import numpy as np @@ -61,6 +64,29 @@ def _configure_processed_features( ) +def _resolve_pairwise_candidates(value: float, n_features: int) -> int: + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError( # noqa: TRY004 - sklearn parameters use ValueError + "pairwise_candidates must be a non-negative int or float" + ) + if not isfinite(float(value)) or value < 0: + raise ValueError("pairwise_candidates must be finite and non-negative") + if isinstance(value, Integral): + return int(value) + return ceil(float(value) * n_features) + + +def _validate_tao_pair_scale(value: float) -> float: + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError( # noqa: TRY004 - sklearn parameters use ValueError + "tao_pair_scale must be finite and non-negative" + ) + scale = float(value) + if not isfinite(scale) or scale < 0: + raise ValueError("tao_pair_scale must be finite and non-negative") + return scale + + class _IdentityLabelEncoder(LabelEncoder): """``LabelEncoder`` for targets already encoded as ``0 .. n_classes - 1``. @@ -94,15 +120,29 @@ class BaseShapeCART(BaseEstimator): Subclasses own the native backend handle (``_est``) and validation rules. """ + _tao_refined_: bool = False + @property def feature_importances_(self) -> np.ndarray: """Normalized per-feature importances from the fitted tree. Length matches the number of logical features passed to ``fit`` (one-to-one with :attr:`processed_features_`). Available only after - training. + training without TAO refinement. """ check_is_fitted(self, attributes=("_est",)) + if getattr(self, "_tao_refined_", False): + raise AttributeError( + "feature_importances_ is unavailable after TAO refinement; " + "use permutation importance on held-out data instead." + ) + if getattr(self._est, "has_pair_nodes", False): + warnings.warn( + "Pair-node impurity gain is split equally between both features; " + "this attribution is only a bookkeeping convention.", + UserWarning, + stacklevel=2, + ) return np.asarray(self._est.feature_importance, dtype=np.float64).ravel() @property @@ -128,6 +168,8 @@ def _normalize_tree_export(tree: dict) -> dict: for node in tree.get("nodes", []): if node.get("is_leaf", True): continue + if node.get("routing_kind") == "pair": + continue if node.get("is_categorical"): b2p = node.get("bin_to_partition") if not b2p: @@ -163,12 +205,12 @@ class SGTClassifier(ClassifierMixin, BaseShapeCART): """Shape Generalized Tree classifier. A decision tree where each internal node applies a learnable, axis-aligned - *shape function* to a single feature rather than a single threshold. The - shape function is itself an inner tree (univariate, depth-limited) that - partitions the feature's value range into ``num_partitions`` bins; the - outer tree then routes samples through those bins to grow the overall - classifier. Training is performed by the native ShapeCART C++ trainer - exposed through ``ClassificationShapeGeneralizedTree``. + *shape function*. By default the shape function is a depth-limited inner + tree over one logical feature; ``pairwise_candidates > 0`` also lets it use + an ordinary axis-aligned CART over two logical features. The outer tree + routes the resulting bins into ``num_partitions`` children. Training is + performed by the native ShapeCART C++ trainer exposed through + ``ClassificationShapeGeneralizedTree``. The estimator follows the ``scikit-learn`` ``ClassifierMixin`` contract and is compatible with sklearn pipelines, cross-validators, and metaestimators. @@ -223,6 +265,15 @@ class SGTClassifier(ClassifierMixin, BaseShapeCART): - ``"log2"``: use ``max(1, int(log2(n_features)))`` columns. String values are case-insensitive in the native binding. + pairwise_candidates : int or float, default=0 + Maximum retained feature pairs fitted per node. An integer is an + absolute limit; a float resolves to ``ceil(value * n_logical_features)``. + Zero preserves univariate-only training. + pairwise_penalty : float, default=0.0 + Non-negative penalty added when comparing a fitted pair with the best + univariate candidate. + tao_pair_scale : float, default=1.1 + Multiplier applied to ``tao_lambda`` for pair routers during TAO. class_weight : dict, list of dict, or None, default=None Per-class multipliers. A single mapping applies to every output; for multi-output ``y`` of shape ``(n_samples, n_outputs)``, pass a list of @@ -244,7 +295,8 @@ class SGTClassifier(ClassifierMixin, BaseShapeCART): feature_importances_ : ndarray of shape (n_logical_features,) Normalized impurity-based importances from the fitted tree. Index ``i`` corresponds to :attr:`processed_features_` entry ``i`` (same - order as the logical features passed to the native trainer). + order as the logical features passed to the native trainer). Available + only when the model has not undergone TAO refinement. processed_features_ : ProcessedFeatures Resolved logical features used at :meth:`fit`. ``features[i]`` and ``logical_names[i]`` align with ``feature_importances_[i]``. @@ -307,9 +359,12 @@ def __init__( coordinate_descent_smart_init: bool = True, random_state: int | None = 42, max_features: float | str | None = None, + pairwise_candidates: float = 0, + pairwise_penalty: float = 0.0, class_weight: Mapping[Any, float] | Sequence[Mapping[Any, float]] | None = None, tao_n_runs: int = 10, tao_lambda: float = 0.0, + tao_pair_scale: float = 1.1, ) -> None: """Store hyperparameters; training happens in :meth:`fit`.""" self.criterion = criterion @@ -327,10 +382,13 @@ def __init__( self.coordinate_descent_smart_init = bool(coordinate_descent_smart_init) self.random_state = random_state self.max_features = max_features + self.pairwise_candidates = pairwise_candidates + self.pairwise_penalty = pairwise_penalty self.class_weight = class_weight self.tao_n_runs = tao_n_runs self.tao_lambda = tao_lambda + self.tao_pair_scale = tao_pair_scale self._est: Any = None self._le: Any = None self.classes_: Any | None = None @@ -448,7 +506,7 @@ def fit( self.n_features_in_ = X.shape[1] if column_names is None: column_names = _column_names_from_X(X) - self.feature_names_in_: np.ndarray | None = ( + self.feature_names_in_ = ( np.asarray(column_names, dtype=object) if column_names is not None else None ) @@ -459,6 +517,16 @@ def fit( column_names=column_names, ) self._processed_features = processed_features + resolved_pairwise_candidates = _resolve_pairwise_candidates( + self.pairwise_candidates, len(processed_features.features) + ) + if ( + not isinstance(self.pairwise_penalty, Real) + or not isfinite(float(self.pairwise_penalty)) + or self.pairwise_penalty < 0 + ): + raise ValueError("pairwise_penalty must be finite and non-negative") + tao_pair_scale = _validate_tao_pair_scale(self.tao_pair_scale) outer_depth = 0 if self.max_depth is None else int(self.max_depth) outer_leaves = 0 if self.max_leaf_nodes is None else int(self.max_leaf_nodes) @@ -484,6 +552,8 @@ def fit( bool(self.coordinate_descent_smart_init), int(42 if self.random_state is None else self.random_state), self.max_features, + resolved_pairwise_candidates, + float(self.pairwise_penalty), ) X32 = np.ascontiguousarray(X, dtype=np.float32) @@ -491,6 +561,7 @@ def fit( self._est.fit( X32, y_u, sample_weight=sw, features=processed_features.to_native() ) + self._tao_refined_ = False if self.tao_n_runs > 0: from sgtlearn.tao import TAO_refine @@ -503,6 +574,7 @@ def fit( check_input=check_input, n_runs=self.tao_n_runs, lambda_=self.tao_lambda, + tao_pair_scale=tao_pair_scale, ) return self @@ -575,8 +647,9 @@ class SGTRegressor(RegressorMixin, BaseShapeCART): """Shape Generalized Tree regressor. Regression analogue of :class:`SGTClassifier`. Each internal node applies a - learnable shape function (an inner univariate tree) to a single feature, - and the outer tree routes samples through the resulting bins. Training is + learnable shape function: an inner univariate tree by default, or an + ordinary axis-aligned two-feature CART when bivariate branching is enabled. + The outer tree routes samples through the resulting bins. Training is performed by the native ShapeCART C++ trainer exposed through ``RegressionShapeGeneralizedTree``. @@ -624,6 +697,15 @@ class SGTRegressor(RegressorMixin, BaseShapeCART): max_features : int, float, {"sqrt", "log2"} or None, default=None Per-split feature subsampling. Same semantics as :class:`SGTClassifier`. + pairwise_candidates : int or float, default=0 + Maximum retained feature pairs fitted per node. An integer is an + absolute limit; a float resolves to ``ceil(value * n_logical_features)``. + Zero preserves univariate-only training. + pairwise_penalty : float, default=0.0 + Non-negative penalty applied only when comparing fitted pair and + univariate candidates. + tao_pair_scale : float, default=1.1 + Multiplier applied to ``tao_lambda`` for pair routers during TAO. Attributes ---------- @@ -634,7 +716,8 @@ class SGTRegressor(RegressorMixin, BaseShapeCART): feature_importances_ : ndarray of shape (n_logical_features,) Normalized impurity-based importances from the fitted tree. Index ``i`` corresponds to :attr:`processed_features_` entry ``i`` (same - order as the logical features passed to the native trainer). + order as the logical features passed to the native trainer). Available + only when the model has not undergone TAO refinement. processed_features_ : ProcessedFeatures Resolved logical features used at :meth:`fit`. ``features[i]`` and ``logical_names[i]`` align with ``feature_importances_[i]``. @@ -694,8 +777,11 @@ def __init__( coordinate_descent_smart_init: bool = True, random_state: int | None = 42, max_features: float | str | None = None, + pairwise_candidates: float = 0, + pairwise_penalty: float = 0.0, tao_n_runs: int = 10, tao_lambda: float = 0.0, + tao_pair_scale: float = 1.1, ) -> None: self.criterion = criterion self.num_partitions = int(num_partitions) @@ -712,8 +798,11 @@ def __init__( self.coordinate_descent_smart_init = bool(coordinate_descent_smart_init) self.random_state = random_state self.max_features = max_features + self.pairwise_candidates = pairwise_candidates + self.pairwise_penalty = pairwise_penalty self.tao_n_runs = tao_n_runs self.tao_lambda = tao_lambda + self.tao_pair_scale = tao_pair_scale self._est: Any = None self.n_outputs_: int = 1 self.n_features_in_: int | None = None @@ -775,7 +864,7 @@ def fit( self.n_features_in_ = X.shape[1] if column_names is None: column_names = _column_names_from_X(X) - self.feature_names_in_: np.ndarray | None = ( + self.feature_names_in_ = ( np.asarray(column_names, dtype=object) if column_names is not None else None ) @@ -786,6 +875,16 @@ def fit( column_names=column_names, ) self._processed_features = processed_features + resolved_pairwise_candidates = _resolve_pairwise_candidates( + self.pairwise_candidates, len(processed_features.features) + ) + if ( + not isinstance(self.pairwise_penalty, Real) + or not isfinite(float(self.pairwise_penalty)) + or self.pairwise_penalty < 0 + ): + raise ValueError("pairwise_penalty must be finite and non-negative") + tao_pair_scale = _validate_tao_pair_scale(self.tao_pair_scale) outer_depth = 0 if self.max_depth is None else int(self.max_depth) outer_leaves = 0 if self.max_leaf_nodes is None else int(self.max_leaf_nodes) @@ -810,6 +909,8 @@ def fit( bool(self.coordinate_descent_smart_init), int(42 if self.random_state is None else self.random_state), self.max_features, + resolved_pairwise_candidates, + float(self.pairwise_penalty), ) X32 = np.ascontiguousarray(X, dtype=np.float32) @@ -821,6 +922,7 @@ def fit( sample_weight=sw, features=processed_features.to_native(), ) + self._tao_refined_ = False if self.tao_n_runs > 0: from sgtlearn.tao import TAO_refine @@ -833,6 +935,7 @@ def fit( check_input=check_input, n_runs=self.tao_n_runs, lambda_=self.tao_lambda, + tao_pair_scale=tao_pair_scale, ) return self diff --git a/sgtlearn/ensemble/_random_sgforest.py b/sgtlearn/ensemble/_random_sgforest.py index 27d2067..f443d16 100644 --- a/sgtlearn/ensemble/_random_sgforest.py +++ b/sgtlearn/ensemble/_random_sgforest.py @@ -2,6 +2,7 @@ from __future__ import annotations +import warnings from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from numbers import Integral @@ -99,8 +100,11 @@ def __init__( bootstrap: bool = True, max_samples: float | None = None, random_state: int | np.random.RandomState | None = None, + pairwise_candidates: float = 0, + pairwise_penalty: float = 0.0, tao_n_runs: int = 10, tao_lambda: float = 0.0, + tao_pair_scale: float = 1.1, n_jobs: int | None = None, verbose: int = 0, ) -> None: @@ -122,8 +126,11 @@ def __init__( self.bootstrap = bool(bootstrap) self.max_samples = max_samples self.random_state = random_state + self.pairwise_candidates = pairwise_candidates + self.pairwise_penalty = pairwise_penalty self.tao_n_runs = int(tao_n_runs) self.tao_lambda = float(tao_lambda) + self.tao_pair_scale = tao_pair_scale self.n_jobs = n_jobs self.verbose = int(verbose) @@ -143,8 +150,11 @@ def _tree_kwargs(self) -> dict[str, Any]: "coordinate_descent_patience": self.coordinate_descent_patience, "coordinate_descent_smart_init": self.coordinate_descent_smart_init, "max_features": self.max_features, + "pairwise_candidates": self.pairwise_candidates, + "pairwise_penalty": self.pairwise_penalty, "tao_n_runs": self.tao_n_runs, "tao_lambda": self.tao_lambda, + "tao_pair_scale": self.tao_pair_scale, } @abstractmethod @@ -254,19 +264,37 @@ def tree_factory(tree_seed: int, kw: dict[str, Any]) -> Any: def _tree_feature_importances_matrix(self) -> np.ndarray: check_is_fitted(self, attributes=("estimators_",)) - return np.stack( - [ - np.asarray(est.feature_importances_, dtype=np.float64) - for est in self.estimators_ - ] + if any(getattr(est, "_tao_refined_", False) for est in self.estimators_): + raise AttributeError( + "feature importances are unavailable after TAO refinement; " + "use permutation importance on held-out data instead." + ) + has_pair_nodes = any( + getattr(getattr(est, "_est", None), "has_pair_nodes", False) + for est in self.estimators_ ) + if has_pair_nodes: + warnings.warn( + "Pair-node impurity gain is attributed equally to both features.", + UserWarning, + stacklevel=2, + ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + return np.stack( + [ + np.asarray(est.feature_importances_, dtype=np.float64) + for est in self.estimators_ + ] + ) @property def mean_feature_importances_(self) -> np.ndarray: """Mean per-logical-feature importances across fitted base trees. Aligned with :attr:`processed_features_` (same order as each tree's - ``feature_importances_``). Available only after :meth:`fit`. + ``feature_importances_``). Available only after :meth:`fit` without + TAO refinement. """ return self._tree_feature_importances_matrix().mean(axis=0) @@ -275,7 +303,8 @@ def std_feature_importance_(self) -> np.ndarray: """Per-logical-feature standard deviation of importances across trees. Population std (``ddof=0``) over base estimators; aligned with - :attr:`mean_feature_importances_`. Available only after :meth:`fit`. + :attr:`mean_feature_importances_`. Available only after :meth:`fit` + without TAO refinement. """ return self._tree_feature_importances_matrix().std(axis=0) diff --git a/sgtlearn/ensemble/random_sgforest_classifier.py b/sgtlearn/ensemble/random_sgforest_classifier.py index 67f9618..5a998ab 100644 --- a/sgtlearn/ensemble/random_sgforest_classifier.py +++ b/sgtlearn/ensemble/random_sgforest_classifier.py @@ -49,6 +49,15 @@ class RandomSGForestClassifier(ClassifierMixin, RandomSGForest): Per-split feature subsampling for each base tree. Defaults to ``"sqrt"`` to follow ``RandomForestClassifier`` convention. See :class:`sgtlearn.SGTClassifier` for the full semantics. + pairwise_candidates : int or float, default=0 + Maximum retained feature pairs fitted per node. An integer is an + absolute limit; a float resolves to ``ceil(value * n_logical_features)``. + Zero preserves univariate-only training. + pairwise_penalty : float, default=0.0 + Non-negative penalty added when comparing a fitted pair with the best + univariate candidate. + tao_pair_scale : float, default=1.1 + Multiplier applied to ``tao_lambda`` for pair routers during TAO. bootstrap : bool, default=True If ``True``, each tree is fit on a bootstrap resample (with replacement) of the training set. If ``False``, every tree is fit on @@ -83,10 +92,11 @@ class RandomSGForestClassifier(ClassifierMixin, RandomSGForest): Number of features seen during :meth:`fit`. mean_feature_importances_ : ndarray of shape (n_logical_features,) Mean of per-tree :attr:`~sgtlearn.SGTClassifier.feature_importances_`, - aligned with :attr:`processed_features_`. + aligned with :attr:`processed_features_`. Unavailable after TAO. std_feature_importance_ : ndarray of shape (n_logical_features,) Population standard deviation of per-tree importances across the forest (same alignment as :attr:`mean_feature_importances_`). + Unavailable after TAO. processed_features_ : ProcessedFeatures Logical features resolved once and shared by every base tree. @@ -131,8 +141,11 @@ def __init__( max_samples: float | None = None, random_state: int | np.random.RandomState | None = None, class_weight: Mapping[Any, float] | Sequence[Mapping[Any, float]] | None = None, + pairwise_candidates: float = 0, + pairwise_penalty: float = 0.0, tao_n_runs: int = 10, tao_lambda: float = 0.0, + tao_pair_scale: float = 1.1, n_jobs: int | None = None, verbose: int = 0, ) -> None: @@ -156,8 +169,11 @@ def __init__( bootstrap=bootstrap, max_samples=max_samples, random_state=random_state, + pairwise_candidates=pairwise_candidates, + pairwise_penalty=pairwise_penalty, tao_n_runs=tao_n_runs, tao_lambda=tao_lambda, + tao_pair_scale=tao_pair_scale, n_jobs=n_jobs, verbose=verbose, ) diff --git a/sgtlearn/ensemble/random_sgforest_regressor.py b/sgtlearn/ensemble/random_sgforest_regressor.py index 8daabee..0fe7507 100644 --- a/sgtlearn/ensemble/random_sgforest_regressor.py +++ b/sgtlearn/ensemble/random_sgforest_regressor.py @@ -44,6 +44,15 @@ class RandomSGForestRegressor(RegressorMixin, RandomSGForest): Per-split feature subsampling for each base tree. Defaults to ``"sqrt"`` to follow ``RandomForestRegressor`` convention. See :class:`sgtlearn.SGTRegressor` for the full semantics. + pairwise_candidates : int or float, default=0 + Maximum retained feature pairs fitted per node. An integer is an + absolute limit; a float resolves to ``ceil(value * n_logical_features)``. + Zero preserves univariate-only training. + pairwise_penalty : float, default=0.0 + Non-negative penalty added when comparing a fitted pair with the best + univariate candidate. + tao_pair_scale : float, default=1.1 + Multiplier applied to ``tao_lambda`` for pair routers during TAO. bootstrap : bool, default=True If ``True``, each tree is fit on a bootstrap resample (with replacement) of the training set. If ``False``, every tree is fit on @@ -70,10 +79,11 @@ class RandomSGForestRegressor(RegressorMixin, RandomSGForest): Number of features seen during :meth:`fit`. mean_feature_importances_ : ndarray of shape (n_logical_features,) Mean of per-tree :attr:`~sgtlearn.SGTRegressor.feature_importances_`, - aligned with :attr:`processed_features_`. + aligned with :attr:`processed_features_`. Unavailable after TAO. std_feature_importance_ : ndarray of shape (n_logical_features,) Population standard deviation of per-tree importances across the forest (same alignment as :attr:`mean_feature_importances_`). + Unavailable after TAO. processed_features_ : ProcessedFeatures Logical features resolved once and shared by every base tree. @@ -117,8 +127,11 @@ def __init__( bootstrap: bool = True, max_samples: float | None = None, random_state: int | np.random.RandomState | None = None, + pairwise_candidates: float = 0, + pairwise_penalty: float = 0.0, tao_n_runs: int = 10, tao_lambda: float = 0.0, + tao_pair_scale: float = 1.1, n_jobs: int | None = None, verbose: int = 0, ) -> None: @@ -141,8 +154,11 @@ def __init__( bootstrap=bootstrap, max_samples=max_samples, random_state=random_state, + pairwise_candidates=pairwise_candidates, + pairwise_penalty=pairwise_penalty, tao_n_runs=tao_n_runs, tao_lambda=tao_lambda, + tao_pair_scale=tao_pair_scale, n_jobs=n_jobs, verbose=verbose, ) diff --git a/sgtlearn/tao.py b/sgtlearn/tao.py index 84c6b77..ad4657b 100644 --- a/sgtlearn/tao.py +++ b/sgtlearn/tao.py @@ -24,7 +24,12 @@ effective_sample_weight_classification, normalize_sample_weight, ) -from sgtlearn.base import BaseShapeCART, SGTClassifier, SGTRegressor +from sgtlearn.base import ( + BaseShapeCART, + SGTClassifier, + SGTRegressor, + _validate_tao_pair_scale, +) from sgtlearn.ensemble._random_sgforest import RandomSGForest from sgtlearn.ensemble.random_sgforest_classifier import RandomSGForestClassifier from sgtlearn.ensemble.random_sgforest_regressor import RandomSGForestRegressor @@ -160,6 +165,7 @@ def TAO_refine( sample_weight: np.ndarray | None = None, n_runs: int = 10, lambda_: float = 0.0, + tao_pair_scale: float = 1.1, check_input: bool = True, n_jobs: int | None = None, ) -> TaoModel: @@ -179,7 +185,11 @@ def TAO_refine( internal node, a non-constant routing rule must beat the dummy rule by more than ``lambda_ * n_samples`` in weighted reward units (equivalently ``lambda_ * n_samples / n_care`` on the mean care reward). + Pair routers pay ``tao_pair_scale`` times this cost; dummy routers pay zero. + After any call with ``n_runs > 0``, impurity feature importances are + unavailable; use held-out permutation importance instead. """ + tao_pair_scale = _validate_tao_pair_scale(tao_pair_scale) targets = _tao_targets(model) X, y = _validate_X_y(model, X, y, check_input=check_input) X32, y_native, sw = _prepare_tao_arrays(model, X, y, sample_weight) @@ -191,14 +201,30 @@ def TAO_refine( if len(targets) == 1 or n_jobs_eff == 1: for tree in targets: TreeAlternatingOptimization( - tree._est, X32, y_native, sw, n_runs=n_runs, lambda_=lambda_ + tree._est, + X32, + y_native, + sw, + n_runs=n_runs, + lambda_=lambda_, + tao_pair_scale=tao_pair_scale, ) else: Parallel(n_jobs=n_jobs_eff, prefer="threads")( delayed(TreeAlternatingOptimization)( - tree._est, X32, y_native, sw, n_runs=n_runs, lambda_=lambda_ + tree._est, + X32, + y_native, + sw, + n_runs=n_runs, + lambda_=lambda_, + tao_pair_scale=tao_pair_scale, ) for tree in targets ) + if n_runs > 0: + for tree in targets: + tree._tao_refined_ = True + return model diff --git a/tests/test_pairwise_categorical_missing.py b/tests/test_pairwise_categorical_missing.py new file mode 100644 index 0000000..176ecdb --- /dev/null +++ b/tests/test_pairwise_categorical_missing.py @@ -0,0 +1,180 @@ +"""Public acceptance tests for categorical and joint-missing pair routing.""" + +from __future__ import annotations + +from collections.abc import Callable + +import numpy as np +import pytest + +from sgtlearn import SGTClassifier, SGTRegressor + + +def test_classifier_continuous_categorical_pair_routes_joint_interaction() -> None: + states = np.array( + [ + [-1.0, 1.0, 0.0], + [-1.0, 0.0, 1.0], + [1.0, 1.0, 0.0], + [1.0, 0.0, 1.0], + ] + ) + counts = [40, 30, 30, 28] + X = np.repeat(states, counts, axis=0) + y = np.repeat([0, 1, 1, 0], counts) + + model = SGTClassifier( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y, feature_dict={0: [0], 1: [1, 2]}) + + assert model.score(X, y) == 1.0 + root = model.tree_export()["nodes"][0] + assert root["routing_kind"] == "pair" + assert root["features"] == [0, 1, 2] + assert root["pair_axes"] == [ + { + "logical_feature": 0, + "kind": "continuous", + "columns": [0], + "categories": [], + "catchall": None, + }, + { + "logical_feature": 1, + "kind": "categorical", + "columns": [1, 2], + "categories": [1, 2], + "catchall": "missing", + }, + ] + + +@pytest.mark.parametrize("estimator", [SGTClassifier, SGTRegressor]) +def test_continuous_pair_routes_each_joint_missing_state(estimator: type) -> None: + levels = [-1.0, 1.0, np.nan] + states = np.array([(first, second) for first in levels for second in levels]) + counts = [40, 31, 30, 29, 28, 27, 26, 25, 24] + X = np.repeat(states, counts, axis=0) + y = np.repeat(np.arange(len(states)), counts) + + model = estimator( + num_partitions=9, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=5, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + np.testing.assert_array_equal(model.predict(states), np.arange(len(states))) + inner = model.tree_export()["nodes"][0]["pair_inner_tree"] + assert all("missing" in node for node in inner if not node["is_leaf"]) + assert any( + not inner[node["missing"]]["is_leaf"] + for node in inner + if not node["is_leaf"] + ), "a missing edge must remain splittable by the other feature" + + +def _mixed_states() -> tuple[np.ndarray, dict[int, list[int]], list[str]]: + continuous = [-1.0, 1.0, np.nan] + categorical = [[1.0, 0.0], [0.0, 1.0], [0.0, 0.0]] + return ( + np.array([[value, *category] for value in continuous for category in categorical]), + {0: [0], 1: [1, 2]}, + ["continuous", "categorical"], + ) + + +def _categorical_states() -> tuple[np.ndarray, dict[int, list[int]], list[str]]: + categories = [[1.0, 0.0], [0.0, 1.0], [0.0, 0.0]] + return ( + np.array([[*first, *second] for first in categories for second in categories]), + {0: [0, 1], 1: [2, 3]}, + ["categorical", "categorical"], + ) + + +@pytest.mark.parametrize("estimator", [SGTClassifier, SGTRegressor]) +@pytest.mark.parametrize("make_case", [_mixed_states, _categorical_states]) +def test_categorical_pairs_route_no_active_as_axis_specific_missing( + estimator: type, + make_case: Callable[[], tuple[np.ndarray, dict[int, list[int]], list[str]]], +) -> None: + states, feature_dict, expected_kinds = make_case() + counts = [40, 31, 30, 29, 28, 27, 26, 25, 24] + X = np.repeat(states, counts, axis=0) + y = np.repeat(np.arange(len(states)), counts) + + model = estimator( + num_partitions=9, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=5, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y, feature_dict=feature_dict) + + np.testing.assert_array_equal(model.predict(states), np.arange(len(states))) + root = model.tree_export()["nodes"][0] + assert root["routing_kind"] == "pair" + assert [axis["kind"] for axis in root["pair_axes"]] == expected_kinds + assert all( + axis["catchall"] == "missing" + for axis in root["pair_axes"] + if axis["kind"] == "categorical" + ) + + +@pytest.mark.parametrize("estimator", [SGTClassifier, SGTRegressor]) +def test_missing_only_pair_bin_is_not_rejected_by_inner_min_leaf( + estimator: type, +) -> None: + finite = np.array( + [[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]] + ) + X = np.vstack([np.repeat(finite, [40, 30, 30, 28], axis=0), [np.nan, -1.0]]) + y = np.concatenate([np.repeat([0, 1, 1, 0], [40, 30, 30, 28]), [2]]) + + model = estimator( + num_partitions=3, + max_depth=1, + inner_min_samples_leaf=10, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert model.predict(np.array([[np.nan, -1.0]])).item() == 2 + + +@pytest.mark.parametrize("estimator", [SGTClassifier, SGTRegressor]) +def test_pair_predict_time_missing_uses_majority_inner_route(estimator: type) -> None: + states = np.array( + [[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]] + ) + X = np.repeat(states, [5, 20, 30, 40], axis=0) + y = np.repeat([0, 1, 1, 0], [5, 20, 30, 40]) + model = estimator( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert model.tree_export()["nodes"][0]["routing_kind"] == "pair" + np.testing.assert_array_equal( + model.predict(np.array([[-1.0, np.nan], [1.0, np.nan]])), + model.predict(np.array([[-1.0, 1.0], [1.0, 1.0]])), + ) diff --git a/tests/test_pairwise_classifier.py b/tests/test_pairwise_classifier.py new file mode 100644 index 0000000..889516e --- /dev/null +++ b/tests/test_pairwise_classifier.py @@ -0,0 +1,197 @@ +"""Public acceptance tests for opt-in classifier pair routing.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from sgtlearn import SGTClassifier +from sgtlearn import SGTRegressor + + +@pytest.mark.parametrize("estimator", [SGTClassifier, SGTRegressor]) +def test_pairwise_candidates_zero_preserves_legacy_predictions(estimator) -> None: + X = np.array([[-2.0, 0.0], [-1.0, 1.0], [1.0, -1.0], [2.0, 0.0]]) + y = np.array([0, 0, 1, 1]) if estimator is SGTClassifier else np.array([0.0, 0.0, 1.0, 1.0]) + common = dict(max_depth=2, tao_n_runs=0, random_state=0) + implicit = estimator(**common).fit(X, y) + explicit = estimator(pairwise_candidates=0, **common).fit(X, y) + np.testing.assert_array_equal(implicit.predict(X), explicit.predict(X)) + + +@pytest.mark.parametrize("estimator", [SGTClassifier, SGTRegressor]) +@pytest.mark.parametrize("value", [True, -1, np.nan, np.inf, "one"]) +def test_pairwise_candidates_rejects_invalid_values(estimator, value) -> None: + y = np.array([0, 1, 0, 1]) if estimator is SGTClassifier else np.array([0.0, 1.0, 0.0, 1.0]) + with pytest.raises(ValueError): + estimator(pairwise_candidates=value, tao_n_runs=0).fit(np.zeros((4, 2)), y) + + +@pytest.mark.parametrize("estimator", [SGTClassifier, SGTRegressor]) +@pytest.mark.parametrize("value", [-1.0, np.nan, np.inf, "one"]) +def test_pairwise_penalty_rejects_invalid_values(estimator, value) -> None: + y = np.array([0, 1, 0, 1]) if estimator is SGTClassifier else np.array([0.0, 1.0, 0.0, 1.0]) + with pytest.raises(ValueError): + estimator(pairwise_penalty=value, tao_n_runs=0).fit(np.zeros((4, 2)), y) + + +@pytest.mark.parametrize("estimator", [SGTClassifier, SGTRegressor]) +def test_pair_candidates_are_restricted_to_max_features_subset(estimator) -> None: + X = np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]) + y = np.array([0, 1, 1, 0]) if estimator is SGTClassifier else np.array([0.0, 1.0, 1.0, 0.0]) + model = estimator( + max_features=1, pairwise_candidates=1, max_depth=1, + inner_max_depth=2, inner_max_leaf_nodes=4, tao_n_runs=0, random_state=0, + ).fit(X, y) + assert model.tree_export()["nodes"][0].get("routing_kind") != "pair" + + +def test_exact_final_tie_favors_univariate_route() -> None: + X = np.zeros((8, 2)) + y = np.array([0, 1] * 4) + model = SGTClassifier( + pairwise_candidates=1, max_depth=1, inner_max_depth=2, + inner_max_leaf_nodes=4, tao_n_runs=0, random_state=0, + ).fit(X, y) + assert model.tree_export()["nodes"][0].get("routing_kind") != "pair" + + +def test_continuous_pair_captures_xor_and_exports_native_router() -> None: + quadrants = np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]) + counts = [40, 30, 30, 28] + X = np.repeat(quadrants, counts, axis=0) + y = np.repeat(np.array([0, 1, 1, 0]), counts) + + baseline = SGTClassifier(max_depth=1, tao_n_runs=0, random_state=0).fit(X, y) + paired = SGTClassifier( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert paired.score(X, y) == 1.0 + assert paired.score(X, y) > baseline.score(X, y) + + root = paired.tree_export()["nodes"][0] + assert root["routing_kind"] == "pair" + assert root["pair_features"] == [0, 1] + assert root["features"] == [0, 1] + assert root["pair_inner_tree"] + assert len(root["pair_leaf_bins"]) == len(root["bin_to_partition"]) + + with pytest.warns(UserWarning, match="equally"): + np.testing.assert_allclose(paired.feature_importances_, [0.5, 0.5]) + + +@pytest.mark.parametrize("criterion", ["gini", "entropy"]) +def test_pair_search_keeps_zero_gain_marginals_for_balanced_xor( + criterion: str, +) -> None: + states = np.array( + [[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]] + ) + X = np.repeat(states, 40, axis=0) + y = np.repeat([0, 1, 1, 0], 40) + + model = SGTClassifier( + criterion=criterion, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert model.tree_export()["nodes"][0]["routing_kind"] == "pair" + assert model.score(X, y) == 1.0 + + +@pytest.mark.parametrize("budget", [0.3, 99]) +def test_float_and_excess_pair_budgets_fit_available_interaction( + budget: int | float, +) -> None: + states = np.array( + [[-1.0, -1.0, 0.0], [-1.0, 1.0, 0.0], + [1.0, -1.0, 0.0], [1.0, 1.0, 0.0]] + ) + X = np.repeat(states, 40, axis=0) + y = np.repeat([0, 1, 1, 0], 40) + model = SGTClassifier( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=budget, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert model.tree_export()["nodes"][0]["pair_features"] == [0, 1] + assert model.score(X, y) == 1.0 + + +def test_pair_ranking_tie_uses_logical_feature_indices() -> None: + base = np.array( + [[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]] + ) + X = np.repeat(np.column_stack([base, base[:, 1]]), 40, axis=0) + y = np.repeat([0, 1, 1, 0], 40) + model = SGTClassifier( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert model.tree_export()["nodes"][0]["pair_features"] == [0, 1] + + +def test_pairwise_penalty_switches_selection_without_blocking_univariate() -> None: + states = np.array( + [[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]] + ) + X = np.repeat(states, [40, 10, 30, 5], axis=0) + y = np.repeat([0, 1, 1, 0], [40, 10, 30, 5]) + common = dict( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ) + + pair = SGTClassifier(pairwise_penalty=0.0, **common).fit(X, y) + univariate = SGTClassifier(pairwise_penalty=0.5, **common).fit(X, y) + + assert pair.tree_export()["nodes"][0]["routing_kind"] == "pair" + assert univariate.tree_export()["nodes"][0].get("routing_kind") != "pair" + assert univariate.tree_export()["num_nodes"] > 1 + + +def test_continuous_pair_supports_three_way_routing() -> None: + X = np.tile( + np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]), + (32, 1), + ) + y = np.tile(np.array([0, 1, 1, 2]), 32) + + paired = SGTClassifier( + num_partitions=3, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert paired.score(X, y) == 1.0 + root = paired.tree_export()["nodes"][0] + assert root["routing_kind"] == "pair" + assert len(root["children"]) == 3 diff --git a/tests/test_pairwise_regression.py b/tests/test_pairwise_regression.py new file mode 100644 index 0000000..4305d2e --- /dev/null +++ b/tests/test_pairwise_regression.py @@ -0,0 +1,92 @@ +"""Public acceptance tests for opt-in regression pair routing.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from sgtlearn import SGTRegressor + + +def _three_way_quadrants() -> tuple[np.ndarray, np.ndarray, np.ndarray]: + X = np.tile( + np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]), + (32, 1), + ) + return ( + X, + np.tile(np.array([1.0, 0.0, 0.0, 2.0]), 32), + np.tile(np.array([2.0, 1.0, 1.0, 2.0]), 32), + ) + + +@pytest.mark.parametrize("criterion", ["squared_error", "absolute_error"]) +def test_continuous_pair_regression_uses_three_way_router(criterion: str) -> None: + X, y, sample_weight = _three_way_quadrants() + + baseline = SGTRegressor( + criterion=criterion, num_partitions=3, max_depth=1, tao_n_runs=0, random_state=0 + ).fit(X, y, sample_weight=sample_weight) + paired = SGTRegressor( + criterion=criterion, + num_partitions=3, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y, sample_weight=sample_weight) + + paired_error = np.mean(np.abs(paired.predict(X) - y)) + baseline_error = np.mean(np.abs(baseline.predict(X) - y)) + root = paired.tree_export()["nodes"][0] + assert root["routing_kind"] == "pair", root + assert paired_error == 0.0 + assert paired_error < baseline_error + + assert root["pair_features"] == [0, 1] + assert root["features"] == [0, 1] + assert root["pair_inner_tree"] + assert len(root["children"]) == 3 + assert all(0 <= part < 3 for part in root["bin_to_partition"]) + + +@pytest.mark.parametrize("criterion", ["squared_error", "absolute_error"]) +def test_continuous_pair_regression_preserves_multioutput_contract( + criterion: str, +) -> None: + X, y0, sample_weight = _three_way_quadrants() + y = np.column_stack([y0, 10.0 + y0]) + + paired = SGTRegressor( + criterion=criterion, + num_partitions=3, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y, sample_weight=sample_weight) + + np.testing.assert_allclose(paired.predict(X), y, rtol=0, atol=0) + root = paired.tree_export()["nodes"][0] + assert root["routing_kind"] == "pair" + assert len(root["children"]) == 3 + + +def test_regression_pair_importances_warn_and_split_gain_equally() -> None: + X, y, sample_weight = _three_way_quadrants() + model = SGTRegressor( + num_partitions=3, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y, sample_weight=sample_weight) + + with pytest.warns(UserWarning, match="equally"): + np.testing.assert_allclose(model.feature_importances_, [0.5, 0.5]) diff --git a/tests/test_plot_helpers.py b/tests/test_plot_helpers.py index 9009671..224ff96 100644 --- a/tests/test_plot_helpers.py +++ b/tests/test_plot_helpers.py @@ -9,7 +9,7 @@ import numpy as np from sklearn.datasets import make_classification from sgtlearn import SGTClassifier -from sgtlearn._export import _merge_routing_regions, _route_samples, _compute_layout_leafcounter, _draw_leaf_text, _draw_internal_panel, _draw_arrow_edge +from sgtlearn._export import _merge_routing_regions, _pair_switch_boundaries, _route_samples, _compute_layout_leafcounter, _draw_leaf_text, _draw_internal_panel, _draw_arrow_edge from tests.constants import TEST_TAO_N_RUNS @@ -91,6 +91,20 @@ def test_merge_x_min_greater_than_first_threshold_clamps_left_edge(): assert len(regions) == 2 +def test_pair_switch_boundaries_only_keeps_partition_changes(): + cells = [(0.0, 1.0, 0.5), (1.0, 2.0, 1.5), (2.0, 3.0, 2.5)] + + x_partitions = np.array([[0, 0], [1, 1], [1, 1]]) + assert _pair_switch_boundaries(cells, x_partitions, axis=0) == [1.0] + + y_partitions = np.array([[0, 1, 1], [0, 1, 1]]) + assert _pair_switch_boundaries(cells, y_partitions, axis=1) == [1.0] + + both_partitions = np.array([[0, 0, 1], [0, 1, 1], [1, 1, 1]]) + assert _pair_switch_boundaries(cells, both_partitions, axis=0) == [1.0, 2.0] + assert _pair_switch_boundaries(cells, both_partitions, axis=1) == [1.0, 2.0] + + def _fitted_clf(): X, y = make_classification(n_samples=200, n_features=4, random_state=0) return SGTClassifier( @@ -143,6 +157,40 @@ def test_route_samples_dtype_indices_are_int(): assert arr.dtype.kind in ("i", "u") +def test_route_samples_replays_pair_missing_edges() -> None: + tree = { + "root_index": 0, + "nodes": [ + { + "id": 0, "is_leaf": False, "routing_kind": "pair", + "features": [0, 1], "children": [1, 2, 3], + "bin_to_partition": [0, 1, 2, 2, 2], + "pair_axes": [ + {"kind": "continuous", "columns": [0]}, + {"kind": "continuous", "columns": [1]}, + ], + "pair_inner_tree": [ + {"id": 0, "is_leaf": False, "axis": 0, "kind": "continuous", "feature": 0, "threshold": 0.0, "left": 1, "right": 2, "missing": 3}, + {"id": 1, "is_leaf": True, "bin": 0}, + {"id": 2, "is_leaf": True, "bin": 1}, + {"id": 3, "is_leaf": False, "axis": 1, "kind": "continuous", "feature": 1, "threshold": 0.0, "left": 4, "right": 5, "missing": 6}, + {"id": 4, "is_leaf": True, "bin": 2}, + {"id": 5, "is_leaf": True, "bin": 3}, + {"id": 6, "is_leaf": True, "bin": 4}, + ], + }, + {"id": 1, "is_leaf": True, "children": []}, + {"id": 2, "is_leaf": True, "children": []}, + {"id": 3, "is_leaf": True, "children": []}, + ], + } + X = np.array([[-1.0, 1.0], [1.0, 1.0], [np.nan, -1.0], [np.nan, 1.0], [np.nan, np.nan]]) + reach = _route_samples(tree, X) + assert reach[1].tolist() == [0] + assert reach[2].tolist() == [1] + assert reach[3].tolist() == [2, 3, 4] + + def _toy_tree() -> dict: """Hand-rolled tree dict matching tree_export()'s shape for layout tests. diff --git a/tests/test_plot_tree.py b/tests/test_plot_tree.py index 5071cbe..974af91 100644 --- a/tests/test_plot_tree.py +++ b/tests/test_plot_tree.py @@ -221,3 +221,106 @@ def test_plot_tree_leaf_uses_partition_color(fitted_classifier): f"{expected_p0}, {expected_p1}" ) plt.close("all") + + +def test_plot_tree_pair_heatmap_reuses_exported_router_and_axes(): + states = np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]) + X = np.repeat(states, [40, 30, 30, 28], axis=0) + y = np.repeat([0, 1, 1, 0], [40, 30, 30, 28]) + est = SGTClassifier( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + fig, ax = plt.subplots() + artists = plot_tree( + est, + X=X, + feature_names=["first", "second"], + precision=3, + ax=ax, + ) + panels = [artist for artist in artists if hasattr(artist, "patches")] + assert ax in fig.axes + assert panels and len(panels[0].patches) == 4 + assert not panels[0].collections + assert {panel.get_label() for panel in panels} >= { + "pair-x-histogram", + "pair-y-histogram", + } + assert panels[0].get_xlabel() == "first" + assert panels[0].get_ylabel() == "second" + assert [tick.get_text() for tick in panels[0].get_xticklabels()] == ["0.000"] + assert [tick.get_text() for tick in panels[0].get_yticklabels()] == ["0.000"] + plt.close(fig) + + +@pytest.mark.parametrize("pair_kind", ["mixed", "categorical"]) +def test_plot_tree_pair_heatmap_renders_categories_and_missing_cells(pair_kind): + categories = [[1.0, 0.0], [0.0, 1.0], [0.0, 0.0]] + if pair_kind == "mixed": + states = np.array( + [[value, *category] for value in [-1.0, 1.0, np.nan] + for category in categories] + ) + feature_dict = {0: [0], 1: [1, 2]} + else: + states = np.array( + [[*first, *second] for first in categories for second in categories] + ) + feature_dict = {0: [0, 1], 1: [2, 3]} + X = np.repeat(states, [40, 31, 30, 29, 28, 27, 26, 25, 24], axis=0) + y = np.repeat(np.arange(9), [40, 31, 30, 29, 28, 27, 26, 25, 24]) + est = SGTClassifier( + num_partitions=9, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=5, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y, feature_dict=feature_dict) + + fig, ax = plt.subplots() + artists = plot_tree(est, X=X, cmap="tab10", ax=ax) + panel = next(artist for artist in artists if hasattr(artist, "patches")) + assert len(panel.patches) == 9 # 3 × 3, including both missing margins/corner + assert len({patch.get_facecolor() for patch in panel.patches}) == 9 + widths = [patch.get_width() for patch in panel.patches] + heights = [patch.get_height() for patch in panel.patches] + assert min(widths) < 0.5 * max(widths) + assert min(heights) < 0.5 * max(heights) + x_labels = [tick.get_text() for tick in panel.get_xticklabels()] + y_labels = [tick.get_text() for tick in panel.get_yticklabels()] + assert "NaN" in x_labels + assert "NaN" in y_labels + if pair_kind == "mixed": + assert any(label not in {"", "NaN"} for label in x_labels) + plt.close(fig) + + +def test_plot_tree_pair_heatmap_renders_without_training_data(): + states = np.array( + [[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]] + ) + X = np.repeat(states, [40, 30, 30, 28], axis=0) + y = np.repeat([0, 1, 1, 0], [40, 30, 30, 28]) + est = SGTClassifier( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + fig, ax = plt.subplots() + artists = plot_tree(est, ax=ax) + panel = next(artist for artist in artists if hasattr(artist, "patches")) + assert panel.patches + assert "NaN" in [tick.get_text() for tick in panel.get_xticklabels()] + assert "NaN" in [tick.get_text() for tick in panel.get_yticklabels()] + plt.close(fig) diff --git a/tests/test_random_sgforest_pairwise.py b/tests/test_random_sgforest_pairwise.py new file mode 100644 index 0000000..f9b7f08 --- /dev/null +++ b/tests/test_random_sgforest_pairwise.py @@ -0,0 +1,78 @@ +"""Public pairwise-parameter behavior for random SGT forests.""" + +from __future__ import annotations + +import numpy as np +import pytest +import warnings + +from sgtlearn import RandomSGForestClassifier, RandomSGForestRegressor + + +def test_classifier_forest_propagates_pairwise_options_and_fits_interaction() -> None: + quadrants = np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]) + counts = [40, 30, 30, 28] + X = np.repeat(quadrants, counts, axis=0) + y = np.repeat(np.array([0, 1, 1, 0]), counts) + + forest = RandomSGForestClassifier( + n_estimators=2, + bootstrap=False, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + pairwise_penalty=0.25, + max_features=None, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert forest.score(X, y) == 1.0 + assert all(tree.pairwise_candidates == 1 for tree in forest.estimators_) + assert all(tree.pairwise_penalty == 0.25 for tree in forest.estimators_) + with pytest.warns(UserWarning, match="equally") as caught: + forest.mean_feature_importances_ + assert len(caught) == 1 + + +def test_classifier_forest_pairwise_zero_does_not_warn_for_importances() -> None: + X = np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]) + y = np.array([0, 1, 1, 0]) + forest = RandomSGForestClassifier( + n_estimators=1, bootstrap=False, max_depth=1, tao_n_runs=0, random_state=0 + ).fit(X, y) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + forest.mean_feature_importances_ + assert len(caught) == 0 + + +def test_regressor_forest_accepts_pairwise_options() -> None: + forest = RandomSGForestRegressor(pairwise_candidates=2, pairwise_penalty=0.5) + assert forest.get_params()["pairwise_candidates"] == 2 + assert forest.get_params()["pairwise_penalty"] == 0.5 + + +def test_regressor_forest_emits_one_pair_importance_warning() -> None: + states = np.array( + [[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]] + ) + X = np.tile(states, (32, 1)) + y = np.tile([1.0, 0.0, 0.0, 2.0], 32) + forest = RandomSGForestRegressor( + n_estimators=2, + bootstrap=False, + max_features=None, + num_partitions=3, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + with pytest.warns(UserWarning, match="equally") as caught: + forest.mean_feature_importances_ + assert len(caught) == 1 diff --git a/tests/test_tao.py b/tests/test_tao.py index a8e4153..7b39df1 100644 --- a/tests/test_tao.py +++ b/tests/test_tao.py @@ -160,6 +160,22 @@ def _sample_weights(n_samples: int, seed: int) -> np.ndarray: return rng.uniform(0.5, 2.0, size=n_samples) +def test_feature_importances_are_unavailable_after_tao_and_reset_on_refit() -> None: + X, y = load_iris(return_X_y=True) + clf = SGTClassifier(tao_n_runs=0, random_state=0).fit(X, y) + + tao.TAO_refine(clf, X, y, n_runs=0) + assert clf.feature_importances_.shape == (X.shape[1],) + + tao.TAO_refine(clf, X, y, n_runs=1) + + with pytest.raises(AttributeError, match="unavailable after TAO"): + clf.feature_importances_ + + clf.fit(X, y) + assert clf.feature_importances_.shape == (X.shape[1],) + + def _classification_training_score( tree: SGTClassifier, X: np.ndarray, @@ -534,3 +550,158 @@ def test_tao_forest_refine_mutates_in_place() -> None: assert result is forest assert [est._est for est in forest.estimators_] == handles_before + + +def _tao_pair_interaction_data() -> tuple[np.ndarray, np.ndarray]: + quadrants = np.array( + [[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]] + ) + counts = [40, 10, 30, 5] + return np.repeat(quadrants, counts, axis=0), np.repeat([0, 1, 1, 0], counts) + + +def test_tao_reconsiders_retained_classifier_pair() -> None: + X, y = _tao_pair_interaction_data() + clf = SGTClassifier( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + pairwise_penalty=1.0, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert clf.tree_export()["nodes"][0].get("routing_kind") != "pair" + assert clf.score(X, y) == pytest.approx(70 / 85) + + tao.TAO_refine(clf, X, y, n_runs=1, lambda_=0.0, tao_pair_scale=1.1) + + root = clf.tree_export()["nodes"][0] + assert root["routing_kind"] == "pair" + assert root["pair_features"] == [0, 1] + assert len(root["bin_sample_counts"]) == len(root["bin_to_partition"]) + assert len(root["bin_counts"]) == len(root["bin_to_partition"]) + assert sum(root["bin_sample_counts"]) == X.shape[0] + with pytest.raises(AttributeError, match="unavailable after TAO"): + clf.feature_importances_ + assert clf.score(X, y) == 1.0 + + +def test_tao_makes_forest_feature_importances_unavailable() -> None: + X, y = _tao_pair_interaction_data() + forest = RandomSGForestClassifier( + n_estimators=1, + bootstrap=False, + max_features=None, + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + pairwise_penalty=1.0, + tao_n_runs=0, + random_state=0, + n_jobs=1, + ).fit(X, y) + + tao.TAO_refine(forest, X, y, n_runs=1, lambda_=0.0) + + for attr in ("mean_feature_importances_", "std_feature_importance_"): + with pytest.raises(AttributeError, match="unavailable after TAO"): + getattr(forest, attr) + + +def test_tao_accepts_improving_retained_regression_pair_multioutput() -> None: + X, labels = _tao_pair_interaction_data() + y = np.column_stack([labels.astype(float), 10.0 + labels]) + reg = SGTRegressor( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + pairwise_penalty=1.0, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + assert reg.tree_export()["nodes"][0].get("routing_kind") != "pair" + assert not np.array_equal(reg.predict(X), y) + + tao.TAO_refine(reg, X, y, n_runs=1, lambda_=0.0) + + root = reg.tree_export()["nodes"][0] + assert root["routing_kind"] == "pair" + assert root["pair_features"] == [0, 1] + assert len(root["bin_sample_counts"]) == len(root["bin_to_partition"]) + assert len(root["bin_counts"]) == len(root["bin_to_partition"]) + assert sum(root["bin_sample_counts"]) == X.shape[0] + np.testing.assert_array_equal(reg.predict(X), y) + + +def test_tao_pair_scale_changes_pair_vs_dummy_choice() -> None: + X, y = _tao_pair_interaction_data() + + def fit_pair() -> SGTClassifier: + return SGTClassifier( + max_depth=1, + inner_max_depth=2, + inner_max_leaf_nodes=4, + pairwise_candidates=1, + tao_n_runs=0, + random_state=0, + ).fit(X, y) + + default_scale = fit_pair() + high_scale = fit_pair() + tao.TAO_refine( + default_scale, X, y, n_runs=1, lambda_=0.3, tao_pair_scale=1.1 + ) + tao.TAO_refine( + high_scale, X, y, n_runs=1, lambda_=0.3, tao_pair_scale=2.0 + ) + + assert default_scale.tree_export()["nodes"][0]["routing_kind"] == "pair" + assert default_scale.score(X, y) == 1.0 + assert high_scale.tree_export()["nodes"][0].get("routing_kind") != "pair" + assert high_scale.score(X, y) == pytest.approx(45 / 85) + + +@pytest.mark.parametrize( + ("forest_cls", "target"), + [ + (RandomSGForestClassifier, lambda y: y), + (RandomSGForestRegressor, lambda y: y.astype(float)), + ], +) +def test_tao_pair_scale_defaults_and_forwards_through_forests( + forest_cls, target +) -> None: + assert SGTClassifier().get_params()["tao_pair_scale"] == 1.1 + assert SGTRegressor().get_params()["tao_pair_scale"] == 1.1 + assert forest_cls().get_params()["tao_pair_scale"] == 1.1 + + X, y = _tao_pair_interaction_data() + forest = forest_cls( + n_estimators=2, + bootstrap=False, + max_features=None, + pairwise_candidates=1, + tao_n_runs=0, + tao_pair_scale=1.7, + random_state=0, + n_jobs=1, + ).fit(X, target(y)) + + assert forest.tao_pair_scale == 1.7 + assert all(tree.tao_pair_scale == 1.7 for tree in forest.estimators_) + + +@pytest.mark.parametrize("bad_scale", [-1.0, np.inf, np.nan]) +def test_tao_pair_scale_rejects_invalid_values(bad_scale: float) -> None: + X, y = _tao_pair_interaction_data() + with pytest.raises(ValueError, match="tao_pair_scale"): + SGTClassifier(tao_n_runs=0, tao_pair_scale=bad_scale).fit(X, y) + + clf = SGTClassifier(tao_n_runs=0).fit(X, y) + with pytest.raises(ValueError, match="tao_pair_scale"): + tao.TAO_refine(clf, X, y, tao_pair_scale=bad_scale) diff --git a/tests/test_tree_export.py b/tests/test_tree_export.py index 4101971..34315a8 100644 --- a/tests/test_tree_export.py +++ b/tests/test_tree_export.py @@ -7,6 +7,7 @@ from sklearn.exceptions import NotFittedError from sgtlearn import SGTClassifier, SGTRegressor +from sgtlearn._export import _route_samples from tests.constants import TEST_TAO_N_RUNS @@ -129,3 +130,36 @@ def test_classifier_multiway_partitions(): assert len(n["children"]) <= 3 for p in n["bin_to_partition"]: assert 0 <= p < 3 + + +def test_pair_classifier_export_replays_predictions(): + states = np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]) + X = np.repeat(states, [40, 30, 30, 28], axis=0) + y = np.repeat([0, 1, 1, 0], [40, 30, 30, 28]) + est = SGTClassifier(max_depth=1, inner_max_depth=2, inner_max_leaf_nodes=4, + pairwise_candidates=1, tao_n_runs=0, random_state=0).fit(X, y) + tree = est.tree_export() + assert tree["nodes"][0]["routing_kind"] == "pair" + assert "nan_prediction_partition" not in tree["nodes"][0] + reach = _route_samples(tree, X) + replay = np.empty(X.shape[0], dtype=int) + for leaf in (node for node in tree["nodes"] if node["is_leaf"]): + replay[reach[leaf["id"]]] = np.argmax(leaf["class_counts"][0]) + np.testing.assert_array_equal(replay, est.predict(X)) + + +def test_pair_regressor_export_replays_predictions(): + states = np.array([[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, 1.0]]) + X = np.tile(states, (32, 1)) + y = np.tile([0.0, 1.0, 2.0, 0.0], 32) + est = SGTRegressor(num_partitions=3, max_depth=1, inner_max_depth=2, + inner_max_leaf_nodes=4, pairwise_candidates=1, + tao_n_runs=0, random_state=0).fit(X, y) + tree = est.tree_export() + assert tree["nodes"][0]["routing_kind"] == "pair" + assert "nan_prediction_partition" not in tree["nodes"][0] + reach = _route_samples(tree, X) + replay = np.empty(X.shape[0]) + for leaf in (node for node in tree["nodes"] if node["is_leaf"]): + replay[reach[leaf["id"]]] = leaf["value"] + np.testing.assert_allclose(replay, est.predict(X), rtol=0, atol=0)