feat: add soft gradient boosting (#95)

* update code

* update doc

* update test

* update test

* update example

* update example

Author: xuyxu

CHANGELOG.rst

@@ -17,7 +17,7 @@ Changelog
 
 Ver 0.1.*
 ---------
-
+* |Feature| |API| Add :class:`SoftGradientBoostingClassifier` and :class:`SoftGradientBoostingRegressor` | `@xuyxu <https://github.com/xuyxu>`__
 * |Feature| |API| Support using dataloader with multiple input | `@xuyxu <https://github.com/xuyxu>`__
 * |Fix| Fix missing functionality of ``use_reduction_sum`` for :meth:`fit` of Gradient Boosting | `@xuyxu <https://github.com/xuyxu>`__
 * |Enhancement| Relax :mod:`tensorboard` as a soft dependency | `@xuyxu <https://github.com/xuyxu>`__

README.rst

@@ -100,6 +100,8 @@ Supported Ensemble
 +------------------------------+------------+---------------------------+
 | Fast Geometric Ensemble [6]_ | Sequential |     fast_geometric.py     |
 +------------------------------+------------+---------------------------+
+|  Soft Gradient Boosting [7]_ |  Parallel  | soft_gradient_boosting.py |
++------------------------------+------------+---------------------------+
 
 Dependencies
 ------------
@@ -123,6 +125,8 @@ Reference
 
 .. [6] Garipov, Timur, et al. Loss Surfaces, Mode Connectivity, and Fast Ensembling of DNNs. NeurIPS, 2018.
 
+.. [7] Feng, Ji, et al. Soft Gradient Boosting Machine. ArXiv, 2020.
+
 .. _pytorch: https://pytorch.org/
 
 .. _pypi: https://pypi.org/project/torchensemble/

docs/parameters.rst

@@ -165,3 +165,24 @@ FastGeometricRegressor
 
 .. autoclass:: torchensemble.fast_geometric.FastGeometricRegressor
     :members:
+
+Soft Gradient Boosting
+----------------------
+
+In soft gradient boosting, all base estimators could be simultaneously fitted,
+while achieving the similar boosting improvements as in gradient boosting.
+
+Reference:
+    J. Feng, Y.-X. Xu et al., Soft Gradient Boosting Machine, ArXiv, 2020.
+
+SoftGradientBoostingClassifier
+******************************
+
+.. autoclass:: torchensemble.soft_gradient_boosting.SoftGradientBoostingClassifier
+    :members:
+
+SoftGradientBoostingRegressor
+*****************************
+
+.. autoclass:: torchensemble.soft_gradient_boosting.SoftGradientBoostingRegressor
+    :members:

examples/classification_cifar10_cnn.py

@@ -12,6 +12,7 @@
 from torchensemble.bagging import BaggingClassifier
 from torchensemble.gradient_boosting import GradientBoostingClassifier
 from torchensemble.snapshot_ensemble import SnapshotEnsembleClassifier
+from torchensemble.soft_gradient_boosting import SoftGradientBoostingClassifier
 
 from torchensemble.utils.logging import set_logger
 
@@ -228,5 +229,34 @@ def forward(self, x):
         )
     )
 
+    # SoftGradientBoostingClassifier
+    model = SoftGradientBoostingClassifier(
+        estimator=LeNet5, n_estimators=n_estimators, cuda=True
+    )
+
+    # Set the optimizer
+    model.set_optimizer("Adam", lr=lr, weight_decay=weight_decay)
+
+    # Training
+    tic = time.time()
+    model.fit(train_loader, epochs=epochs)
+    toc = time.time()
+    training_time = toc - tic
+
+    # Evaluating
+    tic = time.time()
+    testing_acc = model.evaluate(test_loader)
+    toc = time.time()
+    evaluating_time = toc - tic
+
+    records.append(
+        (
+            "SoftGradientBoostingClassifier",
+            training_time,
+            evaluating_time,
+            testing_acc,
+        )
+    )
+
     # Print results on different ensemble methods
     display_records(records, logger)

torchensemble/__init__.py

@@ -12,6 +12,8 @@
 from .adversarial_training import AdversarialTrainingRegressor
 from .fast_geometric import FastGeometricClassifier
 from .fast_geometric import FastGeometricRegressor
+from .soft_gradient_boosting import SoftGradientBoostingClassifier
+from .soft_gradient_boosting import SoftGradientBoostingRegressor
 
 
 __all__ = [
@@ -29,4 +31,6 @@
     "AdversarialTrainingRegressor",
     "FastGeometricClassifier",
     "FastGeometricRegressor",
+    "SoftGradientBoostingClassifier",
+    "SoftGradientBoostingRegressor",
 ]