Add generics describing what fitted expressions compute

NAMESPACE

@@ -125,6 +125,69 @@ S3method(tidypredict_n_trees,lgb.Booster)
 S3method(tidypredict_n_trees,randomForest)
 S3method(tidypredict_n_trees,ranger)
 S3method(tidypredict_n_trees,xgb.Booster)
+S3method(tidypredict_normalized,C5.0)
+S3method(tidypredict_normalized,H2OBinomialModel)
+S3method(tidypredict_normalized,H2OMultinomialModel)
+S3method(tidypredict_normalized,H2ORegressionModel)
+S3method(tidypredict_normalized,LiblineaR)
+S3method(tidypredict_normalized,bagger)
+S3method(tidypredict_normalized,default)
+S3method(tidypredict_normalized,party)
+S3method(tidypredict_normalized,pm_bart)
+S3method(tidypredict_normalized,pm_catboost)
+S3method(tidypredict_normalized,pm_lgb)
+S3method(tidypredict_normalized,pm_multiclass_regression)
+S3method(tidypredict_normalized,pm_naive_bayes)
+S3method(tidypredict_normalized,pm_nnet)
+S3method(tidypredict_normalized,pm_nullmodel_classification)
+S3method(tidypredict_normalized,pm_regression)
+S3method(tidypredict_normalized,pm_tree)
+S3method(tidypredict_normalized,pm_xgb)
+S3method(tidypredict_normalized,rpart)
+S3method(tidypredict_normalized,rqs)
+S3method(tidypredict_outcome_levels,C5.0)
+S3method(tidypredict_outcome_levels,H2OBinomialModel)
+S3method(tidypredict_outcome_levels,H2OMultinomialModel)
+S3method(tidypredict_outcome_levels,H2ORegressionModel)
+S3method(tidypredict_outcome_levels,LiblineaR)
+S3method(tidypredict_outcome_levels,bagger)
+S3method(tidypredict_outcome_levels,default)
+S3method(tidypredict_outcome_levels,glm)
+S3method(tidypredict_outcome_levels,ksvm)
+S3method(tidypredict_outcome_levels,lognet)
+S3method(tidypredict_outcome_levels,party)
+S3method(tidypredict_outcome_levels,pm_bart)
+S3method(tidypredict_outcome_levels,pm_catboost)
+S3method(tidypredict_outcome_levels,pm_lgb)
+S3method(tidypredict_outcome_levels,pm_multiclass_regression)
+S3method(tidypredict_outcome_levels,pm_naive_bayes)
+S3method(tidypredict_outcome_levels,pm_nnet)
+S3method(tidypredict_outcome_levels,pm_nullmodel_classification)
+S3method(tidypredict_outcome_levels,pm_regression)
+S3method(tidypredict_outcome_levels,pm_tree)
+S3method(tidypredict_outcome_levels,pm_xgb)
+S3method(tidypredict_outcome_levels,rpart)
+S3method(tidypredict_outcome_levels,rqs)
+S3method(tidypredict_output_type,C5.0)
+S3method(tidypredict_output_type,H2OBinomialModel)
+S3method(tidypredict_output_type,H2OMultinomialModel)
+S3method(tidypredict_output_type,H2ORegressionModel)
+S3method(tidypredict_output_type,LiblineaR)
+S3method(tidypredict_output_type,bagger)
+S3method(tidypredict_output_type,default)
+S3method(tidypredict_output_type,party)
+S3method(tidypredict_output_type,pm_bart)
+S3method(tidypredict_output_type,pm_catboost)
+S3method(tidypredict_output_type,pm_lgb)
+S3method(tidypredict_output_type,pm_multiclass_regression)
+S3method(tidypredict_output_type,pm_naive_bayes)
+S3method(tidypredict_output_type,pm_nnet)
+S3method(tidypredict_output_type,pm_nullmodel_classification)
+S3method(tidypredict_output_type,pm_regression)
+S3method(tidypredict_output_type,pm_tree)
+S3method(tidypredict_output_type,pm_xgb)
+S3method(tidypredict_output_type,rpart)
+S3method(tidypredict_output_type,rqs)
 S3method(tidypredict_test,"_xgb.Booster")
 S3method(tidypredict_test,C5.0)
 S3method(tidypredict_test,H2OBinomialModel)
@@ -201,6 +264,9 @@ export(tidypredict_fit)
 export(tidypredict_interval)
 export(tidypredict_load)
 export(tidypredict_n_trees)
+export(tidypredict_normalized)
+export(tidypredict_outcome_levels)
+export(tidypredict_output_type)
 export(tidypredict_save)
 export(tidypredict_sql)
 export(tidypredict_sql_interval)

NEWS.md

@@ -1,7 +1,13 @@
 # tidypredict (development version)
 
+- New generics describe what a model's fitted expressions compute, which the expressions themselves do not say: `tidypredict_output_type()` returns one of `"numeric"`, `"prob"`, `"decision"` or `"class"`, `tidypredict_outcome_levels()` returns the outcome levels in model order, and `tidypredict_normalized()` reports whether per-level probabilities already sum to one. See `?tidypredict_metadata`. (#435)
+
+- The distinctions these generics record cannot be recovered from the shape of a `tidypredict_fit()` result. A `LiblineaR` SVM classifier and a `LiblineaR` logistic regression both return a single expression, but the first is an uncalibrated decision value whose sign picks the class, so thresholding it at 0.5 as though it were a probability is wrong. A multiclass probability list and a `quantreg::rq()` fit with several `tau` are both named lists of expressions of the same length. (#435)
+
 - Fixed `cforest()` models failing with "'language' object cannot be coerced to type 'symbol'" under partykit 1.3-0. That release added a shim to partykit's methods that identifies the caller with `as.name()`, which errors when the generic is reached as `partykit::gettree()`. (#434)
 
+- `tidypredict_output_type()`, `tidypredict_outcome_levels()` and `tidypredict_normalized()` describe what a model's fitted expressions actually compute: a number, a probability, an uncalibrated decision value or a class label, which outcome levels they are named for, and whether a per-level list already sums to one. None of it is recoverable from the shape of the result, so a caller generating code from `tidypredict_fit()` no longer has to keep its own list of which backend produces which shape. See `?tidypredict_metadata`. (#433)
+
 - New generics expose the pieces `tidypredict_fit()` is assembled from, so that packages generating their own code from a fitted model can reuse tidypredict's parsing: `tidypredict_trees()` returns per-tree expressions, `tidypredict_class_trees()` returns per-tree expressions for each outcome level, `tidypredict_class_exprs()` returns one finished expression per outcome level, and `tidypredict_n_trees()` returns the number of trees. See `?tidypredict_extractors`. (#433)
 
 - The eleven `.extract_*()` functions are deprecated in favour of those generics. They were exported but documented as internal, and each is now a thin wrapper that warns. Two of them change return type under the new names: `.extract_earth_multiclass()` and `.extract_glmnet_multiclass()` returned deparsed strings, while `tidypredict_class_exprs()` returns language objects like every other extractor. (#433)

R/model-C5.0.R

@@ -1092,3 +1092,25 @@ build_tree_formula.pm_tree_C5.0 <- function(model) {
   }
   c50_with_na_descent(build_tree_formula_single(model), model$tree_info)
 }
+
+# Output metadata ---------------------------------
+
+# C5.0 is classification only, and the fit votes the trees into a single class
+# label rather than a probability.
+#' @export
+tidypredict_output_type.C5.0 <- function(x, ...) {
+  rlang::check_dots_empty()
+  "class"
+}
+
+#' @export
+tidypredict_outcome_levels.C5.0 <- function(x, ...) {
+  rlang::check_dots_empty()
+  x$levels
+}
+
+#' @export
+tidypredict_normalized.C5.0 <- function(x, ...) {
+  rlang::check_dots_empty()
+  NA
+}

R/model-bagger.R

@@ -197,3 +197,33 @@ bagger_check_model <- function(model, call = rlang::caller_env()) {
 build_tree_formula.pm_tree_bagger <- function(model) {
   bagger_build_formula(model)
 }
+
+# Output metadata ---------------------------------
+
+# `bagger_classes()` is the same mode signal `parse_model.bagger()` uses:
+# `NULL` for regression, the outcome levels for classification. A classification
+# ensemble averages class probabilities and then picks the largest, so the fit
+# is a class label rather than a probability.
+#' @export
+tidypredict_output_type.bagger <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  if (is.null(bagger_classes(x))) {
+    return("numeric")
+  }
+  "class"
+}
+
+#' @export
+tidypredict_outcome_levels.bagger <- function(x, ...) {
+  rlang::check_dots_empty()
+  bagger_classes(x)
+}
+
+#' @export
+tidypredict_normalized.bagger <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  # A single expression either way, so there are no per-level values to sum.
+  NA
+}

R/model-bart.R

@@ -257,3 +257,24 @@ tidypredict_test.bart <- function(
     y_scale = diff(range(model$y))
   )
 }
+
+# Output metadata ---------------------------------
+
+# {dbarts} models are supported for regression only.
+#' @export
+tidypredict_output_type.pm_bart <- function(x, ...) {
+  rlang::check_dots_empty()
+  "numeric"
+}
+
+#' @export
+tidypredict_outcome_levels.pm_bart <- function(x, ...) {
+  rlang::check_dots_empty()
+  NULL
+}
+
+#' @export
+tidypredict_normalized.pm_bart <- function(x, ...) {
+  rlang::check_dots_empty()
+  NA
+}

R/model-catboost.R

@@ -919,3 +919,43 @@ tidypredict_n_trees.catboost.Model <- function(x, ...) {
   # round, so `niter` is the number of rounds rather than the number of trees.
   length(tidypredict_trees(x))
 }
+
+# Output metadata ---------------------------------
+
+# The same objective groups `build_fit_formula_catboost_nested()` switches on.
+catboost_parsed_objective <- function(x) {
+  x$general$params$objective %||% "RMSE"
+}
+
+#' @export
+tidypredict_output_type.pm_catboost <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  objective <- catboost_parsed_objective(x)
+  if (
+    objective %in%
+      c(catboost_multiclass_objectives, catboost_sigmoid_objectives)
+  ) {
+    return("prob")
+  }
+  "numeric"
+}
+
+#' @export
+tidypredict_outcome_levels.pm_catboost <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  # CatBoost is fit on integer labels, and the multiclass expressions come back
+  # named `class_0`, `class_1` and so on, which are positions, not levels.
+  NULL
+}
+
+#' @export
+tidypredict_normalized.pm_catboost <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  if (catboost_parsed_objective(x) %in% catboost_multiclass_objectives) {
+    return(TRUE)
+  }
+  NA
+}

R/model-glm.R

@@ -29,3 +29,23 @@ te_interval_glm <- function(parsedmodel, interval = 0.95) {
   }
   intervals
 }
+
+# Output metadata ---------------------------------
+
+# The parsed form is coefficients only, so the response levels have to come off
+# the fitted object. A binomial glm fit on a 0/1 numeric records none, which is
+# the "did not retain the levels" case.
+#' @export
+tidypredict_outcome_levels.glm <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  if (!identical(x$family$family, "binomial")) {
+    return(NULL)
+  }
+
+  response <- stats::model.frame(x)[[1]]
+  if (!is.factor(response)) {
+    return(NULL)
+  }
+  levels(response)
+}

R/model-glmnet.R

@@ -210,3 +210,14 @@ tidypredict_class_exprs.multnet <- function(x, ..., penalty = NULL) {
   names(eqs) <- class_names
   eqs
 }
+
+# Output metadata ---------------------------------
+
+# A `multnet` fit parses to a multiclass parsed model, which carries its own
+# levels. A binary `lognet` parses to a single logistic expression, so its
+# levels only exist on the fitted object.
+#' @export
+tidypredict_outcome_levels.lognet <- function(x, ...) {
+  rlang::check_dots_empty()
+  as.character(x$classnames)
+}

R/model-h2o.R

@@ -208,3 +208,66 @@ tidypredict_test.H2OMultinomialModel <- function(
 h2o_test_results <- function(base, te, threshold) {
   test_results_numeric(base, te, threshold)
 }
+
+# Output metadata ---------------------------------
+
+# H2O models have no `parse_model()` method, so the metadata has to be read off
+# the model handle. The three classes are exactly the three modes.
+#' @export
+tidypredict_output_type.H2ORegressionModel <- function(x, ...) {
+  rlang::check_dots_empty()
+  "numeric"
+}
+
+#' @export
+tidypredict_outcome_levels.H2ORegressionModel <- function(x, ...) {
+  rlang::check_dots_empty()
+  NULL
+}
+
+#' @export
+tidypredict_normalized.H2ORegressionModel <- function(x, ...) {
+  rlang::check_dots_empty()
+  NA
+}
+
+#' @export
+tidypredict_output_type.H2OBinomialModel <- function(x, ...) {
+  rlang::check_dots_empty()
+  "prob"
+}
+
+#' @export
+tidypredict_outcome_levels.H2OBinomialModel <- function(x, ...) {
+  rlang::check_dots_empty()
+  h2o_response_domain(x)
+}
+
+#' @export
+tidypredict_normalized.H2OBinomialModel <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  # `tidypredict_fit()` returns the probability of the second domain level
+  # only, so there are no per-level values to sum.
+  NA
+}
+
+#' @export
+tidypredict_output_type.H2OMultinomialModel <- function(x, ...) {
+  rlang::check_dots_empty()
+  "prob"
+}
+
+#' @export
+tidypredict_outcome_levels.H2OMultinomialModel <- function(x, ...) {
+  rlang::check_dots_empty()
+  h2o_response_domain(x)
+}
+
+#' @export
+tidypredict_normalized.H2OMultinomialModel <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  # `expr_softmax()` over the domain.
+  TRUE
+}

R/model-ksvm.R

@@ -5,6 +5,10 @@
 #' @export
 parse_model.ksvm <- function(model) parse_model_ksvm(model)
 
+ksvm_is_classification <- function(model) {
+  kernlab::type(model) %in% c("C-svc", "nu-svc", "C-bsvc")
+}
+
 parse_model_ksvm <- function(model, call = rlang::caller_env()) {
   acceptable_formula(model)
 
@@ -19,7 +23,7 @@ parse_model_ksvm <- function(model, call = rlang::caller_env()) {
   }
 
   svm_type <- kernlab::type(model)
-  is_classification <- svm_type %in% c("C-svc", "nu-svc", "C-bsvc")
+  is_classification <- ksvm_is_classification(model)
   is_regression <- svm_type %in% c("eps-svr", "nu-svr", "eps-bsvr")
   if (!is_classification && !is_regression) {
     cli::cli_abort(
@@ -287,3 +291,21 @@ tidypredict_test.ksvm <- function(
     model@kcall
   )
 }
+
+# Output metadata ---------------------------------
+
+# `parse_model_ksvm()` folds Platt scaling into the glm logit machinery, so
+# `tidypredict_output_type.pm_regression()` already reports "prob" for a
+# classifier and "numeric" for an SVR. Only the levels need the fitted object.
+#' @export
+tidypredict_outcome_levels.ksvm <- function(x, ...) {
+  rlang::check_dots_empty()
+
+  # `lev()` holds the sorted response values for a regression, so read the mode
+  # off `type()` the way `parse_model_ksvm()` does. A classifier is binary
+  # because `parse_model_ksvm()` rejects anything else.
+  if (!ksvm_is_classification(x)) {
+    return(NULL)
+  }
+  as.character(kernlab::lev(x))
+}