Upgrade xgboost to 3.2.0

.github/workflows/precompile.yml

@@ -56,6 +56,7 @@ jobs:
         with:
           files: |
             cache/*.tar.gz
+            cache/*.tar.gz.sha256
 
   macos:
     runs-on: macos-15
@@ -87,3 +88,4 @@ jobs:
         with:
           files: |
             ${{ 'cache/*aarch64*.tar.gz' }}
+            ${{ 'cache/*aarch64*.tar.gz.sha256' }}

CHANGELOG.md

@@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## MORE UNRELEASED
+
+### Added
+
+- Nothing
+
+### Updated
+
+- Updated the bundled XGBoost release from 3.1.3 to 3.2.0.
+- Model persistence now uses XGBoost's stable JSON/UBJ model format while retaining support for loading legacy serialized snapshots.
+- Custom-gradient training now uses `XGBoosterTrainOneIter`, replacing the deprecated `XGBoosterBoostOneIter` API.
+- XGBoost C API revision checks now ignore formatting-only declaration changes and validate symbols exported by the built shared library.
+- Incremental builds now refresh the packaged XGBoost library instead of nesting the new library under an existing `priv/lib` directory.
+- Fix precompiled nif download; tags now X.Y.Z not vX.Y.Z.
+- Remove older nif versions from precompiled list.
+- Ensure checksum SHA's are part of the precompiled packages.
+
+### Removed
+
+- Nothing.
+
 ## 0.10.3
 
 ### Added

Makefile

@@ -7,15 +7,19 @@ TEMP ?= $(HOME)/.cache
 MIX_ENV ?= dev
 XGBOOST_CACHE ?= $(TEMP)/exgboost
 XGBOOST_GIT_REPO ?= https://github.com/dmlc/xgboost.git
-# v3.1.3 tagged release
-XGBOOST_GIT_REV ?= v3.1.3
-OLD_XGBOOST_GIT_REV ?= v3.0.5
+
+# Use tagged releases in the checks below.
+XGBOOST_GIT_REV ?= v3.2.0
+OLD_XGBOOST_GIT_REV ?= v3.1.3
 NEW_XGBOOST_GIT_REV ?= $(XGBOOST_GIT_REV)
+
 XGBOOST_NS = xgboost-$(XGBOOST_GIT_REV)
 XGBOOST_DIR = $(XGBOOST_CACHE)/$(XGBOOST_NS)
 XGBOOST_LIB_DIR = $(XGBOOST_DIR)/build/xgboost
 XGBOOST_LIB_DIR_FLAG = $(XGBOOST_LIB_DIR)/exgboost.ok
 
+.PHONY: check-xgboost-c-api compare-xgboost-c-api clean
+
 # Set build type based on MIX_ENV
 ifeq ($(MIX_ENV), prod)
 	CMAKE_BUILD_TYPE = Release
@@ -54,13 +58,13 @@ else
 endif
 
 $(EXGBOOST_SO): $(EXGBOOST_CACHE_SO)
-	@mkdir -p $(PRIV_DIR)
-	cp -a $(abspath $(EXGBOOST_CACHE_LIB_DIR)) $(EXGBOOST_LIB_DIR) ; \
+	@mkdir -p $(EXGBOOST_LIB_DIR)
+	cp -a $(abspath $(EXGBOOST_CACHE_LIB_DIR))/. $(EXGBOOST_LIB_DIR)/ ; \
 	cp -a $(abspath $(EXGBOOST_CACHE_SO)) $(EXGBOOST_SO) ;
 
 $(EXGBOOST_CACHE_SO): $(XGBOOST_LIB_DIR_FLAG) $(C_SRCS)
-	@mkdir -p cache
-	cp -R $(XGBOOST_LIB_DIR) $(EXGBOOST_CACHE_LIB_DIR)
+	@mkdir -p $(EXGBOOST_CACHE_LIB_DIR)
+	cp -R $(XGBOOST_LIB_DIR)/. $(EXGBOOST_CACHE_LIB_DIR)/
 	cp $(XGBOOST_DIR)/lib/$(LIBXGBOOST) $(EXGBOOST_CACHE_LIB_DIR)
 	$(CC) $(CFLAGS) $(wildcard $(EXGBOOST_DIR)/src/*.c) $(LDFLAGS) -o $(EXGBOOST_CACHE_SO)
 	$(POST_INSTALL)
@@ -85,7 +89,9 @@ $(XGBOOST_LIB_DIR_FLAG): $(XGBOOST_DIR)/.git
 	touch $(XGBOOST_LIB_DIR_FLAG)
 
 check-xgboost-c-api: $(XGBOOST_LIB_DIR_FLAG)
-	./scripts/check_xgboost_c_api.sh "$(XGBOOST_LIB_DIR)/include"
+	./scripts/check_xgboost_c_api.sh \
+		"$(XGBOOST_LIB_DIR)/include" \
+		"$(XGBOOST_LIB_DIR)/lib/$(LIBXGBOOST)"
 
 compare-xgboost-c-api:
 	@set -eu; \

c/exgboost/include/booster.h

@@ -9,7 +9,7 @@ ERL_NIF_TERM EXGBoosterSlice(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]
 ERL_NIF_TERM EXGBoosterSetParam(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 ERL_NIF_TERM EXGBoosterGetNumFeature(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 ERL_NIF_TERM EXGBoosterUpdateOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
-ERL_NIF_TERM EXGBoosterBoostOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+ERL_NIF_TERM EXGBoosterTrainOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 ERL_NIF_TERM EXGBoosterEvalOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 ERL_NIF_TERM EXGBoosterGetAttrNames(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 ERL_NIF_TERM EXGBoosterGetAttr(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);

c/exgboost/src/booster.c

@@ -220,18 +220,18 @@ ERL_NIF_TERM EXGBoosterUpdateOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TER
 END:
   return ret;
 }
-ERL_NIF_TERM EXGBoosterBoostOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
-  ErlNifBinary grad_bin;
-  ErlNifBinary hess_bin;
+
+ERL_NIF_TERM EXGBoosterTrainOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
   BoosterHandle booster;
   BoosterHandle **booster_resource = NULL;
   DMatrixHandle dtrain;
   DMatrixHandle **dtrain_resource = NULL;
-  float *grad = NULL;
-  float *hess = NULL;
-  unsigned grad_len = 0;
-  unsigned hess_len = 0;
-  bst_ulong len;
+  ERL_NIF_TERM grad_binary, grad_typestr, grad_shape, grad_readonly;
+  ERL_NIF_TERM hess_binary, hess_typestr, hess_shape, hess_readonly;
+  char *grad = NULL;
+  char *hess = NULL;
+  const char *error_msg = NULL;
+  int iteration = 0;
   ERL_NIF_TERM ret = -1;
   int result = -1;
   if (4 != argc) {
@@ -248,29 +248,48 @@ ERL_NIF_TERM EXGBoosterBoostOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM
     goto END;
   }
   dtrain = *dtrain_resource;
-  if (!enif_inspect_binary(env, argv[2], &grad_bin)) {
-    ret = exg_error(env, "Grad must be a binary");
+
+  if (!exg_get_array_interface_tuple(env, argv[2], &grad_binary, &grad_typestr, &grad_shape,
+                                     &grad_readonly, &error_msg)) {
+    ret = exg_error(env, error_msg);
+    goto END;
+  }
+  if (!exg_build_array_interface_json(env, grad_binary, grad_typestr, grad_shape, grad_readonly,
+                                      &grad, &error_msg)) {
+    ret = exg_error(env, error_msg);
+    goto END;
+  }
+
+  if (!exg_get_array_interface_tuple(env, argv[3], &hess_binary, &hess_typestr, &hess_shape,
+                                     &hess_readonly, &error_msg)) {
+    ret = exg_error(env, error_msg);
     goto END;
   }
-  if (!enif_inspect_binary(env, argv[3], &hess_bin)) {
-    ret = exg_error(env, "Hess must be a binary");
+  if (!exg_build_array_interface_json(env, hess_binary, hess_typestr, hess_shape, hess_readonly,
+                                      &hess, &error_msg)) {
+    ret = exg_error(env, error_msg);
     goto END;
   }
-  grad = (float *)grad_bin.data;
-  hess = (float *)hess_bin.data;
-  grad_len = grad_bin.size / sizeof(float);
-  hess_len = hess_bin.size / sizeof(float);
-  if (grad_len != hess_len) {
-    ret = exg_error(env, "Grad and Hess must have the same length");
+
+  result = XGBoosterBoostedRounds(booster, &iteration);
+  if (result != 0) {
+    ret = exg_error(env, XGBGetLastError());
     goto END;
   }
-  result = XGBoosterBoostOneIter(booster, dtrain, grad, hess, (bst_ulong)grad_len);
+
+  result = XGBoosterTrainOneIter(booster, dtrain, iteration, grad, hess);
   if (result == 0) {
     ret = ok_atom(env);
   } else {
     ret = exg_error(env, XGBGetLastError());
   }
 END:
+  if (grad != NULL) {
+    enif_free(grad);
+  }
+  if (hess != NULL) {
+    enif_free(hess);
+  }
   return ret;
 }
 

c/exgboost/src/exgboost.c

@@ -57,7 +57,7 @@ static ErlNifFunc nif_funcs[] = {
     {"booster_set_param", 3, EXGBoosterSetParam},
     {"booster_get_num_feature", 1, EXGBoosterGetNumFeature},
     {"booster_update_one_iter", 3, EXGBoosterUpdateOneIter, ERL_NIF_DIRTY_JOB_CPU_BOUND},
-    {"booster_boost_one_iter", 4, EXGBoosterBoostOneIter, ERL_NIF_DIRTY_JOB_CPU_BOUND},
+    {"booster_boost_one_iter", 4, EXGBoosterTrainOneIter, ERL_NIF_DIRTY_JOB_CPU_BOUND},
     {"booster_eval_one_iter", 4, EXGBoosterEvalOneIter, ERL_NIF_DIRTY_JOB_CPU_BOUND},
     {"booster_get_attr_names", 1, EXGBoosterGetAttrNames},
     {"booster_get_attr", 2, EXGBoosterGetAttr},

lib/exgboost/booster.ex

@@ -27,23 +27,21 @@ defmodule EXGBoost.Booster do
   A Booster can be serialized to a file using `EXGBoost.Booster.save` and loaded from a file
   using `EXGBoost.Booster.load`. The file format can be specified using the `:format` option
   which can be either `:json` or `:ubj`. The default is `:json`. If the file already exists, it will
-  be overwritten by default.  Boosters can either be serialized to a file or to a binary string.
-  Boosters can be serialized in three different ways: configuration only, configuration and model, or
-  model only. Any function that uses the `to` and `from` `buffer` functions will serialize the Booster
-  to a binary string. The `to` and `from` `file` functions will serialize the Booster to a file.
-  Functions named with `weights` will serialize the model weights only. Functions named with `config` will
-  serialize the configuration only. Functions that specify `model` will serialize both the model weights
-  and the configuration.
+  be overwritten by default. Boosters can either be serialized to a file or to a binary string.
+  Model and weight outputs use XGBoost's stable model-I/O representation so they can be loaded by
+  later XGBoost releases. Configuration output contains the internal training configuration and is
+  intended for use with the same XGBoost version.
 
   ### Output Formats
   - `file` - Save to a file.
   - `buffer` - Save to a binary string.
 
   ### Output Contents
-  - `config` - Save the configuration only.
-  - `weights` - Save the model weights only.
-  - `model` - Save both the model weights and the configuration.
+  - `config` - Save the internal training configuration only.
+  - `weights` - Save the portable model representation (retained as a compatibility alias).
+  - `model` - Save the portable model representation, including trees and the objective.
   """
+  alias EXGBoost.ArrayInterface
   alias EXGBoost.DMatrix
   alias EXGBoost.Internal
   alias EXGBoost.NIF
@@ -220,7 +218,10 @@ defmodule EXGBoost.Booster do
           |> then(&File.write!(filepath, &1))
 
         :model ->
-          EXGBoost.NIF.booster_serialize_to_buffer(booster.ref)
+          EXGBoost.NIF.booster_save_model_to_buffer(
+            booster.ref,
+            Jason.encode!(%{format: opts[:format]})
+          )
           |> Internal.unwrap!()
           |> then(&File.write!(filepath, &1))
 
@@ -233,7 +234,11 @@ defmodule EXGBoost.Booster do
           EXGBoost.NIF.booster_save_json_config(booster.ref) |> Internal.unwrap!()
 
         :model ->
-          EXGBoost.NIF.booster_serialize_to_buffer(booster.ref) |> Internal.unwrap!()
+          EXGBoost.NIF.booster_save_model_to_buffer(
+            booster.ref,
+            Jason.encode!(%{format: opts[:format]})
+          )
+          |> Internal.unwrap!()
 
         :weights ->
           EXGBoost.NIF.booster_save_model_to_buffer(
@@ -360,8 +365,8 @@ defmodule EXGBoost.Booster do
     EXGBoost.NIF.booster_boost_one_iter(
       booster.ref,
       dmatrix.ref,
-      Nx.to_binary(grad),
-      Nx.to_binary(hess)
+      grad |> ArrayInterface.from_tensor() |> ArrayInterface.to_tuple(),
+      hess |> ArrayInterface.from_tensor() |> ArrayInterface.to_tuple()
     )
   end
 

lib/exgboost/nif.ex

@@ -282,11 +282,16 @@ defmodule EXGBoost.NIF do
     do: :erlang.nif_error(:not_implemented)
 
   @doc """
-  Update the model, by directly specify gradient and second order gradient, this can be used to replace UpdateOneIter, to support customized loss function
+  Update the model with a custom gradient and second-order gradient.
 
-  Grad and hess must be binaries of Nx.Tensor float32
+  Grad and hess must be array-interface tuples for float32 tensors.
   """
-  @spec booster_boost_one_iter(booster_reference(), dmatrix_reference(), binary(), binary()) ::
+  @spec booster_boost_one_iter(
+          booster_reference(),
+          dmatrix_reference(),
+          array_interface_tuple(),
+          array_interface_tuple()
+        ) ::
           :ok | {:error, String.t()}
   def booster_boost_one_iter(_booster_handle, _dmatrix_handle, _grad, _hess),
     do: :erlang.nif_error(:not_implemented)

mix.exs

@@ -9,12 +9,12 @@ defmodule EXGBoost.MixProject do
       version: @version,
       make_precompiler: {:nif, CCPrecompiler},
       make_precompiler_url:
-        "https://github.com/iperks/exgboost/releases/download/v#{@version}/@{artefact_filename}",
+        "https://github.com/iperks/exgboost/releases/download/#{@version}/@{artefact_filename}",
       make_precompiler_priv_paths: ["libexgboost.*", "lib"],
       # NIF Versions correspond to OTP Releases
       # https://github.com/erlang/otp/blob/d3aa6c044c3927f011fb76ac087d5ce0e814954c/erts/emulator/beam/erl_nif.h#L57
       make_precompiler_nif_versions: [
-        versions: ["2.15", "2.16", "2.17", "2.18"]
+        versions: ["2.17", "2.18"]
       ],
       elixir: "~> 1.17",
       start_permanent: Mix.env() == :prod,

scripts/check_xgboost_c_api.sh

@@ -24,7 +24,15 @@ extract_signatures() {
       }
     }
   ' "$header" \
-    | sed -E 's/[[:space:]]+/ /g; s/^[[:space:]]+//; s/[[:space:]]+$//' \
+    | sed -E '
+        s/[[:space:]]+/ /g
+        s/^[[:space:]]+//
+        s/[[:space:]]+$//
+        s/[[:space:]]*\*[[:space:]]*/\*/g
+        s/[[:space:]]*,[[:space:]]*/,/g
+        s/\([[:space:]]*/(/g
+        s/[[:space:]]*\)/)/g
+      ' \
     | awk '
         {
           if (match($0, /XG[A-Za-z0-9_]+[[:space:]]*\(/)) {
@@ -163,15 +171,21 @@ if (( ${#missing_in_header[@]} > 0 )); then
 fi
 
 if [[ -n "$shared_lib" && -f "$shared_lib" && "$(command -v nm || true)" != "" ]]; then
+  declare -A exported_map=()
+
   if [[ "$(uname -s)" == "Darwin" ]]; then
-    mapfile -t exported_symbols < <(nm -gU "$shared_lib" | awk '{print $3}' | sed 's/^_//' | sort -u)
+    while IFS= read -r symbol; do
+      [[ -n "$symbol" ]] && exported_map["$symbol"]=1
+    done < <(nm -gU "$shared_lib" | awk '{print $3}' | sed 's/^_//' | sort -u)
   else
-    mapfile -t exported_symbols < <(nm -D --defined-only "$shared_lib" | awk '{print $3}' | sort -u)
+    while IFS= read -r symbol; do
+      [[ -n "$symbol" ]] && exported_map["$symbol"]=1
+    done < <(nm -D --defined-only "$shared_lib" | awk '{print $3}' | sort -u)
   fi
 
   missing_in_lib=()
   for symbol in "${used_symbols[@]}"; do
-    if ! printf '%s\n' "${exported_symbols[@]}" | grep -qx "$symbol"; then
+    if [[ -z "${exported_map[$symbol]+x}" ]]; then
       missing_in_lib+=("$symbol")
     fi
   done

test/exgboost_test.exs

@@ -332,6 +332,10 @@ defmodule EXGBoostTest do
 
     EXGBoost.write_model(booster, "test")
     assert File.exists?("test.json")
+
+    assert %{"learner" => _learner, "version" => _version} =
+             "test.json" |> File.read!() |> Jason.decode!()
+
     bst = EXGBoost.read_model("test.json")
     assert is_struct(bst, EXGBoost.Booster)
     File.rm!("test.json")
@@ -395,10 +399,44 @@ defmodule EXGBoostTest do
 
     buffer = EXGBoost.dump_model(booster)
     assert is_binary(buffer)
+    assert %{"learner" => _learner, "version" => _version} = Jason.decode!(buffer)
+
     bst = EXGBoost.load_model(buffer)
     assert is_struct(bst, EXGBoost.Booster)
   end
 
+  test "load_model accepts legacy serialized snapshots", context do
+    nrows = :rand.uniform(10)
+    ncols = :rand.uniform(10)
+    {x, new_key} = Nx.Random.normal(context[:key], 0, 1, shape: {nrows, ncols})
+    {y, _new_key} = Nx.Random.normal(new_key, 0, 1, shape: {nrows})
+
+    booster =
+      EXGBoost.train(x, y,
+        num_boost_rounds: 2,
+        tree_method: :hist,
+        eval_metric: :rmse
+      )
+
+    snapshot =
+      EXGBoost.NIF.booster_serialize_to_buffer(booster.ref)
+      |> EXGBoost.Internal.unwrap!()
+
+    assert %EXGBoost.Booster{} = EXGBoost.load_model(snapshot)
+  end
+
+  test "boost trains with custom gradient and Hessian" do
+    x = Nx.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], type: {:f, 32})
+    y = Nx.tensor([0.0, 1.0, 0.0], type: {:f, 32})
+    dmatrix = DMatrix.from_tensor(x, y, format: :dense)
+    booster = Booster.booster(dmatrix, tree_method: :hist)
+    gradient = Nx.tensor([0.1, -0.2, 0.1], type: {:f, 32})
+    hessian = Nx.tensor([1.0, 1.0, 1.0], type: {:f, 32})
+
+    assert :ok = Booster.boost(booster, dmatrix, gradient, hessian)
+    assert Booster.get_boosted_rounds(booster) == 1
+  end
+
   test "load_model accepts model artifacts produced by dump_weights", context do
     nrows = :rand.uniform(10)
     ncols = :rand.uniform(10)