diff --git a/.github/workflows/precompile.yml b/.github/workflows/precompile.yml index b19d338..c2694b1 100644 --- a/.github/workflows/precompile.yml +++ b/.github/workflows/precompile.yml @@ -56,6 +56,7 @@ jobs: with: files: | cache/*.tar.gz + cache/*.tar.gz.sha256 macos: runs-on: macos-15 @@ -87,3 +88,4 @@ jobs: with: files: | ${{ 'cache/*aarch64*.tar.gz' }} + ${{ 'cache/*aarch64*.tar.gz.sha256' }} diff --git a/CHANGELOG.md b/CHANGELOG.md index acaa365..50a6b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## MORE UNRELEASED + +### Added + +- Nothing + +### Updated + +- Updated the bundled XGBoost release from 3.1.3 to 3.2.0. +- Model persistence now uses XGBoost's stable JSON/UBJ model format while retaining support for loading legacy serialized snapshots. +- Custom-gradient training now uses `XGBoosterTrainOneIter`, replacing the deprecated `XGBoosterBoostOneIter` API. +- XGBoost C API revision checks now ignore formatting-only declaration changes and validate symbols exported by the built shared library. +- Incremental builds now refresh the packaged XGBoost library instead of nesting the new library under an existing `priv/lib` directory. +- Fix precompiled nif download; tags now X.Y.Z not vX.Y.Z. +- Remove older nif versions from precompiled list. +- Ensure checksum SHA's are part of the precompiled packages. + +### Removed + +- Nothing. + ## 0.10.3 ### Added diff --git a/Makefile b/Makefile index 78aaf55..7433946 100644 --- a/Makefile +++ b/Makefile @@ -7,15 +7,19 @@ TEMP ?= $(HOME)/.cache MIX_ENV ?= dev XGBOOST_CACHE ?= $(TEMP)/exgboost XGBOOST_GIT_REPO ?= https://github.com/dmlc/xgboost.git -# v3.1.3 tagged release -XGBOOST_GIT_REV ?= v3.1.3 -OLD_XGBOOST_GIT_REV ?= v3.0.5 + +# Use tagged releases in the checks below. +XGBOOST_GIT_REV ?= v3.2.0 +OLD_XGBOOST_GIT_REV ?= v3.1.3 NEW_XGBOOST_GIT_REV ?= $(XGBOOST_GIT_REV) + XGBOOST_NS = xgboost-$(XGBOOST_GIT_REV) XGBOOST_DIR = $(XGBOOST_CACHE)/$(XGBOOST_NS) XGBOOST_LIB_DIR = $(XGBOOST_DIR)/build/xgboost XGBOOST_LIB_DIR_FLAG = $(XGBOOST_LIB_DIR)/exgboost.ok +.PHONY: check-xgboost-c-api compare-xgboost-c-api clean + # Set build type based on MIX_ENV ifeq ($(MIX_ENV), prod) CMAKE_BUILD_TYPE = Release @@ -54,13 +58,13 @@ else endif $(EXGBOOST_SO): $(EXGBOOST_CACHE_SO) - @mkdir -p $(PRIV_DIR) - cp -a $(abspath $(EXGBOOST_CACHE_LIB_DIR)) $(EXGBOOST_LIB_DIR) ; \ + @mkdir -p $(EXGBOOST_LIB_DIR) + cp -a $(abspath $(EXGBOOST_CACHE_LIB_DIR))/. $(EXGBOOST_LIB_DIR)/ ; \ cp -a $(abspath $(EXGBOOST_CACHE_SO)) $(EXGBOOST_SO) ; $(EXGBOOST_CACHE_SO): $(XGBOOST_LIB_DIR_FLAG) $(C_SRCS) - @mkdir -p cache - cp -R $(XGBOOST_LIB_DIR) $(EXGBOOST_CACHE_LIB_DIR) + @mkdir -p $(EXGBOOST_CACHE_LIB_DIR) + cp -R $(XGBOOST_LIB_DIR)/. $(EXGBOOST_CACHE_LIB_DIR)/ cp $(XGBOOST_DIR)/lib/$(LIBXGBOOST) $(EXGBOOST_CACHE_LIB_DIR) $(CC) $(CFLAGS) $(wildcard $(EXGBOOST_DIR)/src/*.c) $(LDFLAGS) -o $(EXGBOOST_CACHE_SO) $(POST_INSTALL) @@ -85,7 +89,9 @@ $(XGBOOST_LIB_DIR_FLAG): $(XGBOOST_DIR)/.git touch $(XGBOOST_LIB_DIR_FLAG) check-xgboost-c-api: $(XGBOOST_LIB_DIR_FLAG) - ./scripts/check_xgboost_c_api.sh "$(XGBOOST_LIB_DIR)/include" + ./scripts/check_xgboost_c_api.sh \ + "$(XGBOOST_LIB_DIR)/include" \ + "$(XGBOOST_LIB_DIR)/lib/$(LIBXGBOOST)" compare-xgboost-c-api: @set -eu; \ diff --git a/c/exgboost/include/booster.h b/c/exgboost/include/booster.h index 88946a3..619f43b 100644 --- a/c/exgboost/include/booster.h +++ b/c/exgboost/include/booster.h @@ -9,7 +9,7 @@ ERL_NIF_TERM EXGBoosterSlice(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] ERL_NIF_TERM EXGBoosterSetParam(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM EXGBoosterGetNumFeature(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM EXGBoosterUpdateOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); -ERL_NIF_TERM EXGBoosterBoostOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM EXGBoosterTrainOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM EXGBoosterEvalOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM EXGBoosterGetAttrNames(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); ERL_NIF_TERM EXGBoosterGetAttr(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); diff --git a/c/exgboost/src/booster.c b/c/exgboost/src/booster.c index d401d49..c8f0b96 100644 --- a/c/exgboost/src/booster.c +++ b/c/exgboost/src/booster.c @@ -220,18 +220,18 @@ ERL_NIF_TERM EXGBoosterUpdateOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TER END: return ret; } -ERL_NIF_TERM EXGBoosterBoostOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary grad_bin; - ErlNifBinary hess_bin; + +ERL_NIF_TERM EXGBoosterTrainOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { BoosterHandle booster; BoosterHandle **booster_resource = NULL; DMatrixHandle dtrain; DMatrixHandle **dtrain_resource = NULL; - float *grad = NULL; - float *hess = NULL; - unsigned grad_len = 0; - unsigned hess_len = 0; - bst_ulong len; + ERL_NIF_TERM grad_binary, grad_typestr, grad_shape, grad_readonly; + ERL_NIF_TERM hess_binary, hess_typestr, hess_shape, hess_readonly; + char *grad = NULL; + char *hess = NULL; + const char *error_msg = NULL; + int iteration = 0; ERL_NIF_TERM ret = -1; int result = -1; if (4 != argc) { @@ -248,29 +248,48 @@ ERL_NIF_TERM EXGBoosterBoostOneIter(ErlNifEnv *env, int argc, const ERL_NIF_TERM goto END; } dtrain = *dtrain_resource; - if (!enif_inspect_binary(env, argv[2], &grad_bin)) { - ret = exg_error(env, "Grad must be a binary"); + + if (!exg_get_array_interface_tuple(env, argv[2], &grad_binary, &grad_typestr, &grad_shape, + &grad_readonly, &error_msg)) { + ret = exg_error(env, error_msg); + goto END; + } + if (!exg_build_array_interface_json(env, grad_binary, grad_typestr, grad_shape, grad_readonly, + &grad, &error_msg)) { + ret = exg_error(env, error_msg); + goto END; + } + + if (!exg_get_array_interface_tuple(env, argv[3], &hess_binary, &hess_typestr, &hess_shape, + &hess_readonly, &error_msg)) { + ret = exg_error(env, error_msg); goto END; } - if (!enif_inspect_binary(env, argv[3], &hess_bin)) { - ret = exg_error(env, "Hess must be a binary"); + if (!exg_build_array_interface_json(env, hess_binary, hess_typestr, hess_shape, hess_readonly, + &hess, &error_msg)) { + ret = exg_error(env, error_msg); goto END; } - grad = (float *)grad_bin.data; - hess = (float *)hess_bin.data; - grad_len = grad_bin.size / sizeof(float); - hess_len = hess_bin.size / sizeof(float); - if (grad_len != hess_len) { - ret = exg_error(env, "Grad and Hess must have the same length"); + + result = XGBoosterBoostedRounds(booster, &iteration); + if (result != 0) { + ret = exg_error(env, XGBGetLastError()); goto END; } - result = XGBoosterBoostOneIter(booster, dtrain, grad, hess, (bst_ulong)grad_len); + + result = XGBoosterTrainOneIter(booster, dtrain, iteration, grad, hess); if (result == 0) { ret = ok_atom(env); } else { ret = exg_error(env, XGBGetLastError()); } END: + if (grad != NULL) { + enif_free(grad); + } + if (hess != NULL) { + enif_free(hess); + } return ret; } diff --git a/c/exgboost/src/exgboost.c b/c/exgboost/src/exgboost.c index 94f07a4..e9bc834 100644 --- a/c/exgboost/src/exgboost.c +++ b/c/exgboost/src/exgboost.c @@ -57,7 +57,7 @@ static ErlNifFunc nif_funcs[] = { {"booster_set_param", 3, EXGBoosterSetParam}, {"booster_get_num_feature", 1, EXGBoosterGetNumFeature}, {"booster_update_one_iter", 3, EXGBoosterUpdateOneIter, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"booster_boost_one_iter", 4, EXGBoosterBoostOneIter, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"booster_boost_one_iter", 4, EXGBoosterTrainOneIter, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"booster_eval_one_iter", 4, EXGBoosterEvalOneIter, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"booster_get_attr_names", 1, EXGBoosterGetAttrNames}, {"booster_get_attr", 2, EXGBoosterGetAttr}, diff --git a/lib/exgboost/booster.ex b/lib/exgboost/booster.ex index adbdac4..dbfe5fc 100644 --- a/lib/exgboost/booster.ex +++ b/lib/exgboost/booster.ex @@ -27,23 +27,21 @@ defmodule EXGBoost.Booster do A Booster can be serialized to a file using `EXGBoost.Booster.save` and loaded from a file using `EXGBoost.Booster.load`. The file format can be specified using the `:format` option which can be either `:json` or `:ubj`. The default is `:json`. If the file already exists, it will - be overwritten by default. Boosters can either be serialized to a file or to a binary string. - Boosters can be serialized in three different ways: configuration only, configuration and model, or - model only. Any function that uses the `to` and `from` `buffer` functions will serialize the Booster - to a binary string. The `to` and `from` `file` functions will serialize the Booster to a file. - Functions named with `weights` will serialize the model weights only. Functions named with `config` will - serialize the configuration only. Functions that specify `model` will serialize both the model weights - and the configuration. + be overwritten by default. Boosters can either be serialized to a file or to a binary string. + Model and weight outputs use XGBoost's stable model-I/O representation so they can be loaded by + later XGBoost releases. Configuration output contains the internal training configuration and is + intended for use with the same XGBoost version. ### Output Formats - `file` - Save to a file. - `buffer` - Save to a binary string. ### Output Contents - - `config` - Save the configuration only. - - `weights` - Save the model weights only. - - `model` - Save both the model weights and the configuration. + - `config` - Save the internal training configuration only. + - `weights` - Save the portable model representation (retained as a compatibility alias). + - `model` - Save the portable model representation, including trees and the objective. """ + alias EXGBoost.ArrayInterface alias EXGBoost.DMatrix alias EXGBoost.Internal alias EXGBoost.NIF @@ -220,7 +218,10 @@ defmodule EXGBoost.Booster do |> then(&File.write!(filepath, &1)) :model -> - EXGBoost.NIF.booster_serialize_to_buffer(booster.ref) + EXGBoost.NIF.booster_save_model_to_buffer( + booster.ref, + Jason.encode!(%{format: opts[:format]}) + ) |> Internal.unwrap!() |> then(&File.write!(filepath, &1)) @@ -233,7 +234,11 @@ defmodule EXGBoost.Booster do EXGBoost.NIF.booster_save_json_config(booster.ref) |> Internal.unwrap!() :model -> - EXGBoost.NIF.booster_serialize_to_buffer(booster.ref) |> Internal.unwrap!() + EXGBoost.NIF.booster_save_model_to_buffer( + booster.ref, + Jason.encode!(%{format: opts[:format]}) + ) + |> Internal.unwrap!() :weights -> EXGBoost.NIF.booster_save_model_to_buffer( @@ -360,8 +365,8 @@ defmodule EXGBoost.Booster do EXGBoost.NIF.booster_boost_one_iter( booster.ref, dmatrix.ref, - Nx.to_binary(grad), - Nx.to_binary(hess) + grad |> ArrayInterface.from_tensor() |> ArrayInterface.to_tuple(), + hess |> ArrayInterface.from_tensor() |> ArrayInterface.to_tuple() ) end diff --git a/lib/exgboost/nif.ex b/lib/exgboost/nif.ex index be5429a..1f55db4 100644 --- a/lib/exgboost/nif.ex +++ b/lib/exgboost/nif.ex @@ -282,11 +282,16 @@ defmodule EXGBoost.NIF do do: :erlang.nif_error(:not_implemented) @doc """ - Update the model, by directly specify gradient and second order gradient, this can be used to replace UpdateOneIter, to support customized loss function + Update the model with a custom gradient and second-order gradient. - Grad and hess must be binaries of Nx.Tensor float32 + Grad and hess must be array-interface tuples for float32 tensors. """ - @spec booster_boost_one_iter(booster_reference(), dmatrix_reference(), binary(), binary()) :: + @spec booster_boost_one_iter( + booster_reference(), + dmatrix_reference(), + array_interface_tuple(), + array_interface_tuple() + ) :: :ok | {:error, String.t()} def booster_boost_one_iter(_booster_handle, _dmatrix_handle, _grad, _hess), do: :erlang.nif_error(:not_implemented) diff --git a/mix.exs b/mix.exs index c596dc5..7913dd6 100644 --- a/mix.exs +++ b/mix.exs @@ -9,12 +9,12 @@ defmodule EXGBoost.MixProject do version: @version, make_precompiler: {:nif, CCPrecompiler}, make_precompiler_url: - "https://github.com/iperks/exgboost/releases/download/v#{@version}/@{artefact_filename}", + "https://github.com/iperks/exgboost/releases/download/#{@version}/@{artefact_filename}", make_precompiler_priv_paths: ["libexgboost.*", "lib"], # NIF Versions correspond to OTP Releases # https://github.com/erlang/otp/blob/d3aa6c044c3927f011fb76ac087d5ce0e814954c/erts/emulator/beam/erl_nif.h#L57 make_precompiler_nif_versions: [ - versions: ["2.15", "2.16", "2.17", "2.18"] + versions: ["2.17", "2.18"] ], elixir: "~> 1.17", start_permanent: Mix.env() == :prod, diff --git a/scripts/check_xgboost_c_api.sh b/scripts/check_xgboost_c_api.sh index 6ed3fa5..d6fa27f 100755 --- a/scripts/check_xgboost_c_api.sh +++ b/scripts/check_xgboost_c_api.sh @@ -24,7 +24,15 @@ extract_signatures() { } } ' "$header" \ - | sed -E 's/[[:space:]]+/ /g; s/^[[:space:]]+//; s/[[:space:]]+$//' \ + | sed -E ' + s/[[:space:]]+/ /g + s/^[[:space:]]+// + s/[[:space:]]+$// + s/[[:space:]]*\*[[:space:]]*/\*/g + s/[[:space:]]*,[[:space:]]*/,/g + s/\([[:space:]]*/(/g + s/[[:space:]]*\)/)/g + ' \ | awk ' { if (match($0, /XG[A-Za-z0-9_]+[[:space:]]*\(/)) { @@ -163,15 +171,21 @@ if (( ${#missing_in_header[@]} > 0 )); then fi if [[ -n "$shared_lib" && -f "$shared_lib" && "$(command -v nm || true)" != "" ]]; then + declare -A exported_map=() + if [[ "$(uname -s)" == "Darwin" ]]; then - mapfile -t exported_symbols < <(nm -gU "$shared_lib" | awk '{print $3}' | sed 's/^_//' | sort -u) + while IFS= read -r symbol; do + [[ -n "$symbol" ]] && exported_map["$symbol"]=1 + done < <(nm -gU "$shared_lib" | awk '{print $3}' | sed 's/^_//' | sort -u) else - mapfile -t exported_symbols < <(nm -D --defined-only "$shared_lib" | awk '{print $3}' | sort -u) + while IFS= read -r symbol; do + [[ -n "$symbol" ]] && exported_map["$symbol"]=1 + done < <(nm -D --defined-only "$shared_lib" | awk '{print $3}' | sort -u) fi missing_in_lib=() for symbol in "${used_symbols[@]}"; do - if ! printf '%s\n' "${exported_symbols[@]}" | grep -qx "$symbol"; then + if [[ -z "${exported_map[$symbol]+x}" ]]; then missing_in_lib+=("$symbol") fi done diff --git a/test/exgboost_test.exs b/test/exgboost_test.exs index 18fe3e6..b3fea71 100644 --- a/test/exgboost_test.exs +++ b/test/exgboost_test.exs @@ -332,6 +332,10 @@ defmodule EXGBoostTest do EXGBoost.write_model(booster, "test") assert File.exists?("test.json") + + assert %{"learner" => _learner, "version" => _version} = + "test.json" |> File.read!() |> Jason.decode!() + bst = EXGBoost.read_model("test.json") assert is_struct(bst, EXGBoost.Booster) File.rm!("test.json") @@ -395,10 +399,44 @@ defmodule EXGBoostTest do buffer = EXGBoost.dump_model(booster) assert is_binary(buffer) + assert %{"learner" => _learner, "version" => _version} = Jason.decode!(buffer) + bst = EXGBoost.load_model(buffer) assert is_struct(bst, EXGBoost.Booster) end + test "load_model accepts legacy serialized snapshots", context do + nrows = :rand.uniform(10) + ncols = :rand.uniform(10) + {x, new_key} = Nx.Random.normal(context[:key], 0, 1, shape: {nrows, ncols}) + {y, _new_key} = Nx.Random.normal(new_key, 0, 1, shape: {nrows}) + + booster = + EXGBoost.train(x, y, + num_boost_rounds: 2, + tree_method: :hist, + eval_metric: :rmse + ) + + snapshot = + EXGBoost.NIF.booster_serialize_to_buffer(booster.ref) + |> EXGBoost.Internal.unwrap!() + + assert %EXGBoost.Booster{} = EXGBoost.load_model(snapshot) + end + + test "boost trains with custom gradient and Hessian" do + x = Nx.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], type: {:f, 32}) + y = Nx.tensor([0.0, 1.0, 0.0], type: {:f, 32}) + dmatrix = DMatrix.from_tensor(x, y, format: :dense) + booster = Booster.booster(dmatrix, tree_method: :hist) + gradient = Nx.tensor([0.1, -0.2, 0.1], type: {:f, 32}) + hessian = Nx.tensor([1.0, 1.0, 1.0], type: {:f, 32}) + + assert :ok = Booster.boost(booster, dmatrix, gradient, hessian) + assert Booster.get_boosted_rounds(booster) == 1 + end + test "load_model accepts model artifacts produced by dump_weights", context do nrows = :rand.uniform(10) ncols = :rand.uniform(10)