Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Calculate base_score based on input labels for mae. (#8107)

Browse Source 
Fit an intercept as base score for abs loss.
This commit is contained in:
Jiaming Yuan 2022-09-20 20:53:54 +08:00 committed by GitHub
parent 4f42aa5f12
commit fffb1fca52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 999 additions and 343 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
amalgamation/xgboost-all0.cc
View File

@ -75,19 +75,20 @@
#include "../src/collective/communicator.cc"
// common
#include "../src/common/common.cc"
#include "../src/common/column_matrix.cc"
#include "../src/common/random.cc"
#include "../src/common/charconv.cc"
#include "../src/common/timer.cc"
#include "../src/common/quantile.cc"
#include "../src/common/host_device_vector.cc"
#include "../src/common/column_matrix.cc"
#include "../src/common/common.cc"
#include "../src/common/hist_util.cc"
#include "../src/common/host_device_vector.cc"
#include "../src/common/io.cc"
#include "../src/common/json.cc"
#include "../src/common/numeric.cc"
#include "../src/common/pseudo_huber.cc"
#include "../src/common/quantile.cc"
#include "../src/common/random.cc"
#include "../src/common/survival_util.cc"
#include "../src/common/threading_utils.cc"
#include "../src/common/timer.cc"
#include "../src/common/version.cc"
// c_api

30
include/xgboost/learner.h
View File

@ -8,10 +8,9 @@
#ifndef XGBOOST_LEARNER_H_
#define XGBOOST_LEARNER_H_
#include <dmlc/any.h>
#include <xgboost/base.h>
#include <xgboost/feature_map.h>
#include <xgboost/generic_parameters.h>
#include <xgboost/generic_parameters.h> // Context
#include <xgboost/host_device_vector.h>
#include <xgboost/model.h>
#include <xgboost/predictor.h>
@ -274,7 +273,7 @@ class Learner : public Model, public Configurable, public dmlc::Serializable {
/**
* \brief Return the context object of this Booster.
*/
virtual GenericParameter const* Ctx() const = 0;
virtual Context const* Ctx() const = 0;
/*!
* \brief Get configuration arguments currently stored by the learner
* \return Key-value pairs representing configuration arguments
@ -289,7 +288,7 @@ class Learner : public Model, public Configurable, public dmlc::Serializable {
/*! \brief The evaluation metrics used to evaluate the model. */
std::vector<std::unique_ptr<Metric> > metrics_;
/*! \brief Training parameter. */
GenericParameter generic_parameters_;
Context ctx_;
};
struct LearnerModelParamLegacy;
@ -298,8 +297,14 @@ struct LearnerModelParamLegacy;
* \brief Basic Model Parameters, used to describe the booster.
*/
struct LearnerModelParam {
/* \brief global bias */
bst_float base_score { 0.5f };
private:
/**
* \brief Global bias, this is just a scalar value but can be extended to vector when we
* support multi-class and multi-target.
*/
linalg::Tensor<float, 1> base_score_;
public:
/* \brief number of features */
uint32_t num_feature { 0 };
/* \brief number of classes, if it is multi-class classification */
@ -310,7 +315,18 @@ struct LearnerModelParam {
LearnerModelParam() = default;
// As the old `LearnerModelParamLegacy` is still used by binary IO, we keep
// this one as an immutable copy.
LearnerModelParam(LearnerModelParamLegacy const& user_param, float base_margin, ObjInfo t);
LearnerModelParam(Context const* ctx, LearnerModelParamLegacy const& user_param,
linalg::Tensor<float, 1> base_margin, ObjInfo t);
LearnerModelParam(LearnerModelParamLegacy const& user_param, ObjInfo t);
LearnerModelParam(bst_feature_t n_features, linalg::Tensor<float, 1> base_margin,
uint32_t n_groups)
: base_score_{std::move(base_margin)}, num_feature{n_features}, num_output_group{n_groups} {}
linalg::TensorView<float const, 1> BaseScore(Context const* ctx) const;
linalg::TensorView<float const, 1> BaseScore(int32_t device) const;
void Copy(LearnerModelParam const& that);
/* \brief Whether this parameter is initialized with LearnerModelParamLegacy. */
bool Initialized() const { return num_feature != 0; }
};

56
include/xgboost/linalg.h
View File

@ -8,6 +8,7 @@
#include <dmlc/endian.h>
#include <xgboost/base.h>
#include <xgboost/generic_parameters.h>
#include <xgboost/host_device_vector.h>
#include <xgboost/json.h>
#include <xgboost/span.h>
@ -16,6 +17,7 @@
#include <cassert>
#include <limits>
#include <string>
#include <tuple>
#include <type_traits>
#include <utility>
#include <vector>
@ -213,6 +215,22 @@ LINALG_HD decltype(auto) constexpr Apply(Fn &&f, Tup &&t) {
constexpr auto kSize = std::tuple_size<Tup>::value;
return Apply(std::forward<Fn>(f), std::forward<Tup>(t), std::make_index_sequence<kSize>{});
}
/**
* C++ 17 conjunction
*/
template <class...>
struct Conjunction : std::true_type {};
template <class B1>
struct Conjunction<B1> : B1 {};
template <class B1, class... Bn>
struct Conjunction<B1, Bn...> : std::conditional_t<bool(B1::value), Conjunction<Bn...>, B1> {};
template <typename... Index>
using IsAllIntegral = Conjunction<std::is_integral<std::remove_reference_t<Index>>...>;
template <typename... Index>
using EnableIfIntegral = std::enable_if_t<IsAllIntegral<Index...>::value>;
} // namespace detail
/**
@ -406,7 +424,7 @@ class TensorView {
*
* \endcode
*/
template <typename... Index>
template <typename... Index, detail::EnableIfIntegral<Index...> * = nullptr>
LINALG_HD T &operator()(Index &&...index) {
static_assert(sizeof...(index) <= kDim, "Invalid index.");
size_t offset = detail::Offset<0ul>(stride_, 0ul, std::forward<Index>(index)...);
@ -416,7 +434,7 @@ class TensorView {
/**
* \brief Index the tensor to obtain a scalar value.
*/
template <typename... Index>
template <typename... Index, detail::EnableIfIntegral<Index...> * = nullptr>
LINALG_HD T const &operator()(Index &&...index) const {
static_assert(sizeof...(index) <= kDim, "Invalid index.");
size_t offset = detail::Offset<0ul>(stride_, 0ul, std::forward<Index>(index)...);
@ -656,7 +674,7 @@ class Tensor {
}
if (device >= 0) {
data_.SetDevice(device);
data_.DevicePointer(); // Pull to device;
data_.ConstDevicePointer(); // Pull to device;
}
CHECK_EQ(data_.Size(), detail::CalcSize(shape_));
}
@ -702,12 +720,29 @@ class Tensor {
}
template <typename I, int32_t D>
explicit Tensor(std::initializer_list<T> data, I const (&shape)[D], int32_t device) {
explicit Tensor(std::initializer_list<T> data, I const (&shape)[D],
int32_t device = Context::kCpuId) {
auto &h_vec = data_.HostVector();
h_vec = data;
// shape
this->Initialize(shape, device);
}
/**
* \brief Index operator. Not thread safe, should not be used in performance critical
* region. For more efficient indexing, consider getting a view first.
*/
template <typename... Index>
T &operator()(Index &&...idx) {
return this->HostView()(std::forward<Index>(idx)...);
}
/**
* \brief Index operator. Not thread safe, should not be used in performance critical
* region. For more efficient indexing, consider getting a view first.
*/
template <typename... Index>
T const &operator()(Index &&...idx) const {
return this->HostView()(std::forward<Index>(idx)...);
}
/**
* \brief Get a \ref TensorView for this tensor.
@ -761,7 +796,7 @@ class Tensor {
*
* If the total size is changed, then data in this tensor is no longer valid.
*/
template <typename... S>
template <typename... S, detail::EnableIfIntegral<S...> * = nullptr>
void Reshape(S &&...s) {
static_assert(sizeof...(S) <= kDim, "Invalid shape.");
detail::ReshapeImpl<0>(shape_, std::forward<S>(s)...);
@ -777,15 +812,20 @@ class Tensor {
*
* If the total size is changed, then data in this tensor is no longer valid.
*/
template <int32_t D>
void Reshape(size_t (&shape)[D]) {
template <size_t D>
void Reshape(common::Span<size_t const, D> shape) {
static_assert(D <= kDim, "Invalid shape.");
std::copy(shape, shape + D, this->shape_);
std::copy(shape.data(), shape.data() + D, this->shape_);
std::fill(shape_ + D, shape_ + kDim, 1);
auto n = detail::CalcSize(shape_);
data_.Resize(n);
}
template <size_t D>
void Reshape(size_t (&shape)[D]) {
this->Reshape(common::Span<size_t const, D>{shape});
}
/**
* \brief Set device ordinal for this tensor.
*/

12
include/xgboost/objective.h
View File

@ -27,7 +27,10 @@ class RegTree;
/*! \brief interface of objective function */
class ObjFunction : public Configurable {
protected:
GenericParameter const* ctx_;
Context const* ctx_;
public:
static constexpr float DefaultBaseScore() { return 0.5f; }
public:
/*! \brief virtual destructor */
@ -75,6 +78,13 @@ class ObjFunction : public Configurable {
virtual bst_float ProbToMargin(bst_float base_score) const {
return base_score;
}
/**
* \brief Make initialize estimation of prediction.
*
* \param info MetaInfo that contains label.
* \param base_score Output estimation.
*/
virtual void InitEstimation(MetaInfo const& info, linalg::Tensor<float, 1>* base_score) const;
/*!
* \brief Return task of this objective.
*/

7
include/xgboost/predictor.h
View File

@ -102,13 +102,10 @@ class PredictionContainer {
*/
class Predictor {
protected:
/*
* \brief Runtime parameters.
*/
GenericParameter const* ctx_;
Context const* ctx_;
public:
explicit Predictor(GenericParameter const* ctx) : ctx_{ctx} {}
explicit Predictor(Context const* ctx) : ctx_{ctx} {}
virtual ~Predictor() = default;

4
src/common/algorithm.h
View File

@ -1,7 +1,8 @@
/*!
* Copyright 2022 by XGBoost Contributors
*/
#pragma once
#ifndef XGBOOST_COMMON_ALGORITHM_H_
#define XGBOOST_COMMON_ALGORITHM_H_
#include <algorithm> // std::upper_bound
#include <cinttypes> // std::size_t
@ -14,3 +15,4 @@ auto SegmentId(It first, It last, Idx idx) {
}
} // namespace common
} // namespace xgboost
#endif // XGBOOST_COMMON_ALGORITHM_H_

12
src/common/common.h
View File

@ -265,6 +265,7 @@ struct OptionalWeights {
explicit OptionalWeights(float w) : dft{w} {}
XGBOOST_DEVICE float operator[](size_t i) const { return weights.empty() ? dft : weights[i]; }
auto Empty() const { return weights.empty(); }
};
/**
@ -276,7 +277,7 @@ XGBOOST_DEVICE size_t LastOf(size_t group, Indexable const &indptr) {
}
/**
* @brief A CRTP (curiously recurring template pattern) helper function.
* \brief A CRTP (curiously recurring template pattern) helper function.
*
* https://www.fluentcpp.com/2017/05/19/crtp-helper/
*
@ -284,7 +285,7 @@ XGBOOST_DEVICE size_t LastOf(size_t group, Indexable const &indptr) {
* 1. Makes "crtp" explicit in the inheritance structure of a CRTP base class.
* 2. Avoids having to `static_cast` in a lot of places.
*
* @tparam T The derived class in a CRTP hierarchy.
* \tparam T The derived class in a CRTP hierarchy.
*/
template <typename T>
struct Crtp {
@ -292,6 +293,13 @@ struct Crtp {
T const &Underlying() const { return static_cast<T const &>(*this); }
};
/**
* \brief C++17 std::as_const
*/
template <typename T>
typename std::add_const<T>::type &AsConst(T &v) noexcept { // NOLINT(runtime/references)
return v;
}
} // namespace common
} // namespace xgboost
#endif // XGBOOST_COMMON_COMMON_H_

26
src/common/linalg_op.h
View File

@ -4,6 +4,7 @@
#ifndef XGBOOST_COMMON_LINALG_OP_H_
#define XGBOOST_COMMON_LINALG_OP_H_
#include <type_traits>
#include <cstdint> // std::int32_t
#include "common.h"
#include "threading_utils.h"
@ -59,6 +60,31 @@ void ElementWiseKernel(GenericParameter const* ctx, linalg::TensorView<T, D> t,
ElementWiseKernelHost(t, ctx->Threads(), fn);
}
#endif // !defined(XGBOOST_USE_CUDA)
template <typename T, std::int32_t kDim>
auto cbegin(TensorView<T, kDim> v) { // NOLINT
auto it = common::MakeIndexTransformIter([&](size_t i) -> std::remove_cv_t<T> const& {
return linalg::detail::Apply(v, linalg::UnravelIndex(i, v.Shape()));
});
return it;
}
template <typename T, std::int32_t kDim>
auto cend(TensorView<T, kDim> v) { // NOLINT
return cbegin(v) + v.Size();
}
template <typename T, std::int32_t kDim>
auto begin(TensorView<T, kDim> v) { // NOLINT
auto it = common::MakeIndexTransformIter(
[&](size_t i) -> T& { return linalg::detail::Apply(v, linalg::UnravelIndex(i, v.Shape())); });
return it;
}
template <typename T, std::int32_t kDim>
auto end(TensorView<T, kDim> v) { // NOLINT
return begin(v) + v.Size();
}
} // namespace linalg
} // namespace xgboost
#endif // XGBOOST_COMMON_LINALG_OP_H_

28
src/common/numeric.cc Normal file
View File

@ -0,0 +1,28 @@
/*!
* Copyright 2022 by XGBoost Contributors
*/
#include "numeric.h"
#include <numeric> // std::accumulate
#include <type_traits> // std::is_same
#include "threading_utils.h" // MemStackAllocator, ParallelFor, DefaultMaxThreads
#include "xgboost/generic_parameters.h" // Context
#include "xgboost/host_device_vector.h" // HostDeviceVector
namespace xgboost {
namespace common {
double Reduce(Context const* ctx, HostDeviceVector<float> const& values) {
if (ctx->IsCPU()) {
auto const& h_values = values.ConstHostVector();
MemStackAllocator<double, DefaultMaxThreads()> result_tloc(ctx->Threads(), 0);
ParallelFor(h_values.size(), ctx->Threads(),
[&](auto i) { result_tloc[omp_get_thread_num()] += h_values[i]; });
auto result = std::accumulate(result_tloc.cbegin(), result_tloc.cend(), 0.0);
static_assert(std::is_same<decltype(result), double>::value, "");
return result;
}
return cuda::Reduce(ctx, values);
}
} // namespace common
} // namespace xgboost

25
src/common/numeric.cu Normal file
View File

@ -0,0 +1,25 @@
/*!
* Copyright 2022 by XGBoost Contributors
*/
#include <thrust/execution_policy.h>
#include <thrust/functional.h> // thrust:plus
#include "device_helpers.cuh" // dh::Reduce, safe_cuda, dh::XGBCachingDeviceAllocator
#include "numeric.h"
#include "xgboost/generic_parameters.h" // Context
#include "xgboost/host_device_vector.h" // HostDeviceVector
namespace xgboost {
namespace common {
namespace cuda {
double Reduce(Context const* ctx, HostDeviceVector<float> const& values) {
values.SetDevice(ctx->gpu_id);
auto const d_values = values.ConstDeviceSpan();
dh::XGBCachingDeviceAllocator<char> alloc;
auto res = dh::Reduce(thrust::cuda::par(alloc), d_values.data(),
d_values.data() + d_values.size(), 0.0, thrust::plus<double>{});
return res;
}
} // namespace cuda
} // namespace common
} // namespace xgboost

26
src/common/numeric.h
View File

@ -8,8 +8,10 @@
#include <iterator> // std::iterator_traits
#include <vector>
#include "threading_utils.h"
#include "xgboost/generic_parameters.h"
#include "common.h" // AssertGPUSupport
#include "threading_utils.h" // MemStackAllocator, DefaultMaxThreads
#include "xgboost/generic_parameters.h" // Context
#include "xgboost/host_device_vector.h" // HostDeviceVector
namespace xgboost {
namespace common {
@ -18,8 +20,8 @@ namespace common {
* \brief Run length encode on CPU, input must be sorted.
*/
template <typename Iter, typename Idx>
void RunLengthEncode(Iter begin, Iter end, std::vector<Idx> *p_out) {
auto &out = *p_out;
void RunLengthEncode(Iter begin, Iter end, std::vector<Idx>* p_out) {
auto& out = *p_out;
out = std::vector<Idx>{0};
size_t n = std::distance(begin, end);
for (size_t i = 1; i < n; ++i) {
@ -45,7 +47,7 @@ void PartialSum(int32_t n_threads, InIt begin, InIt end, T init, OutIt out_it) {
auto n = static_cast<size_t>(std::distance(begin, end));
const size_t batch_threads =
std::max(static_cast<size_t>(1), std::min(n, static_cast<size_t>(n_threads)));
common::MemStackAllocator<T, 128> partial_sums(batch_threads);
MemStackAllocator<T, DefaultMaxThreads()> partial_sums(batch_threads);
size_t block_size = n / batch_threads;
@ -90,6 +92,20 @@ void PartialSum(int32_t n_threads, InIt begin, InIt end, T init, OutIt out_it) {
}
exc.Rethrow();
}
namespace cuda {
double Reduce(Context const* ctx, HostDeviceVector<float> const& values);
#if !defined(XGBOOST_USE_CUDA)
inline double Reduce(Context const*, HostDeviceVector<float> const&) {
AssertGPUSupport();
return 0;
}
#endif // !defined(XGBOOST_USE_CUDA)
} // namespace cuda
/**
* \brief Reduction with summation.
*/
double Reduce(Context const* ctx, HostDeviceVector<float> const& values);
} // namespace common
} // namespace xgboost

47
src/common/stats.cu Normal file
View File

@ -0,0 +1,47 @@
/*!
* Copyright 2022 by XGBoost Contributors
*/
#include <thrust/iterator/counting_iterator.h> // thrust::make_counting_iterator
#include "common.h" // common::OptionalWeights
#include "device_helpers.cuh" // dh::MakeTransformIterator, tcbegin, tcend
#include "stats.cuh" // common::SegmentedQuantile, common::SegmentedWeightedQuantile
#include "xgboost/generic_parameters.h" // Context
#include "xgboost/host_device_vector.h" // HostDeviceVector
#include "xgboost/linalg.h" // linalg::TensorView, UnravelIndex, Apply
namespace xgboost {
namespace common {
namespace cuda {
float Median(Context const* ctx, linalg::TensorView<float const, 2> t,
common::OptionalWeights weights) {
HostDeviceVector<size_t> segments{0, t.Size()};
segments.SetDevice(ctx->gpu_id);
auto d_segments = segments.ConstDeviceSpan();
auto val_it = dh::MakeTransformIterator<float>(
thrust::make_counting_iterator(0ul), [=] XGBOOST_DEVICE(size_t i) {
return linalg::detail::Apply(t, linalg::UnravelIndex(i, t.Shape()));
});
HostDeviceVector<float> quantile{0};
quantile.SetDevice(ctx->gpu_id);
if (weights.Empty()) {
common::SegmentedQuantile(ctx, 0.5, dh::tcbegin(d_segments), dh::tcend(d_segments), val_it,
val_it + t.Size(), &quantile);
} else {
CHECK_NE(t.Shape(1), 0);
auto w_it = dh::MakeTransformIterator<float>(thrust::make_counting_iterator(0ul),
[=] XGBOOST_DEVICE(size_t i) {
auto sample_idx = i / t.Shape(1);
return weights[sample_idx];
});
common::SegmentedWeightedQuantile(ctx, 0.5, dh::tcbegin(d_segments), dh::tcend(d_segments),
val_it, val_it + t.Size(), w_it, w_it + t.Size(), &quantile);
}
CHECK_EQ(quantile.Size(), 1);
return quantile.HostVector().front();
}
} // namespace cuda
} // namespace common
} // namespace xgboost

41
src/common/stats.h
View File

@ -8,7 +8,8 @@
#include <limits>
#include <vector>
#include "common.h"
#include "common.h" // AssertGPUSupport
#include "xgboost/generic_parameters.h"
#include "xgboost/linalg.h"
namespace xgboost {
@ -90,6 +91,44 @@ float WeightedQuantile(double alpha, Iter begin, Iter end, WeightIter weights) {
idx = std::min(idx, static_cast<size_t>(n - 1));
return val(idx);
}
namespace cuda {
float Median(Context const* ctx, linalg::TensorView<float const, 2> t,
common::OptionalWeights weights);
#if !defined(XGBOOST_USE_CUDA)
inline float Median(Context const*, linalg::TensorView<float const, 2>, common::OptionalWeights) {
AssertGPUSupport();
return 0;
}
#endif // !defined(XGBOOST_USE_CUDA)
} // namespace cuda
inline float Median(Context const* ctx, linalg::Tensor<float, 2> const& t,
HostDeviceVector<float> const& weights) {
if (!ctx->IsCPU()) {
weights.SetDevice(ctx->gpu_id);
auto opt_weights = OptionalWeights(weights.ConstDeviceSpan());
auto t_v = t.View(ctx->gpu_id);
return cuda::Median(ctx, t_v, opt_weights);
}
auto opt_weights = OptionalWeights(weights.ConstHostSpan());
auto t_v = t.HostView();
auto iter = common::MakeIndexTransformIter(
[&](size_t i) { return linalg::detail::Apply(t_v, linalg::UnravelIndex(i, t_v.Shape())); });
float q{0};
if (opt_weights.Empty()) {
q = common::Quantile(0.5, iter, iter + t_v.Size());
} else {
CHECK_NE(t_v.Shape(1), 0);
auto w_it = common::MakeIndexTransformIter([&](size_t i) {
auto sample_idx = i / t_v.Shape(1);
return opt_weights[sample_idx];
});
q = common::WeightedQuantile(0.5, iter, iter + t_v.Size(), w_it);
}
return q;
}
} // namespace common
} // namespace xgboost
#endif // XGBOOST_COMMON_STATS_H_

15
src/common/threading_utils.h
View File

@ -8,6 +8,7 @@
#include <dmlc/omp.h>
#include <algorithm>
#include <cstdint> // std::int32_t
#include <limits>
#include <type_traits> // std::is_signed
#include <vector>
@ -253,7 +254,7 @@ inline int32_t OmpGetNumThreads(int32_t n_threads) {
* MaxStackSize, it will be allocated inside the stack. Otherwise, it will be
* heap-allocated.
*/
template <typename T, size_t MaxStackSize>
template <typename T, std::size_t MaxStackSize>
class MemStackAllocator {
public:
explicit MemStackAllocator(size_t required_size) : required_size_(required_size) {
@ -278,11 +279,23 @@ class MemStackAllocator {
T& operator[](size_t i) { return ptr_[i]; }
T const& operator[](size_t i) const { return ptr_[i]; }
auto data() const { return ptr_; } // NOLINT
auto data() { return ptr_; } // NOLINT
std::size_t size() const { return required_size_; } // NOLINT
auto cbegin() const { return data(); } // NOLINT
auto cend() const { return data() + size(); } // NOLINT
private:
T* ptr_ = nullptr;
size_t required_size_;
T stack_mem_[MaxStackSize];
};
/**
* \brief Constant that can be used for initializing static thread local memory.
*/
std::int32_t constexpr DefaultMaxThreads() { return 128; }
} // namespace common
} // namespace xgboost

4
src/data/array_interface.h
View File

@ -345,8 +345,8 @@ struct ToDType<int64_t> {
};
#if !defined(XGBOOST_USE_CUDA)
inline void ArrayInterfaceHandler::SyncCudaStream(int64_t stream) { common::AssertGPUSupport(); }
inline bool ArrayInterfaceHandler::IsCudaPtr(void const *ptr) { return false; }
inline void ArrayInterfaceHandler::SyncCudaStream(int64_t) { common::AssertGPUSupport(); }
inline bool ArrayInterfaceHandler::IsCudaPtr(void const *) { return false; }
#endif // !defined(XGBOOST_USE_CUDA)
/**

17
src/gbm/gblinear.cc
View File

@ -161,9 +161,10 @@ class GBLinear : public GradientBooster {
uint32_t layer_begin, uint32_t) override {
LinearCheckLayer(layer_begin);
const int ngroup = model_.learner_model_param->num_output_group;
auto base_score = learner_model_param_->BaseScore(ctx_);
for (int gid = 0; gid < ngroup; ++gid) {
this->Pred(inst, dmlc::BeginPtr(*out_preds), gid,
learner_model_param_->base_score);
this->Pred(inst, dmlc::BeginPtr(*out_preds), gid, base_score(0));
}
}
@ -184,6 +185,7 @@ class GBLinear : public GradientBooster {
contribs.resize(p_fmat->Info().num_row_ * ncolumns * ngroup);
// make sure contributions is zeroed, we could be reusing a previously allocated one
std::fill(contribs.begin(), contribs.end(), 0);
auto base_score = learner_model_param_->BaseScore(ctx_);
// start collecting the contributions
for (const auto &batch : p_fmat->GetBatches<SparsePage>()) {
// parallel over local batch
@ -202,8 +204,8 @@ class GBLinear : public GradientBooster {
}
// add base margin to BIAS
p_contribs[ncolumns - 1] =
model_.Bias()[gid] + ((base_margin.Size() != 0) ? base_margin(row_idx, gid)
: learner_model_param_->base_score);
model_.Bias()[gid] +
((base_margin.Size() != 0) ? base_margin(row_idx, gid) : base_score(0));
}
});
}
@ -268,10 +270,12 @@ class GBLinear : public GradientBooster {
monitor_.Start("PredictBatchInternal");
model_.LazyInitModel();
std::vector<bst_float> &preds = *out_preds;
auto base_margin = p_fmat->Info().base_margin_.View(GenericParameter::kCpuId);
auto base_margin = p_fmat->Info().base_margin_.View(Context::kCpuId);
// start collecting the prediction
const int ngroup = model_.learner_model_param->num_output_group;
preds.resize(p_fmat->Info().num_row_ * ngroup);
auto base_score = learner_model_param_->BaseScore(Context::kCpuId);
for (const auto &page : p_fmat->GetBatches<SparsePage>()) {
auto const& batch = page.GetView();
// output convention: nrow * k, where nrow is number of rows
@ -285,8 +289,7 @@ class GBLinear : public GradientBooster {
const size_t ridx = page.base_rowid + i;
// loop over output groups
for (int gid = 0; gid < ngroup; ++gid) {
float margin =
(base_margin.Size() != 0) ? base_margin(ridx, gid) : learner_model_param_->base_score;
float margin = (base_margin.Size() != 0) ? base_margin(ridx, gid) : base_score(0);
this->Pred(batch[i], &preds[ridx * ngroup], gid, margin);
}
});

19
src/gbm/gbtree.cc
View File

@ -638,13 +638,12 @@ void GPUDartPredictInc(common::Span<float> out_predts,
}
#endif
void GPUDartInplacePredictInc(common::Span<float> out_predts,
common::Span<float> predts, float tree_w,
size_t n_rows, float base_score,
bst_group_t n_groups,
bst_group_t group)
void GPUDartInplacePredictInc(common::Span<float> /*out_predts*/, common::Span<float> /*predts*/,
float /*tree_w*/, size_t /*n_rows*/,
linalg::TensorView<float const, 1> /*base_score*/,
bst_group_t /*n_groups*/, bst_group_t /*group*/)
#if defined(XGBOOST_USE_CUDA)
; // NOLINT
; // NOLINT
#else
{
common::AssertGPUSupport();
@ -850,15 +849,17 @@ class Dart : public GBTree {
size_t n_rows = p_fmat->Info().num_row_;
if (predts.predictions.DeviceIdx() != Context::kCpuId) {
p_out_preds->predictions.SetDevice(predts.predictions.DeviceIdx());
auto base_score = model_.learner_model_param->BaseScore(predts.predictions.DeviceIdx());
GPUDartInplacePredictInc(p_out_preds->predictions.DeviceSpan(),
predts.predictions.DeviceSpan(), w, n_rows,
model_.learner_model_param->base_score, n_groups, group);
predts.predictions.DeviceSpan(), w, n_rows, base_score, n_groups,
group);
} else {
auto base_score = model_.learner_model_param->BaseScore(Context::kCpuId);
auto& h_predts = predts.predictions.HostVector();
auto& h_out_predts = p_out_preds->predictions.HostVector();
common::ParallelFor(n_rows, ctx_->Threads(), [&](auto ridx) {
const size_t offset = ridx * n_groups + group;
h_out_predts[offset] += (h_predts[offset] - model_.learner_model_param->base_score) * w;
h_out_predts[offset] += (h_predts[offset] - base_score(0)) * w;
});
}
}

11
src/gbm/gbtree.cu
View File

@ -31,13 +31,14 @@ void GPUDartPredictInc(common::Span<float> out_predts,
});
}
void GPUDartInplacePredictInc(common::Span<float> out_predts,
common::Span<float> predts, float tree_w,
size_t n_rows, float base_score,
bst_group_t n_groups, bst_group_t group) {
void GPUDartInplacePredictInc(common::Span<float> out_predts, common::Span<float> predts,
float tree_w, size_t n_rows,
linalg::TensorView<float const, 1> base_score, bst_group_t n_groups,
bst_group_t group) {
CHECK_EQ(base_score.Size(), 1);
dh::LaunchN(n_rows, [=] XGBOOST_DEVICE(size_t ridx) {
const size_t offset = ridx * n_groups + group;
out_predts[offset] += (predts[offset] - base_score) * tree_w;
out_predts[offset] += (predts[offset] - base_score(0)) * tree_w;
});
}
} // namespace gbm

344
src/learner.cc
View File

@ -4,47 +4,48 @@
* \brief Implementation of learning algorithm.
* \author Tianqi Chen
*/
#include "xgboost/learner.h"
#include <dmlc/any.h>
#include <dmlc/io.h>
#include <dmlc/parameter.h>
#include <dmlc/thread_local.h>
#include <atomic>
#include <mutex>
#include <algorithm>
#include <atomic>
#include <iomanip>
#include <limits>
#include <limits> // std::numeric_limits
#include <memory>
#include <mutex>
#include <sstream>
#include <string>
#include <stack>
#include <string>
#include <utility>
#include <vector>
#include "dmlc/any.h"
#include "common/charconv.h"
#include "common/common.h"
#include "common/io.h"
#include "common/linalg_op.h"
#include "common/observer.h"
#include "common/random.h"
#include "common/threading_utils.h"
#include "common/timer.h"
#include "common/version.h"
#include "xgboost/base.h"
#include "xgboost/c_api.h"
#include "xgboost/data.h"
#include "xgboost/model.h"
#include "xgboost/predictor.h"
#include "xgboost/feature_map.h"
#include "xgboost/gbm.h"
#include "xgboost/generic_parameters.h"
#include "xgboost/host_device_vector.h"
#include "xgboost/json.h"
#include "xgboost/learner.h"
#include "xgboost/logging.h"
#include "xgboost/metric.h"
#include "xgboost/model.h"
#include "xgboost/objective.h"
#include "xgboost/parameter.h"
#include "common/common.h"
#include "common/io.h"
#include "common/observer.h"
#include "common/random.h"
#include "common/timer.h"
#include "common/charconv.h"
#include "common/version.h"
#include "common/threading_utils.h"
#include "xgboost/predictor.h"
namespace {
@ -85,26 +86,29 @@ struct LearnerModelParamLegacy : public dmlc::Parameter<LearnerModelParamLegacy>
uint32_t minor_version;
uint32_t num_target{1};
int32_t base_score_estimated{0};
/*! \brief reserved field */
int reserved[26];
int reserved[25];
/*! \brief constructor */
LearnerModelParamLegacy() {
std::memset(this, 0, sizeof(LearnerModelParamLegacy));
base_score = 0.5f;
base_score = ObjFunction::DefaultBaseScore();
num_target = 1;
major_version = std::get<0>(Version::Self());
minor_version = std::get<1>(Version::Self());
base_score_estimated = 0;
static_assert(sizeof(LearnerModelParamLegacy) == 136,
"Do not change the size of this struct, as it will break binary IO.");
}
// Skip other legacy fields.
Json ToJson() const {
Object obj;
char floats[NumericLimits<float>::kToCharsSize];
auto ret = to_chars(floats, floats + NumericLimits<float>::kToCharsSize, base_score);
CHECK(ret.ec == std::errc());
obj["base_score"] =
std::string{floats, static_cast<size_t>(std::distance(floats, ret.ptr))};
CHECK(ret.ec == std::errc{});
obj["base_score"] = std::string{floats, static_cast<size_t>(std::distance(floats, ret.ptr))};
char integers[NumericLimits<int64_t>::kToCharsSize];
ret = to_chars(integers, integers + NumericLimits<int64_t>::kToCharsSize,
@ -136,10 +140,14 @@ struct LearnerModelParamLegacy : public dmlc::Parameter<LearnerModelParamLegacy>
}
this->Init(m);
std::string str = get<String const>(j_param.at("base_score"));
from_chars(str.c_str(), str.c_str() + str.size(), base_score);
// It can only be estimated during the first training, we consider it estimated afterward
base_score_estimated = 1;
}
inline LearnerModelParamLegacy ByteSwap() const {
LearnerModelParamLegacy ByteSwap() const {
LearnerModelParamLegacy x = *this;
dmlc::ByteSwap(&x.base_score, sizeof(x.base_score), 1);
dmlc::ByteSwap(&x.num_feature, sizeof(x.num_feature), 1);
@ -149,14 +157,30 @@ struct LearnerModelParamLegacy : public dmlc::Parameter<LearnerModelParamLegacy>
dmlc::ByteSwap(&x.major_version, sizeof(x.major_version), 1);
dmlc::ByteSwap(&x.minor_version, sizeof(x.minor_version), 1);
dmlc::ByteSwap(&x.num_target, sizeof(x.num_target), 1);
dmlc::ByteSwap(&x.base_score_estimated, sizeof(x.base_score_estimated), 1);
dmlc::ByteSwap(x.reserved, sizeof(x.reserved[0]), sizeof(x.reserved) / sizeof(x.reserved[0]));
return x;
}
template <typename Container>
Args UpdateAllowUnknown(Container const& kwargs) {
// Detect whether user has made their own base score.
if (std::find_if(kwargs.cbegin(), kwargs.cend(),
[](auto const& kv) { return kv.first == "base_score"; }) != kwargs.cend()) {
base_score_estimated = true;
}
if (std::find_if(kwargs.cbegin(), kwargs.cend(), [](auto const& kv) {
return kv.first == "base_score_estimated";
}) != kwargs.cend()) {
LOG(FATAL) << "`base_score_estimated` cannot be specified as hyper-parameter.";
}
return dmlc::Parameter<LearnerModelParamLegacy>::UpdateAllowUnknown(kwargs);
}
// declare parameters
DMLC_DECLARE_PARAMETER(LearnerModelParamLegacy) {
DMLC_DECLARE_FIELD(base_score)
.set_default(0.5f)
.set_default(ObjFunction::DefaultBaseScore())
.describe("Global bias of the model.");
DMLC_DECLARE_FIELD(num_feature)
.set_default(0)
@ -170,12 +194,12 @@ struct LearnerModelParamLegacy : public dmlc::Parameter<LearnerModelParamLegacy>
.set_default(1)
.set_lower_bound(1)
.describe("Number of target for multi-target regression.");
DMLC_DECLARE_FIELD(base_score_estimated).set_default(0);
}
};
LearnerModelParam::LearnerModelParam(LearnerModelParamLegacy const& user_param, float base_margin,
ObjInfo t)
: base_score{base_margin}, num_feature{user_param.num_feature}, task{t} {
LearnerModelParam::LearnerModelParam(LearnerModelParamLegacy const& user_param, ObjInfo t)
: num_feature{user_param.num_feature}, task{t} {
auto n_classes = std::max(static_cast<uint32_t>(user_param.num_class), 1u);
auto n_targets = user_param.num_target;
num_output_group = std::max(n_classes, n_targets);
@ -185,6 +209,53 @@ LearnerModelParam::LearnerModelParam(LearnerModelParamLegacy const& user_param,
<< ", n_targets:" << n_targets;
}
LearnerModelParam::LearnerModelParam(Context const* ctx, LearnerModelParamLegacy const& user_param,
linalg::Tensor<float, 1> base_margin, ObjInfo t)
: LearnerModelParam{user_param, t} {
std::swap(base_score_, base_margin);
// Make sure read access everywhere for thread-safe prediction.
common::AsConst(base_score_).HostView();
if (!ctx->IsCPU()) {
common::AsConst(base_score_).View(ctx->gpu_id);
}
CHECK(common::AsConst(base_score_).Data()->HostCanRead());
}
linalg::TensorView<float const, 1> LearnerModelParam::BaseScore(int32_t device) const {
// multi-class is not yet supported.
CHECK_EQ(base_score_.Size(), 1);
if (device == Context::kCpuId) {
// Make sure that we won't run into race condition.
CHECK(base_score_.Data()->HostCanRead());
return base_score_.HostView();
}
// Make sure that we won't run into race condition.
CHECK(base_score_.Data()->DeviceCanRead());
auto v = base_score_.View(device);
CHECK(base_score_.Data()->HostCanRead()); // make sure read access is not removed.
return v;
}
linalg::TensorView<float const, 1> LearnerModelParam::BaseScore(Context const* ctx) const {
return this->BaseScore(ctx->gpu_id);
}
void LearnerModelParam::Copy(LearnerModelParam const& that) {
base_score_.Reshape(that.base_score_.Shape());
base_score_.Data()->SetDevice(that.base_score_.DeviceIdx());
base_score_.Data()->Copy(*that.base_score_.Data());
common::AsConst(base_score_).HostView();
if (that.base_score_.DeviceIdx() != Context::kCpuId) {
common::AsConst(base_score_).View(that.base_score_.DeviceIdx());
}
CHECK_EQ(base_score_.Data()->DeviceCanRead(), that.base_score_.Data()->DeviceCanRead());
CHECK(base_score_.Data()->HostCanRead());
num_feature = that.num_feature;
num_output_group = that.num_output_group;
task = that.task;
}
struct LearnerTrainParam : public XGBoostParameter<LearnerTrainParam> {
// data split mode, can be row, col, or none.
DataSplitMode dsplit {DataSplitMode::kAuto};
@ -308,8 +379,61 @@ class LearnerConfiguration : public Learner {
LearnerModelParamLegacy mparam_;
LearnerModelParam learner_model_param_;
LearnerTrainParam tparam_;
// Initial prediction.
std::vector<std::string> metric_names_;
/**
* \brief Calculate the `base_score` based on input data.
*
* \param p_fmat The training DMatrix used to estimate the base score.
*/
void InitBaseScore(DMatrix const* p_fmat) {
// Before 1.0.0, we save `base_score` into binary as a transformed value by objective.
// After 1.0.0 we save the value provided by user and keep it immutable instead. To
// keep the stability, we initialize it in binary LoadModel instead of configuration.
// Under what condition should we omit the transformation:
//
// - base_score is loaded from old binary model.
//
// What are the other possible conditions:
//
// - model loaded from new binary or JSON.
// - model is created from scratch.
// - model is configured second time due to change of parameter
CHECK(obj_);
if (!mparam_.base_score_estimated) {
if (p_fmat) {
// We estimate it from input data.
linalg::Tensor<float, 1> base_score;
obj_->InitEstimation(p_fmat->Info(), &base_score);
mparam_.base_score = base_score(0);
CHECK(!std::isnan(mparam_.base_score));
} else {
mparam_.base_score = ObjFunction::DefaultBaseScore();
}
mparam_.base_score_estimated = true;
// Update the shared model parameter
this->ConfigureModelParam();
}
}
// Convert mparam to learner_model_param
void ConfigureModelParam() {
this->ConfigureTargets();
CHECK(obj_);
auto task = obj_->Task();
linalg::Tensor<float, 1> base_score({1}, Ctx()->gpu_id);
auto h_base_score = base_score.HostView();
// transform to margin
h_base_score(0) = obj_->ProbToMargin(mparam_.base_score);
// move it to model param, which is shared with all other components.
learner_model_param_ = LearnerModelParam(Ctx(), mparam_, std::move(base_score), task);
CHECK(learner_model_param_.Initialized());
CHECK_NE(learner_model_param_.BaseScore(Ctx()).Size(), 0);
}
public:
explicit LearnerConfiguration(std::vector<std::shared_ptr<DMatrix> > cache)
: need_configuration_{true} {
@ -329,22 +453,24 @@ class LearnerConfiguration : public Learner {
// Configuration before data is known.
void Configure() override {
// Varient of double checked lock
if (!this->need_configuration_) { return; }
if (!this->need_configuration_) {
return;
}
std::lock_guard<std::mutex> guard(config_lock_);
if (!this->need_configuration_) { return; }
if (!this->need_configuration_) {
return;
}
monitor_.Start("Configure");
auto old_tparam = tparam_;
Args args = {cfg_.cbegin(), cfg_.cend()};
tparam_.UpdateAllowUnknown(args);
auto mparam_backup = mparam_;
mparam_.UpdateAllowUnknown(args);
auto initialized = generic_parameters_.GetInitialised();
auto old_seed = generic_parameters_.seed;
generic_parameters_.UpdateAllowUnknown(args);
auto initialized = ctx_.GetInitialised();
auto old_seed = ctx_.seed;
ctx_.UpdateAllowUnknown(args);
ConsoleLogger::Configure(args);
@ -355,8 +481,8 @@ class LearnerConfiguration : public Learner {
}
// set seed only before the model is initialized
if (!initialized || generic_parameters_.seed != old_seed) {
common::GlobalRandom().seed(generic_parameters_.seed);
if (!initialized || ctx_.seed != old_seed) {
common::GlobalRandom().seed(ctx_.seed);
}
// must precede configure gbm since num_features is required for gbm
@ -364,31 +490,15 @@ class LearnerConfiguration : public Learner {
args = {cfg_.cbegin(), cfg_.cend()}; // renew
this->ConfigureObjective(old_tparam, &args);
auto task = this->ConfigureTargets();
// Before 1.0.0, we save `base_score` into binary as a transformed value by objective.
// After 1.0.0 we save the value provided by user and keep it immutable instead. To
// keep the stability, we initialize it in binary LoadModel instead of configuration.
// Under what condition should we omit the transformation:
//
// - base_score is loaded from old binary model.
//
// What are the other possible conditions:
//
// - model loaded from new binary or JSON.
// - model is created from scratch.
// - model is configured second time due to change of parameter
if (!learner_model_param_.Initialized() || mparam_.base_score != mparam_backup.base_score) {
learner_model_param_ =
LearnerModelParam(mparam_, obj_->ProbToMargin(mparam_.base_score), task);
}
learner_model_param_.task = obj_->Task(); // required by gbm configuration.
this->ConfigureGBM(old_tparam, args);
generic_parameters_.ConfigureGpuId(this->gbm_->UseGPU());
ctx_.ConfigureGpuId(this->gbm_->UseGPU());
this->ConfigureModelParam();
this->ConfigureMetrics(args);
this->need_configuration_ = false;
if (generic_parameters_.validate_parameters) {
if (ctx_.validate_parameters) {
this->ValidateParameters();
}
@ -396,6 +506,11 @@ class LearnerConfiguration : public Learner {
monitor_.Stop("Configure");
}
void CheckModelInitialized() const {
CHECK(learner_model_param_.Initialized()) << "Model not yet initialized.";
CHECK_NE(learner_model_param_.BaseScore(this->Ctx()).Size(), 0);
}
virtual PredictionContainer* GetPredictionCache() const {
return &((*ThreadLocalPredictionCache::Get())[this]);
}
@ -417,7 +532,7 @@ class LearnerConfiguration : public Learner {
auto const& objective_fn = learner_parameters.at("objective");
if (!obj_) {
obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_));
obj_.reset(ObjFunction::Create(tparam_.objective, &ctx_));
}
obj_->LoadConfig(objective_fn);
learner_model_param_.task = obj_->Task();
@ -425,7 +540,7 @@ class LearnerConfiguration : public Learner {
tparam_.booster = get<String>(gradient_booster["name"]);
if (!gbm_) {
gbm_.reset(GradientBooster::Create(tparam_.booster,
&generic_parameters_, &learner_model_param_));
&ctx_, &learner_model_param_));
}
gbm_->LoadConfig(gradient_booster);
@ -441,15 +556,15 @@ class LearnerConfiguration : public Learner {
} else {
metric_names_[i] = get<String>(j_metrics[i]["name"]);
}
metrics_[i] = std::unique_ptr<Metric>(Metric::Create(metric_names_[i], &generic_parameters_));
metrics_[i] = std::unique_ptr<Metric>(Metric::Create(metric_names_[i], &ctx_));
if (!old_serialization) {
metrics_[i]->LoadConfig(j_metrics[i]);
}
}
FromJson(learner_parameters.at("generic_param"), &generic_parameters_);
FromJson(learner_parameters.at("generic_param"), &ctx_);
// make sure the GPU ID is valid in new environment before start running configure.
generic_parameters_.ConfigureGpuId(false);
ctx_.ConfigureGpuId(false);
this->need_configuration_ = true;
}
@ -478,7 +593,7 @@ class LearnerConfiguration : public Learner {
}
learner_parameters["metrics"] = Array(std::move(metrics));
learner_parameters["generic_param"] = ToJson(generic_parameters_);
learner_parameters["generic_param"] = ToJson(ctx_);
}
void SetParam(const std::string& key, const std::string& value) override {
@ -551,7 +666,7 @@ class LearnerConfiguration : public Learner {
return cfg_;
}
GenericParameter const* Ctx() const override { return &generic_parameters_; }
Context const* Ctx() const override { return &ctx_; }
private:
void ValidateParameters() {
@ -654,7 +769,7 @@ class LearnerConfiguration : public Learner {
void ConfigureGBM(LearnerTrainParam const& old, Args const& args) {
if (gbm_ == nullptr || old.booster != tparam_.booster) {
gbm_.reset(GradientBooster::Create(tparam_.booster, &generic_parameters_,
gbm_.reset(GradientBooster::Create(tparam_.booster, &ctx_,
&learner_model_param_));
}
gbm_->Configure(args);
@ -678,7 +793,7 @@ class LearnerConfiguration : public Learner {
cfg_["max_delta_step"] = kMaxDeltaStepDefaultValue;
}
if (obj_ == nullptr || tparam_.objective != old.objective) {
obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_));
obj_.reset(ObjFunction::Create(tparam_.objective, &ctx_));
}
auto& args = *p_args;
args = {cfg_.cbegin(), cfg_.cend()}; // renew
@ -691,7 +806,7 @@ class LearnerConfiguration : public Learner {
return m->Name() != name;
};
if (std::all_of(metrics_.begin(), metrics_.end(), DupCheck)) {
metrics_.emplace_back(std::unique_ptr<Metric>(Metric::Create(name, &generic_parameters_)));
metrics_.emplace_back(std::unique_ptr<Metric>(Metric::Create(name, &ctx_)));
mparam_.contain_eval_metrics = 1;
}
}
@ -703,7 +818,7 @@ class LearnerConfiguration : public Learner {
/**
* Get number of targets from objective function.
*/
ObjInfo ConfigureTargets() {
void ConfigureTargets() {
CHECK(this->obj_);
auto const& cache = this->GetPredictionCache()->Container();
size_t n_targets = 1;
@ -722,7 +837,6 @@ class LearnerConfiguration : public Learner {
} else {
mparam_.num_target = n_targets;
}
return this->obj_->Task();
}
};
@ -754,14 +868,14 @@ class LearnerIO : public LearnerConfiguration {
std::string name = get<String>(objective_fn["name"]);
tparam_.UpdateAllowUnknown(Args{{"objective", name}});
obj_.reset(ObjFunction::Create(name, &generic_parameters_));
obj_.reset(ObjFunction::Create(name, &ctx_));
obj_->LoadConfig(objective_fn);
auto const& gradient_booster = learner.at("gradient_booster");
name = get<String>(gradient_booster["name"]);
tparam_.UpdateAllowUnknown(Args{{"booster", name}});
gbm_.reset(
GradientBooster::Create(tparam_.booster, &generic_parameters_, &learner_model_param_));
GradientBooster::Create(tparam_.booster, &ctx_, &learner_model_param_));
gbm_->LoadModel(gradient_booster);
auto const& j_attributes = get<Object const>(learner.at("attributes"));
@ -791,6 +905,7 @@ class LearnerIO : public LearnerConfiguration {
void SaveModel(Json* p_out) const override {
CHECK(!this->need_configuration_) << "Call Configure before saving model.";
this->CheckModelInitialized();
Version::Save(p_out);
Json& out { *p_out };
@ -826,7 +941,7 @@ class LearnerIO : public LearnerConfiguration {
// About to be deprecated by JSON format
void LoadModel(dmlc::Stream* fi) override {
generic_parameters_.UpdateAllowUnknown(Args{});
ctx_.UpdateAllowUnknown(Args{});
tparam_.Init(std::vector<std::pair<std::string, std::string>>{});
// TODO(tqchen) mark deprecation of old format.
common::PeekableInStream fp(fi);
@ -881,8 +996,8 @@ class LearnerIO : public LearnerConfiguration {
CHECK(fi->Read(&tparam_.objective)) << "BoostLearner: wrong model format";
CHECK(fi->Read(&tparam_.booster)) << "BoostLearner: wrong model format";
obj_.reset(ObjFunction::Create(tparam_.objective, &generic_parameters_));
gbm_.reset(GradientBooster::Create(tparam_.booster, &generic_parameters_,
obj_.reset(ObjFunction::Create(tparam_.objective, &ctx_));
gbm_.reset(GradientBooster::Create(tparam_.booster, &ctx_,
&learner_model_param_));
gbm_->Load(fi);
if (mparam_.contain_extra_attrs != 0) {
@ -925,7 +1040,14 @@ class LearnerIO : public LearnerConfiguration {
}
learner_model_param_ =
LearnerModelParam(mparam_, obj_->ProbToMargin(mparam_.base_score), obj_->Task());
LearnerModelParam(&ctx_, mparam_,
linalg::Tensor<float, 1>{{std::isnan(mparam_.base_score)
? std::numeric_limits<float>::quiet_NaN()
: obj_->ProbToMargin(mparam_.base_score)},
{1},
Context::kCpuId},
obj_->Task());
if (attributes_.find("objective") != attributes_.cend()) {
auto obj_str = attributes_.at("objective");
auto j_obj = Json::Load({obj_str.c_str(), obj_str.size()});
@ -969,6 +1091,8 @@ class LearnerIO : public LearnerConfiguration {
// Save model into binary format. The code is about to be deprecated by more robust
// JSON serialization format.
void SaveModel(dmlc::Stream* fo) const override {
this->CheckModelInitialized();
LearnerModelParamLegacy mparam = mparam_; // make a copy to potentially modify
std::vector<std::pair<std::string, std::string> > extra_attr;
mparam.contain_extra_attrs = 1;
@ -1000,6 +1124,7 @@ class LearnerIO : public LearnerConfiguration {
}
extra_attr.emplace_back("metrics", os.str());
}
std::string header {"binf"};
fo->Write(header.data(), 4);
if (DMLC_IO_NO_ENDIAN_SWAP) {
@ -1022,6 +1147,8 @@ class LearnerIO : public LearnerConfiguration {
}
void Save(dmlc::Stream* fo) const override {
this->CheckModelInitialized();
Json memory_snapshot{Object()};
memory_snapshot["Model"] = Object();
auto& model = memory_snapshot["Model"];
@ -1108,28 +1235,30 @@ class LearnerImpl : public LearnerIO {
}
}
std::vector<std::string> DumpModel(const FeatureMap& fmap,
bool with_stats,
std::vector<std::string> DumpModel(const FeatureMap& fmap, bool with_stats,
std::string format) override {
this->Configure();
this->CheckModelInitialized();
return gbm_->DumpModel(fmap, with_stats, format);
}
Learner *Slice(int32_t begin_layer, int32_t end_layer, int32_t step,
bool *out_of_bound) override {
Learner* Slice(int32_t begin_layer, int32_t end_layer, int32_t step,
bool* out_of_bound) override {
this->Configure();
this->CheckModelInitialized();
CHECK_NE(this->learner_model_param_.num_feature, 0);
CHECK_GE(begin_layer, 0);
auto *out_impl = new LearnerImpl({});
out_impl->learner_model_param_ = this->learner_model_param_;
out_impl->generic_parameters_ = this->generic_parameters_;
auto* out_impl = new LearnerImpl({});
out_impl->learner_model_param_.Copy(this->learner_model_param_);
out_impl->ctx_ = this->ctx_;
auto gbm = std::unique_ptr<GradientBooster>(GradientBooster::Create(
this->tparam_.booster, &out_impl->generic_parameters_,
&out_impl->learner_model_param_));
this->tparam_.booster, &out_impl->ctx_, &out_impl->learner_model_param_));
this->gbm_->Slice(begin_layer, end_layer, step, gbm.get(), out_of_bound);
out_impl->gbm_ = std::move(gbm);
Json config { Object() };
Json config{Object()};
this->SaveConfig(&config);
out_impl->mparam_ = this->mparam_;
out_impl->attributes_ = this->attributes_;
@ -1156,15 +1285,17 @@ class LearnerImpl : public LearnerIO {
monitor_.Start("UpdateOneIter");
TrainingObserver::Instance().Update(iter);
this->Configure();
if (generic_parameters_.seed_per_iteration) {
common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter);
this->InitBaseScore(train.get());
if (ctx_.seed_per_iteration) {
common::GlobalRandom().seed(ctx_.seed * kRandSeedMagic + iter);
}
this->CheckDataSplitMode();
this->ValidateDMatrix(train.get(), true);
auto local_cache = this->GetPredictionCache();
auto& predt = local_cache->Cache(train, generic_parameters_.gpu_id);
auto& predt = local_cache->Cache(train, ctx_.gpu_id);
monitor_.Start("PredictRaw");
this->PredictRaw(train.get(), &predt, true, 0, 0);
@ -1184,14 +1315,18 @@ class LearnerImpl : public LearnerIO {
HostDeviceVector<GradientPair>* in_gpair) override {
monitor_.Start("BoostOneIter");
this->Configure();
if (generic_parameters_.seed_per_iteration) {
common::GlobalRandom().seed(generic_parameters_.seed * kRandSeedMagic + iter);
// Should have been set to default in the first prediction.
CHECK(mparam_.base_score_estimated);
if (ctx_.seed_per_iteration) {
common::GlobalRandom().seed(ctx_.seed * kRandSeedMagic + iter);
}
this->CheckDataSplitMode();
this->ValidateDMatrix(train.get(), true);
auto local_cache = this->GetPredictionCache();
local_cache->Cache(train, generic_parameters_.gpu_id);
local_cache->Cache(train, ctx_.gpu_id);
gbm_->DoBoost(train.get(), in_gpair, &local_cache->Entry(train.get()), obj_.get());
monitor_.Stop("BoostOneIter");
@ -1202,23 +1337,24 @@ class LearnerImpl : public LearnerIO {
const std::vector<std::string>& data_names) override {
monitor_.Start("EvalOneIter");
this->Configure();
this->CheckModelInitialized();
std::ostringstream os;
os.precision(std::numeric_limits<double>::max_digits10);
os << '[' << iter << ']' << std::setiosflags(std::ios::fixed);
if (metrics_.size() == 0 && tparam_.disable_default_eval_metric <= 0) {
metrics_.emplace_back(Metric::Create(obj_->DefaultEvalMetric(), &generic_parameters_));
metrics_.emplace_back(Metric::Create(obj_->DefaultEvalMetric(), &ctx_));
metrics_.back()->Configure({cfg_.begin(), cfg_.end()});
}
auto local_cache = this->GetPredictionCache();
for (size_t i = 0; i < data_sets.size(); ++i) {
std::shared_ptr<DMatrix> m = data_sets[i];
auto &predt = local_cache->Cache(m, generic_parameters_.gpu_id);
auto &predt = local_cache->Cache(m, ctx_.gpu_id);
this->ValidateDMatrix(m.get(), false);
this->PredictRaw(m.get(), &predt, false, 0, 0);
auto &out = output_predictions_.Cache(m, generic_parameters_.gpu_id).predictions;
auto &out = output_predictions_.Cache(m, ctx_.gpu_id).predictions;
out.Resize(predt.predictions.Size());
out.Copy(predt.predictions);
@ -1241,6 +1377,9 @@ class LearnerImpl : public LearnerIO {
static_cast<int>(pred_interactions) +
static_cast<int>(pred_contribs);
this->Configure();
this->InitBaseScore(nullptr);
this->CheckModelInitialized();
CHECK_LE(multiple_predictions, 1) << "Perform one kind of prediction at a time.";
if (pred_contribs) {
gbm_->PredictContribution(data.get(), out_preds, layer_begin, layer_end, approx_contribs);
@ -1251,10 +1390,10 @@ class LearnerImpl : public LearnerIO {
gbm_->PredictLeaf(data.get(), out_preds, layer_begin, layer_end);
} else {
auto local_cache = this->GetPredictionCache();
auto& prediction = local_cache->Cache(data, generic_parameters_.gpu_id);
auto& prediction = local_cache->Cache(data, ctx_.gpu_id);
this->PredictRaw(data.get(), &prediction, training, layer_begin, layer_end);
// Copy the prediction cache to output prediction. out_preds comes from C API
out_preds->SetDevice(generic_parameters_.gpu_id);
out_preds->SetDevice(ctx_.gpu_id);
out_preds->Resize(prediction.predictions.Size());
out_preds->Copy(prediction.predictions);
if (!output_margin) {
@ -1268,8 +1407,10 @@ class LearnerImpl : public LearnerIO {
CHECK(!this->need_configuration_);
return this->gbm_->BoostedRounds();
}
uint32_t Groups() const override {
CHECK(!this->need_configuration_);
this->CheckModelInitialized();
return this->learner_model_param_.num_output_group;
}
@ -1281,6 +1422,9 @@ class LearnerImpl : public LearnerIO {
HostDeviceVector<bst_float>** out_preds, uint32_t iteration_begin,
uint32_t iteration_end) override {
this->Configure();
this->InitBaseScore(nullptr);
this->CheckModelInitialized();
auto& out_predictions = this->GetThreadLocal().prediction_entry;
this->gbm_->InplacePredict(p_m, missing, &out_predictions, iteration_begin, iteration_end);
if (type == PredictionType::kValue) {
@ -1296,6 +1440,8 @@ class LearnerImpl : public LearnerIO {
void CalcFeatureScore(std::string const& importance_type, common::Span<int32_t const> trees,
std::vector<bst_feature_t>* features, std::vector<float>* scores) override {
this->Configure();
this->CheckModelInitialized();
gbm_->FeatureScore(importance_type, trees, features, scores);
}
@ -1315,17 +1461,17 @@ class LearnerImpl : public LearnerIO {
void PredictRaw(DMatrix *data, PredictionCacheEntry *out_preds, bool training,
unsigned layer_begin, unsigned layer_end) const {
CHECK(gbm_ != nullptr) << "Predict must happen after Load or configuration";
this->CheckModelInitialized();
this->ValidateDMatrix(data, false);
gbm_->PredictBatch(data, out_preds, training, layer_begin, layer_end);
}
void ValidateDMatrix(DMatrix* p_fmat, bool is_training) const {
MetaInfo const& info = p_fmat->Info();
info.Validate(generic_parameters_.gpu_id);
info.Validate(ctx_.gpu_id);
auto const row_based_split = [this]() {
return tparam_.dsplit == DataSplitMode::kRow ||
tparam_.dsplit == DataSplitMode::kAuto;
return tparam_.dsplit == DataSplitMode::kRow || tparam_.dsplit == DataSplitMode::kAuto;
};
if (row_based_split()) {
if (is_training) {

1
src/objective/adaptive.h
View File

@ -7,6 +7,7 @@
#include <limits>
#include <vector>
#include "../common/common.h"
#include "rabit/rabit.h"
#include "xgboost/generic_parameters.h"
#include "xgboost/host_device_vector.h"

9
src/objective/objective.cc
View File

@ -1,10 +1,10 @@
/*!
* Copyright 2015 by Contributors
* Copyright 2015-2022 by Contributors
* \file objective.cc
* \brief Registry of all objective functions.
*/
#include <xgboost/objective.h>
#include <dmlc/registry.h>
#include <xgboost/objective.h>
#include <sstream>
@ -31,6 +31,11 @@ ObjFunction* ObjFunction::Create(const std::string& name, GenericParameter const
return pobj;
}
void ObjFunction::InitEstimation(MetaInfo const&, linalg::Tensor<float, 1>* base_score) const {
CHECK(base_score);
base_score->Reshape(1);
(*base_score)(0) = DefaultBaseScore();
}
} // namespace xgboost
namespace xgboost {

37
src/objective/regression_obj.cu
View File

@ -15,7 +15,9 @@
#include "../common/common.h"
#include "../common/linalg_op.h"
#include "../common/numeric.h" // Reduce
#include "../common/pseudo_huber.h"
#include "../common/stats.h"
#include "../common/threading_utils.h"
#include "../common/transform.h"
#include "./regression_loss.h"
@ -37,14 +39,18 @@
namespace xgboost {
namespace obj {
namespace {
void CheckRegInputs(MetaInfo const& info, HostDeviceVector<bst_float> const& preds) {
void CheckInitInputs(MetaInfo const& info) {
CHECK_EQ(info.labels.Shape(0), info.num_row_) << "Invalid shape of labels.";
CHECK_EQ(info.labels.Size(), preds.Size()) << "Invalid shape of labels.";
if (!info.weights_.Empty()) {
CHECK_EQ(info.weights_.Size(), info.num_row_)
<< "Number of weights should be equal to number of data points.";
}
}
void CheckRegInputs(MetaInfo const& info, HostDeviceVector<bst_float> const& preds) {
CheckInitInputs(info);
CHECK_EQ(info.labels.Size(), preds.Size()) << "Invalid shape of labels.";
}
} // anonymous namespace
#if defined(XGBOOST_USE_CUDA)
@ -698,6 +704,33 @@ class MeanAbsoluteError : public ObjFunction {
});
}
void InitEstimation(MetaInfo const& info, linalg::Tensor<float, 1>* base_margin) const override {
CheckInitInputs(info);
base_margin->Reshape(1);
auto out = base_margin->HostView();
double w{0.0};
if (info.weights_.Empty()) {
w = static_cast<double>(info.num_row_);
} else {
w = common::Reduce(ctx_, info.weights_);
}
if (info.num_row_ == 0) {
out(0) = 0;
} else {
// weighted avg
out(0) = common::Median(ctx_, info.labels, info.weights_) * w;
}
// Weighted average base score across all workers
rabit::Allreduce<rabit::op::Sum>(out.Values().data(), out.Values().size());
rabit::Allreduce<rabit::op::Sum>(&w, 1);
std::transform(linalg::cbegin(out), linalg::cend(out), linalg::begin(out),
[w](float v) { return v / w; });
}
void UpdateTreeLeaf(HostDeviceVector<bst_node_t> const& position, MetaInfo const& info,
HostDeviceVector<float> const& prediction, RegTree* p_tree) const override {
if (ctx_->IsCPU()) {

12
src/predictor/cpu_predictor.cc
View File

@ -429,11 +429,12 @@ class CPUPredictor : public Predictor {
}
out_preds->resize(model.learner_model_param->num_output_group *
(model.param.size_leaf_vector + 1));
auto base_score = model.learner_model_param->BaseScore(ctx_)(0);
// loop over output groups
for (uint32_t gid = 0; gid < model.learner_model_param->num_output_group; ++gid) {
(*out_preds)[gid] = PredValue(inst, model.trees, model.tree_info, gid,
&feat_vecs[0], 0, ntree_limit) +
model.learner_model_param->base_score;
(*out_preds)[gid] =
PredValue(inst, model.trees, model.tree_info, gid, &feat_vecs[0], 0, ntree_limit) +
base_score;
}
}
@ -504,7 +505,8 @@ class CPUPredictor : public Predictor {
common::ParallelFor(ntree_limit, n_threads, [&](bst_omp_uint i) {
FillNodeMeanValues(model.trees[i].get(), &(mean_values[i]));
});
auto base_margin = info.base_margin_.View(GenericParameter::kCpuId);
auto base_margin = info.base_margin_.View(Context::kCpuId);
auto base_score = model.learner_model_param->BaseScore(Context::kCpuId)(0);
// start collecting the contributions
for (const auto &batch : p_fmat->GetBatches<SparsePage>()) {
auto page = batch.GetView();
@ -548,7 +550,7 @@ class CPUPredictor : public Predictor {
CHECK_EQ(base_margin.Shape(1), ngroup);
p_contribs[ncolumns - 1] += base_margin(row_idx, gid);
} else {
p_contribs[ncolumns - 1] += model.learner_model_param->base_score;
p_contribs[ncolumns - 1] += base_score;
}
}
});

22
src/predictor/gpu_predictor.cu
View File

@ -511,7 +511,7 @@ void ExtractPaths(
n = d_nodes[n.Parent() + tree_offset];
path_length++;
}
return PathInfo{int64_t(idx), path_length, tree_idx};
return PathInfo{static_cast<int64_t>(idx), path_length, tree_idx};
});
auto end = thrust::copy_if(
thrust::cuda::par(alloc), nodes_transform,
@ -859,12 +859,12 @@ class GPUPredictor : public xgboost::Predictor {
// Add the base margin term to last column
p_fmat->Info().base_margin_.SetDevice(ctx_->gpu_id);
const auto margin = p_fmat->Info().base_margin_.Data()->ConstDeviceSpan();
float base_score = model.learner_model_param->base_score;
dh::LaunchN(
p_fmat->Info().num_row_ * model.learner_model_param->num_output_group,
auto base_score = model.learner_model_param->BaseScore(ctx_);
dh::LaunchN(p_fmat->Info().num_row_ * model.learner_model_param->num_output_group,
[=] __device__(size_t idx) {
phis[(idx + 1) * contributions_columns - 1] +=
margin.empty() ? base_score : margin[idx];
margin.empty() ? base_score(0) : margin[idx];
});
}
@ -918,16 +918,16 @@ class GPUPredictor : public xgboost::Predictor {
// Add the base margin term to last column
p_fmat->Info().base_margin_.SetDevice(ctx_->gpu_id);
const auto margin = p_fmat->Info().base_margin_.Data()->ConstDeviceSpan();
float base_score = model.learner_model_param->base_score;
auto base_score = model.learner_model_param->BaseScore(ctx_);
size_t n_features = model.learner_model_param->num_feature;
dh::LaunchN(
p_fmat->Info().num_row_ * model.learner_model_param->num_output_group,
dh::LaunchN(p_fmat->Info().num_row_ * model.learner_model_param->num_output_group,
[=] __device__(size_t idx) {
size_t group = idx % ngroup;
size_t row_idx = idx / ngroup;
phis[gpu_treeshap::IndexPhiInteractions(
row_idx, ngroup, group, n_features, n_features, n_features)] +=
margin.empty() ? base_score : margin[idx];
phis[gpu_treeshap::IndexPhiInteractions(row_idx, ngroup, group, n_features,
n_features, n_features)] +=
margin.empty() ? base_score(0) : margin[idx];
});
}

7
src/predictor/predictor.cc
View File

@ -80,14 +80,15 @@ void Predictor::InitOutPredictions(const MetaInfo& info, HostDeviceVector<bst_fl
if (ctx_->gpu_id >= 0) {
out_preds->SetDevice(ctx_->gpu_id);
}
if (base_margin->Size() != 0) {
if (!base_margin->Empty()) {
out_preds->Resize(n);
ValidateBaseMarginShape(info.base_margin_, info.num_row_, n_classes);
out_preds->Copy(*base_margin);
} else {
out_preds->Resize(n);
// cannot rely on the Resize to fill as it might skip if the size is already correct.
out_preds->Fill(model.learner_model_param->base_score);
out_preds->Resize(n);
auto base_score = model.learner_model_param->BaseScore(Context::kCpuId)(0);
out_preds->Fill(base_score);
}
}
} // namespace xgboost

10
tests/cpp/common/test_numeric.cc
View File

@ -29,5 +29,15 @@ TEST(Numeric, PartialSum) {
ASSERT_EQ(sol, result);
}
}
TEST(Numeric, Reduce) {
Context ctx;
ASSERT_TRUE(ctx.IsCPU());
HostDeviceVector<float> values(20);
auto& h_values = values.HostVector();
std::iota(h_values.begin(), h_values.end(), 0.0f);
auto sum = Reduce(&ctx, values);
ASSERT_EQ(sum, (values.Size() - 1) * values.Size() / 2);
}
} // namespace common
} // namespace xgboost

15
tests/cpp/common/test_stats.cc
View File

@ -54,5 +54,20 @@ TEST(Stats, WeightedQuantile) {
q = WeightedQuantile(1.0, beg, end, w);
ASSERT_EQ(q, 5);
}
TEST(Stats, Median) {
linalg::Tensor<float, 2> values{{.0f, .0f, 1.f, 2.f}, {4}, Context::kCpuId};
Context ctx;
HostDeviceVector<float> weights;
auto m = Median(&ctx, values, weights);
ASSERT_EQ(m, .5f);
#if defined(XGBOOST_USE_CUDA)
ctx.gpu_id = 0;
ASSERT_FALSE(ctx.IsCPU());
m = Median(&ctx, values, weights);
ASSERT_EQ(m, .5f);
#endif // defined(XGBOOST_USE_CUDA)
}
} // namespace common
} // namespace xgboost

12
tests/cpp/gbm/test_gblinear.cc
View File

@ -19,15 +19,11 @@ namespace gbm {
TEST(GBLinear, JsonIO) {
size_t constexpr kRows = 16, kCols = 16;
LearnerModelParam param;
param.num_feature = kCols;
param.num_output_group = 1;
Context ctx;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
GenericParameter gparam;
gparam.Init(Args{});
std::unique_ptr<GradientBooster> gbm {
CreateTrainedGBM("gblinear", Args{}, kRows, kCols, &param, &gparam) };
std::unique_ptr<GradientBooster> gbm{
CreateTrainedGBM("gblinear", Args{}, kRows, kCols, &mparam, &ctx)};
Json model { Object() };
gbm->SaveModel(&model);
ASSERT_TRUE(IsA<Object>(model));

46
tests/cpp/gbm/test_gbtree.cc
View File

@ -18,15 +18,11 @@ namespace xgboost {
TEST(GBTree, SelectTreeMethod) {
size_t constexpr kCols = 10;
GenericParameter generic_param;
generic_param.UpdateAllowUnknown(Args{});
LearnerModelParam mparam;
mparam.base_score = 0.5;
mparam.num_feature = kCols;
mparam.num_output_group = 1;
Context ctx;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
std::unique_ptr<GradientBooster> p_gbm {
GradientBooster::Create("gbtree", &generic_param, &mparam)};
GradientBooster::Create("gbtree", &ctx, &mparam)};
auto& gbtree = dynamic_cast<gbm::GBTree&> (*p_gbm);
// Test if `tree_method` can be set
@ -45,7 +41,7 @@ TEST(GBTree, SelectTreeMethod) {
ASSERT_EQ(tparam.updater_seq, "grow_quantile_histmaker");
#ifdef XGBOOST_USE_CUDA
generic_param.UpdateAllowUnknown(Args{{"gpu_id", "0"}});
ctx.UpdateAllowUnknown(Args{{"gpu_id", "0"}});
gbtree.Configure({{"tree_method", "gpu_hist"}});
ASSERT_EQ(tparam.updater_seq, "grow_gpu_hist");
gbtree.Configure({{"booster", "dart"}, {"tree_method", "gpu_hist"}});
@ -55,15 +51,11 @@ TEST(GBTree, SelectTreeMethod) {
TEST(GBTree, PredictionCache) {
size_t constexpr kRows = 100, kCols = 10;
GenericParameter generic_param;
generic_param.UpdateAllowUnknown(Args{});
LearnerModelParam mparam;
mparam.base_score = 0.5;
mparam.num_feature = kCols;
mparam.num_output_group = 1;
Context ctx;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
std::unique_ptr<GradientBooster> p_gbm {
GradientBooster::Create("gbtree", &generic_param, &mparam)};
GradientBooster::Create("gbtree", &ctx, &mparam)};
auto& gbtree = dynamic_cast<gbm::GBTree&> (*p_gbm);
gbtree.Configure({{"tree_method", "hist"}});
@ -176,16 +168,11 @@ TEST(GBTree, ChoosePredictor) {
TEST(GBTree, JsonIO) {
size_t constexpr kRows = 16, kCols = 16;
LearnerModelParam mparam;
mparam.num_feature = kCols;
mparam.num_output_group = 1;
mparam.base_score = 0.5;
GenericParameter gparam;
gparam.Init(Args{});
Context ctx;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
std::unique_ptr<GradientBooster> gbm {
CreateTrainedGBM("gbtree", Args{}, kRows, kCols, &mparam, &gparam) };
CreateTrainedGBM("gbtree", Args{}, kRows, kCols, &mparam, &ctx) };
Json model {Object()};
model["model"] = Object();
@ -215,16 +202,11 @@ TEST(GBTree, JsonIO) {
TEST(Dart, JsonIO) {
size_t constexpr kRows = 16, kCols = 16;
LearnerModelParam mparam;
mparam.num_feature = kCols;
mparam.base_score = 0.5;
mparam.num_output_group = 1;
Context ctx;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
GenericParameter gparam;
gparam.Init(Args{});
std::unique_ptr<GradientBooster> gbm {
CreateTrainedGBM("dart", Args{}, kRows, kCols, &mparam, &gparam) };
std::unique_ptr<GradientBooster> gbm{
CreateTrainedGBM("dart", Args{}, kRows, kCols, &mparam, &ctx)};
Json model {Object()};
model["model"] = Object();

11
tests/cpp/helpers.h
View File

@ -451,5 +451,16 @@ class RMMAllocator;
using RMMAllocatorPtr = std::unique_ptr<RMMAllocator, void(*)(RMMAllocator*)>;
RMMAllocatorPtr SetUpRMMResourceForCppTests(int argc, char** argv);
/*
* \brief Make learner model param
*/
inline LearnerModelParam MakeMP(bst_feature_t n_features, float base_score, uint32_t n_groups,
int32_t device = Context::kCpuId) {
size_t shape[1]{1};
LearnerModelParam mparam(n_features, linalg::Tensor<float, 1>{{base_score}, shape, device},
n_groups);
return mparam;
}
} // namespace xgboost
#endif

10
tests/cpp/linear/test_linear.cc
View File

@ -18,10 +18,7 @@ TEST(Linear, Shotgun) {
auto p_fmat = xgboost::RandomDataGenerator(kRows, kCols, 0).GenerateDMatrix();
auto lparam = xgboost::CreateEmptyGenericParam(GPUIDX);
LearnerModelParam mparam;
mparam.num_feature = kCols;
mparam.num_output_group = 1;
mparam.base_score = 0.5;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
{
auto updater = std::unique_ptr<xgboost::LinearUpdater>(
@ -54,10 +51,7 @@ TEST(Linear, coordinate) {
auto p_fmat = xgboost::RandomDataGenerator(kRows, kCols, 0).GenerateDMatrix();
auto lparam = xgboost::CreateEmptyGenericParam(GPUIDX);
LearnerModelParam mparam;
mparam.num_feature = kCols;
mparam.num_output_group = 1;
mparam.base_score = 0.5;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
auto updater = std::unique_ptr<xgboost::LinearUpdater>(
xgboost::LinearUpdater::Create("coord_descent", &lparam));

10
tests/cpp/linear/test_linear.cu
View File

@ -13,15 +13,11 @@ TEST(Linear, GPUCoordinate) {
size_t constexpr kCols = 10;
auto mat = xgboost::RandomDataGenerator(kRows, kCols, 0).GenerateDMatrix();
auto lparam = CreateEmptyGenericParam(GPUIDX);
LearnerModelParam mparam;
mparam.num_feature = kCols;
mparam.num_output_group = 1;
mparam.base_score = 0.5;
auto ctx = CreateEmptyGenericParam(GPUIDX);
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
auto updater = std::unique_ptr<xgboost::LinearUpdater>(
xgboost::LinearUpdater::Create("gpu_coord_descent", &lparam));
xgboost::LinearUpdater::Create("gpu_coord_descent", &ctx));
updater->Configure({{"eta", "1."}});
xgboost::HostDeviceVector<xgboost::GradientPair> gpair(
mat->Info().num_row_, xgboost::GradientPair(-5, 1.0));

25
tests/cpp/predictor/test_cpu_predictor.cc
View File

@ -21,14 +21,11 @@ TEST(CpuPredictor, Basic) {
size_t constexpr kRows = 5;
size_t constexpr kCols = 5;
LearnerModelParam param;
param.num_feature = kCols;
param.base_score = 0.0;
param.num_output_group = 1;
LearnerModelParam mparam{MakeMP(kCols, .0, 1)};
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model = CreateTestModel(&param, &ctx);
gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx);
auto dmat = RandomDataGenerator(kRows, kCols, 0).GenerateDMatrix();
@ -104,14 +101,11 @@ TEST(CpuPredictor, ExternalMemory) {
std::unique_ptr<Predictor> cpu_predictor =
std::unique_ptr<Predictor>(Predictor::Create("cpu_predictor", &lparam));
LearnerModelParam param;
param.base_score = 0;
param.num_feature = dmat->Info().num_col_;
param.num_output_group = 1;
LearnerModelParam mparam{MakeMP(dmat->Info().num_col_, .0, 1)};
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model = CreateTestModel(&param, &ctx);
gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx);
// Test predict batch
PredictionCacheEntry out_predictions;
@ -201,16 +195,11 @@ TEST(CpuPredictor, InplacePredict) {
void TestUpdatePredictionCache(bool use_subsampling) {
size_t constexpr kRows = 64, kCols = 16, kClasses = 4;
LearnerModelParam mparam;
mparam.num_feature = kCols;
mparam.num_output_group = kClasses;
mparam.base_score = 0;
GenericParameter gparam;
gparam.Init(Args{});
LearnerModelParam mparam{MakeMP(kCols, .0, kClasses)};
Context ctx;
std::unique_ptr<gbm::GBTree> gbm;
gbm.reset(static_cast<gbm::GBTree*>(GradientBooster::Create("gbtree", &gparam, &mparam)));
gbm.reset(static_cast<gbm::GBTree*>(GradientBooster::Create("gbtree", &ctx, &mparam)));
std::map<std::string, std::string> cfg;
cfg["tree_method"] = "hist";
cfg["predictor"] = "cpu_predictor";

69
tests/cpp/predictor/test_gpu_predictor.cu
View File

@ -1,5 +1,5 @@
/*!
* Copyright 2017-2020 XGBoost contributors
* Copyright 2017-2022 XGBoost contributors
*/
#include <gtest/gtest.h>
#include <xgboost/c_api.h>
@ -34,14 +34,10 @@ TEST(GPUPredictor, Basic) {
int n_row = i, n_col = i;
auto dmat = RandomDataGenerator(n_row, n_col, 0).GenerateDMatrix();
LearnerModelParam param;
param.num_feature = n_col;
param.num_output_group = 1;
param.base_score = 0.5;
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model = CreateTestModel(&param, &ctx);
Context ctx;
ctx.gpu_id = 0;
LearnerModelParam mparam{MakeMP(n_col, .5, 1, ctx.gpu_id)};
gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx);
// Test predict batch
PredictionCacheEntry gpu_out_predictions;
@ -93,15 +89,12 @@ TEST(GPUPredictor, ExternalMemoryTest) {
std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor", &lparam));
gpu_predictor->Configure({});
LearnerModelParam param;
param.num_feature = 5;
const int n_classes = 3;
param.num_output_group = n_classes;
param.base_score = 0.5;
Context ctx;
ctx.gpu_id = 0;
LearnerModelParam mparam{MakeMP(5, .5, n_classes, ctx.gpu_id)};
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model = CreateTestModel(&param, &ctx, n_classes);
gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx, n_classes);
std::vector<std::unique_ptr<DMatrix>> dmats;
dmats.push_back(CreateSparsePageDMatrix(400));
@ -171,15 +164,10 @@ TEST(GpuPredictor, LesserFeatures) {
TEST(GPUPredictor, ShapStump) {
cudaSetDevice(0);
LearnerModelParam param;
param.num_feature = 1;
param.num_output_group = 1;
param.base_score = 0.5;
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model(&param, &ctx);
Context ctx;
ctx.gpu_id = 0;
LearnerModelParam mparam{MakeMP(1, .5, 1, ctx.gpu_id)};
gbm::GBTreeModel model(&mparam, &ctx);
std::vector<std::unique_ptr<RegTree>> trees;
trees.push_back(std::unique_ptr<RegTree>(new RegTree));
@ -193,24 +181,20 @@ TEST(GPUPredictor, ShapStump) {
auto dmat = RandomDataGenerator(3, 1, 0).GenerateDMatrix();
gpu_predictor->PredictContribution(dmat.get(), &predictions, model);
auto& phis = predictions.HostVector();
auto base_score = mparam.BaseScore(Context::kCpuId)(0);
EXPECT_EQ(phis[0], 0.0);
EXPECT_EQ(phis[1], param.base_score);
EXPECT_EQ(phis[1], base_score);
EXPECT_EQ(phis[2], 0.0);
EXPECT_EQ(phis[3], param.base_score);
EXPECT_EQ(phis[3], base_score);
EXPECT_EQ(phis[4], 0.0);
EXPECT_EQ(phis[5], param.base_score);
EXPECT_EQ(phis[5], base_score);
}
TEST(GPUPredictor, Shap) {
LearnerModelParam param;
param.num_feature = 1;
param.num_output_group = 1;
param.base_score = 0.5;
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model(&param, &ctx);
Context ctx;
ctx.gpu_id = 0;
LearnerModelParam mparam{MakeMP(1, .5, 1, ctx.gpu_id)};
gbm::GBTreeModel model(&mparam, &ctx);
std::vector<std::unique_ptr<RegTree>> trees;
trees.push_back(std::unique_ptr<RegTree>(new RegTree));
@ -258,14 +242,9 @@ TEST(GPUPredictor, PredictLeafBasic) {
std::unique_ptr<Predictor>(Predictor::Create("gpu_predictor", &lparam));
gpu_predictor->Configure({});
LearnerModelParam param;
param.num_feature = kCols;
param.base_score = 0.0;
param.num_output_group = 1;
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model = CreateTestModel(&param, &ctx);
LearnerModelParam mparam{MakeMP(kCols, .0, 1)};
Context ctx;
gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx);
HostDeviceVector<float> leaf_out_predictions;
gpu_predictor->PredictLeaf(dmat.get(), &leaf_out_predictions, model);

21
tests/cpp/predictor/test_predictor.cc
View File

@ -210,11 +210,7 @@ void TestCategoricalPrediction(std::string name) {
size_t constexpr kCols = 10;
PredictionCacheEntry out_predictions;
LearnerModelParam param;
param.num_feature = kCols;
param.num_output_group = 1;
param.base_score = 0.5;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
uint32_t split_ind = 3;
bst_cat_t split_cat = 4;
float left_weight = 1.3f;
@ -222,7 +218,7 @@ void TestCategoricalPrediction(std::string name) {
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model(&param, &ctx);
gbm::GBTreeModel model(&mparam, &ctx);
GBTreeModelForTest(&model, split_ind, split_cat, left_weight, right_weight);
ctx.UpdateAllowUnknown(Args{{"gpu_id", "0"}});
@ -237,27 +233,24 @@ void TestCategoricalPrediction(std::string name) {
predictor->InitOutPredictions(m->Info(), &out_predictions.predictions, model);
predictor->PredictBatch(m.get(), &out_predictions, model, 0);
auto score = mparam.BaseScore(Context::kCpuId)(0);
ASSERT_EQ(out_predictions.predictions.Size(), 1ul);
ASSERT_EQ(out_predictions.predictions.HostVector()[0],
right_weight + param.base_score); // go to right for matching cat
right_weight + score); // go to right for matching cat
row[split_ind] = split_cat + 1;
m = GetDMatrixFromData(row, 1, kCols);
out_predictions.version = 0;
predictor->InitOutPredictions(m->Info(), &out_predictions.predictions, model);
predictor->PredictBatch(m.get(), &out_predictions, model, 0);
ASSERT_EQ(out_predictions.predictions.HostVector()[0],
left_weight + param.base_score);
ASSERT_EQ(out_predictions.predictions.HostVector()[0], left_weight + score);
}
void TestCategoricalPredictLeaf(StringView name) {
size_t constexpr kCols = 10;
PredictionCacheEntry out_predictions;
LearnerModelParam param;
param.num_feature = kCols;
param.num_output_group = 1;
param.base_score = 0.5;
LearnerModelParam mparam{MakeMP(kCols, .5, 1)};
uint32_t split_ind = 3;
bst_cat_t split_cat = 4;
@ -267,7 +260,7 @@ void TestCategoricalPredictLeaf(StringView name) {
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model(&param, &ctx);
gbm::GBTreeModel model(&mparam, &ctx);
GBTreeModelForTest(&model, split_ind, split_cat, left_weight, right_weight);
ctx.gpu_id = 0;

8
tests/cpp/predictor/test_predictor.h
View File

@ -12,11 +12,7 @@ void TestPredictionFromGradientIndex(std::string name, size_t rows, size_t cols,
std::shared_ptr<DMatrix> p_hist) {
constexpr size_t kClasses { 3 };
LearnerModelParam param;
param.num_feature = cols;
param.num_output_group = kClasses;
param.base_score = 0.5;
LearnerModelParam mparam{MakeMP(cols, .5, kClasses)};
auto lparam = CreateEmptyGenericParam(0);
std::unique_ptr<Predictor> predictor =
@ -25,7 +21,7 @@ void TestPredictionFromGradientIndex(std::string name, size_t rows, size_t cols,
GenericParameter ctx;
ctx.UpdateAllowUnknown(Args{});
gbm::GBTreeModel model = CreateTestModel(&param, &ctx, kClasses);
gbm::GBTreeModel model = CreateTestModel(&mparam, &ctx, kClasses);
{
auto p_precise = RandomDataGenerator(rows, cols, 0).GenerateDMatrix();

78
tests/cpp/test_learner.cc
View File

@ -3,8 +3,10 @@
*/
#include <gtest/gtest.h>
#include <xgboost/learner.h>
#include <xgboost/objective.h> // ObjFunction
#include <xgboost/version_config.h>
#include <string> // std::stof, std::string
#include <thread>
#include <vector>
@ -206,8 +208,7 @@ TEST(Learner, MultiThreadedPredict) {
p_dmat->Info().labels.Reshape(kRows);
CHECK_NE(p_dmat->Info().num_col_, 0);
std::shared_ptr<DMatrix> p_data{
RandomDataGenerator{kRows, kCols, 0}.GenerateDMatrix()};
std::shared_ptr<DMatrix> p_data{RandomDataGenerator{kRows, kCols, 0}.GenerateDMatrix()};
CHECK_NE(p_data->Info().num_col_, 0);
std::shared_ptr<Learner> learner{Learner::Create({p_dmat})};
@ -448,4 +449,77 @@ TEST(Learner, MultiTarget) {
EXPECT_THROW({ learner->Configure(); }, dmlc::Error);
}
}
/**
* Test the model initialization sequence is correctly performed.
*/
TEST(Learner, InitEstimation) {
size_t constexpr kCols = 10;
auto Xy = RandomDataGenerator{10, kCols, 0}.GenerateDMatrix(true);
{
std::unique_ptr<Learner> learner{Learner::Create({Xy})};
learner->SetParam("objective", "reg:absoluteerror");
learner->Configure();
HostDeviceVector<float> predt;
learner->Predict(Xy, false, &predt, 0, 0);
auto h_predt = predt.ConstHostSpan();
for (auto v : h_predt) {
ASSERT_EQ(v, ObjFunction::DefaultBaseScore());
}
Json config{Object{}};
learner->SaveConfig(&config);
auto base_score =
std::stof(get<String const>(config["learner"]["learner_model_param"]["base_score"]));
// No base score is estimated yet.
ASSERT_EQ(base_score, ObjFunction::DefaultBaseScore());
}
{
std::unique_ptr<Learner> learner{Learner::Create({Xy})};
learner->SetParam("objective", "reg:absoluteerror");
learner->UpdateOneIter(0, Xy);
HostDeviceVector<float> predt;
learner->Predict(Xy, false, &predt, 0, 0);
auto h_predt = predt.ConstHostSpan();
for (auto v : h_predt) {
ASSERT_NE(v, ObjFunction::DefaultBaseScore());
}
Json config{Object{}};
learner->SaveConfig(&config);
auto base_score =
std::stof(get<String const>(config["learner"]["learner_model_param"]["base_score"]));
ASSERT_NE(base_score, ObjFunction::DefaultBaseScore());
ASSERT_THROW(
{
learner->SetParam("base_score_estimated", "1");
learner->Configure();
},
dmlc::Error);
}
{
std::unique_ptr<Learner> learner{Learner::Create({Xy})};
learner->SetParam("objective", "reg:absoluteerror");
learner->SetParam("base_score", "1.3");
learner->Configure();
HostDeviceVector<float> predt;
learner->Predict(Xy, false, &predt, 0, 0);
auto h_predt = predt.ConstHostSpan();
for (auto v : h_predt) {
ASSERT_FLOAT_EQ(v, 1.3);
}
learner->UpdateOneIter(0, Xy);
Json config{Object{}};
learner->SaveConfig(&config);
auto base_score =
std::stof(get<String const>(config["learner"]["learner_model_param"]["base_score"]));
// no change
ASSERT_FLOAT_EQ(base_score, 1.3);
}
}
} // namespace xgboost

39
tests/cpp/test_serialization.cc
View File

@ -418,6 +418,45 @@ TEST_F(SerializationTest, GPUCoordDescent) {
}
#endif // defined(XGBOOST_USE_CUDA)
class L1SerializationTest : public SerializationTest {};
TEST_F(L1SerializationTest, Exact) {
TestLearnerSerialization({{"booster", "gbtree"},
{"objective", "reg:absoluteerror"},
{"seed", "0"},
{"max_depth", "2"},
{"tree_method", "exact"}},
fmap_, p_dmat_);
}
TEST_F(L1SerializationTest, Approx) {
TestLearnerSerialization({{"booster", "gbtree"},
{"objective", "reg:absoluteerror"},
{"seed", "0"},
{"max_depth", "2"},
{"tree_method", "approx"}},
fmap_, p_dmat_);
}
TEST_F(L1SerializationTest, Hist) {
TestLearnerSerialization({{"booster", "gbtree"},
{"objective", "reg:absoluteerror"},
{"seed", "0"},
{"max_depth", "2"},
{"tree_method", "hist"}},
fmap_, p_dmat_);
}
#if defined(XGBOOST_USE_CUDA)
TEST_F(L1SerializationTest, GpuHist) {
TestLearnerSerialization({{"booster", "gbtree"},
{"objective", "reg:absoluteerror"},
{"seed", "0"},
{"max_depth", "2"},
{"tree_method", "gpu_hist"}},
fmap_, p_dmat_);
}
#endif // defined(XGBOOST_USE_CUDA)
class LogitSerializationTest : public SerializationTest {
protected:

5
tests/python-gpu/test_gpu_updaters.py
View File

@ -208,3 +208,8 @@ class TestGPUUpdaters:
param = dataset.set_params(param)
result = train_result(param, dataset.get_dmat(), 10)
assert tm.non_increasing(result['train'][dataset.metric])
@pytest.mark.skipif(**tm.no_sklearn())
@pytest.mark.parametrize("weighted", [True, False])
def test_adaptive(self, weighted) -> None:
self.cputest.run_adaptive("gpu_hist", weighted)

24
tests/python/test_model_compatibility.py
View File

@ -102,34 +102,38 @@ def run_scikit_model_check(name, path):
@pytest.mark.skipif(**tm.no_sklearn())
def test_model_compatibility():
'''Test model compatibility, can only be run on CI as others don't
"""Test model compatibility, can only be run on CI as others don't
have the credentials.
'''
"""
path = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(path, 'models')
path = os.path.join(path, "models")
zip_path, _ = urllib.request.urlretrieve('https://xgboost-ci-jenkins-artifacts.s3-us-west-2' +
'.amazonaws.com/xgboost_model_compatibility_test.zip')
with zipfile.ZipFile(zip_path, 'r') as z:
if not os.path.exists(path):
zip_path, _ = urllib.request.urlretrieve(
"https://xgboost-ci-jenkins-artifacts.s3-us-west-2"
+ ".amazonaws.com/xgboost_model_compatibility_test.zip"
)
with zipfile.ZipFile(zip_path, "r") as z:
z.extractall(path)
models = [
os.path.join(root, f) for root, subdir, files in os.walk(path)
os.path.join(root, f)
for root, subdir, files in os.walk(path)
for f in files
if f != 'version'
if f != "version"
]
assert models
for path in models:
name = os.path.basename(path)
if name.startswith('xgboost-'):
if name.startswith("xgboost-"):
booster = xgboost.Booster(model_file=path)
run_booster_check(booster, name)
# Do full serialization.
booster = copy.copy(booster)
run_booster_check(booster, name)
elif name.startswith('xgboost_scikit'):
elif name.startswith("xgboost_scikit"):
run_scikit_model_check(name, path)
else:
assert False

71
tests/python/test_updaters.py
View File

@ -1,4 +1,4 @@
from random import choice
import json
from string import ascii_lowercase
from typing import Dict, Any
import testing as tm
@ -397,3 +397,72 @@ class TestTreeMethod:
def test_categorical_missing(self, rows, cols, cats):
self.run_categorical_missing(rows, cols, cats, "approx")
self.run_categorical_missing(rows, cols, cats, "hist")
def run_adaptive(self, tree_method, weighted) -> None:
rng = np.random.RandomState(1994)
from sklearn.datasets import make_regression
from sklearn.utils import stats
n_samples = 256
X, y = make_regression(n_samples, 16, random_state=rng)
if weighted:
w = rng.normal(size=n_samples)
w -= w.min()
Xy = xgb.DMatrix(X, y, weight=w)
base_score = stats._weighted_percentile(y, w, percentile=50)
else:
Xy = xgb.DMatrix(X, y)
base_score = np.median(y)
booster_0 = xgb.train(
{
"tree_method": tree_method,
"base_score": base_score,
"objective": "reg:absoluteerror",
},
Xy,
num_boost_round=1,
)
booster_1 = xgb.train(
{"tree_method": tree_method, "objective": "reg:absoluteerror"},
Xy,
num_boost_round=1,
)
config_0 = json.loads(booster_0.save_config())
config_1 = json.loads(booster_1.save_config())
def get_score(config: Dict) -> float:
return float(config["learner"]["learner_model_param"]["base_score"])
assert get_score(config_0) == get_score(config_1)
raw_booster = booster_1.save_raw(raw_format="deprecated")
booster_2 = xgb.Booster(model_file=raw_booster)
config_2 = json.loads(booster_2.save_config())
assert get_score(config_1) == get_score(config_2)
raw_booster = booster_1.save_raw(raw_format="ubj")
booster_2 = xgb.Booster(model_file=raw_booster)
config_2 = json.loads(booster_2.save_config())
assert get_score(config_1) == get_score(config_2)
booster_0 = xgb.train(
{
"tree_method": tree_method,
"base_score": base_score + 1.0,
"objective": "reg:absoluteerror",
},
Xy,
num_boost_round=1,
)
config_0 = json.loads(booster_0.save_config())
np.testing.assert_allclose(get_score(config_0), get_score(config_1) + 1)
@pytest.mark.skipif(**tm.no_sklearn())
@pytest.mark.parametrize(
"tree_method,weighted", [
("approx", False), ("hist", False), ("approx", True), ("hist", True)
]
)
def test_adaptive(self, tree_method, weighted) -> None:
self.run_adaptive(tree_method, weighted)

45
tests/python/test_with_dask.py
View File

@ -1537,13 +1537,56 @@ class TestWithDask:
@pytest.mark.skipif(**tm.no_dask())
@pytest.mark.gtest
def test_quantile_same_on_all_workers(self) -> None:
self.run_quantile('SameOnAllWorkers')
self.run_quantile("SameOnAllWorkers")
def test_adaptive(self) -> None:
def get_score(config: Dict) -> float:
return float(config["learner"]["learner_model_param"]["base_score"])
def local_test(rabit_args: List[bytes], worker_id: int) -> bool:
with xgb.dask.RabitContext(rabit_args):
if worker_id == 0:
y = np.array([0.0, 0.0, 0.0])
x = np.array([[0.0]] * 3)
else:
y = np.array([1000.0])
x = np.array(
[
[0.0],
]
)
Xy = xgb.DMatrix(x, y)
booster = xgb.train(
{"tree_method": "hist", "objective": "reg:absoluteerror"},
Xy,
num_boost_round=1,
)
config = json.loads(booster.save_config())
base_score = get_score(config)
assert base_score == 250.0
return True
with LocalCluster(n_workers=2, dashboard_address=":0") as cluster:
with Client(cluster) as client:
workers = _get_client_workers(client)
rabit_args = client.sync(
xgb.dask._get_rabit_args, len(workers), None, client
)
futures = []
for i, _ in enumerate(workers):
f = client.submit(local_test, rabit_args, i)
futures.append(f)
results = client.gather(futures)
assert all(results)
def test_n_workers(self) -> None:
with LocalCluster(n_workers=2, dashboard_address=":0") as cluster:
with Client(cluster) as client:
workers = _get_client_workers(client)
from sklearn.datasets import load_breast_cancer
X, y = load_breast_cancer(return_X_y=True)
dX = client.submit(da.from_array, X, workers=[workers[0]]).result()
dy = client.submit(da.from_array, y, workers=[workers[0]]).result()