Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

De-duplicate GPU parameters. (#4454)

Browse Source 
* Only define `gpu_id` and `n_gpus` in `LearnerTrainParam`
* Pass LearnerTrainParam through XGBoost vid factory method.
* Disable all GPU usage when GPU related parameters are not specified (fixes XGBoost choosing GPU over aggressively).
* Test learner train param io.
* Fix gpu pickling.
This commit is contained in:
Jiaming Yuan
2019-05-29 11:55:57 +08:00
committed by GitHub
parent a3fedbeaa8
commit c589eff941
69 changed files with 927 additions and 562 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/objective/hinge.cu
View File

@@ -18,29 +18,12 @@ namespace obj {
DMLC_REGISTRY_FILE_TAG(hinge_obj_gpu);
#endif // defined(XGBOOST_USE_CUDA)
struct HingeObjParam : public dmlc::Parameter<HingeObjParam> {
int n_gpus;
int gpu_id;
DMLC_DECLARE_PARAMETER(HingeObjParam) {
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(GPUSet::kAll)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
.set_default(0)
.describe("gpu to use for objective function evaluation");
}
};
class HingeObj : public ObjFunction {
public:
HingeObj() = default;
void Configure(
const std::vector<std::pair<std::string, std::string> > &args) override {
param_.InitAllowUnknown(args);
devices_ = GPUSet::All(param_.gpu_id, param_.n_gpus);
label_correct_.Resize(devices_.IsEmpty() ? 1 : devices_.Size());
}
const std::vector<std::pair<std::string, std::string> > &args) override {}
void GetGradient(const HostDeviceVector<bst_float> &preds,
const MetaInfo &info,
@@ -57,7 +40,6 @@ class HingeObj : public ObjFunction {
out_gpair->Resize(ndata);
common::Transform<>::Init(
[=] XGBOOST_DEVICE(size_t _idx,
common::Span<int> _label_correct,
common::Span<GradientPair> _out_gpair,
common::Span<const bst_float> _preds,
common::Span<const bst_float> _labels,
@@ -75,8 +57,9 @@ class HingeObj : public ObjFunction {
}
_out_gpair[_idx] = GradientPair(g, h);
},
common::Range{0, static_cast<int64_t>(ndata)}, devices_).Eval(
&label_correct_, out_gpair, &preds, &info.labels_, &info.weights_);
common::Range{0, static_cast<int64_t>(ndata)},
GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, ndata)).Eval(
out_gpair, &preds, &info.labels_, &info.weights_);
}
void PredTransform(HostDeviceVector<bst_float> *io_preds) override {
@@ -84,22 +67,16 @@ class HingeObj : public ObjFunction {
[] XGBOOST_DEVICE(size_t _idx, common::Span<bst_float> _preds) {
_preds[_idx] = _preds[_idx] > 0.0 ? 1.0 : 0.0;
},
common::Range{0, static_cast<int64_t>(io_preds->Size()), 1}, devices_)
common::Range{0, static_cast<int64_t>(io_preds->Size()), 1},
GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, io_preds->Size()))
.Eval(io_preds);
}
const char* DefaultEvalMetric() const override {
return "error";
}
private:
GPUSet devices_;
HostDeviceVector<int> label_correct_;
HingeObjParam param_;
};
// register the objective functions
DMLC_REGISTER_PARAMETER(HingeObjParam);
// register the objective functions
XGBOOST_REGISTER_OBJECTIVE(HingeObj, "binary:hinge")
.describe("Hinge loss. Expects labels to be in [0,1f]")

37
src/objective/multiclass_obj.cu
View File

@@ -13,6 +13,8 @@
#include <algorithm>
#include <limits>
#include <utility>
#include "../common/common.h"
#include "../common/math.h"
#include "../common/transform.h"
@@ -25,18 +27,10 @@ DMLC_REGISTRY_FILE_TAG(multiclass_obj_gpu);
struct SoftmaxMultiClassParam : public dmlc::Parameter<SoftmaxMultiClassParam> {
int num_class;
int n_gpus;
int gpu_id;
// declare parameters
DMLC_DECLARE_PARAMETER(SoftmaxMultiClassParam) {
DMLC_DECLARE_FIELD(num_class).set_lower_bound(1)
.describe("Number of output class in the multi-class classification.");
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(GPUSet::kAll)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
.set_default(0)
.describe("gpu to use for objective function evaluation");
}
};
// TODO(trivialfis): Currently the sharding in softmax is less than ideal
@@ -49,8 +43,6 @@ class SoftmaxMultiClassObj : public ObjFunction {
}
void Configure(const std::vector<std::pair<std::string, std::string> >& args) override {
param_.InitAllowUnknown(args);
devices_ = GPUSet::All(param_.gpu_id, param_.n_gpus);
label_correct_.Resize(devices_.IsEmpty() ? 1 : devices_.Size());
}
void GetGradient(const HostDeviceVector<bst_float>& preds,
const MetaInfo& info,
@@ -63,11 +55,14 @@ class SoftmaxMultiClassObj : public ObjFunction {
const int nclass = param_.num_class;
const auto ndata = static_cast<int64_t>(preds.Size() / nclass);
out_gpair->Shard(GPUDistribution::Granular(devices_, nclass));
info.labels_.Shard(GPUDistribution::Block(devices_));
info.weights_.Shard(GPUDistribution::Block(devices_));
preds.Shard(GPUDistribution::Granular(devices_, nclass));
label_correct_.Shard(GPUDistribution::Block(devices_));
auto devices = GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, preds.Size());
out_gpair->Shard(GPUDistribution::Granular(devices, nclass));
info.labels_.Shard(GPUDistribution::Block(devices));
info.weights_.Shard(GPUDistribution::Block(devices));
preds.Shard(GPUDistribution::Granular(devices, nclass));
label_correct_.Resize(devices.IsEmpty() ? 1 : devices.Size());
label_correct_.Shard(GPUDistribution::Block(devices));
out_gpair->Resize(preds.Size());
label_correct_.Fill(1);
@@ -101,7 +96,7 @@ class SoftmaxMultiClassObj : public ObjFunction {
p = label == k ? p - 1.0f : p;
gpair[idx * nclass + k] = GradientPair(p * wt, h);
}
}, common::Range{0, ndata}, devices_, false)
}, common::Range{0, ndata}, devices, false)
.Eval(out_gpair, &info.labels_, &preds, &info.weights_, &label_correct_);
std::vector<int>& label_correct_h = label_correct_.HostVector();
@@ -126,6 +121,7 @@ class SoftmaxMultiClassObj : public ObjFunction {
const auto ndata = static_cast<int64_t>(io_preds->Size() / nclass);
max_preds_.Resize(ndata);
auto devices = GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, io_preds->Size());
if (prob) {
common::Transform<>::Init(
[=] XGBOOST_DEVICE(size_t _idx, common::Span<bst_float> _preds) {
@@ -133,11 +129,11 @@ class SoftmaxMultiClassObj : public ObjFunction {
_preds.subspan(_idx * nclass, nclass);
common::Softmax(point.begin(), point.end());
},
common::Range{0, ndata}, GPUDistribution::Granular(devices_, nclass))
common::Range{0, ndata}, GPUDistribution::Granular(devices, nclass))
.Eval(io_preds);
} else {
io_preds->Shard(GPUDistribution::Granular(devices_, nclass));
max_preds_.Shard(GPUDistribution::Block(devices_));
io_preds->Shard(GPUDistribution::Granular(devices, nclass));
max_preds_.Shard(GPUDistribution::Block(devices));
common::Transform<>::Init(
[=] XGBOOST_DEVICE(size_t _idx,
common::Span<const bst_float> _preds,
@@ -148,7 +144,7 @@ class SoftmaxMultiClassObj : public ObjFunction {
common::FindMaxIndex(point.cbegin(),
point.cend()) - point.cbegin();
},
common::Range{0, ndata}, devices_, false)
common::Range{0, ndata}, devices, false)
.Eval(io_preds, &max_preds_);
}
if (!prob) {
@@ -162,7 +158,6 @@ class SoftmaxMultiClassObj : public ObjFunction {
bool output_prob_;
// parameter
SoftmaxMultiClassParam param_;
GPUSet devices_;
// Cache for max_preds
HostDeviceVector<bst_float> max_preds_;
HostDeviceVector<int> label_correct_;

6
src/objective/objective.cc
View File

@@ -14,7 +14,7 @@ DMLC_REGISTRY_ENABLE(::xgboost::ObjFunctionReg);
namespace xgboost {
// implement factory functions
ObjFunction* ObjFunction::Create(const std::string& name) {
ObjFunction* ObjFunction::Create(const std::string& name, LearnerTrainParam const* tparam) {
auto *e = ::dmlc::Registry< ::xgboost::ObjFunctionReg>::Get()->Find(name);
if (e == nullptr) {
for (const auto& entry : ::dmlc::Registry< ::xgboost::ObjFunctionReg>::List()) {
@@ -22,7 +22,9 @@ ObjFunction* ObjFunction::Create(const std::string& name) {
}
LOG(FATAL) << "Unknown objective function " << name;
}
return (e->body)();
auto pobj = (e->body)();
pobj->tparam_ = tparam;
return pobj;
}
} // namespace xgboost

93
src/objective/regression_obj.cu
View File

@@ -28,18 +28,10 @@ DMLC_REGISTRY_FILE_TAG(regression_obj_gpu);
struct RegLossParam : public dmlc::Parameter<RegLossParam> {
float scale_pos_weight;
int n_gpus;
int gpu_id;
// declare parameters
DMLC_DECLARE_PARAMETER(RegLossParam) {
DMLC_DECLARE_FIELD(scale_pos_weight).set_default(1.0f).set_lower_bound(0.0f)
.describe("Scale the weight of positive examples by this factor");
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(GPUSet::kAll)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
.set_default(0)
.describe("gpu to use for objective function evaluation");
}
};
@@ -53,8 +45,6 @@ class RegLossObj : public ObjFunction {
void Configure(const std::vector<std::pair<std::string, std::string> >& args) override {
param_.InitAllowUnknown(args);
devices_ = GPUSet::All(param_.gpu_id, param_.n_gpus);
label_correct_.Resize(devices_.IsEmpty() ? 1 : devices_.Size());
}
void GetGradient(const HostDeviceVector<bst_float>& preds,
@@ -67,6 +57,8 @@ class RegLossObj : public ObjFunction {
<< "preds.size=" << preds.Size() << ", label.size=" << info.labels_.Size();
size_t ndata = preds.Size();
out_gpair->Resize(ndata);
auto devices = GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, preds.Size());
label_correct_.Resize(devices.IsEmpty() ? 1 : devices.Size());
label_correct_.Fill(1);
bool is_null_weight = info.weights_.Size() == 0;
@@ -91,7 +83,7 @@ class RegLossObj : public ObjFunction {
_out_gpair[_idx] = GradientPair(Loss::FirstOrderGradient(p, label) * w,
Loss::SecondOrderGradient(p, label) * w);
},
common::Range{0, static_cast<int64_t>(ndata)}, devices_).Eval(
common::Range{0, static_cast<int64_t>(ndata)}, devices).Eval(
&label_correct_, out_gpair, &preds, &info.labels_, &info.weights_);
// copy "label correct" flags back to host
@@ -113,7 +105,8 @@ class RegLossObj : public ObjFunction {
[] XGBOOST_DEVICE(size_t _idx, common::Span<float> _preds) {
_preds[_idx] = Loss::PredTransform(_preds[_idx]);
}, common::Range{0, static_cast<int64_t>(io_preds->Size())},
devices_).Eval(io_preds);
GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, io_preds->Size()))
.Eval(io_preds);
}
float ProbToMargin(float base_score) const override {
@@ -122,7 +115,6 @@ class RegLossObj : public ObjFunction {
protected:
RegLossParam param_;
GPUSet devices_;
};
// register the objective functions
@@ -181,18 +173,10 @@ XGBOOST_REGISTER_OBJECTIVE(GPULogisticRaw, "gpu:binary:logitraw")
// declare parameter
struct PoissonRegressionParam : public dmlc::Parameter<PoissonRegressionParam> {
float max_delta_step;
int n_gpus;
int gpu_id;
DMLC_DECLARE_PARAMETER(PoissonRegressionParam) {
DMLC_DECLARE_FIELD(max_delta_step).set_lower_bound(0.0f).set_default(0.7f)
.describe("Maximum delta step we allow each weight estimation to be." \
" This parameter is required for possion regression.");
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(GPUSet::kAll)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
.set_default(0)
.describe("gpu to use for objective function evaluation");
}
};
@@ -202,8 +186,6 @@ class PoissonRegression : public ObjFunction {
// declare functions
void Configure(const std::vector<std::pair<std::string, std::string> >& args) override {
param_.InitAllowUnknown(args);
devices_ = GPUSet::All(param_.gpu_id, param_.n_gpus);
label_correct_.Resize(devices_.IsEmpty() ? 1 : devices_.Size());
}
void GetGradient(const HostDeviceVector<bst_float>& preds,
@@ -214,6 +196,8 @@ class PoissonRegression : public ObjFunction {
CHECK_EQ(preds.Size(), info.labels_.Size()) << "labels are not correctly provided";
size_t ndata = preds.Size();
out_gpair->Resize(ndata);
auto devices = GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, preds.Size());
label_correct_.Resize(devices.IsEmpty() ? 1 : devices.Size());
label_correct_.Fill(1);
bool is_null_weight = info.weights_.Size() == 0;
@@ -234,7 +218,7 @@ class PoissonRegression : public ObjFunction {
_out_gpair[_idx] = GradientPair{(expf(p) - y) * w,
expf(p + max_delta_step) * w};
},
common::Range{0, static_cast<int64_t>(ndata)}, devices_).Eval(
common::Range{0, static_cast<int64_t>(ndata)}, devices).Eval(
&label_correct_, out_gpair, &preds, &info.labels_, &info.weights_);
// copy "label correct" flags back to host
std::vector<int>& label_correct_h = label_correct_.HostVector();
@@ -249,7 +233,8 @@ class PoissonRegression : public ObjFunction {
[] XGBOOST_DEVICE(size_t _idx, common::Span<bst_float> _preds) {
_preds[_idx] = expf(_preds[_idx]);
},
common::Range{0, static_cast<int64_t>(io_preds->Size())}, devices_)
common::Range{0, static_cast<int64_t>(io_preds->Size())},
GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, io_preds->Size()))
.Eval(io_preds);
}
void EvalTransform(HostDeviceVector<bst_float> *io_preds) override {
@@ -263,7 +248,6 @@ class PoissonRegression : public ObjFunction {
}
private:
GPUSet devices_;
PoissonRegressionParam param_;
HostDeviceVector<int> label_correct_;
};
@@ -279,8 +263,9 @@ XGBOOST_REGISTER_OBJECTIVE(PoissonRegression, "count:poisson")
// cox regression for survival data (negative values mean they are censored)
class CoxRegression : public ObjFunction {
public:
// declare functions
void Configure(const std::vector<std::pair<std::string, std::string> >& args) override {}
void Configure(
const std::vector<std::pair<std::string, std::string> > &args) override {}
void GetGradient(const HostDeviceVector<bst_float>& preds,
const MetaInfo &info,
int iter,
@@ -363,29 +348,11 @@ XGBOOST_REGISTER_OBJECTIVE(CoxRegression, "survival:cox")
.describe("Cox regression for censored survival data (negative labels are considered censored).")
.set_body([]() { return new CoxRegression(); });
struct GammaRegressionParam : public dmlc::Parameter<GammaRegressionParam> {
int n_gpus;
int gpu_id;
DMLC_DECLARE_PARAMETER(GammaRegressionParam) {
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(GPUSet::kAll)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
.set_default(0)
.describe("gpu to use for objective function evaluation");
}
};
// gamma regression
class GammaRegression : public ObjFunction {
public:
// declare functions
void Configure(const std::vector<std::pair<std::string, std::string> >& args) override {
param_.InitAllowUnknown(args);
devices_ = GPUSet::All(param_.gpu_id, param_.n_gpus);
label_correct_.Resize(devices_.IsEmpty() ? 1 : devices_.Size());
}
void Configure(
const std::vector<std::pair<std::string, std::string> > &args) override {}
void GetGradient(const HostDeviceVector<bst_float> &preds,
const MetaInfo &info,
@@ -394,7 +361,9 @@ class GammaRegression : public ObjFunction {
CHECK_NE(info.labels_.Size(), 0U) << "label set cannot be empty";
CHECK_EQ(preds.Size(), info.labels_.Size()) << "labels are not correctly provided";
const size_t ndata = preds.Size();
auto devices = GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, ndata);
out_gpair->Resize(ndata);
label_correct_.Resize(devices.IsEmpty() ? 1 : devices.Size());
label_correct_.Fill(1);
const bool is_null_weight = info.weights_.Size() == 0;
@@ -413,7 +382,7 @@ class GammaRegression : public ObjFunction {
}
_out_gpair[_idx] = GradientPair((1 - y / expf(p)) * w, y / expf(p) * w);
},
common::Range{0, static_cast<int64_t>(ndata)}, devices_).Eval(
common::Range{0, static_cast<int64_t>(ndata)}, devices).Eval(
&label_correct_, out_gpair, &preds, &info.labels_, &info.weights_);
// copy "label correct" flags back to host
@@ -429,7 +398,8 @@ class GammaRegression : public ObjFunction {
[] XGBOOST_DEVICE(size_t _idx, common::Span<bst_float> _preds) {
_preds[_idx] = expf(_preds[_idx]);
},
common::Range{0, static_cast<int64_t>(io_preds->Size())}, devices_)
common::Range{0, static_cast<int64_t>(io_preds->Size())},
GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, io_preds->Size()))
.Eval(io_preds);
}
void EvalTransform(HostDeviceVector<bst_float> *io_preds) override {
@@ -443,13 +413,9 @@ class GammaRegression : public ObjFunction {
}
private:
GPUSet devices_;
GammaRegressionParam param_;
HostDeviceVector<int> label_correct_;
};
// register the objective functions
DMLC_REGISTER_PARAMETER(GammaRegressionParam);
// register the objective functions
XGBOOST_REGISTER_OBJECTIVE(GammaRegression, "reg:gamma")
.describe("Gamma regression for severity data.")
@@ -459,17 +425,9 @@ XGBOOST_REGISTER_OBJECTIVE(GammaRegression, "reg:gamma")
// declare parameter
struct TweedieRegressionParam : public dmlc::Parameter<TweedieRegressionParam> {
float tweedie_variance_power;
int n_gpus;
int gpu_id;
DMLC_DECLARE_PARAMETER(TweedieRegressionParam) {
DMLC_DECLARE_FIELD(tweedie_variance_power).set_range(1.0f, 2.0f).set_default(1.5f)
.describe("Tweedie variance power. Must be between in range [1, 2).");
DMLC_DECLARE_FIELD(n_gpus).set_default(1).set_lower_bound(GPUSet::kAll)
.describe("Number of GPUs to use for multi-gpu algorithms.");
DMLC_DECLARE_FIELD(gpu_id)
.set_lower_bound(0)
.set_default(0)
.describe("gpu to use for objective function evaluation");
}
};
@@ -479,8 +437,6 @@ class TweedieRegression : public ObjFunction {
// declare functions
void Configure(const std::vector<std::pair<std::string, std::string> >& args) override {
param_.InitAllowUnknown(args);
devices_ = GPUSet::All(param_.gpu_id, param_.n_gpus);
label_correct_.Resize(devices_.IsEmpty() ? 1 : devices_.Size());
}
void GetGradient(const HostDeviceVector<bst_float>& preds,
@@ -491,6 +447,9 @@ class TweedieRegression : public ObjFunction {
CHECK_EQ(preds.Size(), info.labels_.Size()) << "labels are not correctly provided";
const size_t ndata = preds.Size();
out_gpair->Resize(ndata);
auto devices = GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, preds.Size());
label_correct_.Resize(devices.IsEmpty() ? 1 : devices.Size());
label_correct_.Fill(1);
const bool is_null_weight = info.weights_.Size() == 0;
@@ -514,7 +473,7 @@ class TweedieRegression : public ObjFunction {
std::exp((1 - rho) * p) + (2 - rho) * expf((2 - rho) * p);
_out_gpair[_idx] = GradientPair(grad * w, hess * w);
},
common::Range{0, static_cast<int64_t>(ndata), 1}, devices_)
common::Range{0, static_cast<int64_t>(ndata), 1}, devices)
.Eval(&label_correct_, out_gpair, &preds, &info.labels_, &info.weights_);
// copy "label correct" flags back to host
@@ -530,7 +489,8 @@ class TweedieRegression : public ObjFunction {
[] XGBOOST_DEVICE(size_t _idx, common::Span<bst_float> _preds) {
_preds[_idx] = expf(_preds[_idx]);
},
common::Range{0, static_cast<int64_t>(io_preds->Size())}, devices_)
common::Range{0, static_cast<int64_t>(io_preds->Size())},
GPUSet::All(tparam_->gpu_id, tparam_->n_gpus, io_preds->Size()))
.Eval(io_preds);
}
@@ -546,7 +506,6 @@ class TweedieRegression : public ObjFunction {
}
private:
GPUSet devices_;
TweedieRegressionParam param_;
HostDeviceVector<int> label_correct_;
};