Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Prevent multiclass Hessian approaching 0 (#3304)

Browse Source 
* Prevent Hessian in multiclass objective becoming zero

* Set default learning rate to 0.5 for "coord_descent" linear updater
This commit is contained in:
Rory Mitchell 2018-05-09 20:25:51 +12:00 committed by GitHub
parent b8a0d66fe6
commit 088bb4b27c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/linear/updater_coordinate.cc
View File

@ -27,7 +27,7 @@ struct CoordinateTrainParam : public dmlc::Parameter<CoordinateTrainParam> {
DMLC_DECLARE_PARAMETER(CoordinateTrainParam) { DMLC_DECLARE_PARAMETER(CoordinateTrainParam) {
DMLC_DECLARE_FIELD(learning_rate) DMLC_DECLARE_FIELD(learning_rate)
.set_lower_bound(0.0f) .set_lower_bound(0.0f)
.set_default(1.0f) .set_default(0.5f)
.describe("Learning rate of each update."); .describe("Learning rate of each update.");
DMLC_DECLARE_FIELD(reg_lambda) DMLC_DECLARE_FIELD(reg_lambda)
.set_lower_bound(0.0f) .set_lower_bound(0.0f)

3
src/objective/multiclass_obj.cc
View File

@ -65,7 +65,8 @@ class SoftmaxMultiClassObj : public ObjFunction {
const bst_float wt = info.GetWeight(i); const bst_float wt = info.GetWeight(i);
for (int k = 0; k < nclass; ++k) { for (int k = 0; k < nclass; ++k) {
bst_float p = rec[k]; bst_float p = rec[k];
const bst_float h = 2.0f * p * (1.0f - p) * wt; const float eps = 1e-16f;
const bst_float h = fmax(2.0f * p * (1.0f - p) * wt, eps);
if (label == k) { if (label == k) {
gpair[i * nclass + k] = GradientPair((p - 1.0f) * wt, h); gpair[i * nclass + k] = GradientPair((p - 1.0f) * wt, h);
} else { } else {

2
tests/cpp/linear/test_linear.cc
View File

@ -31,7 +31,7 @@ TEST(Linear, coordinate) {
mat->InitColAccess(enabled, 1.0f, 1 << 16, false); mat->InitColAccess(enabled, 1.0f, 1 << 16, false);
auto updater = std::unique_ptr<xgboost::LinearUpdater>( auto updater = std::unique_ptr<xgboost::LinearUpdater>(
xgboost::LinearUpdater::Create("coord_descent")); xgboost::LinearUpdater::Create("coord_descent"));
updater->Init({}); updater->Init({{"eta", "1."}});
xgboost::HostDeviceVector<xgboost::GradientPair> gpair( xgboost::HostDeviceVector<xgboost::GradientPair> gpair(
mat->Info().num_row_, xgboost::GradientPair(-5, 1.0)); mat->Info().num_row_, xgboost::GradientPair(-5, 1.0));
xgboost::gbm::GBLinearModel model; xgboost::gbm::GBLinearModel model;