From 7d297b418f288764bf4ca045add59478e27e8961 Mon Sep 17 00:00:00 2001 From: terrytangyuan Date: Mon, 2 Nov 2015 19:57:01 -0600 Subject: [PATCH] Added more thorough test for early stopping (+1 squashed commit) Squashed commits: [4f78cc0] Added test for early stopping (+1 squashed commit) --- tests/python/test_early_stopping.py | 31 +++++--- tests/python/test_with_sklearn.py | 106 +++++++++++++--------------- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/tests/python/test_early_stopping.py b/tests/python/test_early_stopping.py index 6190d6286..ef2cc1263 100644 --- a/tests/python/test_early_stopping.py +++ b/tests/python/test_early_stopping.py @@ -2,18 +2,31 @@ import xgboost as xgb import numpy as np from sklearn.datasets import load_digits from sklearn.cross_validation import KFold, train_test_split +import unittest rng = np.random.RandomState(1994) -def test_early_stopping_nonparallel(): - # digits = load_digits(2) - # X = digits['data'] - # y = digits['target'] - # X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) - # clf = xgb.XGBClassifier() - # clf.fit(X_train, y_train, early_stopping_rounds=10, eval_metric="auc", - # eval_set=[(X_test, y_test)]) - print("This test will be re-visited later. ") +class TestEarlyStopping(unittest.TestCase): + + def test_early_stopping_nonparallel(self): + digits = load_digits(2) + X = digits['data'] + y = digits['target'] + X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) + clf1 = xgb.XGBClassifier() + clf1.fit(X_train, y_train, early_stopping_rounds=5, eval_metric="auc", + eval_set=[(X_test, y_test)]) + clf2 = xgb.XGBClassifier() + clf2.fit(X_train, y_train, early_stopping_rounds=4, eval_metric="auc", + eval_set=[(X_test, y_test)]) + # should be the same + assert clf1.best_score == clf2.best_score + assert clf1.best_score != 1 + # check overfit + clf3 = xgb.XGBClassifier() + clf3.fit(X_train, y_train, early_stopping_rounds=10, eval_metric="auc", + eval_set=[(X_test, y_test)]) + assert clf3.best_score == 1 # TODO: parallel test for early stopping # TODO: comment out for now. Will re-visit later \ No newline at end of file diff --git a/tests/python/test_with_sklearn.py b/tests/python/test_with_sklearn.py index cc62f1c27..3e31ddb65 100644 --- a/tests/python/test_with_sklearn.py +++ b/tests/python/test_with_sklearn.py @@ -4,65 +4,61 @@ from sklearn.cross_validation import KFold, train_test_split from sklearn.metrics import mean_squared_error from sklearn.grid_search import GridSearchCV from sklearn.datasets import load_iris, load_digits, load_boston -import unittest rng = np.random.RandomState(1994) -class TestSklearn(unittest.TestCase): +def test_binary_classification(): + digits = load_digits(2) + y = digits['target'] + X = digits['data'] + kf = KFold(y.shape[0], n_folds=2, shuffle=True, random_state=rng) + for train_index, test_index in kf: + xgb_model = xgb.XGBClassifier().fit(X[train_index],y[train_index]) + preds = xgb_model.predict(X[test_index]) + labels = y[test_index] + err = sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) / float(len(preds)) + assert err < 0.1 - def test_binary_classification(): - digits = load_digits(2) - y = digits['target'] - X = digits['data'] - kf = KFold(y.shape[0], n_folds=2, shuffle=True, random_state=rng) - for train_index, test_index in kf: - xgb_model = xgb.XGBClassifier().fit(X[train_index],y[train_index]) - preds = xgb_model.predict(X[test_index]) - labels = y[test_index] - err = sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) / float(len(preds)) - assert err < 0.1 +def test_multiclass_classification(): + iris = load_iris() + y = iris['target'] + X = iris['data'] + kf = KFold(y.shape[0], n_folds=2, shuffle=True, random_state=rng) + for train_index, test_index in kf: + xgb_model = xgb.XGBClassifier().fit(X[train_index],y[train_index]) + preds = xgb_model.predict(X[test_index]) + # test other params in XGBClassifier().fit + preds2 = xgb_model.predict(X[test_index], output_margin=True, ntree_limit=3) + preds3 = xgb_model.predict(X[test_index], output_margin=True, ntree_limit=0) + preds4 = xgb_model.predict(X[test_index], output_margin=False, ntree_limit=3) + labels = y[test_index] + err = sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) / float(len(preds)) + assert err < 0.4 - def test_multiclass_classification(): - iris = load_iris() - y = iris['target'] - X = iris['data'] - kf = KFold(y.shape[0], n_folds=2, shuffle=True, random_state=rng) - for train_index, test_index in kf: - xgb_model = xgb.XGBClassifier().fit(X[train_index],y[train_index]) - preds = xgb_model.predict(X[test_index]) - # test other params in XGBClassifier().fit - preds2 = xgb_model.predict(X[test_index], output_margin=True, ntree_limit=3) - preds3 = xgb_model.predict(X[test_index], output_margin=True, ntree_limit=0) - preds4 = xgb_model.predict(X[test_index], output_margin=False, ntree_limit=3) - labels = y[test_index] - err = sum(1 for i in range(len(preds)) if int(preds[i]>0.5)!=labels[i]) / float(len(preds)) - assert err < 0.4 - - def test_boston_housing_regression(): - boston = load_boston() - y = boston['target'] - X = boston['data'] - kf = KFold(y.shape[0], n_folds=2, shuffle=True, random_state=rng) - for train_index, test_index in kf: - xgb_model = xgb.XGBRegressor().fit(X[train_index],y[train_index]) - preds = xgb_model.predict(X[test_index]) - # test other params in XGBRegressor().fit - preds2 = xgb_model.predict(X[test_index], output_margin=True, ntree_limit=3) - preds3 = xgb_model.predict(X[test_index], output_margin=True, ntree_limit=0) - preds4 = xgb_model.predict(X[test_index], output_margin=False, ntree_limit=3) - labels = y[test_index] - assert mean_squared_error(preds, labels) < 15 - - def test_parameter_tuning(): - boston = load_boston() - y = boston['target'] - X = boston['data'] - xgb_model = xgb.XGBRegressor() - clf = GridSearchCV(xgb_model, - {'max_depth': [2,4,6], - 'n_estimators': [50,100,200]}, verbose=1) - clf.fit(X,y) - assert clf.best_score_ < 0.7 - assert clf.best_params_ == {'n_estimators': 100, 'max_depth': 4} +def test_boston_housing_regression(): + boston = load_boston() + y = boston['target'] + X = boston['data'] + kf = KFold(y.shape[0], n_folds=2, shuffle=True, random_state=rng) + for train_index, test_index in kf: + xgb_model = xgb.XGBRegressor().fit(X[train_index],y[train_index]) + preds = xgb_model.predict(X[test_index]) + # test other params in XGBRegressor().fit + preds2 = xgb_model.predict(X[test_index], output_margin=True, ntree_limit=3) + preds3 = xgb_model.predict(X[test_index], output_margin=True, ntree_limit=0) + preds4 = xgb_model.predict(X[test_index], output_margin=False, ntree_limit=3) + labels = y[test_index] + assert mean_squared_error(preds, labels) < 25 +def test_parameter_tuning(): + boston = load_boston() + y = boston['target'] + X = boston['data'] + xgb_model = xgb.XGBRegressor() + clf = GridSearchCV(xgb_model, + {'max_depth': [2,4,6], + 'n_estimators': [50,100,200]}, verbose=1) + clf.fit(X,y) + assert clf.best_score_ < 0.7 + assert clf.best_params_ == {'n_estimators': 100, 'max_depth': 4}