365 lines
14 KiB
Python
365 lines
14 KiB
Python
import os
|
|
import tempfile
|
|
import numpy as np
|
|
import xgboost as xgb
|
|
import testing as tm
|
|
import pytest
|
|
from test_dmatrix import set_base_margin_info
|
|
|
|
try:
|
|
import pandas as pd
|
|
except ImportError:
|
|
pass
|
|
|
|
|
|
pytestmark = pytest.mark.skipif(**tm.no_pandas())
|
|
|
|
|
|
dpath = 'demo/data/'
|
|
rng = np.random.RandomState(1994)
|
|
|
|
|
|
class TestPandas:
|
|
def test_pandas(self):
|
|
df = pd.DataFrame([[1, 2., True], [2, 3., False]],
|
|
columns=['a', 'b', 'c'])
|
|
dm = xgb.DMatrix(df, label=pd.Series([1, 2]))
|
|
assert dm.feature_names == ['a', 'b', 'c']
|
|
assert dm.feature_types == ['int', 'float', 'i']
|
|
assert dm.num_row() == 2
|
|
assert dm.num_col() == 3
|
|
np.testing.assert_array_equal(dm.get_label(), np.array([1, 2]))
|
|
|
|
# overwrite feature_names and feature_types
|
|
dm = xgb.DMatrix(df, label=pd.Series([1, 2]),
|
|
feature_names=['x', 'y', 'z'],
|
|
feature_types=['q', 'q', 'q'])
|
|
assert dm.feature_names == ['x', 'y', 'z']
|
|
assert dm.feature_types == ['q', 'q', 'q']
|
|
assert dm.num_row() == 2
|
|
assert dm.num_col() == 3
|
|
|
|
# incorrect dtypes
|
|
df = pd.DataFrame([[1, 2., 'x'], [2, 3., 'y']],
|
|
columns=['a', 'b', 'c'])
|
|
with pytest.raises(ValueError):
|
|
xgb.DMatrix(df)
|
|
|
|
# numeric columns
|
|
df = pd.DataFrame([[1, 2., True], [2, 3., False]])
|
|
dm = xgb.DMatrix(df, label=pd.Series([1, 2]))
|
|
assert dm.feature_names == ['0', '1', '2']
|
|
assert dm.feature_types == ['int', 'float', 'i']
|
|
assert dm.num_row() == 2
|
|
assert dm.num_col() == 3
|
|
np.testing.assert_array_equal(dm.get_label(), np.array([1, 2]))
|
|
|
|
df = pd.DataFrame([[1, 2., 1], [2, 3., 1]], columns=[4, 5, 6])
|
|
dm = xgb.DMatrix(df, label=pd.Series([1, 2]))
|
|
assert dm.feature_names == ['4', '5', '6']
|
|
assert dm.feature_types == ['int', 'float', 'int']
|
|
assert dm.num_row() == 2
|
|
assert dm.num_col() == 3
|
|
|
|
df = pd.DataFrame({'A': ['X', 'Y', 'Z'], 'B': [1, 2, 3]})
|
|
dummies = pd.get_dummies(df)
|
|
# B A_X A_Y A_Z
|
|
# 0 1 1 0 0
|
|
# 1 2 0 1 0
|
|
# 2 3 0 0 1
|
|
result, _, _ = xgb.data._transform_pandas_df(dummies,
|
|
enable_categorical=False)
|
|
exp = np.array([[1., 1., 0., 0.],
|
|
[2., 0., 1., 0.],
|
|
[3., 0., 0., 1.]])
|
|
np.testing.assert_array_equal(result, exp)
|
|
dm = xgb.DMatrix(dummies)
|
|
assert dm.feature_names == ['B', 'A_X', 'A_Y', 'A_Z']
|
|
assert dm.feature_types == ['int', 'int', 'int', 'int']
|
|
assert dm.num_row() == 3
|
|
assert dm.num_col() == 4
|
|
|
|
df = pd.DataFrame({'A=1': [1, 2, 3], 'A=2': [4, 5, 6]})
|
|
dm = xgb.DMatrix(df)
|
|
assert dm.feature_names == ['A=1', 'A=2']
|
|
assert dm.feature_types == ['int', 'int']
|
|
assert dm.num_row() == 3
|
|
assert dm.num_col() == 2
|
|
|
|
df_int = pd.DataFrame([[1, 1.1], [2, 2.2]], columns=[9, 10])
|
|
dm_int = xgb.DMatrix(df_int)
|
|
df_range = pd.DataFrame([[1, 1.1], [2, 2.2]], columns=range(9, 11, 1))
|
|
dm_range = xgb.DMatrix(df_range)
|
|
assert dm_int.feature_names == ['9', '10'] # assert not "9 "
|
|
assert dm_int.feature_names == dm_range.feature_names
|
|
|
|
# test MultiIndex as columns
|
|
df = pd.DataFrame(
|
|
[
|
|
(1, 2, 3, 4, 5, 6),
|
|
(6, 5, 4, 3, 2, 1)
|
|
],
|
|
columns=pd.MultiIndex.from_tuples((
|
|
('a', 1), ('a', 2), ('a', 3),
|
|
('b', 1), ('b', 2), ('b', 3),
|
|
))
|
|
)
|
|
dm = xgb.DMatrix(df)
|
|
assert dm.feature_names == ['a 1', 'a 2', 'a 3', 'b 1', 'b 2', 'b 3']
|
|
assert dm.feature_types == ['int', 'int', 'int', 'int', 'int', 'int']
|
|
assert dm.num_row() == 2
|
|
assert dm.num_col() == 6
|
|
|
|
# test Index as columns
|
|
df = pd.DataFrame([[1, 1.1], [2, 2.2]], columns=pd.Index([1, 2]))
|
|
print(df.columns, isinstance(df.columns, pd.Index))
|
|
Xy = xgb.DMatrix(df)
|
|
np.testing.assert_equal(np.array(Xy.feature_names), np.array(["1", "2"]))
|
|
|
|
def test_slice(self):
|
|
rng = np.random.RandomState(1994)
|
|
rows = 100
|
|
X = rng.randint(3, 7, size=rows)
|
|
X = pd.DataFrame({'f0': X})
|
|
y = rng.randn(rows)
|
|
ridxs = [1, 2, 3, 4, 5, 6]
|
|
m = xgb.DMatrix(X, y)
|
|
sliced = m.slice(ridxs)
|
|
|
|
assert m.feature_types == sliced.feature_types
|
|
|
|
def test_pandas_categorical(self):
|
|
rng = np.random.RandomState(1994)
|
|
rows = 100
|
|
X = rng.randint(3, 7, size=rows)
|
|
X = pd.Series(X, dtype="category")
|
|
X = pd.DataFrame({'f0': X})
|
|
y = rng.randn(rows)
|
|
m = xgb.DMatrix(X, y, enable_categorical=True)
|
|
assert m.feature_types[0] == 'c'
|
|
|
|
X_0 = ["f", "o", "o"]
|
|
X_1 = [4, 3, 2]
|
|
X = pd.DataFrame({"feat_0": X_0, "feat_1": X_1})
|
|
X["feat_0"] = X["feat_0"].astype("category")
|
|
transformed, _, feature_types = xgb.data._transform_pandas_df(
|
|
X, enable_categorical=True
|
|
)
|
|
|
|
assert transformed[:, 0].min() == 0
|
|
|
|
# test missing value
|
|
X = pd.DataFrame({"f0": ["a", "b", np.NaN]})
|
|
X["f0"] = X["f0"].astype("category")
|
|
arr, _, _ = xgb.data._transform_pandas_df(X, enable_categorical=True)
|
|
assert not np.any(arr == -1.0)
|
|
|
|
X = X["f0"]
|
|
y = y[:X.shape[0]]
|
|
with pytest.raises(ValueError, match=r".*enable_categorical.*"):
|
|
xgb.DMatrix(X, y)
|
|
|
|
Xy = xgb.DMatrix(X, y, enable_categorical=True)
|
|
assert Xy.num_row() == 3
|
|
assert Xy.num_col() == 1
|
|
|
|
def test_pandas_sparse(self):
|
|
import pandas as pd
|
|
rows = 100
|
|
X = pd.DataFrame(
|
|
{"A": pd.arrays.SparseArray(np.random.randint(0, 10, size=rows)),
|
|
"B": pd.arrays.SparseArray(np.random.randn(rows)),
|
|
"C": pd.arrays.SparseArray(np.random.permutation(
|
|
[True, False] * (rows // 2)))}
|
|
)
|
|
y = pd.Series(pd.arrays.SparseArray(np.random.randn(rows)))
|
|
dtrain = xgb.DMatrix(X, y)
|
|
booster = xgb.train({}, dtrain, num_boost_round=4)
|
|
predt_sparse = booster.predict(xgb.DMatrix(X))
|
|
predt_dense = booster.predict(xgb.DMatrix(X.sparse.to_dense()))
|
|
np.testing.assert_allclose(predt_sparse, predt_dense)
|
|
|
|
def test_pandas_label(self):
|
|
# label must be a single column
|
|
df = pd.DataFrame({'A': ['X', 'Y', 'Z'], 'B': [1, 2, 3]})
|
|
with pytest.raises(ValueError):
|
|
xgb.data._transform_pandas_df(df, False, None, None, 'label', 'float')
|
|
|
|
# label must be supported dtype
|
|
df = pd.DataFrame({'A': np.array(['a', 'b', 'c'], dtype=object)})
|
|
with pytest.raises(ValueError):
|
|
xgb.data._transform_pandas_df(df, False, None, None, 'label', 'float')
|
|
|
|
df = pd.DataFrame({'A': np.array([1, 2, 3], dtype=int)})
|
|
result, _, _ = xgb.data._transform_pandas_df(df, False, None, None,
|
|
'label', 'float')
|
|
np.testing.assert_array_equal(result, np.array([[1.], [2.], [3.]],
|
|
dtype=float))
|
|
dm = xgb.DMatrix(np.random.randn(3, 2), label=df)
|
|
assert dm.num_row() == 3
|
|
assert dm.num_col() == 2
|
|
|
|
def test_pandas_weight(self):
|
|
kRows = 32
|
|
kCols = 8
|
|
|
|
X = np.random.randn(kRows, kCols)
|
|
y = np.random.randn(kRows)
|
|
w = np.random.uniform(size=kRows).astype(np.float32)
|
|
w_pd = pd.DataFrame(w)
|
|
data = xgb.DMatrix(X, y, w_pd)
|
|
|
|
assert data.num_row() == kRows
|
|
assert data.num_col() == kCols
|
|
|
|
np.testing.assert_array_equal(data.get_weight(), w)
|
|
|
|
def test_base_margin(self):
|
|
set_base_margin_info(pd.DataFrame, xgb.DMatrix, "hist")
|
|
|
|
def test_cv_as_pandas(self):
|
|
dm = xgb.DMatrix(dpath + 'agaricus.txt.train')
|
|
params = {'max_depth': 2, 'eta': 1, 'verbosity': 0,
|
|
'objective': 'binary:logistic', 'eval_metric': 'error'}
|
|
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10)
|
|
assert isinstance(cv, pd.DataFrame)
|
|
exp = pd.Index([u'test-error-mean', u'test-error-std',
|
|
u'train-error-mean', u'train-error-std'])
|
|
assert len(cv.columns.intersection(exp)) == 4
|
|
|
|
# show progress log (result is the same as above)
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10,
|
|
verbose_eval=True)
|
|
assert isinstance(cv, pd.DataFrame)
|
|
exp = pd.Index([u'test-error-mean', u'test-error-std',
|
|
u'train-error-mean', u'train-error-std'])
|
|
assert len(cv.columns.intersection(exp)) == 4
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10,
|
|
verbose_eval=True, show_stdv=False)
|
|
assert isinstance(cv, pd.DataFrame)
|
|
exp = pd.Index([u'test-error-mean', u'test-error-std',
|
|
u'train-error-mean', u'train-error-std'])
|
|
assert len(cv.columns.intersection(exp)) == 4
|
|
|
|
params = {'max_depth': 2, 'eta': 1, 'verbosity': 0,
|
|
'objective': 'binary:logistic', 'eval_metric': 'auc'}
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10, as_pandas=True)
|
|
assert 'eval_metric' in params
|
|
assert 'auc' in cv.columns[0]
|
|
|
|
params = {'max_depth': 2, 'eta': 1, 'verbosity': 0,
|
|
'objective': 'binary:logistic', 'eval_metric': ['auc']}
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10, as_pandas=True)
|
|
assert 'eval_metric' in params
|
|
assert 'auc' in cv.columns[0]
|
|
|
|
params = {'max_depth': 2, 'eta': 1, 'verbosity': 0,
|
|
'objective': 'binary:logistic', 'eval_metric': ['auc']}
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10,
|
|
as_pandas=True, early_stopping_rounds=1)
|
|
assert 'eval_metric' in params
|
|
assert 'auc' in cv.columns[0]
|
|
assert cv.shape[0] < 10
|
|
|
|
params = {'max_depth': 2, 'eta': 1, 'verbosity': 0,
|
|
'objective': 'binary:logistic'}
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10,
|
|
as_pandas=True, metrics='auc')
|
|
assert 'auc' in cv.columns[0]
|
|
|
|
params = {'max_depth': 2, 'eta': 1, 'verbosity': 0,
|
|
'objective': 'binary:logistic'}
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10,
|
|
as_pandas=True, metrics=['auc'])
|
|
assert 'auc' in cv.columns[0]
|
|
|
|
params = {'max_depth': 2, 'eta': 1, 'verbosity': 0,
|
|
'objective': 'binary:logistic', 'eval_metric': ['auc']}
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10,
|
|
as_pandas=True, metrics='error')
|
|
assert 'eval_metric' in params
|
|
assert 'auc' not in cv.columns[0]
|
|
assert 'error' in cv.columns[0]
|
|
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10,
|
|
as_pandas=True, metrics=['error'])
|
|
assert 'eval_metric' in params
|
|
assert 'auc' not in cv.columns[0]
|
|
assert 'error' in cv.columns[0]
|
|
|
|
params = list(params.items())
|
|
cv = xgb.cv(params, dm, num_boost_round=10, nfold=10,
|
|
as_pandas=True, metrics=['error'])
|
|
assert isinstance(params, list)
|
|
assert 'auc' not in cv.columns[0]
|
|
assert 'error' in cv.columns[0]
|
|
|
|
def test_nullable_type(self):
|
|
y = np.random.default_rng(0).random(4)
|
|
|
|
def to_bytes(Xy: xgb.DMatrix) -> bytes:
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
path = os.path.join(tmpdir, "Xy.dmatrix")
|
|
Xy.save_binary(path)
|
|
with open(path, "rb") as fd:
|
|
result = fd.read()
|
|
return result
|
|
|
|
def test_int(dtype) -> bytes:
|
|
arr = pd.DataFrame(
|
|
{"f0": [1, 2, None, 3], "f1": [4, 3, None, 1]}, dtype=dtype
|
|
)
|
|
Xy = xgb.DMatrix(arr, y)
|
|
Xy.feature_types = None
|
|
return to_bytes(Xy)
|
|
|
|
b0 = test_int(np.float32)
|
|
b1 = test_int(pd.Int16Dtype())
|
|
assert b0 == b1
|
|
|
|
def test_bool(dtype) -> bytes:
|
|
arr = pd.DataFrame(
|
|
{"f0": [True, False, None, True], "f1": [False, True, None, True]},
|
|
dtype=dtype,
|
|
)
|
|
Xy = xgb.DMatrix(arr, y)
|
|
Xy.feature_types = None
|
|
return to_bytes(Xy)
|
|
|
|
b0 = test_bool(pd.BooleanDtype())
|
|
b1 = test_bool(bool)
|
|
assert b0 != b1 # None is converted to False with np.bool
|
|
|
|
data = {"f0": [1.0, 2.0, None, 3.0], "f1": [3.0, 2.0, None, 1.0]}
|
|
|
|
arr = np.array([data["f0"], data["f1"]]).T
|
|
Xy = xgb.DMatrix(arr, y)
|
|
Xy.feature_types = None
|
|
Xy.feature_names = None
|
|
from_np = to_bytes(Xy)
|
|
|
|
def test_float(dtype) -> bytes:
|
|
arr = pd.DataFrame(data, dtype=dtype)
|
|
Xy = xgb.DMatrix(arr, y)
|
|
Xy.feature_types = None
|
|
Xy.feature_names = None
|
|
return to_bytes(Xy)
|
|
|
|
b0 = test_float(pd.Float64Dtype())
|
|
b1 = test_float(float)
|
|
assert b0 == b1 # both are converted to NaN
|
|
assert b0 == from_np
|
|
|
|
def test_cat(dtype) -> bytes:
|
|
arr = pd.DataFrame(data, dtype=dtype)
|
|
if dtype is None:
|
|
arr = arr.astype("category")
|
|
Xy = xgb.DMatrix(arr, y, enable_categorical=True)
|
|
Xy.feature_types = None
|
|
return to_bytes(Xy)
|
|
|
|
b0 = test_cat(pd.CategoricalDtype())
|
|
b1 = test_cat(None)
|
|
assert b0 == b1
|