Hendrik/xgboost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support all pandas nullable integer types. (#8480)

Browse Source 
- Enumerate all pandas integer types.
- Tests for `None`, `nan`, and `pd.NA`
This commit is contained in:
Jiaming Yuan
2022-11-28 22:38:16 +08:00
committed by GitHub
parent f2209c1fe4
commit d666ba775e
7 changed files with 145 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python-package/xgboost/core.py
View File

@@ -1068,7 +1068,11 @@ class DMatrix: # pylint: disable=too-many-instance-attributes,too-many-public-m
return ret.value
def num_nonmissing(self) -> int:
"""Get the number of non-missing values in the DMatrix."""
"""Get the number of non-missing values in the DMatrix.
.. versionadded:: 1.7.0
"""
ret = c_bst_ulong()
_check_call(_LIB.XGDMatrixNumNonMissing(self.handle, ctypes.byref(ret)))
return ret.value

83
python-package/xgboost/data.py
View File

@@ -34,7 +34,8 @@ from .core import (
)
DispatchedDataBackendReturnType = Tuple[
ctypes.c_void_p, Optional[FeatureNames], Optional[FeatureTypes]]
ctypes.c_void_p, Optional[FeatureNames], Optional[FeatureTypes]
]
CAT_T = "c"
@@ -217,27 +218,36 @@ def _is_modin_df(data: DataType) -> bool:
_pandas_dtype_mapper = {
'int8': 'int',
'int16': 'int',
'int32': 'int',
'int64': 'int',
'uint8': 'int',
'uint16': 'int',
'uint32': 'int',
'uint64': 'int',
'float16': 'float',
'float32': 'float',
'float64': 'float',
'bool': 'i',
# nullable types
"int8": "int",
"int16": "int",
"int32": "int",
"int64": "int",
"uint8": "int",
"uint16": "int",
"uint32": "int",
"uint64": "int",
"float16": "float",
"float32": "float",
"float64": "float",
"bool": "i",
}
# nullable types
pandas_nullable_mapper = {
"Int8": "int",
"Int16": "int",
"Int32": "int",
"Int64": "int",
"UInt8": "i",
"UInt16": "i",
"UInt32": "i",
"UInt64": "i",
"Float32": "float",
"Float64": "float",
"boolean": "i",
}
_pandas_dtype_mapper.update(pandas_nullable_mapper)
_ENABLE_CAT_ERR = (
"When categorical type is supplied, The experimental DMatrix parameter"
@@ -304,27 +314,27 @@ def is_nullable_dtype(dtype: PandasDType) -> bool:
is_integer_dtype,
)
# dtype: pd.core.arrays.numeric.NumericDtype
nullable_alias = {"Int16", "Int32", "Int64", "Float32", "Float64", "category"}
is_int = is_integer_dtype(dtype) and dtype.name in nullable_alias
is_int = is_integer_dtype(dtype) and dtype.name in pandas_nullable_mapper
# np.bool has alias `bool`, while pd.BooleanDtype has `bzoolean`.
is_bool = is_bool_dtype(dtype) and dtype.name == "boolean"
is_float = is_float_dtype(dtype) and dtype.name in nullable_alias
is_float = is_float_dtype(dtype) and dtype.name in pandas_nullable_mapper
return is_int or is_bool or is_float or is_categorical_dtype(dtype)
def _pandas_cat_null(data: DataFrame) -> DataFrame:
def pandas_cat_null(data: DataFrame) -> DataFrame:
"""Handle categorical dtype and nullable extension types from pandas."""
from pandas.api.types import is_categorical_dtype
# handle category codes and nullable.
cat_columns = [
col
for col, dtype in zip(data.columns, data.dtypes)
if is_categorical_dtype(dtype)
]
nul_columns = [
col for col, dtype in zip(data.columns, data.dtypes) if is_nullable_dtype(dtype)
]
cat_columns = []
nul_columns = []
for col, dtype in zip(data.columns, data.dtypes):
if is_categorical_dtype(dtype):
cat_columns.append(col)
# avoid an unnecessary conversion if possible
elif is_nullable_dtype(dtype):
nul_columns.append(col)
if cat_columns or nul_columns:
# Avoid transformation due to: PerformanceWarning: DataFrame is highly
# fragmented
@@ -333,7 +343,7 @@ def _pandas_cat_null(data: DataFrame) -> DataFrame:
transformed = data
if cat_columns:
# DF doesn't have the cat attribute, so we use apply here
# DF doesn't have the cat attribute, as a result, we use apply here
transformed[cat_columns] = (
transformed[cat_columns]
.apply(lambda x: x.cat.codes)
@@ -343,6 +353,10 @@ def _pandas_cat_null(data: DataFrame) -> DataFrame:
if nul_columns:
transformed[nul_columns] = transformed[nul_columns].astype(np.float32)
# TODO(jiamingy): Investigate the possibility of using dataframe protocol or arrow
# IPC format for pandas so that we can apply the data transformation inside XGBoost
# for better memory efficiency.
return transformed
@@ -357,9 +371,8 @@ def _transform_pandas_df(
from pandas.api.types import is_categorical_dtype, is_sparse
if not all(
dtype.name in _pandas_dtype_mapper
(dtype.name in _pandas_dtype_mapper)
or is_sparse(dtype)
or (is_nullable_dtype(dtype) and not is_categorical_dtype(dtype))
or (is_categorical_dtype(dtype) and enable_categorical)
for dtype in data.dtypes
):
@@ -369,7 +382,7 @@ def _transform_pandas_df(
data, meta, feature_names, feature_types, enable_categorical
)
transformed = _pandas_cat_null(data)
transformed = pandas_cat_null(data)
if meta and len(data.columns) > 1 and meta not in _matrix_meta:
raise ValueError(f"DataFrame for {meta} cannot have multiple columns")
@@ -404,14 +417,12 @@ def _is_pandas_series(data: DataType) -> bool:
def _meta_from_pandas_series(
data: DataType,
name: str,
dtype: Optional[NumpyDType],
handle: ctypes.c_void_p
data: DataType, name: str, dtype: Optional[NumpyDType], handle: ctypes.c_void_p
) -> None:
"""Help transform pandas series for meta data like labels"""
data = data.values.astype('float')
data = data.values.astype("float")
from pandas.api.types import is_sparse
if is_sparse(data):
data = data.to_dense() # type: ignore
assert len(data.shape) == 1 or data.shape[1] == 0 or data.shape[1] == 1

13
python-package/xgboost/testing/__init__.py
View File

@@ -773,6 +773,19 @@ def non_increasing(L: Sequence[float], tolerance: float = 1e-4) -> bool:
return all((y - x) < tolerance for x, y in zip(L, L[1:]))
def predictor_equal(lhs: xgb.DMatrix, rhs: xgb.DMatrix) -> bool:
"""Assert whether two DMatrices contain the same predictors."""
lcsr = lhs.get_data()
rcsr = rhs.get_data()
return all(
(
np.array_equal(lcsr.data, rcsr.data),
np.array_equal(lcsr.indices, rcsr.indices),
np.array_equal(lcsr.indptr, rcsr.indptr),
)
)
def eval_error_metric(predt: np.ndarray, dtrain: xgb.DMatrix) -> Tuple[str, np.float64]:
"""Evaluation metric for xgb.train"""
label = dtrain.get_label()

62
python-package/xgboost/testing/data.py
View File

@@ -1,5 +1,5 @@
"""Utilities for data generation."""
from typing import Generator, Tuple
from typing import Any, Generator, Tuple, Union
import numpy as np
@@ -7,7 +7,7 @@ import numpy as np
def np_dtypes(
n_samples: int, n_features: int
) -> Generator[Tuple[np.ndarray, np.ndarray], None, None]:
"""Generate all supported dtypes from numpy."""
"""Enumerate all supported dtypes from numpy."""
import pandas as pd
rng = np.random.RandomState(1994)
@@ -60,3 +60,61 @@ def np_dtypes(
df_orig = pd.DataFrame(orig)
df = pd.DataFrame(X)
yield df_orig, df
def pd_dtypes() -> Generator:
"""Enumerate all supported pandas extension types."""
import pandas as pd
# Integer
dtypes = [
pd.UInt8Dtype(),
pd.UInt16Dtype(),
pd.UInt32Dtype(),
pd.UInt64Dtype(),
pd.Int8Dtype(),
pd.Int16Dtype(),
pd.Int32Dtype(),
pd.Int64Dtype(),
]
Null: Union[float, None, Any] = np.nan
orig = pd.DataFrame(
{"f0": [1, 2, Null, 3], "f1": [4, 3, Null, 1]}, dtype=np.float32
)
for Null in (np.nan, None, pd.NA):
for dtype in dtypes:
df = pd.DataFrame(
{"f0": [1, 2, Null, 3], "f1": [4, 3, Null, 1]}, dtype=dtype
)
yield orig, df
# Float
Null = np.nan
dtypes = [pd.Float32Dtype(), pd.Float64Dtype()]
orig = pd.DataFrame(
{"f0": [1.0, 2.0, Null, 3.0], "f1": [3.0, 2.0, Null, 1.0]}, dtype=np.float32
)
for Null in (np.nan, None, pd.NA):
for dtype in dtypes:
df = pd.DataFrame(
{"f0": [1.0, 2.0, Null, 3.0], "f1": [3.0, 2.0, Null, 1.0]}, dtype=dtype
)
yield orig, df
# Categorical
orig = orig.astype("category")
for Null in (np.nan, None, pd.NA):
df = pd.DataFrame(
{"f0": [1.0, 2.0, Null, 3.0], "f1": [3.0, 2.0, Null, 1.0]},
dtype=pd.CategoricalDtype(),
)
yield orig, df
# Boolean
for Null in [None, pd.NA]:
data = {"f0": [True, False, Null, True], "f1": [False, True, Null, True]}
# pd.NA is not convertible to bool.
orig = pd.DataFrame(data, dtype=np.bool_ if Null is None else pd.BooleanDtype())
df = pd.DataFrame(data, dtype=pd.BooleanDtype())
yield orig, df