Remove pyarrow workaround. (#7884)

This commit is contained in:
Jiaming Yuan 2022-05-11 20:54:48 +08:00 committed by GitHub
parent 65e6d73b95
commit 8ba4722d04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -2,7 +2,6 @@
# pylint: disable=too-many-return-statements, import-error
'''Data dispatching for DMatrix.'''
import ctypes
from distutils import version
import json
import warnings
import os
@ -590,13 +589,7 @@ def _from_arrow(
if enable_categorical:
raise ValueError("categorical data in arrow is not supported yet.")
major, _, _ = version.StrictVersion(pa.__version__).version
if major == 4:
rb_iter = iter(data.to_batches())
else:
# use_async=True to workaround pyarrow 6.0.1 hang,
# see Modin-3982 and ARROW-15362
rb_iter = iter(data.to_batches(use_async=True))
rb_iter = iter(data.to_batches())
it = record_batch_data_iter(rb_iter)
next_callback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p)(it)
handle = ctypes.c_void_p()