From 739b3f2c5f394ce7a7c848795eaaa1fd61c39795 Mon Sep 17 00:00:00 2001 From: phunterlau Date: Sun, 1 Nov 2015 22:11:11 -0800 Subject: [PATCH] separate setup.py with pip installation, add trouble shooting page --- python-package/build_trouble_shooting.md | 48 ++++++++++++++++++++ python-package/setup.py | 25 +--------- python-package/setup_pip.py | 58 ++++++++++++++++++++++++ python-package/xgboost/libpath.py | 3 +- 4 files changed, 110 insertions(+), 24 deletions(-) create mode 100644 python-package/build_trouble_shooting.md create mode 100644 python-package/setup_pip.py diff --git a/python-package/build_trouble_shooting.md b/python-package/build_trouble_shooting.md new file mode 100644 index 000000000..504575514 --- /dev/null +++ b/python-package/build_trouble_shooting.md @@ -0,0 +1,48 @@ +XGBoost Python Package Troubleshooting +====================== +Windows platform +------------ +The current best solution for installing xgboost on windows machine is building from github. Please go to [windows](/windows/), build with the Visual Studio project file, and install. Additional detailed instruction can be found at this [installation tutorial](https://www.kaggle.com/c/otto-group-product-classification-challenge/forums/t/13043/run-xgboost-from-windows-and-python) from Kaggle Otto Forum. + +`pip install xgboost` is **not** tested nor supported in windows platform for now. + +Linux platform (also Mac OS X in general) +------------ +**Trouble 0**: I see error messages like this when install from github using `python setup.py install`. + + XGBoostLibraryNotFound: Cannot find XGBoost Libarary in the candicate path, did you install compilers and run build.sh in root path? + List of candidates: + /home/dmlc/anaconda/lib/python2.7/site-packages/xgboost-0.4-py2.7.egg/xgboost/libxgboostwrapper.so + /home/dmlc/anaconda/lib/python2.7/site-packages/xgboost-0.4-py2.7.egg/xgboost/../../wrapper/libxgboostwrapper.so + /home/dmlc/anaconda/lib/python2.7/site-packages/xgboost-0.4-py2.7.egg/xgboost/./wrapper/libxgboostwrapper.so + +**Solution 0**: Please check if you have: + +* installed C++ compilers, for example `g++` and `gcc` (Linux) or `clang LLVM` (Mac OS X). Recommended compilers are `g++-5` or newer (Linux and Mac), or `clang` comes with Xcode in Mac OS X. For installting compilers, please refer to your system package management commands, e.g. `apt-get` `yum` or `brew`(Mac). +* compilers in your `$PATH`. Try typing `gcc` and see if your have it in your path. + +**Trouble 1**: I see the same error message in **Trouble 0** when install from `pip install xgboost`. + +**Solution 1**: the problem is the same as in **Trouble 0**, please see **Solution 0**. + +**Trouble 2**: I see this error message when `pip install xgboost`. It says I have `libxgboostwrapper.so` but it is not valid. + + OSError: /home/dmlc/anaconda/lib/python2.7/site-packages/xgboost/./wrapper/libxgboostwrapper.so: invalid ELF header + +**Solution 2**: Solution is as in 0 and 1 by installing `g++` compiler. The reason for this rare error is that, `pip` ships with a pre-compiled `libxgboostwrapper.so` with Mac for placeholder for allowing `setup.py` to find the right lib path. If a system doesn't compile, it may refer to this placeholder lib and fail. This placeholder `libxgboostwrapper.so` will be automatically removed and correctly generated by the compiling on-the-fly for the system. + +**Trouble 3**: My system's `pip` says it can't find a valid `xgboost` installation release on `PyPI`. +**Solution 3**: Some linux system comes with an old `pip` version. Please update to the latest `pip` by following the official installation document at + +**Trouble 4**: I tried `python setup.py install` but it says `setuptools` import fail. +**Solution 4**: Please make sure you have [setuptools](https://pypi.python.org/pypi/setuptools) before installing the python package. + +Mac OS X (specific) +------------ +Most of the troubles and solutions are the same with that in the Linux platform. Mac has the following specific problems. + +**Trouble 0**: I successfully installed `xgboost` using github installation/using `pip install xgboost`. But it runs very slow with only single thread, what is going on? +**Solution 0**: `clang LLVM` compiler on Mac OS X from Xcode doesn't support OpenMP multi-thread. An alternative choice is installing `homebrew` and `brew install g++-5` which provides multi-thread OpenMP support. + +**Trouble 1**: Can I install `clang-omp` for supporting OpenMP without using `gcc`? +**Solution 1**: it is not support and may have linking errors. \ No newline at end of file diff --git a/python-package/setup.py b/python-package/setup.py index 470fe681a..f266e7fb2 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -7,19 +7,6 @@ from setuptools import setup, find_packages #import subprocess sys.path.insert(0, '.') -#build on the fly if install in pip -#otherwise, use build.sh in the parent directory - -#ugly solution since pip version transition and the old pip detection method not -#working. Manually turn on when packing up for pip installation -if False: - if not os.name == 'nt': #if not windows - os.system('sh ./xgboost/build-python.sh') - else: - print('Windows users please use github installation.') - sys.exit() - - CURRENT_DIR = os.path.dirname(__file__) # We can not import `xgboost.libpath` in setup.py directly since xgboost/__init__.py @@ -31,10 +18,8 @@ exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpat LIB_PATH = libpath['find_lib_path']() -#to deploy to pip, please use -#make pythonpack -#python setup.py register sdist upload -#and be sure to test it firstly using "python setup.py register sdist upload -r pypitest" +#Please use setup_pip.py for generating and deploying pip installation +#detailed instruction in setup_pip.py setup(name='xgboost', version=open(os.path.join(CURRENT_DIR, 'xgboost/VERSION')).read().strip(), #version='0.4a23', @@ -47,14 +32,8 @@ setup(name='xgboost', maintainer_email='phunter.lau@gmail.com', zip_safe=False, packages=find_packages(), - #don't need this and don't use this, give everything to MANIFEST.in - #package_dir = {'':'xgboost'}, - #package_data = {'': ['*.txt','*.md','*.sh'], - # } #this will use MANIFEST.in during install where we specify additional files, #this is the golden line include_package_data=True, - #!!! don't use data_files, otherwise install_data process will copy it to - #root directory for some machines, and cause confusions on building data_files=[('xgboost', LIB_PATH)], url='https://github.com/dmlc/xgboost') diff --git a/python-package/setup_pip.py b/python-package/setup_pip.py new file mode 100644 index 000000000..83d907c25 --- /dev/null +++ b/python-package/setup_pip.py @@ -0,0 +1,58 @@ +# pylint: disable=invalid-name, exec-used +"""Setup xgboost package.""" +from __future__ import absolute_import +import sys +import os +from setuptools import setup, find_packages +#import subprocess +sys.path.insert(0, '.') + +#this script is for packing and shipping pip installation +#it builds xgboost code on the fly and packs for pip +#please don't use this file for installing from github + +if not os.name == 'nt': #if not windows, compile and install + os.system('sh ./xgboost/build-python.sh') +else: + print('Windows users please use github installation.') + sys.exit() + +CURRENT_DIR = os.path.dirname(__file__) + +# We can not import `xgboost.libpath` in setup.py directly since xgboost/__init__.py +# import `xgboost.core` and finally will import `numpy` and `scipy` which are setup +# `install_requires`. That's why we're using `exec` here. +libpath_py = os.path.join(CURRENT_DIR, 'xgboost/libpath.py') +libpath = {'__file__': libpath_py} +exec(compile(open(libpath_py, "rb").read(), libpath_py, 'exec'), libpath, libpath) + +LIB_PATH = libpath['find_lib_path']() + +#to deploy to pip, please use +#make pythonpack +#python setup.py register sdist upload +#and be sure to test it firstly using "python setup.py register sdist upload -r pypitest" +setup(name='xgboost', + #version=open(os.path.join(CURRENT_DIR, 'xgboost/VERSION')).read().strip(), + version='0.4a24', + description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), + install_requires=[ + 'numpy', + 'scipy', + ], + maintainer='Hongliang Liu', + maintainer_email='phunter.lau@gmail.com', + zip_safe=False, + packages=find_packages(), + #don't need this and don't use this, give everything to MANIFEST.in + #package_dir = {'':'xgboost'}, + #package_data = {'': ['*.txt','*.md','*.sh'], + # } + #this will use MANIFEST.in during install where we specify additional files, + #this is the golden line + include_package_data=True, + #!!! don't use data_files for creating pip installation, + #otherwise install_data process will copy it to + #root directory for some machines, and cause confusions on building + #data_files=[('xgboost', LIB_PATH)], + url='https://github.com/dmlc/xgboost') diff --git a/python-package/xgboost/libpath.py b/python-package/xgboost/libpath.py index 293719f01..5df72dd3d 100644 --- a/python-package/xgboost/libpath.py +++ b/python-package/xgboost/libpath.py @@ -36,9 +36,10 @@ def find_lib_path(): else: dll_path = [os.path.join(p, 'libxgboostwrapper.so') for p in dll_path] lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)] + #From github issues, most of installation errors come from machines w/o compilers if len(lib_path) == 0 and not os.environ.get('XGBOOST_BUILD_DOC', False): raise XGBoostLibraryNotFound( 'Cannot find XGBoost Libarary in the candicate path, ' + - 'did you run build.sh in root path?\n' + 'did you install compilers and run build.sh in root path?\n' 'List of candidates:\n' + ('\n'.join(dll_path))) return lib_path