From c5a2b795580b2487f66d55cda18a89126dfb4acc Mon Sep 17 00:00:00 2001 From: Hongliang Liu Date: Wed, 10 Aug 2016 05:45:56 -0700 Subject: [PATCH] PyPI (pip installation) setup for 0.6 code (#1445) * force gcc-5 or clang-omp for Mac OS, prepare for pip pack * add sklearn dep, make -j4 * finalize PyPI submission * revert to Xcode clang for passing build #1468 * force to clang, try to solve cmake travis error * remove sklearn dependency --- Makefile | 20 ++++++++++++++++++++ python-package/MANIFEST.in | 11 ++++++++--- python-package/README.rst | 15 +++++++++++---- python-package/prep_pip.sh | 11 +++++++++++ python-package/setup_pip.py | 4 ++-- python-package/xgboost/build-python.sh | 4 ++-- 6 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 python-package/prep_pip.sh diff --git a/Makefile b/Makefile index abe8ccfaa..ac551974d 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,14 @@ ifndef CXX export CXX = $(if $(shell which g++-5),g++-5,g++) endif +# on Mac OS X, force brew gcc-5, since the Xcode c++ fails anyway +# it is useful for pip install compiling-on-the-fly +OS := $(shell uname) +ifeq ($(OS), Darwin) +export CC = $(if $(shell which gcc-5),gcc-5,clang) +export CXX = $(if $(shell which g++-5),g++-5,clang++) +endif + export LDFLAGS= -pthread -lm $(ADD_LDFLAGS) $(DMLC_LDFLAGS) $(PLUGIN_LDFLAGS) export CFLAGS= -std=c++0x -Wall -O3 -msse2 -Wno-unknown-pragmas -funroll-loops -Iinclude $(ADD_CFLAGS) $(PLUGIN_CFLAGS) CFLAGS += -I$(DMLC_CORE)/include -I$(RABIT)/include @@ -151,6 +159,18 @@ pypack: ${XGBOOST_DYLIB} cp ${XGBOOST_DYLIB} python-package/xgboost cd python-package; tar cf xgboost.tar xgboost; cd .. +# create pip installation pack for PyPI +pippack: + $(MAKE) clean_all + rm -rf xgboost-python + cp -r python-package xgboost-python + cp -r Makefile xgboost-python/xgboost/ + cp -r make xgboost-python/xgboost/ + cp -r src xgboost-python/xgboost/ + cp -r include xgboost-python/xgboost/ + cp -r dmlc-core xgboost-python/xgboost/ + cp -r rabit xgboost-python/xgboost/ + # Script to make a clean installable R package. Rpack: $(MAKE) clean_all diff --git a/python-package/MANIFEST.in b/python-package/MANIFEST.in index 04ca379a6..38b1a77a4 100644 --- a/python-package/MANIFEST.in +++ b/python-package/MANIFEST.in @@ -2,10 +2,15 @@ include *.md *.rst recursive-include xgboost * recursive-include xgboost/include * recursive-include xgboost/src * -#exclude pre-compiled .o file for less confusions +recursive-include xgboost/make * +recursive-include xgboost/rabit * +recursive-include xgboost/lib * +recursive-include xgboost/dmlc-core * +#exclude pre-compiled .o and .a file for less confusions +#make sure .a files are all removed for forcing compiling #include the pre-compiled .so is needed as a placeholder #since it will be copy after compiling on the fly -global-exclude xgboost/build/* -global-exclude xgboost/*.o +global-exclude *.o +global-exclude *.a global-exclude *.pyo global-exclude *.pyc diff --git a/python-package/README.rst b/python-package/README.rst index 5b0448e38..a92ac0b8b 100644 --- a/python-package/README.rst +++ b/python-package/README.rst @@ -10,6 +10,13 @@ We are on `PyPI `__ now. For stable version, please install using pip: - ``pip install xgboost`` +- Since this package contains C++ source code, ``pip`` needs a C++ compiler from the system + to compile the source code on-the-fly. Please follow the following instruction for each + supported platform. +- Note for Mac OS X users: please install ``gcc`` from ``brew`` by + ``brew tap homebrew/versions; brew install gcc --without-multilib`` firstly. +- Note for Linux users: please install ``gcc`` by ``sudo apt-get install build-essential`` firstly + or using the corresponding package manager of the system. - Note for windows users: this pip installation may not work on some windows environment, and it may cause unexpected errors. pip installation on windows is currently disabled for further @@ -37,10 +44,10 @@ Examples -------- - Refer also to the walk through example in `demo - folder <../demo/guide-python>`__ -- See also the `example scripts <../demo/kaggle-higgs>`__ for Kaggle + folder `__ +- See also the `example scripts `__ for Kaggle Higgs Challenge, including `speedtest - script <../demo/kaggle-higgs/speedtest.py>`__ on this dataset. + script `__ on this dataset. Note ---- @@ -48,7 +55,7 @@ Note - If you want to build xgboost on Mac OS X with multiprocessing support where clang in XCode by default doesn't support, please install gcc 4.9 or higher using `homebrew `__ - ``brew tap homebrew/versions; brew install gcc49`` + ``brew tap homebrew/versions; brew install gcc --without-multilib`` - If you want to run XGBoost process in parallel using the fork backend for joblib/multiprocessing, you must build XGBoost without support for OpenMP by ``make no_omp=1``. Otherwise, use the forkserver (in diff --git a/python-package/prep_pip.sh b/python-package/prep_pip.sh new file mode 100644 index 000000000..b5b98e00a --- /dev/null +++ b/python-package/prep_pip.sh @@ -0,0 +1,11 @@ +# this script is for preparation for PyPI installation package, +# please don't use it for installing xgboost from github + +# after executing `make pippack`, cd xgboost-python, +#run this script and get the sdist tar.gz in ./dist/ +sh ./xgboost/build-python.sh +cp setup_pip.py setup.py +python setup.py sdist + +#make sure you know what you gonna do, and uncomment the following line +#python setup.py register upload diff --git a/python-package/setup_pip.py b/python-package/setup_pip.py index b5aae4d0c..f5e475bb5 100644 --- a/python-package/setup_pip.py +++ b/python-package/setup_pip.py @@ -34,8 +34,8 @@ LIB_PATH = libpath['find_lib_path']() # and be sure to test it firstly using "python setup.py register sdist upload -r pypitest" setup(name='xgboost', # version=open(os.path.join(CURRENT_DIR, 'xgboost/VERSION')).read().strip(), - version='0.4a30', - description=open(os.path.join(CURRENT_DIR, 'README.rst')).read(), + version='0.6a2', + description='XGBoost Python Package', install_requires=[ 'numpy', 'scipy', diff --git a/python-package/xgboost/build-python.sh b/python-package/xgboost/build-python.sh index 4bec205a2..6baf2b823 100755 --- a/python-package/xgboost/build-python.sh +++ b/python-package/xgboost/build-python.sh @@ -15,14 +15,14 @@ oldpath=`pwd` cd ./xgboost/ #remove the pre-compiled .so and trigger the system's on-the-fly compiling make clean -if make python; then +if make lib/libxgboost.so -j4; then echo "Successfully build multi-thread xgboost" else echo "-----------------------------" echo "Building multi-thread xgboost failed" echo "Start to build single-thread xgboost" make clean - make python no_omp=1 + make lib/libxgboost.so -j4 no_omp=1 echo "Successfully build single-thread xgboost" echo "If you want multi-threaded version" echo "See additional instructions in doc/build.md"