PyPI (pip installation) setup for 0.6 code (#1445)

* force gcc-5 or clang-omp for Mac OS, prepare for pip pack

* add sklearn dep, make -j4

* finalize PyPI submission

* revert to Xcode clang for passing build #1468

* force to clang, try to solve cmake travis error

* remove sklearn dependency

Makefile

@@ -39,6 +39,14 @@ ifndef CXX
 export CXX = $(if $(shell which g++-5),g++-5,g++)
 endif
 
+# on Mac OS X, force brew gcc-5, since the Xcode c++ fails anyway
+# it is useful for pip install compiling-on-the-fly
+OS := $(shell uname)
+ifeq ($(OS), Darwin)
+export CC = $(if $(shell which gcc-5),gcc-5,clang)
+export CXX = $(if $(shell which g++-5),g++-5,clang++)
+endif
+
 export LDFLAGS= -pthread -lm $(ADD_LDFLAGS) $(DMLC_LDFLAGS) $(PLUGIN_LDFLAGS)
 export CFLAGS=  -std=c++0x -Wall -O3 -msse2  -Wno-unknown-pragmas -funroll-loops -Iinclude $(ADD_CFLAGS) $(PLUGIN_CFLAGS)
 CFLAGS += -I$(DMLC_CORE)/include -I$(RABIT)/include
@@ -151,6 +159,18 @@ pypack: ${XGBOOST_DYLIB}
 	cp ${XGBOOST_DYLIB} python-package/xgboost
 	cd python-package; tar cf xgboost.tar xgboost; cd ..
 
+# create pip installation pack for PyPI
+pippack:
+	$(MAKE) clean_all
+	rm -rf xgboost-python
+	cp -r python-package xgboost-python
+	cp -r Makefile xgboost-python/xgboost/
+	cp -r make xgboost-python/xgboost/
+	cp -r src xgboost-python/xgboost/
+	cp -r include xgboost-python/xgboost/
+	cp -r dmlc-core xgboost-python/xgboost/
+	cp -r rabit xgboost-python/xgboost/
+
 # Script to make a clean installable R package.
 Rpack:
 	$(MAKE) clean_all

python-package/MANIFEST.in

@@ -2,10 +2,15 @@ include *.md *.rst
 recursive-include xgboost *
 recursive-include xgboost/include *
 recursive-include xgboost/src *
-#exclude pre-compiled .o file for less confusions
+recursive-include xgboost/make *
+recursive-include xgboost/rabit *
+recursive-include xgboost/lib *
+recursive-include xgboost/dmlc-core *
+#exclude pre-compiled .o and .a file for less confusions
+#make sure .a files are all removed for forcing compiling
 #include the pre-compiled .so is needed as a placeholder
 #since it will be copy after compiling on the fly
-global-exclude xgboost/build/*
-global-exclude xgboost/*.o
+global-exclude *.o
+global-exclude *.a
 global-exclude *.pyo
 global-exclude *.pyc

python-package/README.rst

@@ -10,6 +10,13 @@ We are on `PyPI <https://pypi.python.org/pypi/xgboost>`__ now. For
 stable version, please install using pip:
 
 -  ``pip install xgboost``
+-  Since this package contains C++ source code, ``pip`` needs a C++ compiler from the system
+   to compile the source code on-the-fly. Please follow the following instruction for each
+   supported platform.
+-  Note for Mac OS X users: please install ``gcc`` from ``brew`` by 
+   ``brew tap homebrew/versions; brew install gcc --without-multilib`` firstly.
+-  Note for Linux users: please install ``gcc`` by ``sudo apt-get install build-essential`` firstly
+   or using the corresponding package manager of the system.
 -  Note for windows users: this pip installation may not work on some
    windows environment, and it may cause unexpected errors. pip
    installation on windows is currently disabled for further
@@ -37,10 +44,10 @@ Examples
 --------
 
 -  Refer also to the walk through example in `demo
-   folder <../demo/guide-python>`__
--  See also the `example scripts <../demo/kaggle-higgs>`__ for Kaggle
+   folder <https://github.com/dmlc/xgboost/tree/master/demo/guide-python>`__
+-  See also the `example scripts <https://github.com/dmlc/xgboost/tree/master/demo/kaggle-higgs>`__ for Kaggle
    Higgs Challenge, including `speedtest
-   script <../demo/kaggle-higgs/speedtest.py>`__ on this dataset.
+   script <https://github.com/dmlc/xgboost/tree/master/demo/kaggle-higgs/speedtest.py>`__ on this dataset.
 
 Note
 ----
@@ -48,7 +55,7 @@ Note
 -  If you want to build xgboost on Mac OS X with multiprocessing support
    where clang in XCode by default doesn't support, please install gcc
    4.9 or higher using `homebrew <http://brew.sh/>`__
-   ``brew tap homebrew/versions; brew install gcc49``
+   ``brew tap homebrew/versions; brew install gcc --without-multilib``
 -  If you want to run XGBoost process in parallel using the fork backend
    for joblib/multiprocessing, you must build XGBoost without support
    for OpenMP by ``make no_omp=1``. Otherwise, use the forkserver (in

python-package/prep_pip.sh

@@ -0,0 +1,11 @@
+# this script is for preparation for PyPI installation package, 
+# please don't use it for installing xgboost from github
+
+# after executing `make pippack`, cd xgboost-python,
+#run this script and get the sdist tar.gz in ./dist/
+sh ./xgboost/build-python.sh
+cp setup_pip.py setup.py
+python setup.py sdist
+
+#make sure you know what you gonna do, and uncomment the following line
+#python setup.py register upload

python-package/setup_pip.py

@@ -34,8 +34,8 @@ LIB_PATH = libpath['find_lib_path']()
 # and be sure to test it firstly using "python setup.py register sdist upload -r pypitest"
 setup(name='xgboost',
       # version=open(os.path.join(CURRENT_DIR, 'xgboost/VERSION')).read().strip(),
-      version='0.4a30',
-      description=open(os.path.join(CURRENT_DIR, 'README.rst')).read(),
+      version='0.6a2',
+      description='XGBoost Python Package',
       install_requires=[
           'numpy',
           'scipy',

python-package/xgboost/build-python.sh

@@ -15,14 +15,14 @@ oldpath=`pwd`
 cd ./xgboost/
 #remove the pre-compiled .so and trigger the system's on-the-fly compiling
 make clean
-if make python; then
+if make lib/libxgboost.so -j4; then
     echo "Successfully build multi-thread xgboost"
 else
     echo "-----------------------------"
     echo "Building multi-thread xgboost failed"
     echo "Start to build single-thread xgboost"
     make clean
-    make python no_omp=1
+    make lib/libxgboost.so -j4 no_omp=1
     echo "Successfully build single-thread xgboost"
     echo "If you want multi-threaded version"
     echo "See additional instructions in doc/build.md"