From 56a80f431b9146b98e54a648ce6ff95c1fbb3265 Mon Sep 17 00:00:00 2001 From: Tianqi Chen Date: Fri, 16 Jan 2015 20:56:34 -0800 Subject: [PATCH] check in windows solutions, pass small test in windows --- src/allreduce_robust.cc | 2 +- src/socket.h | 25 +++- tracker/rabit_demo.py | 26 ++-- tracker/rabit_tracker.py | 2 +- windows/.gitignore | 9 ++ windows/basic/basic.vcxproj | 117 +++++++++++++++++ windows/rabit.sln | 44 +++++++ windows/rabit/rabit.vcxproj | 132 ++++++++++++++++++++ windows/rabit_wrapper/rabit_wrapper.vcxproj | 121 ++++++++++++++++++ wrapper/rabit.py | 13 +- 10 files changed, 470 insertions(+), 21 deletions(-) create mode 100644 windows/.gitignore create mode 100644 windows/basic/basic.vcxproj create mode 100644 windows/rabit.sln create mode 100644 windows/rabit/rabit.vcxproj create mode 100644 windows/rabit_wrapper/rabit_wrapper.vcxproj diff --git a/src/allreduce_robust.cc b/src/allreduce_robust.cc index 33fdcd0f0..9b2fed03b 100644 --- a/src/allreduce_robust.cc +++ b/src/allreduce_robust.cc @@ -10,7 +10,7 @@ #define NOMINMAX #include #include -#include "rabit/io.h" +#include #include "rabit/utils.h" #include "rabit/engine.h" #include "rabit/rabit-inl.h" diff --git a/src/socket.h b/src/socket.h index c40cb6a88..c7ffd11ad 100644 --- a/src/socket.h +++ b/src/socket.h @@ -9,6 +9,9 @@ #if defined(_WIN32) #include #include +#ifdef _MSC_VER +#pragma comment(lib, "Ws2_32.lib") +#endif #else #include #include @@ -98,7 +101,7 @@ class Socket { inline static void Startup(void) { #ifdef _WIN32 WSADATA wsa_data; - if (WSAStartup(MAKEWORD(2, 2), &wsa_data) != -1) { + if (WSAStartup(MAKEWORD(2, 2), &wsa_data) == -1) { Socket::Error("Startup"); } if (LOBYTE(wsa_data.wVersion) != 2 || HIBYTE(wsa_data.wVersion) != 2) { @@ -165,17 +168,24 @@ class Socket { sizeof(addr.addr)) == 0) { return port; } - if (errno != EADDRINUSE) { +#if defined(_WIN32) + if (WSAGetLastError() != WSAEADDRINUSE) { + Socket::Error("TryBindHost"); + } +#else + if (errno != EADDRINUSE) { Socket::Error("TryBindHost"); } +#endif } + return -1; } /*! \brief get last error code if any */ inline int GetSockError(void) const { int error = 0; socklen_t len = sizeof(error); - if (getsockopt(sockfd, SOL_SOCKET, SO_ERROR, &error, &len) != 0) { + if (getsockopt(sockfd, SOL_SOCKET, SO_ERROR, reinterpret_cast(&error), &len) != 0) { Error("GetSockError"); } return error; @@ -231,7 +241,7 @@ class TCPSocket : public Socket{ */ inline void SetKeepAlive(bool keepalive) { int opt = static_cast(keepalive); - if (setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &opt, sizeof(opt)) < 0) { + if (setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, reinterpret_cast(&opt), sizeof(opt)) < 0) { Socket::Error("SetKeepAlive"); } } @@ -265,13 +275,14 @@ class TCPSocket : public Socket{ * \return 1 if at mark, 0 if not, -1 if an error occured */ inline int AtMark(void) const { - int atmark; #ifdef _WIN32 + unsigned long atmark; if (ioctlsocket(sockfd, SIOCATMARK, &atmark) != NO_ERROR) return -1; #else + int atmark; if (ioctl(sockfd, SIOCATMARK, &atmark) == -1) return -1; #endif - return atmark; + return static_cast(atmark); } /*! * \brief connect to an address @@ -467,7 +478,9 @@ struct SelectHelper { private: inline static int Select_(int maxfd, fd_set *rfds, fd_set *wfds, fd_set *efds, long timeout) { +#if !defined(_WIN32) utils::Assert(maxfd < FD_SETSIZE, "maxdf must be smaller than FDSETSIZE"); +#endif if (timeout == 0) { return select(maxfd, rfds, wfds, efds, NULL); } else { diff --git a/tracker/rabit_demo.py b/tracker/rabit_demo.py index b99d03c6b..5abd96599 100755 --- a/tracker/rabit_demo.py +++ b/tracker/rabit_demo.py @@ -9,7 +9,10 @@ import os import subprocess from threading import Thread import rabit_tracker as tracker -WRAPPER_PATH = os.path.dirname(__file__) + '/../wrapper' +if os.name == 'nt': + WRAPPER_PATH = os.path.dirname(__file__) + '\\..\\wrapper' +else: + WRAPPER_PATH = os.path.dirname(__file__) + '/../wrapper' parser = argparse.ArgumentParser(description='Rabit script to submit rabit job locally using python subprocess') parser.add_argument('-n', '--nworker', required=True, type=int, @@ -36,19 +39,28 @@ done """ def exec_cmd(cmd, taskid): - if cmd[0].find('/') == -1 and os.path.exists(cmd[0]): + if cmd[0].find('/') == -1 and os.path.exists(cmd[0]) and os.name != 'nt': cmd[0] = './' + cmd[0] cmd = ' '.join(cmd) arg = ' rabit_task_id=%d' % (taskid) cmd = cmd + arg ntrial = 0 while True: - prep = 'PYTHONPATH=\"%s\" ' % WRAPPER_PATH - if args.verbose != 0: - bash = keepalive % (echo % cmd, prep, cmd) + if os.name == 'nt': + prep = 'SET PYTHONPATH=\"%s\"\n' % WRAPPER_PATH + ret = subprocess.call(prep + cmd + ('rabit_num_trial=%d' % ntrial), + shell=True) + if ret == 254: + ntrial += 1 + continue + else: - bash = keepalive % ('', prep, cmd) - ret = subprocess.call(bash, shell=True, executable='bash') + prep = 'PYTHONPATH=\"%s\" ' % WRAPPER_PATH + if args.verbose != 0: + bash = keepalive % (echo % cmd, prep, cmd) + else: + bash = keepalive % ('', prep, cmd) + ret = subprocess.call(bash, shell=True, executable='bash') if ret == 0: if args.verbose != 0: print 'Thread %d exit with 0' % taskid diff --git a/tracker/rabit_tracker.py b/tracker/rabit_tracker.py index e260c19c3..aa21973e5 100644 --- a/tracker/rabit_tracker.py +++ b/tracker/rabit_tracker.py @@ -26,7 +26,7 @@ class ExSocket: sock = self.sock nread = 0 while nread < nbytes: - chunk = self.sock.recv(min(nbytes - nread, 1024), socket.MSG_WAITALL) + chunk = self.sock.recv(min(nbytes - nread, 1024)) nread += len(chunk) res.append(chunk) return ''.join(res) diff --git a/windows/.gitignore b/windows/.gitignore new file mode 100644 index 000000000..38d71ecc9 --- /dev/null +++ b/windows/.gitignore @@ -0,0 +1,9 @@ +*.suo +*.exp +*.sdf +*.exe +ipch +x64 +*.filters +Release +*.user diff --git a/windows/basic/basic.vcxproj b/windows/basic/basic.vcxproj new file mode 100644 index 000000000..5c7c9603a --- /dev/null +++ b/windows/basic/basic.vcxproj @@ -0,0 +1,117 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {A6A95246-EB0A-46BA-9471-5939CB6B0006} + basic + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + Application + false + true + MultiByte + + + + + + + + + + + + + + + + + + + + + Level3 + Disabled + + + true + + + + + Level3 + Disabled + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + Level3 + MaxSpeed + true + true + ..\..\include + + + true + true + true + ..\x64\Release\rabit.lib;%(AdditionalDependencies) + + + + + + + + + \ No newline at end of file diff --git a/windows/rabit.sln b/windows/rabit.sln new file mode 100644 index 000000000..064dfda81 --- /dev/null +++ b/windows/rabit.sln @@ -0,0 +1,44 @@ + +Microsoft Visual Studio Solution File, Format Version 11.00 +# Visual Studio 2010 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "rabit", "rabit\rabit.vcxproj", "{D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "basic", "basic\basic.vcxproj", "{A6A95246-EB0A-46BA-9471-5939CB6B0006}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "rabit_wrapper", "rabit_wrapper\rabit_wrapper.vcxproj", "{2F89A7C5-CA4F-4D77-A728-6702D9F33F9F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}.Debug|Win32.ActiveCfg = Debug|Win32 + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}.Debug|Win32.Build.0 = Debug|Win32 + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}.Debug|x64.ActiveCfg = Debug|x64 + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}.Debug|x64.Build.0 = Debug|x64 + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}.Release|Win32.ActiveCfg = Release|Win32 + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}.Release|Win32.Build.0 = Release|Win32 + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}.Release|x64.ActiveCfg = Release|x64 + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F}.Release|x64.Build.0 = Release|x64 + {A6A95246-EB0A-46BA-9471-5939CB6B0006}.Debug|Win32.ActiveCfg = Debug|Win32 + {A6A95246-EB0A-46BA-9471-5939CB6B0006}.Debug|Win32.Build.0 = Debug|Win32 + {A6A95246-EB0A-46BA-9471-5939CB6B0006}.Debug|x64.ActiveCfg = Debug|Win32 + {A6A95246-EB0A-46BA-9471-5939CB6B0006}.Release|Win32.ActiveCfg = Release|Win32 + {A6A95246-EB0A-46BA-9471-5939CB6B0006}.Release|Win32.Build.0 = Release|Win32 + {A6A95246-EB0A-46BA-9471-5939CB6B0006}.Release|x64.ActiveCfg = Release|x64 + {A6A95246-EB0A-46BA-9471-5939CB6B0006}.Release|x64.Build.0 = Release|x64 + {2F89A7C5-CA4F-4D77-A728-6702D9F33F9F}.Debug|Win32.ActiveCfg = Debug|Win32 + {2F89A7C5-CA4F-4D77-A728-6702D9F33F9F}.Debug|Win32.Build.0 = Debug|Win32 + {2F89A7C5-CA4F-4D77-A728-6702D9F33F9F}.Debug|x64.ActiveCfg = Debug|Win32 + {2F89A7C5-CA4F-4D77-A728-6702D9F33F9F}.Release|Win32.ActiveCfg = Release|Win32 + {2F89A7C5-CA4F-4D77-A728-6702D9F33F9F}.Release|Win32.Build.0 = Release|Win32 + {2F89A7C5-CA4F-4D77-A728-6702D9F33F9F}.Release|x64.ActiveCfg = Release|x64 + {2F89A7C5-CA4F-4D77-A728-6702D9F33F9F}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/windows/rabit/rabit.vcxproj b/windows/rabit/rabit.vcxproj new file mode 100644 index 000000000..36ac12658 --- /dev/null +++ b/windows/rabit/rabit.vcxproj @@ -0,0 +1,132 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {D7B77D06-4F5F-4BD7-B81E-7CC8EBBE684F} + rabit + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + StaticLibrary + false + true + MultiByte + + + + + + + + + + + + + + + + + + + + + Level3 + Disabled + + + true + + + + + Level3 + Disabled + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + Level3 + MaxSpeed + true + true + ..\..\include;%(AdditionalIncludeDirectories) + MultiThreaded + + + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/windows/rabit_wrapper/rabit_wrapper.vcxproj b/windows/rabit_wrapper/rabit_wrapper.vcxproj new file mode 100644 index 000000000..6f2cf9f7e --- /dev/null +++ b/windows/rabit_wrapper/rabit_wrapper.vcxproj @@ -0,0 +1,121 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {2F89A7C5-CA4F-4D77-A728-6702D9F33F9F} + rabit_wrapper + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + DynamicLibrary + false + true + MultiByte + + + DynamicLibrary + false + true + MultiByte + + + + + + + + + + + + + + + + + + + + + Level3 + Disabled + + + true + + + + + Level3 + Disabled + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + ..\..\x64\Release\rabit.lib;%(AdditionalDependencies) + + + + + Level3 + MaxSpeed + true + true + ..\..\include + + + true + true + true + ..\x64\Release\rabit.lib;%(AdditionalDependencies) + + + + + + + + + + + + \ No newline at end of file diff --git a/wrapper/rabit.py b/wrapper/rabit.py index 7fc913084..20ed8abdb 100644 --- a/wrapper/rabit.py +++ b/wrapper/rabit.py @@ -11,10 +11,11 @@ import warnings import numpy as np if os.name == 'nt': - assert False, "Rabit windows is not yet compiled" + WRAPPER_PATH = os.path.dirname(__file__) + '\\..\\windows\\x64\\Release\\' + WRAPPER_SUFFIX = 'dll' else: - WRAPPER_PATH = os.path.dirname(__file__) - + WRAPPER_PATH = os.path.dirname(__file__) + '/' + WRAPPER_SUFFIX = 'so' rbtlib = None # load in xgboost library @@ -24,11 +25,11 @@ def loadlib__(lib = 'standard'): warnings.Warn('rabit.int call was ignored because it has already been initialized', level = 2) return if lib == 'standard': - rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH + '/librabit_wrapper.so') + rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH + 'librabit_wrapper.' + WRAPPER_SUFFIX) elif lib == 'mock': - rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH + '/librabit_wrapper_mock.so') + rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH + 'librabit_wrapper_mock.' + WRAPPER_SUFFIX) elif lib == 'mpi': - rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH + '/librabit_wrapper_mpi.so') + rbtlib = ctypes.cdll.LoadLibrary(WRAPPER_PATH + 'librabit_wrapper_mpi.so' + WRAPPER_SUFFIX) else: raise Exception('unknown rabit lib %s, can be standard, mock, mpi' % lib) rbtlib.RabitGetRank.restype = ctypes.c_int