Enable building with sanitizers. (#3525)

CMakeLists.txt

@@ -14,8 +14,12 @@ option(USE_NCCL "Build using NCCL for multi-GPU. Also requires USE_CUDA")
 option(JVM_BINDINGS "Build JVM bindings" OFF)
 option(GOOGLE_TEST "Build google tests" OFF)
 option(R_LIB "Build shared library for R package" OFF)
+option(USE_SANITIZER "Use santizer flags" OFF)
 set(GPU_COMPUTE_VER "" CACHE STRING
   "Space separated list of compute versions to be built against, e.g. '35 61'")
+set(ENABLED_SANITIZERS "address" "leak" CACHE STRING
+  "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are
+address, leak and thread.")
 
 # Deprecation warning
 if(PLUGIN_UPDATER_GPU)
@@ -43,6 +47,12 @@ if(WIN32 AND MINGW)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libstdc++")
 endif()
 
+# Sanitizer
+if(USE_SANITIZER)
+  include(cmake/Sanitizer.cmake)
+  enable_sanitizers("${ENABLED_SANITIZERS}")
+endif(USE_SANITIZER)
+
 # AVX
 if(USE_AVX)
   if(MSVC)

cmake/Sanitizer.cmake

@@ -0,0 +1,58 @@
+# Set appropriate compiler and linker flags for sanitizers.
+#
+# Usage of this module:
+#  enable_sanitizers("address;leak")
+
+# Add flags
+macro(enable_sanitizer santizer)
+  if(${santizer} MATCHES "address")
+    find_package(ASan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address")
+    link_libraries(${ASan_LIBRARY})
+
+  elseif(${santizer} MATCHES "thread")
+    find_package(TSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread")
+    link_libraries(${TSan_LIBRARY})
+
+  elseif(${santizer} MATCHES "leak")
+    find_package(LSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak")
+    link_libraries(${LSan_LIBRARY})
+
+  else()
+    message(FATAL_ERROR "Santizer ${santizer} not supported.")
+  endif()
+endmacro()
+
+macro(enable_sanitizers SANITIZERS)
+  # Check sanitizers compatibility.
+  # Idealy, we should use if(san IN_LIST SANITIZERS) ... endif()
+  # But I haven't figure out how to make it work.
+  foreach ( _san ${SANITIZERS} )
+    string(TOLOWER ${_san} _san)
+    if (_san MATCHES "thread")
+      if (${_use_other_sanitizers})
+        message(FATAL_ERROR
+          "thread sanitizer is not compatible with ${_san} sanitizer.")
+      endif()
+      set(_use_thread_sanitizer 1)
+    else ()
+      if (${_use_thread_sanitizer})
+        message(FATAL_ERROR
+          "${_san} sanitizer is not compatible with thread sanitizer.")
+      endif()
+      set(_use_other_sanitizers 1)
+    endif()
+  endforeach()
+
+  message("Sanitizers: ${SANITIZERS}")
+
+  foreach( _san ${SANITIZERS} )
+    string(TOLOWER ${_san} _san)
+    enable_sanitizer(${_san})
+  endforeach()
+  message("Sanitizers compile flags: ${SAN_COMPILE_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_COMPILE_FLAGS}")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_COMPILE_FLAGS}")
+endmacro()

cmake/modules/FindASan.cmake

@@ -0,0 +1,13 @@
+set(ASan_LIB_NAME ASan)
+
+find_library(ASan_LIBRARY
+  NAMES libasan.so libasan.so.4
+  PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ASan DEFAULT_MSG
+  ASan_LIBRARY)
+
+mark_as_advanced(
+  ASan_LIBRARY
+  ASan_LIB_NAME)

cmake/modules/FindLSan.cmake

@@ -0,0 +1,13 @@
+set(LSan_LIB_NAME lsan)
+
+find_library(LSan_LIBRARY
+  NAMES liblsan.so liblsan.so.0 liblsan.so.0.0.0
+  PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LSan DEFAULT_MSG
+  LSan_LIBRARY)
+
+mark_as_advanced(
+  LSan_LIBRARY
+  LSan_LIB_NAME)

cmake/modules/FindTSan.cmake

@@ -0,0 +1,13 @@
+set(TSan_LIB_NAME tsan)
+
+find_library(TSan_LIBRARY
+  NAMES libtsan.so libtsan.so.0 libtsan.so.0.0.0
+  PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(TSan DEFAULT_MSG
+  TSan_LIBRARY)
+
+mark_as_advanced(
+  TSan_LIBRARY
+  TSan_LIB_NAME)

doc/contribute.rst

@@ -18,6 +18,7 @@ Everyone is more than welcome to contribute. It is a way to make the project bet
 
 * `Documents`_
 * `Testcases`_
+* `Sanitizers`_
 * `Examples`_
 * `Core Library`_
 * `Python Package`_
@@ -121,6 +122,46 @@ Testcases
 * All the testcases are in `tests <https://github.com/dmlc/xgboost/tree/master/tests>`_.
 * We use python nose for python test cases.
 
+**********
+Sanitizers
+**********
+
+By default, sanitizers are bundled in GCC and Clang/LLVM. One can enable
+sanitizers with GCC >= 4.8 or LLVM >= 3.1, But some distributions might package
+sanitizers separately.  Here is a list of supported sanitizers with
+corresponding library names:
+
+- Address sanitizer: libasan
+- Leak sanitizer:    liblsan
+- Thread sanitizer:  libtsan
+
+Memory sanitizer is exclusive to LLVM, hence not supported in XGBoost.
+
+How to build XGBoost with sanitizers
+====================================
+One can build XGBoost with sanitizer support by specifying -DUSE_SANITIZER=ON.
+By default, address sanitizer and leak sanitizer are used when you turn the
+USE_SANITIZER flag on.  You can always change the default by providing a
+semicolon separated list of sanitizers to ENABLED_SANITIZERS.  Note that thread
+sanitizer is not compatible with the other two sanitizers.
+
+  .. code-block:: bash
+
+    cmake -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak" /path/to/xgboost
+
+How to use sanitizers with CUDA support
+=======================================
+Runing XGBoost on CUDA with address sanitizer (asan) will raise memory error.
+To use asan with CUDA correctly, you need to configure asan via ASAN_OPTIONS
+environment variable:
+
+  .. code-block:: bash
+
+    ASAN_OPTIONS=protect_shadow_gap=0 ../testxgboost
+
+For details, please consult `official documentation <https://github.com/google/sanitizers/wiki>`_ for sanitizers.
+
+
 ********
 Examples
 ********