Add CMake option to run Undefined Behavior Sanitizer (UBSan) (#5211)

* Fix related errors.

Jenkinsfile

@@ -180,10 +180,10 @@ def BuildCPU() {
     ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
     """
     // Sanitizer test
-    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -e ASAN_OPTIONS=symbolize=1 --cap-add SYS_PTRACE'"
+    def docker_extra_params = "CI_DOCKER_EXTRA_PARAMS_INIT='-e ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer -e ASAN_OPTIONS=symbolize=1 -e UBSAN_OPTIONS=print_stacktrace=1:log_path=ubsan_error.log --cap-add SYS_PTRACE'"
     def docker_args = "--build-arg CMAKE_VERSION=3.12"
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address" \
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_SANITIZER=ON -DENABLED_SANITIZERS="address;leak;undefined" \
       -DCMAKE_BUILD_TYPE=Debug -DSANITIZER_PATH=/usr/lib/x86_64-linux-gnu/
     ${docker_extra_params} ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
     """

cmake/Sanitizer.cmake

@@ -4,24 +4,29 @@
 #  enable_sanitizers("address;leak")
 
 # Add flags
-macro(enable_sanitizer santizer)
-  if(${santizer} MATCHES "address")
+macro(enable_sanitizer sanitizer)
+  if(${sanitizer} MATCHES "address")
     find_package(ASan REQUIRED)
     set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=address")
     link_libraries(${ASan_LIBRARY})
 
-  elseif(${santizer} MATCHES "thread")
+  elseif(${sanitizer} MATCHES "thread")
     find_package(TSan REQUIRED)
     set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=thread")
     link_libraries(${TSan_LIBRARY})
 
-  elseif(${santizer} MATCHES "leak")
+  elseif(${sanitizer} MATCHES "leak")
     find_package(LSan REQUIRED)
     set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=leak")
     link_libraries(${LSan_LIBRARY})
 
+  elseif(${sanitizer} MATCHES "undefined")
+    find_package(UBSan REQUIRED)
+    set(SAN_COMPILE_FLAGS "${SAN_COMPILE_FLAGS} -fsanitize=undefined -fno-sanitize-recover=undefined")
+    link_libraries(${UBSan_LIBRARY})
+
   else()
-    message(FATAL_ERROR "Santizer ${santizer} not supported.")
+    message(FATAL_ERROR "Santizer ${sanitizer} not supported.")
   endif()
 endmacro()
 

cmake/modules/FindUBSan.cmake

@@ -0,0 +1,13 @@
+set(UBSan_LIB_NAME UBSan)
+
+find_library(UBSan_LIBRARY
+  NAMES libubsan.so libubsan.so.5 libubsan.so.4 libubsan.so.3 libubsan.so.2 libubsan.so.1 libubsan.so.0
+  PATHS ${SANITIZER_PATH} /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib ${CMAKE_PREFIX_PATH}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(UBSan DEFAULT_MSG
+  UBSan_LIBRARY)
+
+mark_as_advanced(
+  UBSan_LIBRARY
+  UBSan_LIB_NAME)

dmlc-core

@@ -1 +1 @@
-Subproject commit 61bb900ff92545cd59b613331a38aff08577f318
+Subproject commit 552f7de748fbff34f2708b03f930a47ded45d78e

src/common/compressed_iterator.h

@@ -209,7 +209,7 @@ class CompressedIterator {
         (bits_per_byte - ((offset_ + 1) * symbol_bits_)) % bits_per_byte;
     tmp >>= bit_shift;
     // Mask off unneeded bits
-    uint64_t mask = (1 << symbol_bits_) - 1;
+    uint64_t mask = (static_cast<uint64_t>(1) << symbol_bits_) - 1;
     return static_cast<T>(tmp & mask);
   }
 

src/common/hist_util.h

@@ -31,14 +31,17 @@ namespace common {
 template<typename T>
 struct SimpleArray {
   ~SimpleArray() {
-    free(ptr_);
+    std::free(ptr_);
     ptr_ = nullptr;
   }
 
   void resize(size_t n) {
-    T* ptr = static_cast<T*>(malloc(n*sizeof(T)));
-    memcpy(ptr, ptr_, n_ * sizeof(T));
-    free(ptr_);
+    T* ptr = static_cast<T*>(std::malloc(n * sizeof(T)));
+    CHECK(ptr) << "Failed to allocate memory";
+    if (ptr_) {
+      std::memcpy(ptr, ptr_, n_ * sizeof(T));
+      std::free(ptr_);
+    }
     ptr_ = ptr;
     n_ = n;
   }