Add CMake option to run Undefined Behavior Sanitizer (UBSan) (#5211)

* Fix related errors.

src/common/compressed_iterator.h

@@ -209,7 +209,7 @@ class CompressedIterator {
         (bits_per_byte - ((offset_ + 1) * symbol_bits_)) % bits_per_byte;
     tmp >>= bit_shift;
     // Mask off unneeded bits
-    uint64_t mask = (1 << symbol_bits_) - 1;
+    uint64_t mask = (static_cast<uint64_t>(1) << symbol_bits_) - 1;
     return static_cast<T>(tmp & mask);
   }
 

src/common/hist_util.h

@@ -31,14 +31,17 @@ namespace common {
 template<typename T>
 struct SimpleArray {
   ~SimpleArray() {
-    free(ptr_);
+    std::free(ptr_);
     ptr_ = nullptr;
   }
 
   void resize(size_t n) {
-    T* ptr = static_cast<T*>(malloc(n*sizeof(T)));
-    memcpy(ptr, ptr_, n_ * sizeof(T));
-    free(ptr_);
+    T* ptr = static_cast<T*>(std::malloc(n * sizeof(T)));
+    CHECK(ptr) << "Failed to allocate memory";
+    if (ptr_) {
+      std::memcpy(ptr, ptr_, n_ * sizeof(T));
+      std::free(ptr_);
+    }
     ptr_ = ptr;
     n_ = n;
   }