Perform clang-tidy on both cpp and cuda source. (#4034)

* Basic script for using compilation database.

* Add `GENERATE_COMPILATION_DATABASE' to CMake.
* Rearrange CMakeLists.txt.
* Add basic python clang-tidy script.
* Remove modernize-use-auto.
* Add clang-tidy to Jenkins
* Refine logic for correct path detection

In Jenkins, the project root is of form /home/ubuntu/workspace/xgboost_PR-XXXX

* Run clang-tidy in CUDA 9.2 container
* Use clang_tidy container

tests/ci_build/Dockerfile.clang_tidy

@@ -0,0 +1,45 @@
+ARG CUDA_VERSION
+FROM nvidia/cuda:$CUDA_VERSION-devel-ubuntu18.04
+
+# Environment
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install all basic requirements
+RUN \
+    apt-get update && \
+    apt-get install -y tar unzip wget git build-essential cmake python3 python3-pip llvm-7 clang-tidy-7 clang-7
+
+# Set default clang-tidy version
+RUN \
+    update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-7 100 && \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-7 100
+
+# NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
+RUN \
+    export CUDA_SHORT=`echo $CUDA_VERSION | egrep -o '[0-9]+\.[0-9]'` && \
+    if [ "${CUDA_SHORT}" != "10.0" ]; then \
+    wget https://developer.download.nvidia.com/compute/redist/nccl/v2.2/nccl_2.2.13-1%2Bcuda${CUDA_SHORT}_x86_64.txz && \
+    tar xf "nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz" && \
+    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/include/nccl.h /usr/include && \
+    cp nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64/lib/* /usr/lib && \
+    rm -f nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64.txz && \
+    rm -r nccl_2.2.13-1+cuda${CUDA_SHORT}_x86_64; fi
+
+# Install Python packages
+RUN \
+    pip3 install pyyaml
+
+ENV GOSU_VERSION 1.10
+
+# Install lightweight sudo (not bound to TTY)
+RUN set -ex; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu && \
+    gosu nobody true
+
+# Default entry-point to use if running locally
+# It will preserve attributes of created files
+COPY entrypoint.sh /scripts/
+
+WORKDIR /workspace
+ENTRYPOINT ["/scripts/entrypoint.sh"]

tests/ci_build/clang_tidy.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+rm -rf gtest googletest-release-1.7.0
+wget -nc https://github.com/google/googletest/archive/release-1.7.0.zip
+unzip -n release-1.7.0.zip
+mv googletest-release-1.7.0 gtest && cd gtest
+cmake . && make
+mkdir lib && mv libgtest.a lib
+cd ..
+rm -rf release-1.7.0.zip*
+
+python3 tests/ci_build/tidy.py --gtest-path=${PWD}/gtest

tests/ci_build/tidy.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+import subprocess
+import yaml
+import json
+from multiprocessing import Pool, cpu_count
+import shutil
+import os
+import re
+import argparse
+
+
+def call(args):
+    '''Subprocess run wrapper.'''
+    completed = subprocess.run(args, stdout=subprocess.PIPE,
+                               stderr=subprocess.DEVNULL)
+    error_msg = completed.stdout.decode('utf-8')
+    matched = re.match('.*xgboost.*warning.*', error_msg,
+                       re.MULTILINE | re.DOTALL)
+    if matched is None:
+        return_code = 0
+    else:
+        print(error_msg, '\n')
+        return_code = 1
+    return completed.returncode | return_code
+
+
+class ClangTidy(object):
+    '''
+    clang tidy wrapper.
+    Args:
+      gtest_path: Full path of Google Test library.
+      cpp_lint: Run linter on C++ source code.
+      cuda_lint: Run linter on CUDA source code.
+    '''
+    def __init__(self, gtest_path, cpp_lint, cuda_lint):
+        self.gtest_path = gtest_path
+        self.cpp_lint = cpp_lint
+        self.cuda_lint = cuda_lint
+        print('Using Google Test from {}'.format(self.gtest_path))
+        print('Run linter on CUDA: ', self.cuda_lint)
+        print('Run linter on C++:', self.cpp_lint)
+        if not self.cpp_lint and not self.cuda_lint:
+            raise ValueError('Both --cpp and --cuda are set to 0.')
+        self.root_path = os.path.abspath(os.path.curdir)
+        print('Project root:', self.root_path)
+        self.cdb_path = os.path.join(self.root_path, 'cdb')
+
+    def __enter__(self):
+        if os.path.exists(self.cdb_path):
+            shutil.rmtree(self.cdb_path)
+        self._generate_cdb()
+        return self
+
+    def __exit__(self, *args):
+        if os.path.exists(self.cdb_path):
+            shutil.rmtree(self.cdb_path)
+
+    def _generate_cdb(self):
+        '''Run CMake to generate compilation database.'''
+        os.mkdir(self.cdb_path)
+        os.chdir(self.cdb_path)
+        cmake_args = ['cmake', '..', '-DGENERATE_COMPILATION_DATABASE=ON',
+                      '-DGOOGLE_TEST=ON', '-DGTEST_ROOT={}'.format(
+                          self.gtest_path)]
+        if self.cuda_lint:
+            cmake_args.extend(['-DUSE_CUDA=ON', '-DUSE_NCCL=ON'])
+        subprocess.run(cmake_args)
+        os.chdir(self.root_path)
+
+    def _configure_flags(self, path, command):
+        common_args = ['clang-tidy',
+                       # "-header-filter='(xgboost\\/src|xgboost\\/include)'",
+                       '-config='+str(self.clang_tidy)]
+        common_args.append(path)
+        common_args.append('--')
+
+        command = command.split()[1:]  # remove clang/c++/g++
+        # filter out not used flags
+        if '-fuse-ld=gold' in command:
+            command.remove('-fuse-ld=gold')
+        if '-rdynamic' in command:
+            command.remove('-rdynamic')
+        if '-Xcompiler=-fPIC' in command:
+            command.remove('-Xcompiler=-fPIC')
+        if '-Xcompiler=-fPIE' in command:
+            command.remove('-Xcompiler=-fPIE')
+        if '-c' in command:
+            index = command.index('-c')
+            del command[index+1]
+            command.remove('-c')
+        if '-o' in command:
+            index = command.index('-o')
+            del command[index+1]
+            command.remove('-o')
+
+        common_args.extend(command)
+
+        # Two passes, one for device code another for host code.
+        if path.endswith('cu'):
+            args = [common_args.copy(), common_args.copy()]
+            args[0].append('--cuda-host-only')
+            args[1].append('--cuda-device-only')
+        else:
+            args = [common_args.copy()]
+        for a in args:
+            a.append('-Wno-unused-command-line-argument')
+        return args
+
+    def _configure(self):
+        '''Load and configure compile_commands and clang_tidy.'''
+
+        def should_lint(path):
+            if not self.cpp_lint and path.endswith('.cc'):
+                return False
+            return True
+
+        cdb_file = os.path.join(self.cdb_path, 'compile_commands.json')
+        with open(cdb_file, 'r') as fd:
+            self.compile_commands = json.load(fd)
+        tidy_file = os.path.join(self.root_path, '.clang-tidy')
+        with open(tidy_file) as fd:
+            self.clang_tidy = yaml.load(fd)
+        all_files = []
+        for entry in self.compile_commands:
+            path = entry['file']
+            if should_lint(path):
+                print(path)
+                args = self._configure_flags(path, entry['command'])
+                all_files.extend(args)
+        return all_files
+
+    def run(self):
+        '''Run clang-tidy.'''
+        all_files = self._configure()
+        with Pool(cpu_count()) as pool:
+            results = pool.map(call, all_files)
+        passed = True
+        if 1 in results:
+            print('Please correct clang-tidy warnings.')
+            passed = False
+        return passed
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Run clang-tidy.')
+    parser.add_argument('--cpp', type=int, default=1)
+    parser.add_argument('--cuda', type=int, default=1)
+    parser.add_argument('--gtest-path', required=True,
+                        help='Full path of Google Test library directory')
+    args = parser.parse_args()
+    with ClangTidy(args.gtest_path, args.cpp, args.cuda) as linter:
+        passed = linter.run()
+    # Uncomment it once the code base is clang-tidy conformant.
+    # if not passed:
+    #     sys.exit(1)