[CI] Simplify CMake build with modern CMake techniques (#5871)

* [CI] Simplify CMake build

* Make sure that plugins can be built

* [CI] Install lz4 on Mac

R-package/CMakeLists.txt

@@ -6,8 +6,8 @@ file(GLOB_RECURSE R_SOURCES
   ${CMAKE_CURRENT_LIST_DIR}/src/*.c)
 # Use object library to expose symbols
 add_library(xgboost-r OBJECT ${R_SOURCES})
-
-set(R_DEFINITIONS
+target_compile_definitions(xgboost-r
+  PUBLIC
   -DXGBOOST_STRICT_R_MODE=1
   -DXGBOOST_CUSTOMIZE_GLOBAL_PRNG=1
   -DDMLC_LOG_BEFORE_THROW=0
@@ -15,27 +15,23 @@ set(R_DEFINITIONS
   -DDMLC_LOG_CUSTOMIZE=1
   -DRABIT_CUSTOMIZE_MSG_
   -DRABIT_STRICT_CXX98_)
-target_compile_definitions(xgboost-r
-  PRIVATE ${R_DEFINITIONS})
 target_include_directories(xgboost-r
   PRIVATE
   ${LIBR_INCLUDE_DIRS}
   ${PROJECT_SOURCE_DIR}/include
   ${PROJECT_SOURCE_DIR}/dmlc-core/include
   ${PROJECT_SOURCE_DIR}/rabit/include)
+target_link_libraries(xgboost-r PRIVATE ${LIBR_CORE_LIBRARY})
 set_target_properties(
   xgboost-r PROPERTIES
   CXX_STANDARD 14
   CXX_STANDARD_REQUIRED ON
   POSITION_INDEPENDENT_CODE ON)
 
-set(XGBOOST_DEFINITIONS "${XGBOOST_DEFINITIONS};${R_DEFINITIONS}" PARENT_SCOPE)
-set(XGBOOST_OBJ_SOURCES $<TARGET_OBJECTS:xgboost-r> PARENT_SCOPE)
-set(LINKED_LIBRARIES_PRIVATE ${LINKED_LIBRARIES_PRIVATE} ${LIBR_CORE_LIBRARY} PARENT_SCOPE)
-
-if (USE_OPENMP)
-  target_link_libraries(xgboost-r PRIVATE OpenMP::OpenMP_CXX)
-endif ()
+# Get compilation and link flags of xgboost-r and propagate to objxgboost
+target_link_libraries(objxgboost PUBLIC xgboost-r)
+# Add all objects of xgboost-r to objxgboost
+target_sources(objxgboost INTERFACE $<TARGET_OBJECTS:xgboost-r>)
 
 set(LIBR_HOME "${LIBR_HOME}" PARENT_SCOPE)
-set(LIBR_EXECUTABLE "${LIBR_EXECUTABLE}" PARENT_SCOPE)
+set(LIBR_EXECUTABLE "${LIBR_EXECUTABLE}" PARENT_SCOPE)