[R] Robust endian detection in CRAN xgboost build (#5232)

* [R] Robust endian detection in CRAN xgboost build

* Check for external backtrace() lib

* Update Makevars.win

R-package/configure.ac

@@ -4,6 +4,18 @@ AC_PREREQ(2.62)
 
 AC_INIT([xgboost],[0.6-3],[],[xgboost],[])
 
+### Check whether backtrace() is part of libc or the external lib libexecinfo
+AC_MSG_CHECKING([Backtrace lib])
+AC_MSG_RESULT([])
+AC_CHECK_LIB([execinfo], [backtrace], [BACKTRACE_LIB=-lexecinfo], [BACKTRACE_LIB=''])
+
+### Endian detection
+AC_MSG_CHECKING([endian])
+AC_MSG_RESULT([])
+AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include <stdint.h>]], [[const uint16_t endianness = 256; return !!(*(const uint8_t *)&endianness);]])],
+  [ENDIAN_FLAG="-DDMLC_CMAKE_LITTLE_ENDIAN=1"],
+  [ENDIAN_FLAG="-DDMLC_CMAKE_LITTLE_ENDIAN=0"])
+
 OPENMP_CXXFLAGS=""
 
 if test `uname -s` = "Linux"
@@ -26,6 +38,8 @@ then
 fi
 
 AC_SUBST(OPENMP_CXXFLAGS)
+AC_SUBST(ENDIAN_FLAG)
+AC_SUBST(BACKTRACE_LIB)
 AC_CONFIG_FILES([src/Makevars])
 AC_OUTPUT
 

R-package/src/Makevars.in

@@ -17,8 +17,8 @@ endif
 $(foreach v, $(XGB_RFLAGS), $(warning $(v)))
 
 PKG_CPPFLAGS=  -I$(PKGROOT)/include -I$(PKGROOT)/dmlc-core/include -I$(PKGROOT)/rabit/include -I$(PKGROOT) $(XGB_RFLAGS)
-PKG_CXXFLAGS= @OPENMP_CXXFLAGS@ -pthread
-PKG_LIBS = @OPENMP_CXXFLAGS@ -pthread
+PKG_CXXFLAGS= @OPENMP_CXXFLAGS@ @ENDIAN_FLAG@ -pthread
+PKG_LIBS = @OPENMP_CXXFLAGS@ @ENDIAN_FLAG@ @BACKTRACE_LIB@ -pthread
 OBJECTS= ./xgboost_R.o ./xgboost_custom.o ./xgboost_assert.o ./init.o\
          $(PKGROOT)/amalgamation/xgboost-all0.o $(PKGROOT)/amalgamation/dmlc-minimum0.o\
          $(PKGROOT)/rabit/src/engine_empty.o $(PKGROOT)/rabit/src/c_api.o