[JVM-Packages] Add support for detecting musl-based Linux (#7624)

Co-authored-by: Marc Philipp <marc@gradle.com>

jvm-packages/xgboost4j-tester/generate_pom.py

@@ -98,7 +98,7 @@ pom_template = """
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-      <version>4.11</version>
+      <version>4.13.2</version>
       <scope>test</scope>
     </dependency>
     <dependency>

jvm-packages/xgboost4j/pom.xml

@@ -28,7 +28,7 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.13.1</version>
+            <version>4.13.2</version>
             <scope>test</scope>
         </dependency>
         <dependency>

jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/NativeLibLoader.java

@@ -21,7 +21,12 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Locale;
+import java.util.Optional;
+import java.util.stream.Stream;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -34,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
 class NativeLibLoader {
   private static final Log logger = LogFactory.getLog(NativeLibLoader.class);
 
+  private static Path mappedFilesBaseDir = Paths.get("/proc/self/map_files");
+
   /**
    * Supported OS enum.
    */
@@ -41,14 +48,19 @@ class NativeLibLoader {
     WINDOWS("windows"),
     MACOS("macos"),
     LINUX("linux"),
+    LINUX_MUSL("linux-musl"),
     SOLARIS("solaris");
 
     final String name;
 
-    private OS(String name) {
+    OS(String name) {
       this.name = name;
     }
 
+    static void setMappedFilesBaseDir(Path baseDir) {
+      mappedFilesBaseDir = baseDir;
+    }
+
     /**
      * Detects the OS using the system properties.
      * Throws IllegalStateException if the OS is not recognized.
@@ -61,13 +73,47 @@ class NativeLibLoader {
       } else if (os.contains("win")) {
         return WINDOWS;
       } else if (os.contains("nux")) {
-        return LINUX;
+        return isMuslBased() ? LINUX_MUSL : LINUX;
       } else if (os.contains("sunos")) {
         return SOLARIS;
       } else {
         throw new IllegalStateException("Unsupported OS:" + os);
       }
     }
+
+    /**
+     * Checks if the Linux OS is musl based. For this, we check the memory-mapped
+     * filenames and see if one of those contains the string "musl".
+     *
+     * @return true if the Linux OS is musl based, false otherwise.
+     */
+    static boolean isMuslBased() {
+      try (Stream<Path> dirStream = Files.list(mappedFilesBaseDir)) {
+        Optional<String> muslRelatedMemoryMappedFilename = dirStream
+            .map(OS::toRealPath)
+            .filter(s -> s.toLowerCase().contains("musl"))
+            .findFirst();
+
+        muslRelatedMemoryMappedFilename.ifPresent(muslFilename -> {
+          logger.debug("Assuming that detected Linux OS is musl-based, "
+              + "because a memory-mapped file '" + muslFilename + "' was found.");
+        });
+
+        return muslRelatedMemoryMappedFilename.isPresent();
+      } catch (IOException ignored) {
+        // ignored
+      }
+      return false;
+    }
+
+    private static String toRealPath(Path path) {
+      try {
+        return path.toRealPath().toString();
+      } catch (IOException e) {
+        return "";
+      }
+    }
+
   }
 
   /**
@@ -80,7 +126,7 @@ class NativeLibLoader {
 
     final String name;
 
-    private Arch(String name) {
+    Arch(String name) {
       this.name = name;
     }
 
@@ -115,7 +161,7 @@ class NativeLibLoader {
    *   <li>Supported OS: macOS, Windows, Linux, Solaris.</li>
    *   <li>Supported Architectures: x86_64, aarch64, sparc.</li>
    * </ul>
-   * Throws UnsatisfiedLinkError if the library failed to load it's dependencies.
+   * Throws UnsatisfiedLinkError if the library failed to load its dependencies.
    * @throws IOException If the library could not be extracted from the jar.
    */
   static synchronized void initXGBoost() throws IOException {
@@ -129,18 +175,37 @@ class NativeLibLoader {
               platform + "/" + System.mapLibraryName(libName);
           loadLibraryFromJar(libraryPathInJar);
         } catch (UnsatisfiedLinkError ule) {
-          logger.error("Failed to load " + libName + " due to missing native dependencies for " +
+          String failureMessageIncludingOpenMPHint = "Failed to load " + libName + " " +
-              "platform " + platform + ", this is likely due to a missing OpenMP dependency");
+              "due to missing native dependencies for " +
+              "platform " + platform + ", " +
+              "this is likely due to a missing OpenMP dependency";
+
           switch (os) {
             case WINDOWS:
+              logger.error(failureMessageIncludingOpenMPHint);
               logger.error("You may need to install 'vcomp140.dll' or 'libgomp-1.dll'");
               break;
             case MACOS:
-              logger.error("You may need to install 'libomp.dylib', via `brew install libomp`" +
+              logger.error(failureMessageIncludingOpenMPHint);
-                  " or similar");
+              logger.error("You may need to install 'libomp.dylib', via `brew install libomp` " +
+                  "or similar");
               break;
             case LINUX:
+              logger.error(failureMessageIncludingOpenMPHint);
+              logger.error("You may need to install 'libgomp.so' (or glibc) via your package " +
+                  "manager.");
+              logger.error("Alternatively, your Linux OS is musl-based " +
+                  "but wasn't detected as such.");
+              break;
+            case LINUX_MUSL:
+              logger.error(failureMessageIncludingOpenMPHint);
+              logger.error("You may need to install 'libgomp.so' (or glibc) via your package " +
+                  "manager.");
+              logger.error("Alternatively, your Linux OS was wrongly detected as musl-based, " +
+                  "although it is not.");
+              break;
             case SOLARIS:
+              logger.error(failureMessageIncludingOpenMPHint);
               logger.error("You may need to install 'libgomp.so' (or glibc) via your package " +
                   "manager.");
               break;

jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/ArchDetectionTest.java

@@ -0,0 +1,98 @@
+/*
+ Copyright (c) 2014 by Contributors
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package ml.dmlc.xgboost4j.java;
+
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.util.Collection;
+
+import static java.util.Arrays.asList;
+import static junit.framework.TestCase.assertSame;
+import static ml.dmlc.xgboost4j.java.NativeLibLoader.Arch.X86_64;
+import static ml.dmlc.xgboost4j.java.NativeLibLoader.Arch.AARCH64;
+import static ml.dmlc.xgboost4j.java.NativeLibLoader.Arch.SPARC;
+import static ml.dmlc.xgboost4j.java.NativeLibLoader.Arch.detectArch;
+import static org.junit.Assert.assertThrows;
+
+/**
+ * Test cases for {@link NativeLibLoader.Arch}.
+ */
+@RunWith(Enclosed.class)
+public class ArchDetectionTest {
+
+  private static final String OS_ARCH_PROPERTY = "os.arch";
+
+  @RunWith(Parameterized.class)
+  public static class ParameterizedArchDetectionTest {
+
+    private final String osArchValue;
+    private final NativeLibLoader.Arch expectedArch;
+
+    public ParameterizedArchDetectionTest(String osArchValue, NativeLibLoader.Arch expectedArch) {
+      this.osArchValue = osArchValue;
+      this.expectedArch = expectedArch;
+    }
+
+    @Parameters
+    public static Collection<Object[]> data() {
+      return asList(new Object[][]{
+        {"x86_64", X86_64},
+        {"amd64", X86_64},
+        {"aarch64", AARCH64},
+        {"arm64", AARCH64},
+        {"sparc64", SPARC}
+      });
+    }
+
+    @Test
+    public void testArch() {
+      executeAndRestoreProperty(() -> {
+        System.setProperty(OS_ARCH_PROPERTY, osArchValue);
+        assertSame(detectArch(), expectedArch);
+      });
+    }
+  }
+
+  public static class UnsupportedArchDetectionTest {
+
+    @Test
+    public void testUnsupportedArch() {
+      executeAndRestoreProperty(() -> {
+        System.setProperty(OS_ARCH_PROPERTY, "unsupported");
+        assertThrows(IllegalStateException.class, NativeLibLoader.Arch::detectArch);
+      });
+    }
+  }
+
+  private static void executeAndRestoreProperty(Runnable action) {
+    String oldValue = System.getProperty(OS_ARCH_PROPERTY);
+
+    try {
+      action.run();
+    } finally {
+      if (oldValue != null) {
+        System.setProperty(OS_ARCH_PROPERTY, oldValue);
+      } else {
+        System.clearProperty(OS_ARCH_PROPERTY);
+      }
+    }
+  }
+
+}

jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/OsDetectionTest.java

@@ -0,0 +1,123 @@
+/*
+ Copyright (c) 2014 by Contributors
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+package ml.dmlc.xgboost4j.java;
+
+import ml.dmlc.xgboost4j.java.NativeLibLoader.OS;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.util.Collection;
+
+import static java.util.Arrays.asList;
+import static junit.framework.TestCase.assertSame;
+import static ml.dmlc.xgboost4j.java.NativeLibLoader.OS.*;
+import static org.junit.Assert.assertThrows;
+
+/**
+ * Test cases for {@link OS}.
+ */
+@RunWith(Enclosed.class)
+public class OsDetectionTest {
+
+  private static final String OS_NAME_PROPERTY = "os.name";
+
+  @RunWith(Parameterized.class)
+  public static class ParameterizedOSDetectionTest {
+
+    private final String osNameValue;
+    private final OS expectedOS;
+
+    public ParameterizedOSDetectionTest(String osNameValue, OS expectedOS) {
+      this.osNameValue = osNameValue;
+      this.expectedOS = expectedOS;
+    }
+
+    @Parameters
+    public static Collection<Object[]> data() {
+      return asList(new Object[][]{
+        {"windows", WINDOWS},
+        {"mac", MACOS},
+        {"darwin", MACOS},
+        {"linux", LINUX},
+        {"sunos", SOLARIS}
+      });
+    }
+
+    @Test
+    public void getOS() {
+      executeAndRestoreProperty(() -> {
+        System.setProperty(OS_NAME_PROPERTY, osNameValue);
+        assertSame(detectOS(), expectedOS);
+      });
+    }
+  }
+
+  public static class NonParameterizedOSDetectionTest {
+
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder();
+
+    @Test
+    public void testForRegularLinux() throws Exception {
+      setMappedFilesBaseDir(folder.getRoot().toPath());
+      folder.newFile("ld-2.23.so");
+
+      executeAndRestoreProperty(() -> {
+        System.setProperty(OS_NAME_PROPERTY, "linux");
+        assertSame(detectOS(), LINUX);
+      });
+    }
+
+    @Test
+    public void testForMuslBasedLinux() throws Exception {
+      setMappedFilesBaseDir(folder.getRoot().toPath());
+      folder.newFile("ld-musl-x86_64.so.1");
+
+      executeAndRestoreProperty(() -> {
+        System.setProperty(OS_NAME_PROPERTY, "linux");
+        assertSame(detectOS(), LINUX_MUSL);
+      });
+    }
+
+    @Test
+    public void testUnsupportedOs() {
+      executeAndRestoreProperty(() -> {
+        System.setProperty(OS_NAME_PROPERTY, "unsupported");
+        assertThrows(IllegalStateException.class, OS::detectOS);
+      });
+    }
+  }
+
+  private static void executeAndRestoreProperty(Runnable action) {
+    String oldValue = System.getProperty(OS_NAME_PROPERTY);
+
+    try {
+      action.run();
+    } finally {
+      if (oldValue != null) {
+        System.setProperty(OS_NAME_PROPERTY, oldValue);
+      } else {
+        System.clearProperty(OS_NAME_PROPERTY);
+      }
+    }
+  }
+
+}