From 4dafb5fac81cf2b99fefb6fab984116d86410fbd Mon Sep 17 00:00:00 2001 From: Daniel Clausen Date: Sun, 13 Mar 2022 17:37:27 +0100 Subject: [PATCH] [JVM-Packages] Add support for detecting musl-based Linux (#7624) Co-authored-by: Marc Philipp --- jvm-packages/xgboost4j-tester/generate_pom.py | 2 +- jvm-packages/xgboost4j/pom.xml | 2 +- .../dmlc/xgboost4j/java/NativeLibLoader.java | 81 ++++++++++-- .../xgboost4j/java/ArchDetectionTest.java | 98 ++++++++++++++ .../dmlc/xgboost4j/java/OsDetectionTest.java | 123 ++++++++++++++++++ 5 files changed, 296 insertions(+), 10 deletions(-) create mode 100644 jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/ArchDetectionTest.java create mode 100644 jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/OsDetectionTest.java diff --git a/jvm-packages/xgboost4j-tester/generate_pom.py b/jvm-packages/xgboost4j-tester/generate_pom.py index 24a02d341..88f0c57bc 100644 --- a/jvm-packages/xgboost4j-tester/generate_pom.py +++ b/jvm-packages/xgboost4j-tester/generate_pom.py @@ -98,7 +98,7 @@ pom_template = """ junit junit - 4.11 + 4.13.2 test diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml index c911eaa6e..d60a83b29 100644 --- a/jvm-packages/xgboost4j/pom.xml +++ b/jvm-packages/xgboost4j/pom.xml @@ -28,7 +28,7 @@ junit junit - 4.13.1 + 4.13.2 test diff --git a/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/NativeLibLoader.java b/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/NativeLibLoader.java index b8d8be8f6..e6e6542a5 100644 --- a/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/NativeLibLoader.java +++ b/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/NativeLibLoader.java @@ -21,7 +21,12 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Locale; +import java.util.Optional; +import java.util.stream.Stream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,6 +39,8 @@ import org.apache.commons.logging.LogFactory; class NativeLibLoader { private static final Log logger = LogFactory.getLog(NativeLibLoader.class); + private static Path mappedFilesBaseDir = Paths.get("/proc/self/map_files"); + /** * Supported OS enum. */ @@ -41,14 +48,19 @@ class NativeLibLoader { WINDOWS("windows"), MACOS("macos"), LINUX("linux"), + LINUX_MUSL("linux-musl"), SOLARIS("solaris"); final String name; - private OS(String name) { + OS(String name) { this.name = name; } + static void setMappedFilesBaseDir(Path baseDir) { + mappedFilesBaseDir = baseDir; + } + /** * Detects the OS using the system properties. * Throws IllegalStateException if the OS is not recognized. @@ -61,13 +73,47 @@ class NativeLibLoader { } else if (os.contains("win")) { return WINDOWS; } else if (os.contains("nux")) { - return LINUX; + return isMuslBased() ? LINUX_MUSL : LINUX; } else if (os.contains("sunos")) { return SOLARIS; } else { throw new IllegalStateException("Unsupported OS:" + os); } } + + /** + * Checks if the Linux OS is musl based. For this, we check the memory-mapped + * filenames and see if one of those contains the string "musl". + * + * @return true if the Linux OS is musl based, false otherwise. + */ + static boolean isMuslBased() { + try (Stream dirStream = Files.list(mappedFilesBaseDir)) { + Optional muslRelatedMemoryMappedFilename = dirStream + .map(OS::toRealPath) + .filter(s -> s.toLowerCase().contains("musl")) + .findFirst(); + + muslRelatedMemoryMappedFilename.ifPresent(muslFilename -> { + logger.debug("Assuming that detected Linux OS is musl-based, " + + "because a memory-mapped file '" + muslFilename + "' was found."); + }); + + return muslRelatedMemoryMappedFilename.isPresent(); + } catch (IOException ignored) { + // ignored + } + return false; + } + + private static String toRealPath(Path path) { + try { + return path.toRealPath().toString(); + } catch (IOException e) { + return ""; + } + } + } /** @@ -80,7 +126,7 @@ class NativeLibLoader { final String name; - private Arch(String name) { + Arch(String name) { this.name = name; } @@ -115,7 +161,7 @@ class NativeLibLoader { *
  • Supported OS: macOS, Windows, Linux, Solaris.
    • *
  • Supported Architectures: x86_64, aarch64, sparc.
    • * - * Throws UnsatisfiedLinkError if the library failed to load it's dependencies. + * Throws UnsatisfiedLinkError if the library failed to load its dependencies. * @throws IOException If the library could not be extracted from the jar. */ static synchronized void initXGBoost() throws IOException { @@ -129,18 +175,37 @@ class NativeLibLoader { platform + "/" + System.mapLibraryName(libName); loadLibraryFromJar(libraryPathInJar); } catch (UnsatisfiedLinkError ule) { - logger.error("Failed to load " + libName + " due to missing native dependencies for " + - "platform " + platform + ", this is likely due to a missing OpenMP dependency"); + String failureMessageIncludingOpenMPHint = "Failed to load " + libName + " " + + "due to missing native dependencies for " + + "platform " + platform + ", " + + "this is likely due to a missing OpenMP dependency"; + switch (os) { case WINDOWS: + logger.error(failureMessageIncludingOpenMPHint); logger.error("You may need to install 'vcomp140.dll' or 'libgomp-1.dll'"); break; case MACOS: - logger.error("You may need to install 'libomp.dylib', via `brew install libomp`" + - " or similar"); + logger.error(failureMessageIncludingOpenMPHint); + logger.error("You may need to install 'libomp.dylib', via `brew install libomp` " + + "or similar"); break; case LINUX: + logger.error(failureMessageIncludingOpenMPHint); + logger.error("You may need to install 'libgomp.so' (or glibc) via your package " + + "manager."); + logger.error("Alternatively, your Linux OS is musl-based " + + "but wasn't detected as such."); + break; + case LINUX_MUSL: + logger.error(failureMessageIncludingOpenMPHint); + logger.error("You may need to install 'libgomp.so' (or glibc) via your package " + + "manager."); + logger.error("Alternatively, your Linux OS was wrongly detected as musl-based, " + + "although it is not."); + break; case SOLARIS: + logger.error(failureMessageIncludingOpenMPHint); logger.error("You may need to install 'libgomp.so' (or glibc) via your package " + "manager."); break; diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/ArchDetectionTest.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/ArchDetectionTest.java new file mode 100644 index 000000000..137999218 --- /dev/null +++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/ArchDetectionTest.java @@ -0,0 +1,98 @@ +/* + Copyright (c) 2014 by Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package ml.dmlc.xgboost4j.java; + +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import java.util.Collection; + +import static java.util.Arrays.asList; +import static junit.framework.TestCase.assertSame; +import static ml.dmlc.xgboost4j.java.NativeLibLoader.Arch.X86_64; +import static ml.dmlc.xgboost4j.java.NativeLibLoader.Arch.AARCH64; +import static ml.dmlc.xgboost4j.java.NativeLibLoader.Arch.SPARC; +import static ml.dmlc.xgboost4j.java.NativeLibLoader.Arch.detectArch; +import static org.junit.Assert.assertThrows; + +/** + * Test cases for {@link NativeLibLoader.Arch}. + */ +@RunWith(Enclosed.class) +public class ArchDetectionTest { + + private static final String OS_ARCH_PROPERTY = "os.arch"; + + @RunWith(Parameterized.class) + public static class ParameterizedArchDetectionTest { + + private final String osArchValue; + private final NativeLibLoader.Arch expectedArch; + + public ParameterizedArchDetectionTest(String osArchValue, NativeLibLoader.Arch expectedArch) { + this.osArchValue = osArchValue; + this.expectedArch = expectedArch; + } + + @Parameters + public static Collection data() { + return asList(new Object[][]{ + {"x86_64", X86_64}, + {"amd64", X86_64}, + {"aarch64", AARCH64}, + {"arm64", AARCH64}, + {"sparc64", SPARC} + }); + } + + @Test + public void testArch() { + executeAndRestoreProperty(() -> { + System.setProperty(OS_ARCH_PROPERTY, osArchValue); + assertSame(detectArch(), expectedArch); + }); + } + } + + public static class UnsupportedArchDetectionTest { + + @Test + public void testUnsupportedArch() { + executeAndRestoreProperty(() -> { + System.setProperty(OS_ARCH_PROPERTY, "unsupported"); + assertThrows(IllegalStateException.class, NativeLibLoader.Arch::detectArch); + }); + } + } + + private static void executeAndRestoreProperty(Runnable action) { + String oldValue = System.getProperty(OS_ARCH_PROPERTY); + + try { + action.run(); + } finally { + if (oldValue != null) { + System.setProperty(OS_ARCH_PROPERTY, oldValue); + } else { + System.clearProperty(OS_ARCH_PROPERTY); + } + } + } + +} diff --git a/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/OsDetectionTest.java b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/OsDetectionTest.java new file mode 100644 index 000000000..b8ca3d772 --- /dev/null +++ b/jvm-packages/xgboost4j/src/test/java/ml/dmlc/xgboost4j/java/OsDetectionTest.java @@ -0,0 +1,123 @@ +/* + Copyright (c) 2014 by Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package ml.dmlc.xgboost4j.java; + +import ml.dmlc.xgboost4j.java.NativeLibLoader.OS; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import java.util.Collection; + +import static java.util.Arrays.asList; +import static junit.framework.TestCase.assertSame; +import static ml.dmlc.xgboost4j.java.NativeLibLoader.OS.*; +import static org.junit.Assert.assertThrows; + +/** + * Test cases for {@link OS}. + */ +@RunWith(Enclosed.class) +public class OsDetectionTest { + + private static final String OS_NAME_PROPERTY = "os.name"; + + @RunWith(Parameterized.class) + public static class ParameterizedOSDetectionTest { + + private final String osNameValue; + private final OS expectedOS; + + public ParameterizedOSDetectionTest(String osNameValue, OS expectedOS) { + this.osNameValue = osNameValue; + this.expectedOS = expectedOS; + } + + @Parameters + public static Collection data() { + return asList(new Object[][]{ + {"windows", WINDOWS}, + {"mac", MACOS}, + {"darwin", MACOS}, + {"linux", LINUX}, + {"sunos", SOLARIS} + }); + } + + @Test + public void getOS() { + executeAndRestoreProperty(() -> { + System.setProperty(OS_NAME_PROPERTY, osNameValue); + assertSame(detectOS(), expectedOS); + }); + } + } + + public static class NonParameterizedOSDetectionTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testForRegularLinux() throws Exception { + setMappedFilesBaseDir(folder.getRoot().toPath()); + folder.newFile("ld-2.23.so"); + + executeAndRestoreProperty(() -> { + System.setProperty(OS_NAME_PROPERTY, "linux"); + assertSame(detectOS(), LINUX); + }); + } + + @Test + public void testForMuslBasedLinux() throws Exception { + setMappedFilesBaseDir(folder.getRoot().toPath()); + folder.newFile("ld-musl-x86_64.so.1"); + + executeAndRestoreProperty(() -> { + System.setProperty(OS_NAME_PROPERTY, "linux"); + assertSame(detectOS(), LINUX_MUSL); + }); + } + + @Test + public void testUnsupportedOs() { + executeAndRestoreProperty(() -> { + System.setProperty(OS_NAME_PROPERTY, "unsupported"); + assertThrows(IllegalStateException.class, OS::detectOS); + }); + } + } + + private static void executeAndRestoreProperty(Runnable action) { + String oldValue = System.getProperty(OS_NAME_PROPERTY); + + try { + action.run(); + } finally { + if (oldValue != null) { + System.setProperty(OS_NAME_PROPERTY, oldValue); + } else { + System.clearProperty(OS_NAME_PROPERTY); + } + } + } + +}