From dd79ab846f389cf2e9c8802f5f451ee93d485798 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Fri, 3 Feb 2023 22:32:48 -0800 Subject: [PATCH] [CI] Fix failing arm build (#8751) * Always install Conda env into /opt/python; use Mamba * Change ownership of Conda env to buildkite-agent user * Use unique name * Fix --- tests/ci_build/Dockerfile.aarch64 | 10 ++++++---- tests/ci_build/Dockerfile.cpu | 9 +++++---- tests/ci_build/Dockerfile.gpu | 6 +++--- tests/ci_build/Dockerfile.gpu_build_centos7 | 8 ++++---- tests/ci_build/Dockerfile.gpu_build_r_centos7 | 8 ++++---- tests/ci_build/Dockerfile.gpu_jvm | 8 ++++---- tests/ci_build/Dockerfile.jvm | 6 +++--- tests/ci_build/Dockerfile.jvm_cross | 8 ++++---- tests/ci_build/Dockerfile.jvm_gpu_build | 6 +++--- tests/ci_build/Dockerfile.rmm | 6 +++--- tests/ci_build/build_via_cmake.sh | 7 +++++++ 11 files changed, 46 insertions(+), 36 deletions(-) diff --git a/tests/ci_build/Dockerfile.aarch64 b/tests/ci_build/Dockerfile.aarch64 index b7a863239..848b50263 100644 --- a/tests/ci_build/Dockerfile.aarch64 +++ b/tests/ci_build/Dockerfile.aarch64 @@ -10,10 +10,10 @@ RUN \ yum update -y && \ yum install -y devtoolset-7 && \ # Python - wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/4.8.2-1/Miniforge3-4.8.2-1-Linux-aarch64.sh && \ - bash conda.sh -b -p /opt/miniforge-python + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-aarch64.sh && \ + bash conda.sh -b -p /opt/mambaforge -ENV PATH=/opt/miniforge-python/bin:$PATH +ENV PATH=/opt/mambaforge/bin:$PATH ENV CC=/opt/rh/devtoolset-7/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-7/root/usr/bin/c++ ENV CPP=/opt/rh/devtoolset-7/root/usr/bin/cpp @@ -21,7 +21,9 @@ ENV GOSU_VERSION 1.10 # Create new Conda environment COPY conda_env/aarch64_test.yml /scripts/ -RUN conda env create -n aarch64_test --file=/scripts/aarch64_test.yml +RUN mamba create -n aarch64_test && \ + mamba env update -n aarch64_test --file=/scripts/aarch64_test.yml && \ + mamba clean --all # Install lightweight sudo (not bound to TTY) RUN set -ex; \ diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu index d09250b04..c1c20ba37 100644 --- a/tests/ci_build/Dockerfile.cpu +++ b/tests/ci_build/Dockerfile.cpu @@ -15,10 +15,10 @@ RUN \ wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \ bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \ # Python - wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh && \ - bash Mambaforge-Linux-x86_64.sh -b -p /opt/python + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge -ENV PATH=/opt/python/bin:$PATH +ENV PATH=/opt/mambaforge/bin:$PATH ENV CC=gcc-8 ENV CXX=g++-8 ENV CPP=cpp-8 @@ -37,7 +37,8 @@ RUN git clone -b v1.49.1 https://github.com/grpc/grpc.git \ # Create new Conda environment COPY conda_env/linux_cpu_test.yml /scripts/ -RUN mamba env create -n linux_cpu_test --file=/scripts/linux_cpu_test.yml && \ +RUN mamba create -n linux_cpu_test && \ + mamba env update -n linux_cpu_test --file=/scripts/linux_cpu_test.yml && \ mamba clean --all && \ conda run --no-capture-output -n linux_cpu_test pip install buildkite-test-collector diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu index d149638ac..04dc5bcd0 100644 --- a/tests/ci_build/Dockerfile.gpu +++ b/tests/ci_build/Dockerfile.gpu @@ -13,10 +13,10 @@ RUN \ apt-get update && \ apt-get install -y wget unzip bzip2 libgomp1 build-essential openjdk-8-jdk-headless && \ # Python - wget -nv -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3.sh -b -p /opt/python + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge -ENV PATH=/opt/python/bin:$PATH +ENV PATH=/opt/mambaforge/bin:$PATH # Create new Conda environment with cuDF, Dask, and cuPy RUN \ diff --git a/tests/ci_build/Dockerfile.gpu_build_centos7 b/tests/ci_build/Dockerfile.gpu_build_centos7 index 4168e430d..fb27cf4f2 100644 --- a/tests/ci_build/Dockerfile.gpu_build_centos7 +++ b/tests/ci_build/Dockerfile.gpu_build_centos7 @@ -11,9 +11,9 @@ RUN \ yum -y update && \ yum install -y tar unzip wget xz git which ninja-build devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ && \ # Python - wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3.sh -b -p /opt/python && \ - /opt/python/bin/python -m pip install awscli && \ + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge && \ + /opt/mambaforge/bin/python -m pip install awscli && \ # CMake wget -nv -nc https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.sh --no-check-certificate && \ bash cmake-3.18.0-Linux-x86_64.sh --skip-license --prefix=/usr @@ -28,7 +28,7 @@ RUN \ yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \ rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; -ENV PATH=/opt/python/bin:/usr/local/ninja:$PATH +ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:$PATH ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp diff --git a/tests/ci_build/Dockerfile.gpu_build_r_centos7 b/tests/ci_build/Dockerfile.gpu_build_r_centos7 index 54a63a242..ad5f15495 100644 --- a/tests/ci_build/Dockerfile.gpu_build_r_centos7 +++ b/tests/ci_build/Dockerfile.gpu_build_r_centos7 @@ -16,7 +16,7 @@ RUN \ devtoolset-8-gcc-gfortran devtoolset-8-libquadmath-devel \ devtoolset-8-runtime devtoolset-8-libstdc++-devel -ENV PATH=/opt/python/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/3.3.0/bin:$PATH +ENV PATH=/opt/mambaforge/bin:/usr/local/ninja:/opt/software/packages/bin:/opt/R/3.3.0/bin:$PATH ENV LD_LIBRARY_PATH=/opt/software/packages/lib:/opt/R/3.3.0/lib64:$LD_LIBRARY_PATH ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ @@ -32,9 +32,9 @@ RUN \ make -j$(nproc) && \ make install && \ # Python - wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3.sh -b -p /opt/python && \ - /opt/python/bin/python -m pip install auditwheel awscli && \ + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge && \ + /opt/mambaforge/bin/python -m pip install auditwheel awscli && \ # CMake wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \ bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr diff --git a/tests/ci_build/Dockerfile.gpu_jvm b/tests/ci_build/Dockerfile.gpu_jvm index c4eda9150..7883d189e 100644 --- a/tests/ci_build/Dockerfile.gpu_jvm +++ b/tests/ci_build/Dockerfile.gpu_jvm @@ -16,9 +16,9 @@ RUN \ apt-get update && \ apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \ # Python - wget -nv -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3.sh -b -p /opt/python && \ - /opt/python/bin/pip install awscli && \ + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge && \ + /opt/mambaforge/bin/pip install awscli && \ # Maven wget -nv https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \ tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ @@ -28,7 +28,7 @@ RUN \ tar xvf spark-$SPARK_VERSION-bin-hadoop2.7.tgz -C /opt && \ ln -s /opt/spark-$SPARK_VERSION-bin-hadoop2.7 /opt/spark -ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH +ENV PATH=/opt/mambaforge/bin:/opt/spark/bin:/opt/maven/bin:$PATH # Install Python packages RUN \ diff --git a/tests/ci_build/Dockerfile.jvm b/tests/ci_build/Dockerfile.jvm index 9c7001ade..4c5e21203 100644 --- a/tests/ci_build/Dockerfile.jvm +++ b/tests/ci_build/Dockerfile.jvm @@ -9,8 +9,8 @@ RUN \ devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ \ devtoolset-8-runtime devtoolset-8-libstdc++-devel && \ # Python - wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3.sh -b -p /opt/python && \ + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge && \ # CMake wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \ bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \ @@ -19,7 +19,7 @@ RUN \ tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ ln -s /opt/apache-maven-3.6.1/ /opt/maven -ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH +ENV PATH=/opt/mambaforge/bin:/opt/maven/bin:$PATH ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp diff --git a/tests/ci_build/Dockerfile.jvm_cross b/tests/ci_build/Dockerfile.jvm_cross index ca4323615..6d9c5c57f 100644 --- a/tests/ci_build/Dockerfile.jvm_cross +++ b/tests/ci_build/Dockerfile.jvm_cross @@ -13,9 +13,9 @@ RUN \ apt-get update && \ apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \ # Python - wget -nv -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3.sh -b -p /opt/python && \ - /opt/python/bin/pip install awscli && \ + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge && \ + /opt/mambaforge/bin/pip install awscli && \ # Maven wget -nv https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \ tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ @@ -25,7 +25,7 @@ RUN \ tar xvf spark-$SPARK_VERSION-bin-hadoop2.7.tgz -C /opt && \ ln -s /opt/spark-$SPARK_VERSION-bin-hadoop2.7 /opt/spark -ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH +ENV PATH=/opt/mambaforge/bin:/opt/spark/bin:/opt/maven/bin:$PATH # Install Python packages RUN \ diff --git a/tests/ci_build/Dockerfile.jvm_gpu_build b/tests/ci_build/Dockerfile.jvm_gpu_build index cddbb1f65..304db2d52 100644 --- a/tests/ci_build/Dockerfile.jvm_gpu_build +++ b/tests/ci_build/Dockerfile.jvm_gpu_build @@ -11,8 +11,8 @@ RUN \ yum -y update && \ yum install -y tar unzip wget xz git which ninja-build java-1.8.0-openjdk-devel devtoolset-8-gcc devtoolset-8-binutils devtoolset-8-gcc-c++ && \ # Python - wget -nv -nc -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3.sh -b -p /opt/python && \ + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge && \ # CMake wget -nv -nc https://cmake.org/files/v3.14/cmake-3.14.0-Linux-x86_64.sh --no-check-certificate && \ bash cmake-3.14.0-Linux-x86_64.sh --skip-license --prefix=/usr && \ @@ -29,7 +29,7 @@ RUN \ yum -y update && \ yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} -ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH +ENV PATH=/opt/mambaforge/bin:/opt/maven/bin:$PATH ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ ENV CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp diff --git a/tests/ci_build/Dockerfile.rmm b/tests/ci_build/Dockerfile.rmm index 0fbe44865..a1fce9c00 100644 --- a/tests/ci_build/Dockerfile.rmm +++ b/tests/ci_build/Dockerfile.rmm @@ -13,8 +13,8 @@ RUN \ apt-get update && \ apt-get install -y wget unzip bzip2 libgomp1 build-essential ninja-build git && \ # Python - wget -nv -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3.sh -b -p /opt/python + wget -nv -O conda.sh https://github.com/conda-forge/miniforge/releases/download/22.11.1-2/Mambaforge-22.11.1-2-Linux-x86_64.sh && \ + bash conda.sh -b -p /opt/mambaforge # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html) RUN \ @@ -23,7 +23,7 @@ RUN \ apt-get update && \ apt-get install -y --allow-downgrades --allow-change-held-packages libnccl2=${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-dev=${NCCL_VERSION}+cuda${CUDA_SHORT} -ENV PATH=/opt/python/bin:$PATH +ENV PATH=/opt/mambaforge/bin:$PATH # Create new Conda environment with RMM RUN \ diff --git a/tests/ci_build/build_via_cmake.sh b/tests/ci_build/build_via_cmake.sh index 31132a1f7..adf3cf984 100755 --- a/tests/ci_build/build_via_cmake.sh +++ b/tests/ci_build/build_via_cmake.sh @@ -7,6 +7,13 @@ then echo "Activating Conda environment ${conda_env}" shift 1 cmake_args="$@" + + # Workaround for file permission error + if [[ -n $CI_BUILD_UID ]] + then + gosu root chown -R "${CI_BUILD_UID}:${CI_BUILD_GID}" /opt/mambaforge/envs + fi + source activate ${conda_env} cmake_prefix_flag="-DCMAKE_PREFIX_PATH=$CONDA_PREFIX" else