diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64
index 9233494d3..eba5464ff 100644
--- a/Jenkinsfile-win64
+++ b/Jenkinsfile-win64
@@ -137,8 +137,9 @@ def TestWin64GPU(args) {
     conda activate && for /R %%i in (python-package\\dist\\*.whl) DO python -m pip install "%%i"
     """
     echo "Installing Python dependencies..."
+    def cuda_short_ver = args.cuda_target.replaceAll('_', '')
     bat """
-     conda activate && conda install -y hypothesis && conda upgrade scikit-learn pandas numpy hypothesis
+     conda activate && conda install -y hypothesis && conda upgrade scikit-learn pandas numpy hypothesis  && python -m pip install cupy-${cuda_short_ver}
     """
     echo "Running Python tests..."
     bat """
diff --git a/tests/python-gpu/test_gpu_prediction.py b/tests/python-gpu/test_gpu_prediction.py
index 6324e117b..26ae95d1b 100644
--- a/tests/python-gpu/test_gpu_prediction.py
+++ b/tests/python-gpu/test_gpu_prediction.py
@@ -147,6 +147,9 @@ class TestGPUPredict(unittest.TestCase):
             copied_predt = cp.array(booster.predict(d))
             return cp.all(copied_predt == inplace_predt)
 
+        # Don't do this on Windows, see issue #5793
+        if sys.platform.startswith("win"):
+            pytest.skip('Multi-threaded in-place prediction with cuPy is not working on Windows')
         for i in range(10):
             run_threaded_predict(X, rows, predict_dense)