diff --git a/tests/buildkite/build-cpu-arm64.sh b/tests/buildkite/build-cpu-arm64.sh
index 3bbc95472..c826b9154 100755
--- a/tests/buildkite/build-cpu-arm64.sh
+++ b/tests/buildkite/build-cpu-arm64.sh
@@ -19,13 +19,17 @@ $command_wrapper bash -c "cd build && ctest --extra-verbose"
 echo "--- Build binary wheel"
 $command_wrapper bash -c \
   "cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
-$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
-  ${BUILDKITE_COMMIT} ${WHEEL_TAG}
+$command_wrapper python tests/ci_build/rename_whl.py  \
+  --wheel-path python-package/dist/*.whl  \
+  --commit-hash ${BUILDKITE_COMMIT}  \
+  --platform-tag ${WHEEL_TAG}
 
 echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
 $command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
-$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
-  ${BUILDKITE_COMMIT} ${WHEEL_TAG}
+$command_wrapper python tests/ci_build/rename_whl.py  \
+  --wheel-path wheelhouse/*.whl  \
+  --commit-hash ${BUILDKITE_COMMIT}  \
+  --platform-tag ${WHEEL_TAG}
 mv -v wheelhouse/*.whl python-package/dist/
 # Make sure that libgomp.so is vendored in the wheel
 $command_wrapper bash -c \
diff --git a/tests/buildkite/build-cuda-with-rmm.sh b/tests/buildkite/build-cuda-with-rmm.sh
index 559bad8a7..0474f5466 100755
--- a/tests/buildkite/build-cuda-with-rmm.sh
+++ b/tests/buildkite/build-cuda-with-rmm.sh
@@ -36,14 +36,18 @@ $command_wrapper tests/ci_build/build_via_cmake.sh \
 echo "--- Build binary wheel"
 $command_wrapper bash -c \
   "cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
-$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
-  ${BUILDKITE_COMMIT} ${WHEEL_TAG}
+$command_wrapper python tests/ci_build/rename_whl.py  \
+  --wheel-path python-package/dist/*.whl  \
+  --commit-hash ${BUILDKITE_COMMIT}  \
+  --platform-tag ${WHEEL_TAG}
 
 echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
 tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
-$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
-  ${BUILDKITE_COMMIT} ${WHEEL_TAG}
+$command_wrapper python tests/ci_build/rename_whl.py  \
+  --wheel-path wheelhouse/*.whl  \
+  --commit-hash ${BUILDKITE_COMMIT}  \
+  --platform-tag ${WHEEL_TAG}
 mv -v wheelhouse/*.whl python-package/dist/
 # Make sure that libgomp.so is vendored in the wheel
 tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
diff --git a/tests/buildkite/build-cuda.sh b/tests/buildkite/build-cuda.sh
index 5abc5ca5a..f08502c31 100755
--- a/tests/buildkite/build-cuda.sh
+++ b/tests/buildkite/build-cuda.sh
@@ -35,14 +35,18 @@ $command_wrapper tests/ci_build/build_via_cmake.sh \
 echo "--- Build binary wheel"
 $command_wrapper bash -c \
   "cd python-package && rm -rf dist/* && pip wheel --no-deps -v . --wheel-dir dist/"
-$command_wrapper python tests/ci_build/rename_whl.py python-package/dist/*.whl \
-  ${BUILDKITE_COMMIT} ${WHEEL_TAG}
+$command_wrapper python tests/ci_build/rename_whl.py  \
+  --wheel-path python-package/dist/*.whl  \
+  --commit-hash ${BUILDKITE_COMMIT}  \
+  --platform-tag ${WHEEL_TAG}
 
 echo "--- Audit binary wheel to ensure it's compliant with manylinux2014 standard"
 tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
   --plat ${WHEEL_TAG} python-package/dist/*.whl
-$command_wrapper python tests/ci_build/rename_whl.py wheelhouse/*.whl \
-  ${BUILDKITE_COMMIT} ${WHEEL_TAG}
+$command_wrapper python tests/ci_build/rename_whl.py  \
+  --wheel-path wheelhouse/*.whl  \
+  --commit-hash ${BUILDKITE_COMMIT}  \
+  --platform-tag ${WHEEL_TAG}
 mv -v wheelhouse/*.whl python-package/dist/
 # Make sure that libgomp.so is vendored in the wheel
 tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
@@ -54,6 +58,15 @@ if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
 then
   aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
     --acl public-read --no-progress
+
+  # Generate the meta info which includes xgboost version and the commit info
+  $command_wrapper python tests/ci_build/format_wheel_meta.py \
+    --wheel-path python-package/dist/*.whl  \
+    --commit-hash ${BUILDKITE_COMMIT}  \
+    --platform-tag ${WHEEL_TAG}  \
+    --meta-path python-package/dist/
+  aws s3 cp python-package/dist/meta.json s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
+    --acl public-read --no-progress
 fi
 echo "-- Stash C++ test executable (testxgboost)"
 buildkite-agent artifact upload build/testxgboost
diff --git a/tests/buildkite/build-win64-gpu.ps1 b/tests/buildkite/build-win64-gpu.ps1
index 092d4b192..0799d7ac6 100644
--- a/tests/buildkite/build-win64-gpu.ps1
+++ b/tests/buildkite/build-win64-gpu.ps1
@@ -29,7 +29,10 @@ conda activate
 & pip wheel --no-deps -v . --wheel-dir dist/
 Get-ChildItem . -Filter dist/*.whl |
 Foreach-Object {
-  & python ../tests/ci_build/rename_whl.py $_.FullName $Env:BUILDKITE_COMMIT win_amd64
+  & python ../tests/ci_build/rename_whl.py `
+    --wheel-path $_.FullName `
+    --commit-hash $Env:BUILDKITE_COMMIT `
+    --platform-tag win_amd64
   if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
 }
 
@@ -44,7 +47,7 @@ if ( $is_release_branch -eq 1 ) {
   Get-ChildItem . -Filter python-package/dist/*.whl |
   Foreach-Object {
     & aws s3 cp python-package/dist/$_ s3://xgboost-nightly-builds/$Env:BUILDKITE_BRANCH/ `
-    --acl public-read --no-progress
+      --acl public-read --no-progress
     if ($LASTEXITCODE -ne 0) { throw "Last command failed" }
   }
 }
diff --git a/tests/ci_build/build_python_wheels.sh b/tests/ci_build/build_python_wheels.sh
index 912fee482..03a9bc1d6 100644
--- a/tests/ci_build/build_python_wheels.sh
+++ b/tests/ci_build/build_python_wheels.sh
@@ -67,4 +67,7 @@ fi
 
 python -m pip install cibuildwheel
 python -m cibuildwheel python-package --output-dir wheelhouse
-python tests/ci_build/rename_whl.py wheelhouse/*.whl ${commit_id} ${wheel_tag}
+python tests/ci_build/rename_whl.py  \
+  --wheel-path wheelhouse/*.whl  \
+  --commit-hash ${commit_id}  \
+  --platform-tag ${wheel_tag}
diff --git a/tests/ci_build/format_wheel_meta.py b/tests/ci_build/format_wheel_meta.py
new file mode 100644
index 000000000..735c6a971
--- /dev/null
+++ b/tests/ci_build/format_wheel_meta.py
@@ -0,0 +1,58 @@
+"""
+Script to generate meta.json to store metadata for a nightly build of
+XGBoost Python package.
+"""
+import json
+import pathlib
+from argparse import ArgumentParser
+
+
+def main(args):
+    wheel_path = pathlib.Path(args.wheel_path).expanduser().resolve()
+    if not wheel_path.exists():
+        raise ValueError(f"Wheel cannot be found at path {wheel_path}")
+    if not wheel_path.is_file():
+        raise ValueError(f"Path {wheel_path} is not a valid file")
+    wheel_dir, wheel_name = wheel_path.parent, wheel_path.name
+
+    meta_path = pathlib.Path(args.meta_path)
+    if not meta_path.exists():
+        raise ValueError(f"Path {meta_path} does not exist")
+    if not meta_path.is_dir():
+        raise ValueError(f"Path {meta_path} is not a valid directory")
+
+    tokens = wheel_name.split("-")
+    assert len(tokens) == 5
+    version = tokens[1].split("+")[0]
+
+    meta_info = {
+        "wheel_name": wheel_name,
+        "platform_tag": args.platform_tag,
+        "version": version,
+        "commit_id": args.commit_hash,
+    }
+    with open(meta_path / "meta.json", "w") as f:
+        json.dump(meta_info, f, indent=4)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(
+        description="Format meta.json encoding the latest nightly version of the Python wheel"
+    )
+    parser.add_argument(
+        "--wheel-path", type=str, required=True, help="Path to the wheel"
+    )
+    parser.add_argument(
+        "--commit-hash", type=str, required=True, help="Git commit hash"
+    )
+    parser.add_argument(
+        "--platform-tag",
+        type=str,
+        required=True,
+        help="Platform tag (e.g. manylinux2014_x86_64)",
+    )
+    parser.add_argument(
+        "--meta-path", type=str, required=True, help="Directory to place meta.json"
+    )
+    parsed_args = parser.parse_args()
+    main(parsed_args)
diff --git a/tests/ci_build/rename_whl.py b/tests/ci_build/rename_whl.py
index 2da7db8de..62005f58e 100644
--- a/tests/ci_build/rename_whl.py
+++ b/tests/ci_build/rename_whl.py
@@ -1,39 +1,59 @@
-import os
-import sys
-
-from test_utils import DirectoryExcursion
-
-if len(sys.argv) != 4:
-    print("Usage: {} [wheel to rename] [commit id] [platform tag]".format(sys.argv[0]))
-    sys.exit(1)
+import pathlib
+from argparse import ArgumentParser
 
 
-whl_path = sys.argv[1]
-commit_id = sys.argv[2]
-platform_tag = sys.argv[3]
+def main(args):
+    wheel_path = pathlib.Path(args.wheel_path).expanduser().resolve()
+    if not wheel_path.exists():
+        raise ValueError(f"Wheel cannot be found at path {wheel_path}")
+    if not wheel_path.is_file():
+        raise ValueError(f"Path {wheel_path} is not a valid file")
+    wheel_dir, wheel_name = wheel_path.parent, wheel_path.name
 
-dirname, basename = os.path.dirname(whl_path), os.path.basename(whl_path)
-
-with DirectoryExcursion(dirname):
-    tokens = basename.split("-")
+    tokens = wheel_name.split("-")
     assert len(tokens) == 5
     version = tokens[1].split("+")[0]
     keywords = {
         "pkg_name": tokens[0],
         "version": version,
-        "commit_id": commit_id,
-        "platform_tag": platform_tag,
+        "commit_id": args.commit_hash,
+        "platform_tag": args.platform_tag,
     }
-    new_name = "{pkg_name}-{version}+{commit_id}-py3-none-{platform_tag}.whl".format(
-        **keywords
+    new_wheel_name = (
+        "{pkg_name}-{version}+{commit_id}-py3-none-{platform_tag}.whl".format(
+            **keywords
+        )
     )
-    print("Renaming {} to {}...".format(basename, new_name))
-    if os.path.isfile(new_name):
-        os.remove(new_name)
-    os.rename(basename, new_name)
+    new_wheel_path = wheel_dir / new_wheel_name
+    print(f"Renaming {wheel_name} to {new_wheel_name}...")
+    if new_wheel_path.is_file():
+        new_wheel_path.unlink()
+    wheel_path.rename(new_wheel_path)
 
-    filesize = os.path.getsize(new_name) / 1024 / 1024  # MB
-    print(f"Wheel size: {filesize}")
+    filesize = new_wheel_path.stat().st_size / 1024 / 1024  # MiB
+    print(f"Wheel size: {filesize:.2f} MiB")
 
-    msg = f"Limit of wheel size set by PyPI is exceeded. {new_name}: {filesize}"
-    assert filesize <= 300, msg
+    if filesize > 300:
+        raise RuntimeError(
+            f"Limit of wheel size set by PyPI is exceeded. {new_wheel_name}: {filesize:.2f} MiB"
+        )
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(
+        description="Format a Python wheel's name using the git commit hash and platform tag"
+    )
+    parser.add_argument(
+        "--wheel-path", type=str, required=True, help="Path to the wheel"
+    )
+    parser.add_argument(
+        "--commit-hash", type=str, required=True, help="Git commit hash"
+    )
+    parser.add_argument(
+        "--platform-tag",
+        type=str,
+        required=True,
+        help="Platform tag (e.g. manylinux2014_x86_64)",
+    )
+    parsed_args = parser.parse_args()
+    main(parsed_args)