From 8dc1e4b3ea6f1b440c3806c04f18157e23fbd7a9 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Tue, 21 Mar 2023 09:22:11 -0700 Subject: [PATCH] Improve doxygen (#8959) * Remove Sphinx build from GH Action * Build Doxygen as part of RTD build * Add jQuery --- .github/workflows/main.yml | 37 ------- doc/c++.rst | 2 +- doc/c.rst | 2 +- doc/conf.py | 191 ++++++++++++++++++++++--------------- 4 files changed, 114 insertions(+), 118 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac50b744b..ab2a58fe9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -156,40 +156,3 @@ jobs: xgboost \ cpp \ include src python-package - - sphinx: - runs-on: ubuntu-latest - name: Build docs using Sphinx - steps: - - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 - with: - submodules: 'true' - - uses: actions/setup-python@7f80679172b057fc5e90d70d197929d454754a5a # v4.3.0 - with: - python-version: "3.8" - architecture: 'x64' - - name: Install system packages - run: | - sudo apt-get install -y --no-install-recommends graphviz doxygen ninja-build - python -m pip install wheel setuptools awscli - python -m pip install -r doc/requirements.txt - - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_') - - name: Run Sphinx - run: | - make -C doc html - env: - SPHINX_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} - READTHEDOCS: "True" - - - name: Publish - run: | - tar cvjf ${{ steps.extract_branch.outputs.branch }}.tar.bz2 doxygen/doc_doxygen/ - python -m awscli s3 cp ./${{ steps.extract_branch.outputs.branch }}.tar.bz2 s3://xgboost-docs/doxygen/ --acl public-read - if: github.ref == 'refs/heads/master' || contains(github.ref, 'refs/heads/release_') - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_IAM_S3_UPLOADER }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_IAM_S3_UPLOADER }} diff --git a/doc/c++.rst b/doc/c++.rst index 4a045fc42..ce30bbefa 100644 --- a/doc/c++.rst +++ b/doc/c++.rst @@ -8,5 +8,5 @@ As a result it's changing quite often and we don't maintain its stability. Alon plugin system (see ``plugin/example`` in XGBoost's source tree), users can utilize some existing c++ headers for gaining more access to the internal of XGBoost. -* `C++ interface documentation (latest master branch) `_ +* `C++ interface documentation (latest master branch) <./dev/files.html>`_ * `C++ interface documentation (last stable release) `_ diff --git a/doc/c.rst b/doc/c.rst index 02581b874..d63e779e1 100644 --- a/doc/c.rst +++ b/doc/c.rst @@ -10,7 +10,7 @@ simply look at function comments in ``include/xgboost/c_api.h``. The reference i to sphinx with the help of breathe, which doesn't contain links to examples but might be easier to read. For the original doxygen pages please visit: -* `C API documentation (latest master branch) `_ +* `C API documentation (latest master branch) <./dev/c__api_8h.html>`_ * `C API documentation (last stable release) `_ *************** diff --git a/doc/conf.py b/doc/conf.py index 7d585e420..73fe48acc 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -13,53 +13,106 @@ # serve to show the default. import os import re +import shutil import subprocess import sys +import tarfile import urllib.request +import warnings from subprocess import call from urllib.error import HTTPError from sh.contrib import git -git_branch = os.getenv('SPHINX_GIT_BRANCH', default=None) +CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +PROJECT_ROOT = os.path.normpath(os.path.join(CURR_PATH, os.path.pardir)) +TMP_DIR = os.path.join(CURR_PATH, "tmp") +DOX_DIR = "doxygen" + + +def run_doxygen(): + """Run the doxygen make command in the designated folder.""" + curdir = os.path.normpath(os.path.abspath(os.path.curdir)) + if os.path.exists(TMP_DIR): + print(f"Delete directory {TMP_DIR}") + shutil.rmtree(TMP_DIR) + else: + print(f"Create directory {TMP_DIR}") + os.mkdir(TMP_DIR) + try: + os.chdir(PROJECT_ROOT) + if not os.path.exists(DOX_DIR): + os.mkdir(DOX_DIR) + os.chdir(os.path.join(PROJECT_ROOT, DOX_DIR)) + print( + "Build doxygen at {}".format( + os.path.join(PROJECT_ROOT, DOX_DIR, "doc_doxygen") + ) + ) + subprocess.check_call(["cmake", "..", "-DBUILD_C_DOC=ON", "-GNinja"]) + subprocess.check_call(["ninja", "doc_doxygen"]) + + src = os.path.join(PROJECT_ROOT, DOX_DIR, "doc_doxygen", "html") + dest = os.path.join(TMP_DIR, "dev") + print(f"Copy directory {src} -> {dest}") + shutil.copytree(src, dest) + except OSError as e: + sys.stderr.write("doxygen execution failed: %s" % e) + finally: + os.chdir(curdir) + + +def is_readthedocs_build(): + if os.environ.get("READTHEDOCS", None) == "True": + return True + warnings.warn( + "Skipping Doxygen build... You won't have documentation for C/C++ functions. " + "Set environment variable READTHEDOCS=True if you want to build Doxygen. " + "(If you do opt in, make sure to install Doxygen, Graphviz, CMake, and C++ compiler " + "on your system.)" + ) + return False + + +if is_readthedocs_build(): + run_doxygen() + + +git_branch = os.getenv("SPHINX_GIT_BRANCH", default=None) if not git_branch: # If SPHINX_GIT_BRANCH environment variable is not given, run git # to determine branch name git_branch = [ - re.sub(r'origin/', '', x.lstrip(' ')) for x in str( - git.branch('-r', '--contains', 'HEAD')).rstrip('\n').split('\n') + re.sub(r"origin/", "", x.lstrip(" ")) + for x in str(git.branch("-r", "--contains", "HEAD")).rstrip("\n").split("\n") ] - git_branch = [x for x in git_branch if 'HEAD' not in x] + git_branch = [x for x in git_branch if "HEAD" not in x] else: git_branch = [git_branch] -print('git_branch = {}'.format(git_branch[0])) +print("git_branch = {}".format(git_branch[0])) try: filename, _ = urllib.request.urlretrieve( - 'https://s3-us-west-2.amazonaws.com/xgboost-docs/{}.tar.bz2'.format( - git_branch[0])) - call( - 'if [ -d tmp ]; then rm -rf tmp; fi; mkdir -p tmp/jvm; cd tmp/jvm; tar xvf {}' - .format(filename), - shell=True) + f"https://s3-us-west-2.amazonaws.com/xgboost-docs/{git_branch[0]}.tar.bz2" + ) + if not os.path.exists(TMP_DIR): + print(f"Create directory {TMP_DIR}") + os.mkdir(TMP_DIR) + jvm_doc_dir = os.path.join(TMP_DIR, "jvm") + if os.path.exists(jvm_doc_dir): + print(f"Delete directory {jvm_doc_dir}") + shutil.rmtree(jvm_doc_dir) + print(f"Create directory {jvm_doc_dir}") + os.mkdir(jvm_doc_dir) + + with tarfile.open(filename, "r:bz2") as t: + t.extractall(jvm_doc_dir) except HTTPError: - print('JVM doc not found. Skipping...') -try: - filename, _ = urllib.request.urlretrieve( - 'https://s3-us-west-2.amazonaws.com/xgboost-docs/doxygen/{}.tar.bz2'. - format(git_branch[0])) - call( - 'mkdir -p tmp/dev; cd tmp/dev; tar xvf {}; mv doc_doxygen/html/* .; rm -rf doc_doxygen' - .format(filename), - shell=True) -except HTTPError: - print('C API doc not found. Skipping...') + print("JVM doc not found. Skipping...") # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) -PROJECT_ROOT = os.path.normpath(os.path.join(CURR_PATH, os.path.pardir)) libpath = os.path.join(PROJECT_ROOT, "python-package/") sys.path.insert(0, libpath) sys.path.insert(0, CURR_PATH) @@ -82,50 +135,56 @@ release = xgboost.__version__ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones extensions = [ - 'matplotlib.sphinxext.plot_directive', - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx.ext.mathjax', - 'sphinx.ext.intersphinx', + "matplotlib.sphinxext.plot_directive", + "sphinxcontrib.jquery", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.mathjax", + "sphinx.ext.intersphinx", "sphinx_gallery.gen_gallery", - 'breathe', - 'recommonmark' + "breathe", + "recommonmark", ] sphinx_gallery_conf = { # path to your example scripts "examples_dirs": ["../demo/guide-python", "../demo/dask", "../demo/aft_survival"], # path to where to save gallery generated output - "gallery_dirs": ["python/examples", "python/dask-examples", "python/survival-examples"], + "gallery_dirs": [ + "python/examples", + "python/dask-examples", + "python/survival-examples", + ], "matplotlib_animations": True, } autodoc_typehints = "description" -graphviz_output_format = 'png' -plot_formats = [('svg', 300), ('png', 100), ('hires.png', 300)] +graphviz_output_format = "png" +plot_formats = [("svg", 300), ("png", 100), ("hires.png", 300)] plot_html_show_source_link = False plot_html_show_formats = False # Breathe extension variables -DOX_DIR = "doxygen" -breathe_projects = { - "xgboost": os.path.join(PROJECT_ROOT, DOX_DIR, "doc_doxygen/xml") -} +breathe_projects = {} +if is_readthedocs_build(): + breathe_projects = { + "xgboost": os.path.join(PROJECT_ROOT, DOX_DIR, "doc_doxygen/xml") + } breathe_default_project = "xgboost" # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: -source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] # The encoding of source files. # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -134,7 +193,7 @@ master_doc = 'index' # Usually you set "language" from the command line for these cases. language = "en" -autoclass_content = 'both' +autoclass_content = "both" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -144,8 +203,10 @@ autoclass_content = 'both' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] -html_extra_path = ['./tmp'] +exclude_patterns = ["_build"] +html_extra_path = [] +if is_readthedocs_build(): + html_extra_path = [TMP_DIR] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -163,7 +224,7 @@ html_extra_path = ['./tmp'] # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -186,27 +247,24 @@ html_logo = "https://raw.githubusercontent.com/dmlc/dmlc.github.io/master/img/lo html_css_files = ["css/custom.css"] -html_sidebars = { - '**': ['logo-text.html', 'globaltoc.html', 'searchbox.html'] -} +html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "searchbox.html"]} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Output file base name for HTML help builder. -htmlhelp_basename = project + 'doc' +htmlhelp_basename = project + "doc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = { -} +latex_elements = {} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, '%s.tex' % project, project, author, 'manual'), + (master_doc, "%s.tex" % project, project, author, "manual"), ] intersphinx_mapping = { @@ -221,30 +279,5 @@ intersphinx_mapping = { } -# hook for doxygen -def run_doxygen(): - """Run the doxygen make command in the designated folder.""" - curdir = os.path.normpath(os.path.abspath(os.path.curdir)) - try: - os.chdir(PROJECT_ROOT) - if not os.path.exists(DOX_DIR): - os.mkdir(DOX_DIR) - os.chdir(os.path.join(PROJECT_ROOT, DOX_DIR)) - subprocess.check_call(["cmake", "..", "-DBUILD_C_DOC=ON", "-GNinja"]) - subprocess.check_call(["ninja", "doc_doxygen"]) - except OSError as e: - sys.stderr.write("doxygen execution failed: %s" % e) - finally: - os.chdir(curdir) - - -def generate_doxygen_xml(app): - """Run the doxygen make commands if we're on the ReadTheDocs server""" - read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True' - if read_the_docs_build: - run_doxygen() - - def setup(app): - app.add_css_file('custom.css') - app.connect("builder-inited", generate_doxygen_xml) + app.add_css_file("custom.css")