new doc

doc/guide.md

@@ -352,7 +352,7 @@ Rabit is a portable library that can run on multiple platforms.
 
 #### Customize Tracker Script
 You can also modify the tracker script to allow rabit to run on other platforms. To do so, refer to existing
-tracker scripts, such as [../tracker/rabit_hadoop.py](../tracker/rabit_hadoop.py) and [../tracker/rabit_mpi.py](https://github.com/dmlc/rabit/blob/master/tracker/rabit_mpi.py) to get a sense of how it is done.
+tracker scripts, such as [../tracker/rabit_yarn.py](../tracker/rabit_yarn.py) and [../tracker/rabit_mpi.py](https://github.com/dmlc/rabit/blob/master/tracker/rabit_mpi.py) to get a sense of how it is done.
 
 You will need to implement a platform dependent submission function with the following definition
 ```python

doc/sphinx_util.py

@@ -2,6 +2,7 @@
 """Helper utilty function for customization."""
 import sys
 import os
+import docutils
 import subprocess
 
 if os.environ.get('READTHEDOCS', None) == 'True':
@@ -10,37 +11,26 @@ if os.environ.get('READTHEDOCS', None) == 'True':
                     'mv recommonmark/recommonmark recom', shell=True)
 
 sys.path.insert(0, os.path.abspath('..'))
-from recom import parser
+from recom import parser, transform
 
-class MarkdownParser(parser.CommonMarkParser):
+class MarkdownParser(docutils.parsers.Parser):
     github_doc_root = None
-    doc_suffix = set(['md', 'rst'])
 
-    @staticmethod
-    def remap_url(url):
-        if MarkdownParser.github_doc_root is None or url is None:
-            return url
-        if url.startswith('#'):
-            return url
-        arr = url.split('#', 1)
-        ssuffix = arr[0].rsplit('.', 1)
 
-        if len(ssuffix) == 2 and (ssuffix[-1] in MarkdownParser.doc_suffix
-                                  and arr[0].find('://') == -1):
-            arr[0] = arr[0][:-3] + '.html'
-            return '#'.join(arr)
-        else:
-            return MarkdownParser.github_doc_root + url
+    def __init__(self):
+        self.parser = parser.CommonMarkParser()
 
-    def reference(self, block):
-        block.destination = remap_url(block.destination)
-        return super(MarkdownParser, self).reference(block)
+    def parse(self, inputstring, document):
+        self.parser.parse(inputstring, document)
+        transform.AutoStructify.url_resolver = [resolve_url]
+        for trans in self.get_transforms():
+            transform.AutoStructify(document).apply()
 
-# inplace modify the function in recommonmark module to allow link remap
-old_ref = parser.reference
+    def get_transforms(self):
+        return [transform.AutoStructify]
 
-def reference(block):
-    block.destination = MarkdownParser.remap_url(block.destination)
-    return old_ref(block)
-
-parser.reference = reference
+def resolve_url(url):
+    if MarkdownParser.github_doc_root is None or url is None:
+        return url
+    else:
+        return MarkdownParser.github_doc_root + url