feat: relative cross-references

Fixes mkdocstrings#27

- Move subsitution to from collect to render
- Warn w/ source location about missing reference after relative path resolution
- Work around issue with bad source information in python 3.7
- Add debug logging
- Fix bug in regular expression

Author: analog-cbarber

src/mkdocstrings_handlers/python/crossref.py

@@ -1,10 +1,23 @@
+#  Copyright (c) 2022.   Analog Devices Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 """Support for translating compact relative crossreferences in docstrings."""
-# author: Christopher Barber, Analog Devices | Analog Garage
 
 from __future__ import annotations
 
 import re
-from typing import List, Optional
+import sys
+from typing import Callable, List, Optional, Union, cast
 
 from griffe.dataclasses import Docstring, Object
 from mkdocstrings.loggers import get_logger
@@ -13,6 +26,9 @@
 
 logger = get_logger(__name__)
 
+# line numbers from griffe are not reliable before python 3.8; this may eventually be fixed...
+_supports_linenums = sys.version_info >= (3, 8)
+
 
 def _re_or(*exps: str) -> str:
     """Construct an "or" regular expression from a sequence of regular expressions.
@@ -41,7 +57,7 @@ def _re_named(name: str, exp: str, optional: bool = False) -> str:
     return f"(?P<{name}>{exp}){optchar}"
 
 
-_RE_REL_CROSSREF = re.compile(r"\[(.+?)\]\[([\.^\(][^\]]*?|[^\]]*?\.)\]")
+_RE_REL_CROSSREF = re.compile(r"\[([^\[\]]+?)\]\[([\.^\(][^\]]*?|[^\]]*?\.)\]")
 """Regular expression that matches relative cross-reference expressions in doc-string.
 
 This will match a cross reference where the path expression either ends in '.'
@@ -79,6 +95,10 @@ def _re_named(name: str, exp: str, optional: bool = False) -> str:
 """Regular expression that matches a qualified python identifier."""
 
 
+def _always_ok(_ref: str) -> bool:
+    return True
+
+
 class _RelativeCrossrefProcessor:
     """
     A callable object that substitutes relative cross-reference expressions.
@@ -93,13 +113,15 @@ class _RelativeCrossrefProcessor:
     _cur_offset: int
     _cur_ref_parts: List[str]
     _ok: bool
+    _check_ref: Union[Callable[[str], bool], Callable[[str], bool]]
 
-    def __init__(self, doc: Docstring):
+    def __init__(self, doc: Docstring, checkref: Optional[Callable[[str], bool]] = None):
         self._doc = doc
         self._cur_match = None
         self._cur_input = ""
         self._cur_offset = 0
         self._cur_ref_parts = []
+        self._check_ref = checkref or _always_ok
         self._ok = True
 
     def __call__(self, match: re.Match) -> str:
@@ -117,13 +139,23 @@ def __call__(self, match: re.Match) -> str:
             self._process_append_from_title(ref_match, title)
 
         if self._ok:
-            result = f"[{title}][{'.'.join(self._cur_ref_parts)}]"
+            new_ref = ".".join(self._cur_ref_parts)
+            logger.debug(
+                "cross-reference substitution\nin %s:\n[%s][%s] -> [...][%s]",  # noqa: WPS323
+                cast(Object, self._doc.parent).canonical_path,
+                title,
+                ref,
+                new_ref,
+            )
+            if not self._check_ref(new_ref):
+                self._error(f"Cannot load reference '{new_ref}'")
+            result = f"[{title}][{new_ref}]"
         else:
             result = match.group(0)
 
         return result
 
-    def _start_match(self, match: re.Match):
+    def _start_match(self, match: re.Match) -> None:
         self._cur_match = match
         self._cur_offset = match.start(0)
         self._cur_input = match[0]
@@ -143,7 +175,7 @@ def _process_append_from_title(self, ref_match: re.Match, title_text: str) -> No
                 return
             self._cur_ref_parts.append(id_from_title)
 
-    def _process_parent_specifier(self, ref_match: re.Match):
+    def _process_parent_specifier(self, ref_match: re.Match) -> None:
         if not ref_match.group("parent"):
             return
 
@@ -196,11 +228,11 @@ def _process_up_specifier(self, obj: Object, ref_match: re.Match) -> Optional[Ob
             level = len(ref_match.group("up"))
             rel_obj = obj
             for _ in range(level):
-                if rel_obj.parent is None:
+                if rel_obj.parent is not None:
+                    rel_obj = rel_obj.parent
+                else:
                     self._error(f"'{ref_match.group('up')}' has too many levels for {obj.canonical_path}")
                     break
-                else:
-                    rel_obj = rel_obj.parent
         return rel_obj
 
     def _error(self, msg: str) -> None:
@@ -219,9 +251,10 @@ def _error(self, msg: str) -> None:
             # recognize that this is a navigable location it can highlight.
             prefix = f"file://{parent.filepath}:"
             line = doc.lineno
-            if line is not None:
-                # Add line offset to match in docstring
-                line += doc.value.count("\n", 0, self._cur_offset)
+            if line is not None:  # pragma: no branch
+                if _supports_linenums:  # pragma: no branch
+                    # Add line offset to match in docstring
+                    line += doc.value.count("\n", 0, self._cur_offset)
                 prefix += f"{line}:"
                 # It would be nice to add the column as well, but we cannot determine
                 # that without knowing how much the doc string was unindented.
@@ -232,17 +265,19 @@ def _error(self, msg: str) -> None:
         self._ok = False
 
 
-def substitute_relative_crossrefs(obj: Object):
+def substitute_relative_crossrefs(obj: Object, checkref: Optional[Callable[[str], bool]] = None) -> None:
     """Recursively expand relative cross-references in all docstrings in tree.
 
     Arguments:
-        obj: root object. The object's docstring will be be processed as well
-            as all of its children recursively.
+        obj: a Griffe [Object][griffe.dataclasses.] whose docstrings should be modified
+        checkref: optional function to check whether computed cross-reference is valid.
+            Should return True if valid, False if not valid.
     """
     doc = obj.docstring
-    if doc:
-        doc.value = _RE_REL_CROSSREF.sub(_RelativeCrossrefProcessor(doc), doc.value)
+
+    if doc is not None:
+        doc.value = _RE_REL_CROSSREF.sub(_RelativeCrossrefProcessor(doc, checkref=checkref), doc.value)
 
     for member in obj.members.values():
-        if isinstance(member, Object):
-            substitute_relative_crossrefs(member)
+        if isinstance(member, Object):  # pragma: no branch
+            substitute_relative_crossrefs(member, checkref=checkref)

src/mkdocstrings_handlers/python/handler.py

@@ -191,11 +191,7 @@ def collect(self, identifier: str, config: dict) -> CollectorItem:  # noqa: D102
                 lines_collection=self._lines_collection,
             )
             try:  # noqa: WPS229
-                module = loader.load_module(module_name)
-
-                if final_config["relative_crossrefs"]:
-                    substitute_relative_crossrefs(module)
-
+                loader.load_module(module_name)
             except ImportError as error:
                 raise CollectionError(str(error)) from error
 
@@ -219,6 +215,9 @@ def collect(self, identifier: str, config: dict) -> CollectorItem:  # noqa: D102
     def render(self, data: CollectorItem, config: dict) -> str:  # noqa: D102 (ignore missing docstring)
         final_config = ChainMap(config, self.default_config)
 
+        if final_config["relative_crossrefs"]:
+            substitute_relative_crossrefs(data, checkref=self._check_ref)
+
         template = self.env.get_template(f"{data.kind.value}.html")
 
         # Heading level is a "state" variable, that will change at each step
@@ -258,6 +257,22 @@ def get_anchors(self, data: CollectorItem) -> list[str]:  # noqa: D102 (ignore m
         except AliasResolutionError:
             return [data.path]
 
+    def _check_ref(self, ref: str) -> bool:
+        """Check for existence of reference.
+
+        Arguments:
+            ref: reference to check
+
+        Returns:
+            true if reference exists
+        """
+        try:
+            self.collect(ref, {})
+        except Exception:  # pylint: disable=broad-except
+            # Only expect a CollectionError but we may as well catch everything.
+            return False
+        return True
+
 
 def get_handler(
     theme: str,  # noqa: W0613 (unused argument config)