diff --git a/CHANGES.rst b/CHANGES.rst
index c36233573a3..4f784c34580 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -89,6 +89,9 @@ Bugs fixed
 * #12494: Fix invalid genindex.html file produced with translated docs
   (regression in 7.1.0).
   Patch by Nicolas Peugnet.
+* #12514: intersphinx: Handle the case where ``intersphinx_cache_limit`` is
+  negative, ensure that it behaves as documented.
+  Patch by Shengyu Zhang.
 
 Testing
 -------
diff --git a/sphinx/ext/intersphinx/_load.py b/sphinx/ext/intersphinx/_load.py
index b458d6a7e18..01caee77895 100644
--- a/sphinx/ext/intersphinx/_load.py
+++ b/sphinx/ext/intersphinx/_load.py
@@ -107,7 +107,10 @@ def fetch_inventory_group(
     app: Sphinx,
     now: int,
 ) -> bool:
-    cache_time = now - app.config.intersphinx_cache_limit * 86400
+    if app.config.intersphinx_cache_limit != -1:
+        cache_time = now - app.config.intersphinx_cache_limit * 86400
+    else:
+        cache_time = None
     failures = []
     try:
         for inv in invs:
@@ -115,7 +118,8 @@ def fetch_inventory_group(
                 inv = posixpath.join(uri, INVENTORY_FILENAME)
             # decide whether the inventory must be read: always read local
             # files; remote ones only if the cache time is expired
-            if '://' not in inv or uri not in cache or cache[uri][1] < cache_time:
+            if '://' not in inv or uri not in cache or \
+                    (cache_time and cache[uri][1] < cache_time):
                 safe_inv_url = _get_safe_url(inv)
                 inv_descriptor = name or 'main_inventory'
                 LOGGER.info(__("loading intersphinx inventory '%s' from %s..."),