Tokens outside text range (#56)

* Use 3.11 everywhere (#38) * v0.6.0 (#42) (#45) * Bump version and add note to README Prepare for release tomorrow and move forward for v0.7.0 and v0.8.0 release * Steal pygments regexes (#34) * test.py * Move to server functions and set up proper tests * Reformat * Reformat * Change to beartype typing * Even more formatting * Remove regex stealer test Runs different locally than on gh runner and not worth the time or effort. * Get docstring areas * Make function work * Add type annotation * format * Add lots of comments but don't remove private type @leycec I tried using the idea you gave for private types but got Union type errors for some reason I couldn't really understand. Thank you so much for your time and thoughts and congrats on the bike ride, thats a long stretch! * Fix a small bug * Improve highlighting functions significantly Now it ignores whitespace at the front of the line! * Stop using private variable * Format for black and ruff * Move docstring tokens up * Update tests * Fix line number for docstring tokens Was 1 behind * Reformat * Bump version * Implement token overwriting (#49) * Display the problem * Get working test * Better overlap checking * Better tests * Sort and remove duplicates * Remove old vestige and format * Move token merging to highlight file * Format * Use overwrite_and_merge_tokens * Use caches (#53) * Cache important functions * Remove old file * Format * Explain python version requirement plans (#54) * Check that all tokens are in text range
salve-org · Jul 7, 2024 · 642df26 · 642df26
1 parent a210242
commit 642df26
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 2 deletions.
diff --git a/salve_ipc/server_functions/highlight.py b/salve_ipc/server_functions/highlight.py
@@ -168,6 +168,26 @@ def overwrite_and_merge_tokens(
     return output_tokens
 
 
+def only_tokens_in_text_range(
+    tokens: list[Token], text_range: tuple[int, int]
+) -> list[Token]:
+    # We create a new list becase lists are pass by reference
+    output_tokens: list[Token] = []
+
+    for token in tokens:
+        token_lineno: int = token[0][0]
+        minimum_line: int = text_range[0]
+        maximum_line: int = text_range[1]
+
+        if token_lineno < minimum_line or token_lineno > maximum_line:
+            continue
+
+        output_tokens.append(token)
+
+    output_tokens = merge_tokens(output_tokens)
+    return output_tokens
+
+
 @cache
 def get_new_token_type(old_token: str) -> str:
     """Turns pygments token types into a generic predefined Token"""
@@ -513,5 +533,5 @@ def get_highlights(
         # if there are not hidden chars we don't want to needlessly compute this
         new_tokens += find_hidden_chars(split_text, text_range[0])
 
-    new_tokens = merge_tokens(new_tokens)
+    new_tokens = only_tokens_in_text_range(new_tokens, text_range)
     return new_tokens
diff --git a/tests/test_ipc.py b/tests/test_ipc.py
@@ -28,7 +28,9 @@ def test_IPC():
         expected_keywords=[],
         current_word="thid",
     )
-    context.request(HIGHLIGHT, file="test", language="python")
+    context.request(
+        HIGHLIGHT, file="test", language="python", text_range=(1, 18)
+    )
     context.request(EDITORCONFIG, file_path=__file__)
     context.request(
         DEFINITION,

diff --git a/tests/testing_file1.py b/tests/testing_file1.py
@@ -16,3 +16,6 @@ def __init__(self):
 
 Foo()
 # https://www.google.com
+"""
+test
+"""
-Original file line number
+Diff line change
@@ Expand Up / @@ -16,3 +16,6 @@ def __init__(self): @@
     Foo()
     # https://www.google.com
+    """
+    test
+    """