Fix tokenization of Korean chars fix: #1877

hankcs · Feb 24, 2024 · 785bed5 · 785bed5
1 parent beab770
commit 785bed5
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/hanlp/components/tokenizers/transformer.py b/hanlp/components/tokenizers/transformer.py
@@ -147,7 +147,7 @@ def tag_to_span(self, batch_tags, batch: dict):
                             tags[i - 1] = 'B'
                         elif prev_tag == 'E':
                             tags[i - 1] = 'M'
-                        tags[i] = 'M'
+                        tags[i] = tag = 'M'
                     offset = e
                     prev_tag = tag
         for tags in batch_tags:

diff --git a/hanlp/version.py b/hanlp/version.py
@@ -2,7 +2,7 @@
 # Author: hankcs
 # Date: 2019-12-28 19:26
 
-__version__ = '2.1.0-beta.55'
+__version__ = '2.1.0-beta.56'
 """HanLP version"""