支持𩽾𩾌(ān kāng)之类的补充字符集 fix #1564

hankcs · Jan 31, 2021 · b9a899b · b9a899b · hanlpbot · Jan 31, 2021
1 parent 2577426
commit b9a899b
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 4 deletions.
diff --git a/data/dictionary/pinyin/pinyin.txt b/data/dictionary/pinyin/pinyin.txt
@@ -30724,4 +30724,5 @@
 龥=yue4
 重启=chong2,qi3
 还款=huan2,kuan3
-侠传=xia2,zhuan4
+侠传=xia2,zhuan4
+𩽾𩾌=an1,kang1
diff --git a/src/main/java/com/hankcs/hanlp/dictionary/py/PinyinDictionary.java b/src/main/java/com/hankcs/hanlp/dictionary/py/PinyinDictionary.java
@@ -180,15 +180,17 @@ protected static List<Pinyin> segLongest(char[] charArray, AhoCorasickDoubleArra
     protected static List<Pinyin> segLongest(char[] charArray, AhoCorasickDoubleArrayTrie<Pinyin[]> trie, boolean remainNone)
     {
         final Pinyin[][] wordNet = new Pinyin[charArray.length][];
+        final int[] lengths = new int[charArray.length];
         trie.parseText(charArray, new AhoCorasickDoubleArrayTrie.IHit<Pinyin[]>()
         {
             @Override
             public void hit(int begin, int end, Pinyin[] value)
             {
                 int length = end - begin;
-                if (wordNet[begin] == null || length > wordNet[begin].length)
+                if (length > lengths[begin])
                 {
-                    wordNet[begin] = length == 1 ? new Pinyin[]{value[0]} : value;
+                    wordNet[begin] = value;
+                    lengths[begin] = length;
                 }
             }
         });
@@ -208,7 +210,7 @@ public void hit(int begin, int end, Pinyin[] value)
             {
                 pinyinList.add(pinyin);
             }
-            offset += wordNet[offset].length;
+            offset += lengths[offset];
         }
         return pinyinList;
     }

diff --git a/src/test/java/com/hankcs/hanlp/dictionary/py/PinyinDictionaryTest.java b/src/test/java/com/hankcs/hanlp/dictionary/py/PinyinDictionaryTest.java
@@ -0,0 +1,17 @@
+package com.hankcs.hanlp.dictionary.py;
+
+import com.hankcs.hanlp.HanLP;
+import junit.framework.TestCase;
+
+import java.util.Arrays;
+
+public class PinyinDictionaryTest extends TestCase
+{
+
+    public void testGet()
+    {
+        System.out.println(Arrays.toString(PinyinDictionary.get("鼖")));
+        System.out.println(PinyinDictionary.convertToPinyin("\uD867\uDF7E\uD867\uDF8C"));
+        System.out.println(HanLP.convertToPinyinList("\uD867\uDF7E\uD867\uDF8C"));
+    }
+}