diff --git a/budoux/models/ja.json b/budoux/models/ja.json index bd8e9ed9..be0dcd5a 100644 --- a/budoux/models/ja.json +++ b/budoux/models/ja.json @@ -1 +1 @@ -{"UW3":{"、":4348,"。":5774,"の":2805,"に":3412,"は":3917,"を":4831,"が":3367,"と":2731,"も":2425,"る":2467,"で":2971,"っ":-1660,"て":2130,"た":1553,"ら":988,"な":2090,"う":1829,"く":2398,"い":1428,"り":930,"］":1899,"，":3184,"か":1651,"や":1993,"！":2989,"ば":1754,"？":2991,"．":1683,"…":2655,"ど":1651,"）":3167,"・":869,"ー":-80,"ま":437,"へ":2062,"☆":2405,"つ":1127,"日":762,"♪":2363,"ず":1539,"ろ":907,"笑":2631,"だ":834,"」":1192,"ん":640,"京":-1393,"間":751,"人":621,"電":-1382,"ゃ":1213,"今":1036,"中":889,"分":879,"真":687,"ぐ":1457,"お":-1098,"す":727,"々":1291,"０":-941,"度":1129,"き":574,"む":1275,"ッ":-1105,"「":-1181,"イ":-397,"然":817,"大":-866,"ぶ":1234,"観":-937,"後":912,"都":-406,"ス":-507,"１":-565,"え":648,"［":-971,"夜":1216,"体":843,"変":236,"昔":1491,"け":252,"〜":379,"フ":-796,"一":-228,"月":653,"光":-247,"年":322,"あ":442,"水":378,"』":663,"ｏ":-601,"超":719,"技":315,"★":600,"果":503,"係":445,"方":451,"れ":139,"ぁ":498,"語":499,"＞":487,"グ":238,"題":597,"味":321,"学":-262,"み":329,"ド":-317,"自":-345,"し":147,"ぞ":492,"シ":-401,"ン":-65,"：":255,"ち":186,"機":-330,"カ":-256,"メ":-52,"色":327,"び":228,"出":-299,"ア":-253,"ぱ":372,"前":219,"”":326,"ね":296,"頭":349,"感":206,"不":-205,"レ":-172,"堂":294,"想":184,"生":-120,"じ":127,"ざ":131,"屋":118,"　":126,"ぼ":130,"プ":-100,"皆":138,"神":-76,"め":14,"パ":-49,"最":-54,"気":36,"週":23,"台":74,"こ":8,"来":10},"UW4":{"の":-3224,"、":-6660,"。":-6198,"て":-3362,"に":-3623,"で":-2470,"は":-2973,"る":-4288,"が":-2613,"っ":-3432,"を":-3751,"か":-2331,"な":-2099,"と":-2643,"た":-1629,"ら":-3927,"こ":-139,"し":-1823,"い":-919,"り":-3457,"れ":-3736,"う":-1772,"だ":-1999,"ん":-2892,"ー":-2482,"そ":70,"き":-1894,"も":-2239,"お":873,"く":-2573,"ま":-1471,"「":2187,"す":-1090,"思":1338,"私":1727,"」":-4609,"京":620,"人":531,"よ":-631,"一":502,"け":-3026,"，":-3819,"１":763,"！":-3989,"行":375,"見":479,"大":578,"さ":-1372,"使":943,"・":-539,"（":2188,"今":1191,"ろ":-2740,"え":-1506,"食":421,"時":487,"み":-926,"ン":-1800,"…":-2661,"何":843,"じ":-2231,"気":417,"わ":-1222,"ル":-1103,"２":636,"自":766,"ど":-986,"？":-2796,"最":1377,"．":-2593,"ば":-2156,"つ":-980,"べ":-2333,"あ":656,"ち":-1561,"携":318,"め":-1439,"帯":-1170,"ず":-1788,"や":-1048,"笑":-2614,"せ":-1306,"）":-2958,"ッ":-1249,"』":-2986,"ひ":530,"ゃ":-2030,"］":-1206,"ト":-335,"［":894,"ね":-1333,"僕":1097,"感":207,"３":80,"不":732,"ぁ":-1189,"ょ":-1353,"０":-243,"”":-888,"間":-349,"ぎ":-1014,"リ":-615,"風":431,"実":474,"イ":-226,"場":409,"買":451,"多":433,"少":456,"タ":-491,"♪":-1359,"〜":-1076,"以":-59,"書":324,"無":218,"ス":-199,"飲":583,"高":216,"祭":310,"☆":-1011,"ご":148,"楽":117,"子":-406,"び":-864,"合":-772,"ｏ":-555,"ぼ":-782,"乗":141,"小":450,"新":352,"街":438,"全":272,"的":-536,"夏":456,"『":572,"ぶ":-762,"ュ":-476,"ぜ":-755,"込":-685,"回":-642,"知":17,"相":306,"ぐ":-594,"誰":517,"言":255,"ぞ":-681,"へ":-451,"げ":-578,"物":-188,"流":417,"三":257,"キ":132,"二":183,"５":127,"道":103,"前":-70,"必":128,"結":207,"オ":131,"づ":-397,"話":149,"味":62,"通":-163,"南":445,"グ":-182,"デ":104,"年":-227,"四":151,"換":-112,"参":170,"真":-261,"百":150,"む":-173,"写":62,"東":112,"フ":-47,"事":28,"＾":-170,"友":23,"ラ":-46,"返":-54,"修":66,"ほ":-24,"ム":-54,"安":44,"★":-87,"素":23,"秋":62,"甘":18,"−":-10,"押":5},"UW5":{"。":-1472,"、":-844,"な":-428,"で":-863,"っ":1064,"き":745,"う":285,"に":-245,"が":-403,"す":-622,"と":-547,"し":-281,"る":1,"く":-129,"ん":929,"え":585,"は":-808,"べ":1479,"を":-507,"帯":384,"ー":55,"・":-941,"ろ":-24,"れ":526,"い":195,"わ":552,"め":660,"の":-462,"つ":1209,"た":418,"か":188,"あ":-254,"じ":650,"も":-218,"ち":580,"日":534,"そ":-248,"て":205,"」":-1038,"思":-666,"だ":-358,"！":-404,"こ":142,"々":878,"ト":-493,"］":-477,"ご":135,"ず":592,"年":583,"ル":-389,"せ":-280,"ぎ":620,"利":490,"分":134,"フ":41,"む":511,"み":162,"使":-672,"ょ":451,"体":-879,"り":616,"料":482,"電":-161,"ゃ":238,"手":-20,"田":408,"要":119,"前":189,"寺":-282,"．":-114,"ど":-157,"さ":-126,"よ":-51,"一":-118,"ま":5,"？":-217,"び":162,"通":-101,"　":-367,"タ":72,"学":40,"ぼ":394,"張":256,"先":-267,"へ":-178,"ば":115,"山":68,"言":-113,"的":-82,"味":46,"今":-151,"行":-47,"当":67,"…":-24,"葉":32,"テ":-72,"Ｎ":131,"ｏ":26,"条":10,"１":-6,"）":-1},"UW2":{"、":-1403,"。":-1207,"の":-610,"を":-1780,"に":-983,"は":-837,"る":-547,"も":-843,"か":-70,"と":-326,"ん":456,"ま":188,"て":-706,"し":119,"で":-795,"が":-987,"っ":223,"一":1003,"さ":524,"よ":446,"す":250,"く":-396,"う":-448,"結":1747,"少":536,"毎":1640,"お":-664,"最":693,"だ":375,"同":427,"そ":-491,"な":-355,"多":533,"全":1003,"り":-609,"え":218,"関":1034,"＾":590,"い":-156,"ほ":436,"や":-196,"ぜ":750,"ひ":-1391,"ぱ":510,"「":-483,"水":598,"目":-746,"話":11,"ら":-100,"以":1441,"パ":457,"我":-1261,"メ":-147,"題":1010,"ね":339,"つ":-241,"時":32,"本":284,"今":265,"ば":-340,"，":-602,"ー":118,"こ":-158,"手":395,"当":425,"．":-470,"的":148,"あ":-378,"良":326,"イ":-92,"世":-950,"効":547,"裏":-973,"絶":792,"料":214,"［":-206,"取":-272,"ゃ":105,"女":-458,"た":240,"容":370,"］":-129,"々":106,"ラ":-214,"…":198,"生":134,"正":401,"立":-316,"み":-54,"実":41,"打":223,"電":95,"車":106,"早":208,"百":-250,"乗":-184,"高":83,"感":56,"落":-234,"２":-87,"３":-20,"笑":-186,"文":116,"先":68,"和":89,"大":23,"万":-120,"友":93,"サ":-59,"三":39,"前":18,"甘":39,"違":23,"見":-1},"UW6":{"に":132,"。":-271,"う":34,"で":111,"を":-69,"て":-26,"が":687,"の":46,"は":137,"し":136,"な":198,"さ":480,"た":-142,"り":178,"と":-155,"ー":-142,"寺":487,"ル":106,"や":610,"い":475,"ご":69,"ぱ":1103,"０":498,"っ":139,"か":131,"ん":234,"・":-393,"け":-103,"ろ":37,"み":300,"め":481,"せ":384,"ま":124,"神":653,"込":890,"気":313,"ば":214,"す":48,"分":572,"思":-291,"く":110,"送":-1074,"カ":246,"京":-220,"こ":-30,"、":48,"出":238,"パ":-434,"御":721,"的":121,"過":650,"ピ":395,"よ":34,"わ":58,"つ":66,"］":-72,"イ":-58,"ど":-30,"ょ":-125,"メ":78,"：":20,"れ":156,"き":25,"び":83,"田":199,"焼":150,"お":55,"日":-11,"時":63,"周":246,"る":23,"生":-59,"参":111,"Ｓ":-89,"方":-67,"間":-36,"不":44,"ナ":40,"今":44,"へ":-40,"じ":13,"切":23,"だ":378},"BW2":{"とい":350,"てい":-1168,"ない":-2635,"です":-2464,"ので":-1090,"であ":-2694,"とこ":-1011,"、と":-915,"にも":-1103,"でも":-1713,"もし":-274,"てお":-2014,"とし":81,"には":-842,"帯電":-1366,"、そ":-1357,"てみ":-1486,"など":-1274,"のよ":-582,"たい":-1546,"はな":-280,"くな":-1318,"のだ":-532,"でし":-1509,"の前":-871,"らな":-318,"ると":-740,"でき":-1369,"いも":513,"・・":-1136,"もあ":-544,"では":-1324,"にな":-268,"どこ":-805,"した":-559,"はい":-700,"のも":-638,"とも":-479,"てあ":-1304,"うど":-928,"いた":-627,"のが":-581,"はあ":-1114,"てき":-1052,"のは":-514,"人が":837,"にし":-243,"とか":-632,"もの":-378,"はま":-423,"もう":-625,"ばい":-458,"でな":-817,"るの":-436,"ても":-433,"てく":-523,"なか":-323,"いま":-195,"らい":-345,"いと":-335,"日本":-470,"いい":-296,"、清":-623,"の１":-726,"、１":-805,"たも":259,"のか":-173,"はず":-253,"なあ":-597,"てほ":-321,"いつ":-210,"のみ":-346,"と同":-281,"スで":434,"なく":-246,"はじ":-265,"いな":-96,"いし":-177,"やす":-226,"かな":-122,"ーな":281,"とき":-98,"たち":-98,"るよ":-67,"うい":-80,"なこ":-61,"合い":57,"ろう":-23,"たし":-49,"すご":-53,"らし":-6,"もち":-10,"なぁ":-1},"BW3":{"もの":3151,"いう":571,"よう":-1084,"とこ":1684,"こと":1109,"いい":917,"とき":1917,"でも":1416,"して":336,"かけ":2656,"その":1744,"とい":-944,"この":1367,"もう":2130,"そし":-356,"なっ":-350,"もら":2614,"かか":1734,"ころ":-295,"ちょ":1606,"メー":140,"した":769,"ため":802,"はし":2095,"しま":-83,"なる":223,"とて":1365,"ある":-222,"・・":-1075,"から":-684,"そう":-545,"さん":-1458,"まし":-649,"どう":584,"いろ":356,"よく":1065,"より":-952,"とも":777,"もん":725,"かっ":-851,"ただ":1217,"いて":-524,"すぐ":1186,"でき":541,"はま":1006,"これ":908,"すご":865,"しい":-539,"とっ":534,"いか":372,"なく":-82,"ひと":668,"ない":211,"はず":471,"こそ":-780,"どこ":771,"まだ":766,"まあ":856,"わか":674,"いた":346,"だけ":-650,"おり":-1022,"ます":-297,"だっ":-776,"しゃ":578,"みた":-622,"まっ":-58,"また":44,"考え":354,"いの":-391,"とり":468,"楽し":317,"いら":508,"あと":306,"かつ":367,"たい":-193,"たま":365,"もっ":338,"もと":574,"つい":303,"京都":71,"そこ":218,"った":-250,"たっ":399,"祭り":274,"中で":207,"だろ":-490,"うま":173,"なん":92,"で、":164,"ずっ":377,"され":-218,"出来":-237,"ルー":278,"なか":79,"以前":253,"てき":275,"つけ":243,"いる":92,"行き":-234,"書い":168,"のも":232,"なに":199,"つな":230,"かも":-136,"うか":-74,"しく":-131,"きる":-120,"まぁ":93,"写真":84,"と思":-62,"どの":44,"ませ":-32,"です":-15,"うち":10,"まま":-76,"はや":5},"BW1":{"いう":556,"から":1447,"ない":846,"こと":-1367,"とか":1502,"しい":760,"では":-226,"うな":330,"とが":-1201,"もう":1592,"るの":-1215,"。・":2221,"かも":-775,"でも":680,"より":830,"んな":186,"んで":-578,"れた":809,"が、":1342,"たい":382,"とき":1319,"には":536,"てい":-751,"最近":910,"たの":-1145,"たら":735,"もの":-802,"少し":1285,"ても":396,"うに":-545,"だけ":547,"った":136,"とい":-865,"その":-1160,"たと":-826,"同じ":1303,"ただ":1029,"にも":451,"こそ":962,"もし":1199,"れて":-675,"この":-990,"なら":1087,"ると":471,"は、":313,"よう":-1124,"思い":-1085,"なぜ":1330,"で、":595,"い、":-78,"のは":752,"いの":-541,"０円":532,"いた":124,"さん":237,"のが":769,"、と":588,"た．":799,"普段":740,"の時":943,"大変":590,"くて":589,"と、":448,"のも":471,"とは":-154,"パン":458,"て、":234,"の後":839,"ので":215,"まり":334,"いで":-601,"えば":431,"の間":819,"やら":707,"とで":472,"うの":-307,"てる":379,"る人":443,"ため":538,"使い":-662,"との":287,"れば":266,"ころ":277,"けど":180,"の食":608,"使う":322,"一番":293,"現在":503,"買い":-596,"なの":-150,"たま":812,"に、":442,"ぱり":415,"んと":-52,"りと":250,"だし":327,"いも":-423,"！！":138,"まだ":225,"的な":174,"。」":296,"んだ":198,"まま":638,"も、":157,"うち":183,"る。":75,"いる":20,"べき":328,"ねー":457,"です":121,"＾＾":194,"わけ":234,"実際":248,"クル":220,"すが":-70,"して":-40,"取り":-164,"どう":78,"気が":150,"にか":238,"きな":37,"める":-99,"よく":49,"うと":-152,"電車":136,"える":-40,"こう":-77,"じゃ":-106,"か、":-58,"とも":-6,"な、":-53,"見て":-36,"らく":49,"きの":-61,"、今":23,"てた":26,"ケー":-52,"ろう":-19,"らと":-19,"じて":13,"料理":10,"いか":-1},"UW1":{"い":-21,"な":-227,"そ":190,"す":276,"に":-264,"で":-92,"の":134,"ー":99,"が":211,"と":-236,"て":38,"は":-469,"ち":347,"し":-83,"携":-74,"だ":208,"ス":-344,"ろ":230,"［":-519,"く":51,"ん":156,"機":400,"寺":472,"ン":160,"＾":595,"う":296,"り":-239,"か":-56,"安":-675,"あ":-274,"外":159,"何":290,"大":129,"自":297,"つ":37,"０":37,"チ":307,"っ":63,"効":464,"ひ":123,"人":84,"ざ":371,"行":-110,"．":-286,"３":-262,"を":-185,"や":-57,"小":186,"不":-297,"ｏ":-401,"、":-30,"た":-202,"ラ":96,"「":-76,"居":248,"（":-118,"ュ":91,"リ":42,"様":267,"好":62,"見":53,"ツ":111,"生":88,"さ":-27,"！":24,"国":-162,"オ":68,"ボ":148,"？":23,"庭":97,"料":40,"連":-84,"パ":49,"よ":-34},"TW3":{"という":683,"ている":-1400,"ところ":-871,"、ある":-2226,"ること":-195,"てしま":-1451,"として":268,"らない":-1104,"すこと":695,"である":-918,"と言っ":-1082,"はない":-297,"になっ":-118,"ていた":-630,"、また":-917,"ていう":516,"はそれ":-653,"であっ":-693,"となっ":281,"にして":-188,"ばいい":-405,"でした":-222,"ういう":-181,"もいい":-334,"とした":-269,"］京都":-183,"、なん":-11,"いくら":10},"TW2":{"ではな":-1286,"気に入":-2820,"かもし":-932,"んでい":-1786,"そもそ":-1208,"とがあ":-858,"その後":-795,"それ以":231,"でもあ":-930,"しかな":408,"うにな":-218,"からな":-412,"ともあ":-487,"でもな":-707,"ってこ":-80,"ないと":-58,"えてい":39,"とがで":-116,"しかし":-14,"してお":-1},"TW1":{"という":598,"しかし":2016,"・・・":1431,"ような":1157,"かなり":1813,"やはり":1666,"ながら":1297,"ない、":-1181,"かなか":833,"ました":-377,"なんと":-640,"いたい":810,"ことも":-222,"ている":98,"そして":513,"っかり":640,"それは":-272,"ひとつ":380,"くらい":179,"ていて":197,"として":206,"ように":-152,"に来て":329,"ちろん":250,"ろいろ":257,"００円":111,"くさん":139,"、それ":24,"いった":85,"とんど":89,"ょうど":93,"メール":31,"らない":23,"ことは":43},"TW4":{"の京都":2904,"ところ":1260,"くらい":1780,"かなり":1886,"ことが":-823,"ことに":-917,"って、":1789,"メール":341,"なんか":473,"もちろ":1682,"ない。":227,"ように":-917,"こと。":683,"そうで":-971,"しかも":1161,"した。":-704,"いろい":702,"ソフト":747,"しかし":964,"ことで":171,"うどん":585,"でも、":680,"いかな":627,"ような":-1079,"だから":556,"ことも":-477,"ことを":303,"なんて":-336,"しない":445,"いえば":463,"わけで":-225,"ことは":-301,"ありま":-185,"まって":-248,"しても":-331,"なかな":403,"やって":248,"らしい":374,"いっぱ":246,"観光地":184,"いる。":117,"ないと":93,"言って":-89,"なんと":98,"してき":67,"おいし":75,"いけな":-32,"すると":35,"すごい":23,"あたり":-52}} \ No newline at end of file +{"UW3":{"。":6728,"、":4793,"を":5613,"に":3832,"の":3728,"は":4247,"が":4149,"と":2957,"る":2755,"く":2852,"も":3216,"て":2448,"で":3460,"た":2223,"っ":-1955,"い":1914,"う":1964,"方":2774,"な":2289,"ら":1418,"］":2179,"か":1931,"り":1064,"，":3562,"！":3285,"ー":-113,"？":3707,"や":2168,"ば":1685,"ど":1948,"．":2081,"・":1008,"つ":1318,"…":2476,"ご":-2411,"）":2663,"部":1626,"☆":2714,"上":540,"」":1657,"ッ":-2007,"れ":375,"♪":2559,"ず":2032,"以":-1912,"ん":704,"だ":895,"へ":2163,"ろ":1051,"ク":342,"電":-1749,"笑":2419,"今":1104,"ま":-484,"分":1162,"中":1012,"ぐ":2032,"京":-1416,"人":495,"お":-1350,"イ":-592,"利":-713,"０":-863,"き":491,"ぁ":1313,"間":727,"す":959,"ン":-313,"フ":-1354,"ゃ":1253,"ス":-1019,"「":-1412,"日":750,"度":1104,"後":1273,"シ":-1228,"行":-808,"々":1430,"ア":-842,"ド":-1098,"当":-343,"観":-1024,"ぶ":1465,"リ":-626,"大":-904,"え":751,"夜":1560,"』":1445,"１":-635,"秋":1420,"態":-1193,"昔":1693,"体":1037,"む":1166,"空":796,"［":-1062,"変":398,"学":-641,"皆":1406,"月":621,"題":1106,"機":-775,"然":591,"あ":783,"超":1029,"〜":526,"ぞ":831,"非":849,"ち":463,"光":-218,"係":555,"年":352,"み":312,"由":478,"＞":696,"感":452,"プ":-431,"ｏ":-603,"番":290,"ズ":344,"色":657,"し":224,"意":-369,"び":341,"入":-491,"け":157,"所":622,"自":-411,"場":-59,"メ":-203,"　":311,"指":-356,"気":69,"頭":788,"着":-302,"別":286,"出":-175,"ぱ":455,"カ":-121,"朝":316,"”":313,"品":139,"果":565,"一":-71,"語":196,"程":262,"応":275,"寺":135,"有":-208,"生":-143,"神":-190,"想":209,"不":-171,"店":200,"：":111,"グ":38,"レ":-105,"料":-113,"屋":71,"パ":-87,"水":70,"サ":79,"減":86,"回":28,"バ":-12,"５":-4},"UW4":{"。":-7454,"の":-4026,"て":-3886,"が":-3308,"、":-7522,"る":-5031,"に":-4106,"は":-3340,"で":-3076,"っ":-4215,"を":-4779,"あ":564,"と":-2905,"う":-2139,"た":-1616,"ま":-1127,"な":-1887,"り":-4104,"ご":551,"し":-1826,"れ":-4215,"こ":-496,"く":-3009,"ら":-4434,"か":-2652,"そ":152,"き":-2034,"だ":-1794,"ー":-2920,"い":-1560,"み":-1078,"行":374,"空":2550,"ん":-3604,"も":-2695,"す":-1087,"」":-5550,"さ":-1889,"思":1398,"め":-1842,"け":-3319,"「":2416,"や":-1292,"時":593,"人":598,"一":541,"京":686,"私":1919,"，":-4270,"表":902,"つ":-1274,"！":-4521,"ッ":-2351,"・":-625,"よ":-553,"冷":2511,"１":770,"ン":-2142,"え":-1671,"持":119,"使":1139,"わ":-1549,"方":-831,"来":435,"（":2512,"ゆ":-2377,"ろ":-2910,"ル":-1250,"手":246,"…":-2234,"じ":-2235,"？":-3518,"何":1232,"ど":-1238,"［":1883,"見":508,"べ":-2886,"）":-4231,"大":531,"本":-1158,"彼":1311,"笑":-3521,"当":271,"．":-2893,"日":288,"せ":-1291,"ば":-2305,"今":1115,"イ":-501,"ち":-2020,"気":410,"入":633,"自":804,"ず":-1738,"２":553,"最":1132,"用":-721,"帯":-1198,"お":755,"有":910,"携":394,"食":175,"ゃ":-2476,"間":-350,"多":1041,"ひ":1304,"中":62,"］":-1461,"』":-3295,"ね":-1755,"白":570,"ぁ":-1446,"水":337,"風":759,"コ":329,"ぎ":-1380,"合":-1101,"リ":-588,"０":-400,"僕":974,"感":216,"”":-842,"ょ":-1617,"小":722,"ぼ":-959,"公":685,"３":84,"楽":380,"♪":-1513,"〜":-1208,"☆":-1315,"全":523,"道":276,"二":520,"ス":-228,"子":-685,"書":380,"ド":-58,"不":535,"高":245,"特":763,"祭":408,"返":-363,"態":-544,"づ":-1053,"実":477,"定":-355,"少":444,"話":355,"買":433,"他":388,"ぜ":-896,"街":536,"目":133,"ｏ":-565,"的":-562,"込":-726,"ぞ":-956,"ュ":-549,"ぐ":-787,"サ":147,"無":172,"東":517,"夏":546,"現":229,"前":-155,"ぶ":-685,"場":121,"換":-349,"げ":-637,"タ":-256,"デ":205,"年":-410,"び":-543,"ほ":-202,"物":-209,"フ":-202,"言":146,"券":-589,"へ":-432,"ト":-159,"外":180,"ざ":-354,"秋":360,"難":227,"ョ":-220,"味":93,"流":325,"写":295,"　":-221,"新":91,"ラ":-110,"店":99,"参":199,"５":52,"＾":-328,"美":167,"々":-202,"三":85,"開":59,"押":210,"相":97,"甘":106,"安":114,"回":-147,"関":-151,"取":-117,"出":-33,"勢":-12,"ム":-52,"−":-57,"６":24,"百":32,"Ｃ":24,"屋":-16,"オ":8,"家":16,"ィ":-12,"ク":-4},"UW5":{"。":-2064,"、":-1010,"で":-892,"ま":-99,"す":-474,"な":-458,"っ":997,"る":217,"て":222,"は":-855,"が":-518,"き":1186,"し":-365,"を":-637,"あ":-552,"日":790,"こ":424,"に":-611,"え":686,"だ":-539,"い":363,"そ":-296,"ろ":-280,"ん":1035,"べ":1778,"・":-1055,"と":-630,"み":332,"か":223,"ざ":1423,"帯":371,"わ":580,"れ":534,"の":-607,"ら":233,"つ":1267,"ち":794,"う":383,"め":689,"ル":-478,"じ":642,"さ":-302,"」":-1120,"合":456,"よ":-273,"ず":996,"り":353,"ぼ":1417,"く":460,"年":975,"電":-290,"せ":-306,"冷":1220,"た":203,"使":-1266,"！":-437,"思":-810,"］":-483,"も":-128,"利":356,"学":191,"意":739,"ぐ":179,"バ":-257,"ト":-355,"当":439,"分":201,"々":563,"体":-1129,"場":469,"む":664,"券":-832,"見":-249,"イ":189,"来":-128,"料":584,"寺":-493,"手":173,"ぎ":262,"．":-169,"タ":284,"ば":302,"水":-221,"通":-189,"行":-215,"先":-737,"一":-212,"田":358,"続":-686,"本":41,"ー":24,"ア":-172,"ょ":338,"？":-244,"び":222,"ｏ":191,"へ":-279,"要":114,"何":-342,"最":-104,"上":-38,"け":38,"　":-234,"都":43,"成":93,"的":-118,"ラ":59,"中":88,"ン":-25,"（":-97,"Ｎ":235,"７":-151,"効":87,"枚":95,"１":-41,"ゃ":20,"有":-57,"京":-41,"オ":-102,"降":28,"生":-12,"）":-20,"ュ":-4},"UW2":{"。":-1435,"、":-1422,"を":-2559,"が":-1164,"は":-1025,"っ":253,"に":-1053,"以":2191,"れ":6,"か":-87,"て":-1038,"で":-850,"全":1718,"ん":588,"の":-678,"そ":-342,"ゆ":1826,"め":94,"や":171,"と":-790,"く":-676,"う":-389,"も":-824,"一":1094,"あ":-137,"こ":84,"さ":563,"利":22,"だ":329,"ご":-701,"毎":1711,"す":65,"結":2020,"少":607,"彼":771,"最":450,"き":-546,"同":501,"よ":599,"機":1124,"人":227,"え":118,"ほ":667,"い":-98,"大":361,"り":-391,"お":-412,"「":-480,"本":699,"手":768,"意":558,"関":892,"京":-332,"目":-1046,"る":-265,"ぜ":780,"な":-421,"着":-459,"ケ":-265,"題":869,"思":-34,"我":-1566,"ひ":-1571,"多":282,"上":-219,"た":-140,"ー":281,"ぱ":403,"ど":67,"水":310,"振":-1294,"ば":-375,"夕":692,"つ":-293,"絶":1290,"裏":-1205,"寺":-454,"友":327,"名":308,"容":912,"今":280,"み":-131,"フ":-298,"入":-364,"ン":138,"イ":-180,"高":345,"ま":-151,"ら":-57,"ス":123,"取":-279,"料":53,"世":-929,"パ":295,"電":310,"ｒ":1144,"々":180,"，":-377,"打":427,"＾":149,"落":-586,"プ":-252,"冷":-564,"車":221,"和":397,"用":-220,"甘":553,"的":174,"下":230,"選":254,"極":690,"知":-368,"ラ":-235,"重":319,"良":164,"メ":-47,"実":126,"時":98,"立":-357,"］":-114,"相":242,"：":201,"乗":-222,"女":-225,"格":351,"Ｈ":-196,"ィ":-50,"感":67,"正":275,"当":105,"［":-37,"け":29,"早":100,"式":33,"ル":8,"サ":-74,"４":45,"５":-44,"川":49,"残":40,"三":20,"見":-8,"カ":-4,"ゃ":4,"明":4},"UW6":{"が":458,"。":-261,"で":251,"入":1042,"し":496,"は":244,"ご":-343,"あ":-434,"に":259,"空":1574,"て":-48,"だ":87,"の":74,"め":1567,"な":197,"さ":523,"う":-76,"券":1647,"や":728,"り":471,"ま":172,"ろ":319,"い":346,"を":168,"か":239,"ん":363,"ら":50,"ル":173,"っ":269,"ク":-297,"彼":-1380,"け":-161,"０":657,"寺":710,"態":407,"・":-672,"、":143,"込":694,"ば":364,"場":-799,"：":881,"く":196,"ゆ":571,"る":83,"気":448,"ー":-8,"タ":425,"ぱ":888,"的":471,"メ":254,"わ":215,"ピ":833,"ず":-550,"ケ":-466,"過":936,"送":-1043,"シ":-165,"イ":-256,"カ":294,"も":-76,"効":-690,"れ":131,"焼":761,"た":-5,"来":-262,"方":-301,"つ":66,"機":251,"思":-43,"出":201,"印":379,"こ":51,"店":254,"自":167,"ぎ":322,"ゃ":-136,"着":-113,"参":278,"間":-73,"不":176,"ぞ":227,"周":337,"パ":-119,"…":-96,"ン":-25,"ょ":-138,"回":-46,"力":-75,"び":53,"１":4,"練":36,"生":-24,"Ｎ":-48,"Ｓ":-20,"９":8},"BW2":{"たと":1506,"とい":288,"とお":-3916,"のみ":-2778,"てい":-1173,"であ":-3424,"ない":-2833,"です":-3140,"たま":-2384,"ので":-905,"いた":-1757,"でき":-2098,"うや":-1836,"にも":-1242,"には":-963,"でも":-1866,"はな":-294,"くな":-1808,"、そ":-1730,"、と":-636,"るよ":-1166,"帯電":-1447,"てお":-2417,"ただ":-3137,"のよ":-616,"でし":-1972,"また":-2915,"ると":-559,"など":-1328,"の機":-1328,"のは":-1154,"くし":-1455,"もあ":-956,"いま":-603,"てみ":-1631,"てあ":-2192,"たい":-1203,"もし":-546,"はで":398,"いつ":-1249,"がと":-484,"のだ":-995,"すご":-1286,"はい":-737,"うど":-1333,"人が":1448,"いも":544,"にな":-270,"・・":-1236,"てき":-1175,"時間":-828,"の前":-673,"のも":-751,"はあ":-1272,"どこ":-929,"では":-1166,"くだ":-995,"とな":111,"れた":-801,"なく":-804,"た時":453,"らな":-87,"、な":-216,"とか":-910,"にし":-414,"てく":-749,"でな":-1185,"る場":78,"のか":-354,"とも":-533,"した":-580,"もな":-186,"とこ":-385,"いし":-588,"ード":-559,"たよ":-582,"いよ":-794,"はず":-658,"ばい":-551,"ても":-255,"いい":-353,"とき":-281,"ては":27,"もつ":-372,"なか":-225,"のが":-360,"の１":-795,"くさ":-134,"はじ":-435,"もの":-147,"、１":-856,"ーな":657,"、清":-495,"と同":-436,"にで":-377,"、ま":-262,"しい":-318,"いな":-109,"の方":319,"やす":-414,"たも":143,"がな":82,"つの":388,"ませ":-121,"とす":-220,"］京":-271,"よう":-85,"りな":136,"でい":-139,"い人":105,"るの":-109,"ーで":125,"らい":-107,"のま":-45,"なぁ":-53,"てた":-82,"気持":-106,"なり":81,"でに":32,"るで":28,"の後":-32,"今日":-40,"んで":24,"もう":-12,"とは":-4},"BW1":{"以上":1273,"のみ":2658,"れた":1590,"まま":2847,"たま":2261,"いう":338,"から":1803,"よう":-1764,"りが":-1370,"めて":1033,"その":-1196,"白い":2426,"った":161,"とう":1397,"では":-501,"たと":-1243,"ない":827,"とか":1743,"うな":512,"るの":-1611,"こと":-1467,"利用":1155,"でも":756,"とが":-1259,"たの":-1570,"この":-1058,"より":1131,"。・":2639,"のは":1326,"しい":842,"が、":1759,"最近":1593,"かも":-1285,"ック":622,"もう":1776,"ても":636,"んな":369,"たら":943,"んで":-747,"がと":-1484,"もの":-1092,"れて":-1075,"まり":1378,"にも":652,"少し":1499,"まだ":1379,"とい":-1037,"りと":1131,"とき":1494,"ると":205,"同じ":1574,"もつ":1554,"こそ":1334,"さん":451,"で、":726,"は、":527,"ある":205,"だけ":713,"なら":1205,"た．":1789,"て、":840,"れる":473,"には":421,"気に":-66,"思い":-1307,"なぜ":1605,"うや":-1437,"の間":1274,"って":-155,"い、":-292,"もし":1057,"えば":807,"きな":923,"んと":-204,"てる":615,"、と":386,"まる":-282,"たい":222,"いで":-902,"ケー":-854,"ただ":581,"いの":-311,"てい":-581,"るで":373,"とで":938,"くて":684,"いた":369,"いる":181,"まぁ":482,"それ":187,"まで":406,"普段":620,"のが":605,"れば":544,"の後":946,"ージ":970,"しか":49,"のも":609,"なの":-54,"る人":668,"友達":846,"なく":169,"！！":318,"と、":422,"買い":-799,"大変":359,"０円":324,"使い":-756,"、今":407,"に、":573,"める":-345,"現在":575,"ため":581,"気が":370,"クル":478,"んだ":313,"きた":331,"来て":493,"ぱり":435,"でき":-74,"べき":564,"くれ":268,"きり":466,"取り":-293,"。あ":-543,"てた":304,"こう":-328,"てく":-112,"やら":275,"の上":508,"使う":230,"いも":-376,"か、":-238,"。３":372,"けど":166,"るに":-305,"とも":71,"りに":73,"じて":195,"も、":165,"一番":100,"する":-78,"すら":209,"いな":-106,"うと":-114,"など":204,"国人":143,"の時":215,"実際":189,"ろう":-193,"とは":-77,"うち":96,"どう":58,"・・":87,"たり":74,"とに":-57,"は１":180,"うの":-32,"じゃ":-89,"ュー":28,"一つ":-56,"にか":36,"りは":-12,"見て":-4},"UW1":{"お":980,"は":-281,"ま":70,"う":349,"ご":921,"と":102,"る":-32,"ら":110,"な":-311,"上":-1236,"に":-339,"で":-111,"学":179,"い":-69,"り":-86,"、":-28,"バ":452,"当":-977,"が":-164,"電":145,"ち":389,"水":1514,"を":-149,"つ":85,"携":-52,"っ":192,"か":-151,"［":-679,"ゆ":-565,"ー":121,"ん":176,"フ":-161,"ろ":227,"行":-21,"あ":-60,"だ":182,"用":648,"外":230,"０":232,"ス":-301,"空":-552,"間":358,"自":442,"す":163,"し":-7,"ケ":-283,"機":454,"利":-567,"や":-289,"何":307,"ツ":156,"小":306,"入":300,"安":-487,"新":434,"チ":131,"こ":98,"生":269,"さ":-140,"３":-298,"困":671,"イ":258,"ッ":224,"込":96,"庭":580,"問":615,"田":144,"。":-28,"も":-33,"使":-177,"高":204,"券":379,"め":-103,"パ":261,"国":-467,"様":370,"ラ":129,"社":254,"！":110,"振":349,"言":-136,"着":-48,"ペ":353,"み":-44,"．":-49,"思":107,"見":83,"欲":212,"居":173,"メ":94,"？":79,"ざ":158,"撮":-208,"大":33,"連":-100,"長":115,"ょ":-53,"個":-127,"周":40,"進":93,"山":61,"サ":32,"ボ":32,"方":20,"ン":12,"買":-8,"ね":-4},"BW3":{"もの":3658,"あり":1182,"ます":-1304,"いた":591,"いう":798,"その":2051,"いい":1225,"たま":1149,"のみ":2809,"でも":1705,"ある":-359,"よう":-159,"持っ":1377,"とき":1908,"とお":1041,"とい":-1177,"なっ":-544,"なる":406,"ころ":-364,"とう":-430,"かけ":2917,"こと":975,"手に":1163,"以上":-1970,"この":1601,"して":352,"でき":652,"はし":3071,"とて":2260,"から":-1074,"メー":296,"そう":-790,"もら":2683,"ちょ":2013,"かか":1802,"ため":910,"しま":-186,"もう":2313,"・・":-1608,"した":991,"めて":-2007,"より":-1514,"とも":1059,"いろ":772,"もん":1116,"どう":784,"すご":1168,"おり":-1312,"まぁ":1409,"ただ":1447,"きっ":651,"だけ":-1043,"来た":310,"電車":190,"はま":1022,"きま":-890,"しゃ":1379,"しい":-571,"とっ":989,"まし":-902,"よく":815,"さん":-1344,"かっ":-920,"これ":999,"場合":69,"うま":557,"める":-1596,"いて":-473,"ない":260,"すぐ":1151,"です":-540,"やっ":464,"もと":1360,"す。":-277,"いか":537,"中で":435,"わけ":-32,"わか":697,"いる":617,"なか":151,"考え":545,"とこ":787,"いや":897,"また":-494,"こそ":-806,"もっ":729,"みた":-758,"どこ":758,"まだ":374,"お茶":782,"あっ":-84,"パソ":446,"かも":-516,"かつ":730,"なく":184,"とり":708,"うち":225,"まっ":-266,"めっ":368,"いら":687,"まず":464,"出来":-456,"ませ":-464,"だい":421,"なら":-162,"はい":351,"もつ":607,"ここ":647,"ほし":-356,"たい":-308,"つい":283,"つけ":354,"どの":556,"まで":-558,"料理":67,"使っ":250,"行き":-274,"日本":213,"出て":149,"祭り":361,"ルー":408,"楽し":197,"さい":-317,"だろ":-548,"逆に":379,"つな":305,"とあ":326,"お店":197,"イン":186,"書い":164,"ぼく":281,"んは":289,"人は":79,"など":-138,"で、":96,"まり":-225,"みが":-123,"はず":44,"程度":-73,"ずっ":44,"しく":-24,"込ん":-3},"TW4":{"ところ":2298,"くらい":1840,"ような":-2720,"した。":-1639,"できた":2328,"ありま":-1854,"ことが":-955,"かなり":2910,"ます。":-1449,"ように":-1522,"そうで":-1026,"いた。":1051,"だから":995,"ことに":-907,"しかし":1612,"うどん":1004,"メール":369,"もちろ":1602,"だった":-1494,"だって":523,"ことも":-539,"いろい":697,"ようで":-1413,"いかな":786,"あり、":-558,"でも、":675,"ちなみ":1556,"こと。":678,"ことで":263,"ない。":112,"なかな":855,"ソフト":704,"すると":339,"わけで":-339,"いえば":647,"なくな":274,"なんて":-265,"しない":519,"まって":-644,"なんと":485,"ことを":228,"してき":324,"しかも":790,"らしい":546,"あたり":-494,"すごい":197,"ことは":-199,"いくら":180,"なんか":98,"できま":80,"ちょっ":190,"のは、":150,"いいと":-70,"そうい":136,"しても":-90,"観光客":-70,"おいし":74,"いっぱ":98,"いる。":52,"います":-20},"TW3":{"もなく":-1963,"という":835,"ところ":-1715,"として":677,"ている":-1633,"のまま":-1388,"、ある":-2880,"はない":-522,"すこと":762,"てしま":-1446,"らない":-1178,"とした":-1303,"と言っ":-1139,"ていう":588,"はそれ":-915,"である":-808,"になっ":-35,"、なん":-182,"たこと":-95,"であっ":-926,"となっ":309,"もあり":-283,"いとこ":100,"ばいい":-261,"もいい":-455,"であり":-715,"なんで":470,"ういう":-319,"るとき":259,"ていた":-117,"にして":-41,"でした":-136},"TW1":{"という":710,"ような":1395,"のまま":1430,"しかし":2233,"・・・":1265,"持って":-1918,"ながら":1900,"かなり":2064,"。この":-230,"やはり":2088,"思った":877,"もなく":783,"を利用":-1416,"なんと":-712,"そして":819,"、それ":328,"である":375,"ない、":-985,"それは":-305,"ました":-327,"かなか":703,"ている":29,"っかり":775,"くさん":545,"ちろん":463,"ひとつ":586,"はその":-121,"。ただ":628,"すごく":328,"リーグ":349,"ところ":320,"ろいろ":327,"ること":-347,"ょうど":291,"ていて":133,"くらい":83,"なって":-66,"００円":58,"思って":-131,"として":28,"買って":-32},"TW2":{"気に入":-4314,"とがあ":-1364,"ではな":-1140,"んでい":-2225,"うにな":-921,"かもし":-382,"そもそ":-1415,"してい":-533,"しかな":526,"でもあ":-1125,"ともあ":-986,"ったこ":-226,"からな":-580,"えるこ":-465,"してお":-184,"でもな":-388,"ってい":45,"とはい":-16,"ってこ":-20,"ないと":-32,"えてい":4}} \ No newline at end of file diff --git a/data/finetuning/ja/train.txt b/data/finetuning/ja/train.txt index be30d4d8..b5ad995b 100644 --- a/data/finetuning/ja/train.txt +++ b/data/finetuning/ja/train.txt @@ -15,6 +15,17 @@ ようやく▁日が▁暮れた。 やっと▁ようやく▁公開できそうです。 あいつが▁ようやく▁来た。 +夕方▁ようやく▁完成した。 あれが▁入ったのは▁たまたまです。 -たまたま▁手に▁入れる▁ことができた。 +たまたま▁手に▁入れる▁ことが▁できた。 彼が▁たまたま▁持っていた。 +全部▁まとめて▁提出します。 +論点を▁まとめる。 +思った▁とおりに▁書く。 +言われた▁とおりに▁動きます。 +まるで▁水晶の▁ように▁すきとおって▁いた。 +彼の▁すきとおる▁肌 +冷たさを▁もつ▁青い▁空 +当日券のみ▁有効です。 +該当する方▁のみ▁入場できます。 +あの▁青い▁空と▁白い▁雲のみが▁見える。 diff --git a/tests/quality/ja.tsv b/tests/quality/ja.tsv index c2a9ebca..c1335ef7 100644 --- a/tests/quality/ja.tsv +++ b/tests/quality/ja.tsv @@ -4,10 +4,16 @@ init これ以上▁利用する▁場合は▁教えてください。 init 食器は▁そのまま▁入れて▁大丈夫です。 gh152 ダウンロード▁ありがとう▁ございます。 gh152 ご利用▁ありがとう▁ございました。 +gh157 要点を▁まとめる▁必要が▁ある。 gh160 目指すのは▁あらゆる▁人に▁便利な▁ソフトウェア gh160 商品が▁まもなく▁到着します。 gh160 プロジェクトが▁ようやく▁日の▁目を▁見る。 gh160 明け方に▁ようやく▁目覚めると、 -#gh160 明け方▁ようやく▁目覚めると、 +gh160 明け方▁ようやく▁目覚めると、 gh160 これは▁たまたま▁見つけた▁宝物 -#gh160 歩いていて▁たまたま▁目に▁入った▁光景 +gh160 歩いていて▁たまたま▁目に▁入った▁光景 +gh216 あなたの▁意図した▁とおりに▁情報を▁伝える。 +gh220 あの▁イーハトーヴォの▁すきとおった▁風、▁夏でも▁底に▁冷たさを▁もつ▁青い▁そら、▁うつくしい▁森で▁飾られた▁モリーオ市、▁郊外の▁ぎらぎら▁ひかる▁草の▁波。 +gh387 購入された▁お客様のみ▁入れます。 +gh387 購入された▁お客様のみ▁入場できます。 +gh387 パワーのみ▁有効だ diff --git a/tests/test_parser.py b/tests/test_parser.py index b5dd5a9e..da785b8d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -118,7 +118,8 @@ def test_load_default_japanese_parser(self) -> None: '世界中の', '人が', 'アクセスできて', - '使えるようにする', + '使えるように', + 'する', 'ことです。', ])