feat: token prediction (speculative decoding) #767
Annotations
5 errors
|
Run model dependent tests:
test/modelDependent/llama3.1/controlledEvaluate.test.ts#L76
Error: Snapshot `llama 3.1 > controlled evaluate > get probabilities for 3 tokens 1` mismatched
- Expected
+ Received
@@ -7,77 +7,77 @@
},
},
{
"next": {
"probabilities": Map {
- 35308 => 0.5214946269989014,
+ 35308 => 0.47903314232826233,
- 27096 => 0.24320587515830994,
+ 27096 => 0.2675457000732422,
- 11 => 0.022182414308190346,
+ 11 => 0.02197451889514923,
- 198 => 0.011944590136408806,
+ 198 => 0.013969771564006805,
- 374 => 0.008361410349607468,
+ 374 => 0.008969545364379883,
- 863 => 0.008360812440514565,
+ 863 => 0.008655221201479435,
- 1131 => 0.006834662053734064,
+ 1131 => 0.006739677395671606,
- 25 => 0.006243313197046518,
+ 25 => 0.005900361575186253,
- 7940 => 0.00540389958769083,
+ 320 => 0.005746232345700264,
- 1 => 0.005168775096535683,
+ 7940 => 0.005627743434160948,
},
},
},
{
"next": {
"probabilities": Map {
- 927 => 0.9811904430389404,
+ 927 => 0.9799655675888062,
- 198 => 0.0033848676830530167,
+ 198 => 0.0035156032536178827,
- 6288 => 0.0032705331686884165,
+ 6288 => 0.002894952427595854,
- 279 => 0.0006552835111506283,
+ 279 => 0.0007418555323965847,
- 1633 => 0.00031841936288401484,
+ 264 => 0.0003396148094907403,
- 1035 => 0.0003114044084213674,
+ 297 => 0.0003235207695979625,
- 13 => 0.0002916732046287507,
+ 1633 => 0.00031703192507848144,
- 264 => 0.00028948261751793325,
+ 1579 => 0.0003097957232967019,
- 297 => 0.0002833220351021737,
+ 13 => 0.0003070845559705049,
- 720 => 0.00024898265837691724,
+ 1035 => 0.0003007104969583452,
},
"token": 927,
},
},
{
"next": {
- "confidence": 0.9306728839874268,
+ "confidence": 0.9299947023391724,
"token": 279,
},
},
{
"next": {
- "confidence": 0.9597684741020203,
+ "confidence": 0.9565306305885315,
"probabilities": Map {
- 16053 => 0.9597684741020203,
+ 16053 => 0.9565306305885315,
- 1208 => 0.004750591237097979,
+ 1208 => 0.00521566066890955,
- 198 => 0.0031827085185796022,
+ 5679 => 0.003188235918059945,
- 5679 => 0.0029162338469177485,
+ 198 => 0.0030333304312080145,
- 65536 => 0.00197240780107677,
+ 65536 => 0.0021549509838223457,
- 6435 => 0.0009124248754233122,
+ 6435 => 0.0010304288007318974,
- 2697 => 0.0006706250132992864,
+ 21811 => 0.0006608077674172819,
- 720 => 0.0005979162524454296,
+ 32681 => 0.0005957780522294343,
- 21811 => 0.0005516768433153629,
+ 2697 => 0.0005905027501285076,
- 45363 => 0.0005495203076861799,
+ 45363 => 0.0005836125928908587,
},
},
},
{
"next": {
- "confidence": 0.9871460199356079,
+ "confidence": 0.9837724566459656,
"probabilities": Map {
- 5679 => 0.9871460199356079,
+ 5679 => 0.9837724566459656,
- 21811 => 0.001438674982637167,
+ 21811 => 0.0018094592960551381,
- 198 => 0.0009368227329105139,
+ 198 => 0.0010853976709768176,
- 8415 => 0.0007225279696285725,
+ 8415 => 0.0008998656994663179,
- 12875 => 0.00038032486918382347,
+ 4194 => 0.00044961264939047396,
- 4194 => 0.00034695648355409503,
+ 12875 => 0.00040421882295049727,
- 720 => 0.00028149448917247355,
+ 720 => 0.0003244955441914499,
- 14588 => 0.00027612835401669145,
+ 14588 => 0.00032394853769801557,
- 9522 => 0.00024171460245270282,
+ 9522 => 0.000320142979035154,
- 627 => 0.0002042166597675532,
+ 627 => 0.00025715577066875994,
},
"token": 5679,
},
},
]
❯ test/modelDependent/llama3.1/controlledEva
|
|
Run model dependent tests:
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts#L93
Error: Snapshot `llama 3.1 > evaluate with metadata > with probabilities 1` mismatched
- Expected
+ Received
[
{
"probabilities": Map {
- 578 => 0.43072959780693054,
+ 578 => 0.4234016239643097,
- 1115 => 0.13043756783008575,
+ 1115 => 0.1350678652524948,
- 1102 => 0.05168525502085686,
+ 1102 => 0.051406390964984894,
- 763 => 0.04288897663354874,
+ 763 => 0.040449317544698715,
- 1283 => 0.029397012665867805,
+ 1283 => 0.028996290639042854,
- 2100 => 0.029378682374954224,
+ 2100 => 0.02832488901913166,
- 15636 => 0.026268385350704193,
+ 15636 => 0.026462676003575325,
- 2030 => 0.02184896357357502,
+ 2030 => 0.021777881309390068,
- 320 => 0.01690298318862915,
+ 320 => 0.01988242380321026,
- 1628 => 0.011869494803249836,
+ 1628 => 0.012539208866655827,
},
"token": 578,
},
{
"probabilities": Map {
- 16053 => 0.42230042815208435,
+ 16053 => 0.44442975521087646,
- 4062 => 0.30363598465919495,
+ 4062 => 0.2820369601249695,
- 39935 => 0.060395680367946625,
+ 39935 => 0.0518338605761528,
- 2944 => 0.0373028889298439,
+ 2944 => 0.03905597701668739,
- 5679 => 0.023811226710677147,
+ 5679 => 0.023956798017024994,
- 11914 => 0.016298197209835052,
+ 11914 => 0.016141841188073158,
- 2144 => 0.014683431014418602,
+ 2144 => 0.016043849289417267,
- 1121 => 0.006984429899603128,
+ 1121 => 0.006507175974547863,
- 17571 => 0.005794311873614788,
+ 17571 => 0.005837713368237019,
- 3446 => 0.004934381693601608,
+ 3446 => 0.004879034589976072,
},
"token": 16053,
},
{
"probabilities": Map {
- 5679 => 0.9981182813644409,
+ 5679 => 0.9980332255363464,
- 12875 => 0.00015924211766105145,
+ 12875 => 0.00015266593254636973,
- 18964 => 0.00011538491526152939,
+ 39935 => 0.00011596923286560923,
- 39935 => 0.00011497695231810212,
+ 18964 => 0.00010741820005932823,
- 13 => 0.00010490677959751338,
+ 13 => 0.00010332829697290435,
- 627 => 0.00009277161007048562,
+ 627 => 0.0000947250155149959,
- 656 => 0.00006256866618059576,
+ 893 => 0.0000696686256560497,
- 893 => 0.00005633986438624561,
+ 656 => 0.00006114577263360843,
- 198 => 0.00005223735934123397,
+ 374 => 0.000054702515626559034,
- 374 => 0.00005191291347728111,
+ 198 => 0.000054134707170305774,
},
"token": 5679,
},
{
"probabilities": Map {
- 374 => 0.8126624226570129,
+ 374 => 0.8100054264068604,
- 1587 => 0.04815267026424408,
+ 1587 => 0.049722250550985336,
- 596 => 0.024733318015933037,
+ 596 => 0.027551041916012764,
- 1120 => 0.022302960976958275,
+ 3250 => 0.021945547312498093,
- 3250 => 0.02154506742954254,
+ 1120 => 0.02066754922270775,
- 706 => 0.01618366874754429,
+ 706 => 0.016083383932709694,
- 15849 => 0.00869414210319519,
+ 15849 => 0.008394647389650345,
- 1053 => 0.005911793559789658,
+ 1053 => 0.0057802582159638405,
- 55064 => 0.0037806404288858175,
+ 11 => 0.003990540746599436,
- 11 => 0.0036655946169048548,
+ 55064 => 0.003569164779037237,
},
"token": 374,
},
{
"probabilities": Map {
- 2288 => 0.2757589817047119,
+ 2288 => 0.29307958483695984,
- 1120 => 0.16664838790893555,
+ 539 => 0.15234610438346863,
- 539 => 0.15775153040885925,
+ 1120 => 0.1495191901922226,
- 779 => 0.1334378868341446,
+ 779 => 0.14305096864700317,
- 264 => 0.05585397779941559,
+ 264 => 0.056170206516981125,
- 1101 => 0.029216185212135315,
+ 1101 => 0.02853754162788391,
- 16053 => 0.017680438235402107,
+ 16053 => 0.01722031459212303,
- 5042 => 0.015862826257944107,
+ 5042 => 0.015859438106417656,
- 1193 => 0.01458303071558475
|
|
Run model dependent tests:
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts#L271
Error: Snapshot `llama 3.1 > evaluate with metadata > with confidence 1` mismatched
- Expected
+ Received
[
{
- "confidence": 0.43072959780693054,
+ "confidence": 0.4234016239643097,
"token": 578,
},
{
- "confidence": 0.42230042815208435,
+ "confidence": 0.44442975521087646,
"token": 16053,
},
{
- "confidence": 0.9981182813644409,
+ "confidence": 0.9980332255363464,
"token": 5679,
},
{
- "confidence": 0.8126624226570129,
+ "confidence": 0.8100054264068604,
"token": 374,
},
{
- "confidence": 0.2757589817047119,
+ "confidence": 0.29307958483695984,
"token": 2288,
},
{
- "confidence": 0.9066131114959717,
+ "confidence": 0.9030055999755859,
"token": 16053,
},
{
- "confidence": 0.9882798194885254,
+ "confidence": 0.9878785014152527,
"token": 311,
},
{
- "confidence": 0.7492305040359497,
+ "confidence": 0.760230302810669,
"token": 2512,
},
{
- "confidence": 0.9521902799606323,
+ "confidence": 0.948384702205658,
"token": 922,
},
{
- "confidence": 0.6508099436759949,
+ "confidence": 0.683086097240448,
"token": 279,
},
]
❯ test/modelDependent/llama3.1/evaluateWithMetadata.test.ts:271:25
|
|
Run model dependent tests:
test/modelDependent/llama3.1/evaluateWithMetadata.test.ts#L340
Error: Snapshot `llama 3.1 > evaluate with metadata > with probabilities and confidence 1` mismatched
- Expected
+ Received
[
{
- "confidence": 0.43072959780693054,
+ "confidence": 0.4234016239643097,
"probabilities": Map {
- 578 => 0.43072959780693054,
+ 578 => 0.4234016239643097,
- 1115 => 0.13043756783008575,
+ 1115 => 0.1350678652524948,
- 1102 => 0.05168525502085686,
+ 1102 => 0.051406390964984894,
- 763 => 0.04288897663354874,
+ 763 => 0.040449317544698715,
- 1283 => 0.029397012665867805,
+ 1283 => 0.028996290639042854,
- 2100 => 0.029378682374954224,
+ 2100 => 0.02832488901913166,
- 15636 => 0.026268385350704193,
+ 15636 => 0.026462676003575325,
- 2030 => 0.02184896357357502,
+ 2030 => 0.021777881309390068,
- 320 => 0.01690298318862915,
+ 320 => 0.01988242380321026,
- 1628 => 0.011869494803249836,
+ 1628 => 0.012539208866655827,
},
"token": 578,
},
{
- "confidence": 0.42230042815208435,
+ "confidence": 0.44442975521087646,
"probabilities": Map {
- 16053 => 0.42230042815208435,
+ 16053 => 0.44442975521087646,
- 4062 => 0.30363598465919495,
+ 4062 => 0.2820369601249695,
- 39935 => 0.060395680367946625,
+ 39935 => 0.0518338605761528,
- 2944 => 0.0373028889298439,
+ 2944 => 0.03905597701668739,
- 5679 => 0.023811226710677147,
+ 5679 => 0.023956798017024994,
- 11914 => 0.016298197209835052,
+ 11914 => 0.016141841188073158,
- 2144 => 0.014683431014418602,
+ 2144 => 0.016043849289417267,
- 1121 => 0.006984429899603128,
+ 1121 => 0.006507175974547863,
- 17571 => 0.005794311873614788,
+ 17571 => 0.005837713368237019,
- 3446 => 0.004934381693601608,
+ 3446 => 0.004879034589976072,
},
"token": 16053,
},
{
- "confidence": 0.9981182813644409,
+ "confidence": 0.9980332255363464,
"probabilities": Map {
- 5679 => 0.9981182813644409,
+ 5679 => 0.9980332255363464,
- 12875 => 0.00015924211766105145,
+ 12875 => 0.00015266593254636973,
- 18964 => 0.00011538491526152939,
+ 39935 => 0.00011596923286560923,
- 39935 => 0.00011497695231810212,
+ 18964 => 0.00010741820005932823,
- 13 => 0.00010490677959751338,
+ 13 => 0.00010332829697290435,
- 627 => 0.00009277161007048562,
+ 627 => 0.0000947250155149959,
- 656 => 0.00006256866618059576,
+ 893 => 0.0000696686256560497,
- 893 => 0.00005633986438624561,
+ 656 => 0.00006114577263360843,
- 198 => 0.00005223735934123397,
+ 374 => 0.000054702515626559034,
- 374 => 0.00005191291347728111,
+ 198 => 0.000054134707170305774,
},
"token": 5679,
},
{
- "confidence": 0.8126624226570129,
+ "confidence": 0.8100054264068604,
"probabilities": Map {
- 374 => 0.8126624226570129,
+ 374 => 0.8100054264068604,
- 1587 => 0.04815267026424408,
+ 1587 => 0.049722250550985336,
- 596 => 0.024733318015933037,
+ 596 => 0.027551041916012764,
- 1120 => 0.022302960976958275,
+ 3250 => 0.021945547312498093,
- 3250 => 0.02154506742954254,
+ 1120 => 0.02066754922270775,
- 706 => 0.01618366874754429,
+ 706 => 0.016083383932709694,
- 15849 => 0.00869414210319519,
+ 15849 => 0.008394647389650345,
- 1053 => 0.005911793559789658,
+ 1053 => 0.0057802582159638405,
- 55064 => 0.0037806404288858175,
+ 11 => 0.003990540746599436,
- 11 => 0.0036655946169048548,
+ 55064 => 0.003569164779037237,
},
"token": 374,
},
{
- "confidence": 0.2757589817047119,
+ "confidence": 0.29307958483695984,
"probabilities": Map {
- 2288 => 0.2757589817047119,
+ 2288 => 0.29307958483695984,
- 1120 => 0.16664838790893555,
+ 539 => 0.15234610438346863,
- 539 => 0.15775153040885925,
+ 1120 => 0.149519
|
|
Run model dependent tests
Process completed with exit code 1.
|
Loading