From 75fd5117d77415737e337a7fd15ed3317e2ad7a5 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Thu, 7 Sep 2023 12:36:48 +0200 Subject: [PATCH] Add llm support for 0.2 modules along with tests Signed-off-by: Ryan Levick --- abi-conformance/src/lib.rs | 12 +++ abi-conformance/src/test_llm.rs | 103 ++++++++++++++++++++++ adapters/README.md | 2 +- adapters/wasi_snapshot_preview1.spin.wasm | Bin 134037 -> 138357 bytes src/lib.rs | 15 +++- tests/case-helper/src/lib.rs | 4 + tests/rust-case-0.2/src/lib.rs | 5 ++ tests/rust-case-0.8/src/lib.rs | 3 + tests/wit-0.8/llm.wit | 70 +++++++++++++++ tests/wit-0.8/reactor.wit | 1 + tests/wit/llm.wit | 67 ++++++++++++++ wit/llm.wit | 70 +++++++++++++++ wit/spin.wit | 1 + 13 files changed, 348 insertions(+), 5 deletions(-) create mode 100644 abi-conformance/src/test_llm.rs create mode 100644 tests/wit-0.8/llm.wit create mode 100644 tests/wit/llm.wit create mode 100644 wit/llm.wit diff --git a/abi-conformance/src/lib.rs b/abi-conformance/src/lib.rs index ce19c81..67207e2 100644 --- a/abi-conformance/src/lib.rs +++ b/abi-conformance/src/lib.rs @@ -28,6 +28,7 @@ use std::{future::Future, str}; use test_config::Config; use test_http::Http; use test_key_value::KeyValue; +use test_llm::Llm; use test_mysql::Mysql; use test_postgres::Postgres; use test_redis::Redis; @@ -38,6 +39,7 @@ use wasmtime::{ use wasmtime_wasi::preview2::{pipe::WritePipe, Table, WasiCtx, WasiCtxBuilder, WasiView}; pub use test_key_value::KeyValueReport; +pub use test_llm::LlmReport; pub use test_mysql::MysqlReport; pub use test_postgres::PostgresReport; pub use test_redis::RedisReport; @@ -48,6 +50,7 @@ mod test_http; mod test_inbound_http; mod test_inbound_redis; mod test_key_value; +mod test_llm; mod test_mysql; mod test_postgres; mod test_redis; @@ -137,6 +140,11 @@ pub struct Report { /// See [`KeyValueReport`] for details. pub key_value: KeyValueReport, + /// Results of the Spin llm tests + /// + /// See [`LlmReport`] for details. + pub llm: LlmReport, + /// Results of the WASI tests /// /// See [`WasiReport`] for details. @@ -159,6 +167,7 @@ pub async fn test( postgres::add_to_linker(&mut linker, |context| &mut context.postgres)?; mysql::add_to_linker(&mut linker, |context| &mut context.mysql)?; key_value::add_to_linker(&mut linker, |context| &mut context.key_value)?; + llm::add_to_linker(&mut linker, |context| &mut context.llm)?; config::add_to_linker(&mut linker, |context| &mut context.config)?; let pre = linker.instantiate_pre(component)?; @@ -172,6 +181,7 @@ pub async fn test( postgres: test_postgres::test(engine, test_config.clone(), &pre).await?, mysql: test_mysql::test(engine, test_config.clone(), &pre).await?, key_value: test_key_value::test(engine, test_config.clone(), &pre).await?, + llm: test_llm::test(engine, test_config.clone(), &pre).await?, wasi: test_wasi::test(engine, test_config, &pre).await?, }) } @@ -220,6 +230,7 @@ struct Context { postgres: Postgres, mysql: Mysql, key_value: KeyValue, + llm: Llm, config: Config, } @@ -241,6 +252,7 @@ impl Context { postgres: Default::default(), mysql: Default::default(), key_value: Default::default(), + llm: Default::default(), config: Default::default(), } } diff --git a/abi-conformance/src/test_llm.rs b/abi-conformance/src/test_llm.rs new file mode 100644 index 0000000..0c38df5 --- /dev/null +++ b/abi-conformance/src/test_llm.rs @@ -0,0 +1,103 @@ +use std::collections::HashMap; + +use anyhow::{ensure, Result}; +use async_trait::async_trait; +use serde::Serialize; + +use crate::llm; + +/// Report of which key-value functions a module successfully used, if any +#[derive(Serialize, PartialEq, Eq, Debug)] +pub struct LlmReport { + pub infer: Result<(), String>, +} + +#[derive(Default)] +pub struct Llm { + inferences: HashMap<(String, String), String>, + embeddings: HashMap<(String, Vec), Vec>>, +} + +#[async_trait] +impl llm::Host for Llm { + async fn infer( + &mut self, + model: llm::InferencingModel, + prompt: String, + _params: Option, + ) -> wasmtime::Result> { + Ok(self + .inferences + .remove(&(model, prompt.clone())) + .map(|r| llm::InferencingResult { + text: r, + usage: llm::InferencingUsage { + prompt_token_count: 0, + generated_token_count: 0, + }, + }) + .ok_or_else(|| { + llm::Error::RuntimeError(format!( + "expected {:?}, got {:?}", + self.inferences.keys(), + prompt + )) + })) + } + + async fn generate_embeddings( + &mut self, + model: llm::EmbeddingModel, + text: Vec, + ) -> wasmtime::Result> { + Ok(self + .embeddings + .remove(&(model, text.clone())) + .map(|r| llm::EmbeddingsResult { + embeddings: r, + usage: llm::EmbeddingsUsage { + prompt_token_count: 0, + }, + }) + .ok_or_else(|| { + llm::Error::RuntimeError(format!( + "expected {:?}, got {:?}", + self.embeddings.keys(), + text + )) + })) + } +} + +pub(crate) async fn test( + engine: &wasmtime::Engine, + test_config: crate::TestConfig, + pre: &wasmtime::component::InstancePre, +) -> Result { + Ok(LlmReport { + infer: { + let mut store = + crate::create_store_with_context(engine, test_config.clone(), |context| { + context + .llm + .inferences + .insert(("model".into(), "Say hello".into()), "hello".into()); + }); + + crate::run_command( + &mut store, + pre, + &["llm-infer", "model", "Say hello"], + |store| { + ensure!( + store.data().llm.inferences.is_empty(), + "expected module to call `llm::infer` exactly once" + ); + + Ok(()) + }, + ) + .await + }, + }) +} diff --git a/adapters/README.md b/adapters/README.md index a7d4d05..4c9d5bd 100644 --- a/adapters/README.md +++ b/adapters/README.md @@ -5,6 +5,6 @@ The componentize process uses adapters to adapt plain wasm modules to wasi previ * The upstream wasi preview1 adapters for both commands and reactors for use with newer versions of wit-bindgen (v0.5 and above). * These are currently the [v10.0.1 release](https://github.com/bytecodealliance/wasmtime/releases/tag/v10.0.1). * A modified adapter that has knowledge of Spin APIs for use with v0.2 of wit-bindgen which has a different ABI than newer wit-bindgen based modules. - * This is currently built using commit [9555713](https://github.com/rylev/wasmtime/commit/955571392155e428a5f8be585c9a569f1f0b94c7) on the github.com/rylev/wasmtime fork of wasmtime. + * This is currently built using commit [b5d484](https://github.com/rylev/wasmtime/commit/b5d484c6abe040355add59ef3eb8ca1b4d9991e6) on the github.com/rylev/wasmtime fork of wasmtime. * Changes are usually done on the `spin-adapter` branch of that fork. diff --git a/adapters/wasi_snapshot_preview1.spin.wasm b/adapters/wasi_snapshot_preview1.spin.wasm index aef1c71e20e24f7fd07983a7e11822eb14d25c67..98598530ae009c24e7603855e33996485db99500 100755 GIT binary patch delta 22035 zcmeHv33wF6^7z#3W^-+l4N1riy8$FX0tpEr90Ci1f*j$HTf$~{6E??Yv%3MJLIClJ zq6iGopgtrBDjt9W2Jiq>JW&C!ryz$1pd2FN@$>29U)?>kX9Dv3{_lHv-}ei;Q{7Y5 zU0qdOUEMQND@oYTJHxbs$yEh{Ai(Ql$7R}Dt+t;1&8f$K%jUkO+S_`}wn3oM4A7NY zomF+!_TDas&EBoDvMR)8$0|V=bR(|JYPUK~Zfmx+s>Eus*z9F4A$nCEBy5`!x&>QlEG#-YIxISj{&f!1h6`FvxJIKA zbkRbG4jPT1QfakXjRt=b1^O?Y{@su+ozZr#zZ_Wa#X+lL?UP@2s3nhgbZ11PNABP!D+SEqAfaw8j9qy zS}O!6gXp+i6mnZDD+M?0lDAe%s(*mi^}@_phsiy|pYd6V{&kdfDp;2=JDlCAwAm|! zIu2Jmtajm!C{|CoWB2Y%kjquoGp)tQrq%4OcGd}ZMauPdQ?n_O~P?H1u)h+?-~Giu$I>REPS zkrb=z0WXj{K}JA-cj^mp#)|Su4{VgeY$f9seQU3N$iNI4sq)imU$! zbY4W%wF~Fh$I!}Khs4=fatWV6jJKBe`zeH@^X|=^_h*nu3r#ufe?UYroBPG6ca=J_ zxOj%u(1Ttu>dG@kp*%yS zhf#W!Zo70m&K9wS-ZkDW>b6SzQW9W`bRw}^0;|-kjS;@E7qPpKq=gB0w&C6VF6g#0uA1mVMp=wqG>0N{@qix}*JOG% zc$oexWBkL5{&>>V0aqKN0ia$Sp!<&Un;}FAx(yuCeNV5EdLeRih~(%UA0`f<|8$3? zoGJaJHOUSCwz2vvs2S0mN_|w8n zFG8m_9X$xvHvQ2G8-`{{C)%gNr_!;GDN<^GI$|Dg;;9DKA4y{KKsYoc@UGIV`*pN`y=0$Qh><8;B^!pik3y;`?Ts%k&P zPk4W{f9F3WvYL+VeK`3X)z}+Vn4H-Oe+OkY{D-(&+W8LHE=6`};Y#czJ(Jxo<}6jP zac;SBP7OMU!_q-VmK5G~S)d0HxX%?7pIhHeu{q}L4BgN~$;q1`Pd2*d@tF&_CjO9| zO!z{o&PihWjg$WTAX(a;lM0_p2Xl`7hx9r*ucOD>;-r0f)-asT?8OD>H=4#`!V17b+>Z*npmeSjBp^SzyW^zH zo~vYq9YS81vr$onCFxbhB<>`R_N1J}6h0fM*+9f~y@Fhwin}_wHR|fx_Er?#Jh{j3 zT@QI#?`}9I(tAYu@p)*NdUd9#HiY6@kKPAi0i%UC1>Lt2^l6i_k(urlsL45jn=(EaKWxR50-yl(3y_H!vPKrIr z#m^k3adnRFl63OMMRMC;(a69snReK4NSQ#V7kX$OtmB6xMX}jQEg1a zF)!_}SR5S`>0brZdX{XXqW~jqbm|K$eMiA7W40+hQJZUu?5ME=_9hOT3tvg|2PVPS z(!&GS%63AKhK&gLssqZZlYW!Z>}ooIJ%cDSkKUx%Mf2!u{tEiFBzb{05-7<_-pZOG z8E}}K&_P$Jcz1~O)Z&yhZcsoC3L{NwPzY0lqOv`kCgvqlS%vm!$Q> z28MIn4%KR;eYZ}M)!dYS&~jgcYj&t^x3ue4ldnnqWnY8?yE6OMWD2S?Nx{tCsKBD&J&?#`0>lqX16iZm!D&1lL~=h16@2N)P3 zGC#{h$gHtcX@Lk#guw{zv2;!}49$_=9jnLA&W@!zi5fQ&Hc3;*wMSn%e_S&DJ~A!^ ze_tHe3$=7{+@okf58f7JK(aUr(J2(l1~guoC@kKR@6g@M`0?4ApQ%Vvq@|PENKcM0 z5A@i96x}8S+1mAVcyMcLNjp*N8Ev98eWFI5Eh3#QY99O{tjv}L6AN&lyCy28#bkKs zF!ccX!&Jz8W-gfIH!}jKD^pD{)gP1dke8e(-=ox1r>4NSQo+=@@lR5b7^$<@P-m~! zYtXpgo@#2(!*c&^nGT)2j;(u?N)V)jQPL*ldj8tC8I=-lVdrVX++xty{!uY4qKD?TiLAdop00;OW8CGW=uR0hpQIRuiC zXau#VQen1va>^`oj@LsTT2+F~A9M_nwpNA8Q4}-CFf>S9^kjo%QPjs(!*CwAwJXYG z(%HI;TI4#r!e0no_&G50OO%uer%i}S30EM^SVN9c}7pE%D3uIa%|_=ba-?*AX*pf`S` zbFvxEGiI_m%A!y&0iC1G5PUI{&Dj8g)uGK2_zuk5!FK(3Tm#>(bHYWb&mF_jfEL{`D{xLQi_oFPaW^cb->E2(PFOaw5FS;@NOVp-dS_l2 zraq)YoAZ)Wd<=|@U=WOgAxa;Tjn9&HOTXV4w9+=&sJtf)XjFRruBkW#{hYG&ON^no za(yd(s6MVD1N2dP{C{X_MjXcP8|J(LKW($kb&IfBIySEdjyQh45!d_i^R3FMP`ErO zdExDXOJ*0o(jS?xOeE%&lH4<` zg2_`B4r+$nxv&oH;@rZ17#w!FCk4KdhTIdhXK3CJvpvJ8YxrTW+h%))YoRZ_w-d6I zGbKwpb#GhdN)n~?MG@#uG8ZjIlij|kC#vq;q9CKWx-ZyqpZ|IsF!4P0eDzjTi)*fP zLz0io{wFpgJUYSY_DzU%WdhxY@+g?ah4O;ZL|n)gl&_b}@(pf+gS-D-NZ5B-H(UrR zm(g>_ZOcZs^PCOjWQ2Buc{v8$)n|(NhHzt;0q|U3(Cw7kFTVj!NVhJ(8IDTQaszxR z?O0yr%iNEz&~?u68jZYS_tVJheo5h;MP9tJ<~_f)EP#s74ipwfoO;|*>M`uo9hH7x zu^Cy--Ocy!)NATi6{0>5?e<-2o7#*2FSqaytfnE$`Po_0vDNKUxO{M>&&RWq5ZrUn zRX+#Ca!N{DlL1dl{nxZly)1iS9p3Gvatss(&RcQzsA`~~yDZIJLw7Z6*3ezeM{DS= z#_@2_w2^&nA?tx{i+_Ar@pa7C%6F)*>-9*}L4yy<`9~CY=$TU2)5G>h>9*m_qu-|vlohK`%{Jd)Ef-BtrQbL>r)S4_||%ht~S- zBnGUbMl^n%GVTB(>HWtP=fPsDYoR|@njmdkXJiA(l5XD}DrGjb^$sZB5RL=FT?GzE z#|C6-=!~B3fd*<~8yl*q%UB_ouW!6dUQ<0s1L4Z?3r=s&e(@7@(w=;h zPTD7*P$W#xB;?AjPbw^WjwG(qN}4BT49^zO8mOubf)?-z8JdRfsoRlX`%`o-zx)&x z^e0c5Ql4Y8C)6kuh+4e{?JElBezu@Rf}%|$;5ljSrWDl1u1)k=#MhfD1LIGPq7#jx zlj^4xooJ}#CF$86I(ZtO4$?^=%KXjBfIXAvdV27ZdqR<5^5>sbLXqZs_k<$BU;?r%kQ#9$me5FA9RpT0} zq5DIGg^Y0OT$qJxx&bq7^rm>jAaJB`JZnECE!f@em4TZFreGhG8((bBxc^cQ>P9+RzBZ;x_C}3&eWcG8yt`r%$F!ueFOKOtx5O)U z291F&fIhqC>)aAIZ5BIr|69!>{*{3p`CA#H5szl(sfXe=Ms({~U4}+4mS=?OHKXuc zdAl^{73E>X{3vw00bx-q#iQ~!3q*XkVB7RpV*!H-=bq_>p7<@x5QBje#wQrQW$L4G zsYN5xoo1G&?goW}qeSY4)yAB1eWdQV6tTAroY~fCZ!E}>+yQNH=eQ_bJ!tboXfo7e zi*>-5V9U1WUZ1RV{B<;Tj1gk2GE!s`XAfnv$NrAE=eT`;AwJ4qwf`nuHjenz^nG)l z%=<362|soolPeQ_+C}*YxJppbY?7+pS}xNcZ;Jk1Z{@>pQ)_wg?Hw}yxh53l*Z0|L zAyD@$t2+5Tdf*ZmBD14KF1L&C(PN501nQyX7PcOs2Ok{ay90yq(Qdc*r^?IZpEwF; z1oA_Z2z+1@!Pow0-k&CKN!cSN`406Yj!#>Y{-YsO>hr-wdB_W6u)F7fpmD^cgZauHIC!(C4^wdjw!j3+5gz-9 z3N#Rbse~gGe@s=v5l|M=`yVSimo0R7{J{skG5wQb+2y{9H@KUpAx9)Scln(viuRKS z?SO*Dyjmo>Ob=SEJpPXmMSdVe939~XzS$iw&HHS%jKM^>?@NrANp35k0UTG%{0j=3 zOooA#G@g;JN_Lvs6vL1Xou%*i1UB)#2Z7|a0V+j~aq?d>o0FtHN4B?EHXlBYY)bo% z{}dQQ%I7ta=%t%^1?R0Ad!X%*FincbzM zuC)?jU)vTe*T5-M ztmhSz{oxQJ)tpJ?dx%N(S@^=9fCmKlCK6uS8$#$skB9qsNy8Hyyd>fA6<+88IbL*z z2zt>P@B|l+W(`_;QR9PSyaB&vf`qCE!~O(TrjT0qx-d7vk2VX z;1`=Mn%zLR2HQygwk=q^@R)Ln-~d;keXweDQw z3qv4A357P(qp)jzIzu|$0$pJjnRp8fh23QRE$|$?M2d#OFnF0fG8F3Yi`#@@a2{T5 zRNV?=0rrpy!@&xB$t%NQGVCKsBfx=Q(%v@$4n>`3XE*dKO;Ps-DH{nr+VCnc#^6(N zJeAdz>)xR3@~5TGlii4e27YlQB*Zm=N|HvwMA%PiM&al9`^kNyu#NrX=}{Q_?I(vu zL7nW}zrimt`H3Z&Q3T3s#I#m9F-jc`>t*Eg6qz0AkuS%>is0G@$H4tQwMrAsR5#g1 z<3r=%3q=w8be<0H9S;V0lWZCfY48?#b3A+rtI385Fg;>9m0gNnWz>`TMG#N+OoK)6 zFlm?uPmpJ(LL50g32I;+nKT&|z!H9WoE)7D8zWb8(oo}GfgGQNDqF+qVHu-_2l!#h!(Hb)mn#iE`2#sTq9-$%zr6Dw&K}iVR!k~DB zZeoxQp#coiBGi{bVF(p6NR3cFg$nSs87)42(mlck9)g7vTqop{Nzh6YL0>j{n;bEt z6&xUcnxPurCsh`3!5ieH1r_ljy$m1=tdN)f0TUQ@D}VBguRbFf&mQa?B!{hVPdIz( zW6a>@<#~#RSp4zNYvi6%_#3=Pv}J&=;FCdR@J4ncZB{R43t>?zZ3sRs!}vC=T({Fx zJs4k?X9)Oq0?D5N*+^M318m9fP!xk&_b$7J=-#8(42|vr`DO<6LnqSC1`T-GZ-aFy zySVD{1)-j*bve`}plz>1(R@tm%h8cEl8?&40>?;U1+;}rWMTz+f?Z@u1$4#ID;01X z-ixb*qINIRmNMw)Tv7N%X(v?}hVp9NFQL_BT6uGSm z?!;q!_JI@mwhBrR^%gty!5$Xc;c+ZwRzn3|R#Zb#CT|(^_!v>gUiD1TiYO-DBfeA` z!k#hejx}l>a6iCtLTX?ToM=2!0}!J7f$|j@jjCd9@FF=k6Xt~f$Z7?$jW}mP7tP`q z0E`_QAD;y;ht;QmQ*4C^6{;NWYRi@dzNmpI8v)<~kn%vo#Ou|&@ zuA6Ca`jxXQO)ht~-H(Gvx3$Va^RU!9tusU1)sF0nI(Bi)4CvKia+<1KDnSjd0Jf{D zE!Ikvpn;ltRU9j1+pFE#u3868R${dXLX;DQfVnTStxjjP6I?=+&5k(@Y?f@B-BIht z3bhcz?x}<@a6(feR0}$9rWi$35#X2!4lN2Sg5Ip>d1Ruv+#-=r9}-1u(R2VYo*UUR z`#PuyfpSnmI1EsQu*y&kSEWnC2j5DFLq3@*@RY*gLi8AlE^_o(bw1Tvg3wwmG?m^4 z64elt2PZ_^_^(D74rJ4ONP#Hw#(c0rbYs^AD2DtPQ6+?jI<1&?!sVuL7!JZwZFgB= zphi@Qpi>LVZL}L&6HAu71h?GSieg91aJwC{%m;=ygRd4mjiO0Wq&SlO66BKm7vfij z@nqdXn4?Yr&{6j$pS^^}-G>-oh9q@s!0_n9 zXJJa?&X-|H3?!1955V;3o%L#()&@?`{RM;)awtsfL4MGPA&oyAfQdR7MoxYP`mA1< zkffMR#N3=&Hn*HDDYxg0l02(9w^vU0f|C3aiz&xEIBODFb_@)?^C*Slg^NA2-xWaL zy|5s!04Ys*IR)mDlJ0}EN75b*#P%gFo?Xe(e?T^Q<4d?f>^6emh&qPPm2=36&!8Qd z_Bq6m%45(Dr_a)3FiGq_p7l<&$8kl>^FeYShjwEAGy<2f?Iq`M+P-%j+v!1;9f35W z`w2885bI`W`l;E6RQ~gtV-5zcBOXxMto6oCw zAa8c`c`zhT_s#8P@}%}Ek-L8OJanix`=Lg;DQh9 zIlt~|I_1|K-;7^#wMxFPdfHfhP`J$vh_!Z z#r-dX&r|pz4tKtU@KO$!Axth^LduPoka9VzZPg>c&}7Lg#h(&$&%%;|!4k)A^Al38 z;P4a*ujKG*3a{euCkT`2mmxbgr)R0PyR~P}{L-F<*8JSTtBL6{GG25UJ6c2GzPM{= z+`V%d=^tUR&XjM))g>p-Y%a8zOA0My>CceeswA&_VQHb&V$IDhH23OZ8T@xnsreZx z*K&9eg&*VaJPNPl@GBH<;PCGhUeDp4zaacLhusLPHh}DOHvIxgs*S+>qmhLC3aNOs zVE7eG@Fcz5MBe%p(p68vWbS~`;cH0O-ylx4NvXQ$H%Nx3$z#7kF+4-A{D$X1&(hLO zWYhmZs_Ho?@wj@uYBR{o2zv6)sx5$C=pdw$9lwJ?wG~$F-l zx%?+qHF9{-zYyNZ;j}9V?;>5VfTZ3H9|QG?Y2@S;*r|RAj`HHlUE*Fm_M#c*RIjjv zbD8ieOE?F6FyCBK%3f$D-`qZyZZ7M!rn1ew?vweAC#kd9&CWMi^0~zQK{sjcxi?wr zxyf$@-^9#wZwE0UXM%gDiM(^~`ZVE9JNF*XI(NXQCBLL|@B8JP`yepo+=u=d=MDxY zobyXBhbiL@2c(;m)67-#6m_@}w7RXByULqTj;Egcq^Znvp9UqK`^+cr+&`L4JNG#T zC%%C}BuQ3_ajGw11YfkxYB5=L1SaqjF84a}Iz>1NBWYD15+8!4FJU4_z*WCJ9v|Et zBHk??2eRyQjKB^azz8Q-BWA-1Qmhdts!n3#N-&zF`U+cOrT(E}viLO;(Xb7PQpnO! zF-7$ajN$jc3Ki4g+f`xW9ja3>n%BJ$CZ@q@@?n@bRdoiY(8>a02^TvQeg{z~;$oV? z(pv0rJHJN<>svkx@$|l&h0}()JsIjbvM*e0-RV5Ua%u~1rST3XlB~J_vAhMJx*teQ zgs3O_2r&^ZlEMhF5PqapJ=OCoe--ad8!%H`wxiCT~TEAN)UT CS@CNC delta 19924 zcmdTs33yY**7w|`P17KS&*0QK5@C2?f zEV5JtWS6y|A|MLNrYKKbP*HJ1Hn-=>^FK3lZ<3}B;QRmYw~u}#vz(bZbGA8W?#=n> zfOaX-`UqxQ5eR|+d&L%ueJU3(I;^S*8M0?6s63@}8pEE=n&$!Z(x<8@smd!E0O zw(NXMg)q#I6>Zij7T)ln&Qe@uEiWl%-6H~&ngb%sHwUMlglq5-qy2IwvW5z4k%f&K zdL{Toc6jv5au#^iW6v$?=OcF|2iEg8e zmMScTg+e)9kk4k5lKex1D*{T*6%*@YT`7(A4{u%N#d->@#gl}|yj)UhDHa~~W$hFX z)>kd9@eiFjMJKlwn~N;MR3BEeRTnXkX?j+zET%QKmM~pcT2fe;T~cf*$;%UFc$Joy z}OKx_x)l!%%%mVLnb8&7-5#z>e?4&F==j0?=N-Kmp z;Kl0YmYgbKu3WKMigSfWz?b#dCRSGDmP{@d9+mR+omD%aHOcf5gUS7YqCpf6knw?H z8&Nn&<_C&VQNjxlVk@;4C+3tC=UMa9Q84AA@FF=HD29y@UV^}q%8CgkmBqP<<(6El zEgg9*9BR-?W%^|Za)GrL=adVF8vy583WX!mx`2>~qY&yc#8zaXT((JN0YRiRNL1e^ z9D@)SAj`v+oJy3JS0TXOQ<|S%R%t1(7G85g0~;0C<5GvfP&h%}3=+d;3a=w&%L-BE z(izL&06)2^fS&{(xrM9EDe#+QsZOji7gow+PJ^$bh3@~1a|0#QSMXMLS;ZZA%4{8H%N%Mmagg-y4rbaM= zQ84KXT8TnIyvhiJjVk@~(pZDldn@fV%@Ot6q?*uR*eX2~)&W0{Mg-w!ZCJJpurSaL z5E<2p0|c_Mf}k@ACc!A;l;654Q|zVxoNhE;@D=pycu9X*+8v&NMA|;4hZGV~E93p2q1YIGd=w0pdw?j# zWrj(QMpetJd`vN36hjFf8{I9KK^j#i?{q*Y2j`_RtsicbEylU;2Gd<+ ze%eW1$2~YB(w)G4+qRHm;)lZ?$re9brWV`2$x7F0(@xnwTPM;enj$&1)>;*`*!DU2 znP$rN-Geh;aPZZM-CA;9xPVhaprqq!+5)4>sMhb4{!DBw1tmn=X;_``#@|c1>n)qh z$Xq~%rXw?#wQ7x@M_SeXy@>g?odUb1d2O3GJ0I}>g0k5(qrf!14DG~Q(!jEI(xdGc zJ6ndh4%9II(IMN&wS=kw(-Oh-!V#exiikkj0_*cQN2tQY&~*x|$Tm zSQ;cvc{5x}PL6~N(!k`8|6X=p=+u%S)l=$`QXtc2zY6VH?E{oH$5T4Ob?MiXtA`Bs_WMS;NC!sg-&lE@1IZ>uQDa{ z^`!^$e(CvcW|#NN5rK53H&v@Kd>8^qBk@WJeP_-!vH{sh$5hsT0tX>q? zfQ)*i)FO+toJF7XTG}XcoIn=rQPyV6xzmhU8Pp$`nK1({N&n6WL)=2|T_dkS6ukv{ z97Y;JId@j}R&2p?2MDtTG7z-|;eC`{xF5daKN)@}MqGE$+`)+N9gWxfQbq*yqm1aa zAOb#-#w-Yzq<)dGUs|-lENiM==YVUf7PpUFpgRj#SZ`-?4^yFq4fw-0?*!uW4~2Gi zJNj2v_`ZKzL_Rs8oit!TJZ_+9KpgCqo*d8}o{>%u7z%Y#?EM*D`>CA9X?#4T!6{j; z8@i1VF>r&0CS9Xf=1|i)ob{zq6J_7bZDvQ&o;GkJqTg&#Hx#$*LCp~LIfK4(6*u?p z6csn=PVSV$S)<~0NAQqL>O++7Y9}2i2zn;&96HCx#D#4buC=o1@MC_wewj>&!HXCS!q7 zY^|z+3?ZZDsc$c;G6el~sndg9Q8J1id>TdEdqe{fmwA?kb}fyI`1}#d1{gEs9xCGR zj7WqV5{z8vYKU9`w~cHd=znD$+bF7~j-4CgnHkjcLqVj6f|~oLSf+;pHK%`$>ViA$ zl2z{jsLf&rz!-V}aQpmc)>3DIb~W0Jkqt*x;|w%(x)NnSn0(^UF~%U19!y#phdpCw z7}nd*B~N-Tsg0g^F7VUf&?A_R5(H`BLyFGYwMxy6kPJ$VkRe;) zkjo^}{Bc%SmH#Wl$7Zj>ZT^xyR^FxuCAFu~1Kk88mK9Q$jvoroO2@_@bY06s@oeY_ zW-op7DCtdk^k}?nRswJqb@IkZ6B>xh?PqZeID&B>ll+DQcI7m;FY6d^%>4jsZo5VX zoNzy&wdGzuz#jJ+1!VLBi=tsqdAh~UnJ5nImfqcfsQl61%+u?`oc5ah0caOaAU>KW3ooWMkeVz4vZ;KK;6=fSN3BPh$|R3s~PN-6|M_aoGct z7Py)XW*{R`x8<1jP+>j)CA*L+Z}%bvIrrcUxjTW+Et-JH4J_*`B^GOB2a^f776lv~ zOj*F0gPB=80FBAh#R@a+9)T=sd))RMgwUU3$n47n*9NjLF(6s)epIP%V@JMoy+$=L)b~V+U|{-5pD; zXdIwL^}R^34lCM8%c>_M)%>P}p{vnyN;DppeWsM-VR>N61ISj7sgqqL(SsXf547hj z9$GooomJAKupkiga;GVh={npzE!ggdc}VM~Dy(-Eqq~}{*YvT7`s(TV(N~$Z3o0-M z8a2kCN%3IxHRAvNk?}zoj>OG)6|U|1az=#+Tck0wI^#|k&obfJ|IRFn%<=_ZNZF%v zs9KGhtsu{IzAc`oZTa5U%~rIKv66$w&Z&11)hIY)EC;_Zr=9ctSfk)fWH~sQd#@Yt z8*{6Xr`AV$L|tMLAs+Y!r^8=w#JgVo$I`J!6lcvIxVSfE%#*x(%f~#)JAwPoYmInr zAJb0CnHP()H)kHz*bVdMqwFO=o{GG%K3>l=X&7X7e4^g*=&BXmUN=zJn)zk8aQ%oT z1($0&(G1r$45859aRfu$-_c$6kfx0a#~nC%d(H0P?J)~I{>jPCcC4Yx_MbU=2N!q1 zi>PlF_eGSFmJC8p&Rjw{S-YeKd?3BPq#wL1>7_WlnCd7MIU{iNDAlUNlA=}2GfPAF z4!vsXJ8Pb5V02tThMZIskMY_*V{2x|aVjLBV@8>P*oDp77o2X5jz+uEA>s^$a?!7=je7SwRt7u_7KeN-wRTLCmKs!twL>6%qIu zvXaItLsrsw<=RSj2XGpiVL4tYUZv2SnPIty(tQ7_WsMqoCy;`t-5L73S|NEgjSDQR zui&ek}(f=g2TGey)2J=1`JGBROoI)10JzEdO@akdw^#}N~iJrgWdte@)2N*7?s_t^%l zY;-O8q;Va7{#k0l|NU%FWMYpE5%`(4VH7+k?cT5uE=s*NPLqQj`@NNyayjIsTn=5_ zs4&Hg8#~rRC9(M?tBk(=d~yZZx=BIbezDf*3Q3$)F2faT;FTz#u+bX3{be!9!si~s zoqhY zuJ|eLXJ>v22mf-r;-|O=XMPF?-?&5ZQ{00$0b~a5a=vcnr$Dsy<1Tv3`L~}0!}e0- zP#Wbs2D>z32u7tb-iyiOCwpU21^?XJ199%LuYncuRFuzCQNF|b>WLneZ~5k2v%kU3 zEMuP&$P4=w<+Iz|zp}!tI=ZJ0EBYDPs`5n@3obQI!KH|LmRW3&Vh^;!UH#?gGT>mn ztE>N2blJFwbbPT%h{)nY87=vH9HI%m{u*_Ej84P~J!^~iG>QfB8l&f6yr_O&`t6YN z6lIn#4Q>=Sti|n|Cb1X))4#Rnqn85!y^b4)$I~PHNDO=`l^hAh@?%*3Y|q9c0U(>X z(>~7g%cqEoQzp6d;oO6}9&7GrWC6Q2p8%2Fhhh<*Z=00y@0# z-u+rjba88rr@<~M=y-43{HWuOmV?J@WE>B=nUL-}g|E5R4TxjSv;w>=4WsYo=r>4L zPArk<{>05dI(Iss`=lEXWp0&G>>>*I+*9l0r8BA%g)}_4I=!(^=HJzN^3gc?ocQXf(&xa z4)Try!fq|-i#IYz&4m_tvXosLA>F#LTqYNz=ee(_zh(M?UOsXrnLbe5Cq_@#Mix`v zZq!%wUXBACz1gynX|LD;NL(|Z1m$at`x9{_-*ttBQt8E-CW>U}B}zzo%TRFXgfotJ z@kr6tM7uJ{@zjwK`7UOZ5nv?e;A20p_v)6D2smbE4B+6QmldPof($Wgfs0HQS*L|u zsrwgAlqKY#%8qGQHT06!f74Vky%9q(-BWfgL2(9}m<~4JdhPd3m6opZ+3x567EJ4Y zZ=#q=4hy_A5Sl2GF6w^7*#_V4So;4x%v==d@_z-Ex#UUTCd>9N0d_0DKLFlqq6Dpv zSTd zCFiIh3{gLn3^SXkV!1eZq$UxxWR@BpZkqTy9i^~cD!bTJ$?8xk*g<9`Hd*$&cza|< zIk#k<`O0bszO~CPTyFdO5^^aM+Cv@j9}4~90LdB(+u^4#NNr5x)^&ftN|e2pA2A$(tjf6poT!BjKFySA0Eugf8k|Au*$%a|~|+lRrMX z$2&89f&LZx@(KQ$0v}CmCuIoJdOI5i`2Gn7;DwV+N#*H+zrK-? zvW30Iewz4E5U=9Tx}FDV@ficzIR=zhC=?JpO0JHDCu9U@D?`9_atgzLIA(GOL&x1= zsk*-7-~+{FIjUDK8%dAx5C^Z4@#7&1UL(`-ULKZ_F=iMKi^xl6*Z|APu6)=C3yD__ z%pw0OhCs3=2g+ap(dWW!SW6CB5K3a*Kv<1vb0COB<-=3(1RuSGfBl1ht>Is*$wn)T z*RG=2M;KM4b`m@eOUd#A=maas=>o`xd8FMWD1yagvh3^&{j!89WMmNzZb?_n}EuIlP*#%)2G;Ns6eLcqf1Q1M=-;5<@t*px zNoWQ1fPG{{1!g1IM}Db*brJhyveBO(txKW`h4-fGk_z;1liii5hI@!-739KuWKtEx z!c{WA3OeK1qgANPSINyPco6&gPKF`PcGH>S=`VhK@g?ilDlNW*tk%CrTN(2B>l<=# zGBn3wA5VrSWI_L#-cPbuTgbeJVXAz=&8}`8KawF~)i43aO|6D*xLjQ|tjF4zDKH7Y zPE3Izt@tv4N{Y*{E%}H@GrqhD;ykDwz-nF3sqh5A`{d*_xEDUC^P3Lfq5p=C(D@;Q zDFO6XNbGEw?)5Ee74~c)8)idW&rO>FObK-(kHQ$JOP&j6O`YX6v|hyTNr(U+()vlT zf^Xf)<1iA)l|>Ml;wP#Et)|?9zv{MC(6SaYAC#69+bodbDXK*9R|`rv=4zlVdeU_< z+;>j^h4!9UQBj&`DK9T6hYT+_@M^(6sY(zjP#}4EG4w^vY5Nl7kx5G+RBZs$ICAAI zjIaB03H;QV_?|+=>`IDHfi)ss4Td~Rc~NyqaXS4eeFw}MkW*1ot`4sI?i5`0hY<4d zhcMpv#xym}8vtvj-2{QuegRSBR+tz;-V7Bz>K?raqx3L{%(?+_WYaZB6x$8umHuCY zj)>Qxg_t;yx7>dnuc6zMnj6rOY`G52#15lqhmqX44ryZ22wv%b163iJLOm#U97jx| z7)MV30RhD8CbSbf4dyK+e}Ffca1(_ph3xtXdXrN(p@Z1jM#{c}4ASE}NQN$?^gE~& zQzz0k6FICAwIu0#l;t#%c^TRe{r3TCLcRy!$H# za5XvAL zC(uJrV)_Y~-)FId_!9N6NU#2mmhR$! zEWY$xze0j|e-5v-{SCn$$mf+szdmM@IIwOG<(` zsym-ee+y!JW;M)5GP-^`k}-ZfJBgBvWUM0>$wL`@+9z*8T=+QWyd>H7v?Sx@rLWwA zgpOwC6eSbtXDG>OEI~;w%TLnQ;+mc$uhHxz`8+vEP~=2=auTacZju5_O=4&>$&s3* zP{~YE&##5q|B&rpK7N?3vtnsKFE$y8#qCn~AH zJS8E9%7%GLsxVDSSPMBd3C%3>up>)Jb^RnIQ!qzK$jqtEIZCG0OHnetL57kU&Iw9p zI`Wg$fKz^wS@qJB%y!OBGRKje1ap(nY$ORhJxxy4BTlJF9(Bn~^4QXIVt1H_pY2FG zi29{(iHT$~7T^hNYfJWk7y$EG@g^-UV8uiciwjw?0E+}f9KMM*KFJy{)8Zmp4D6he zX3p)DhaVGiQqv}M>9d$Yj8h@R5>{M=Mbc4)GrUI|pJI){YAh}#wQ8|lKvHU+rK2S^ zH6<@K&61MbXBlfIi`0lEq2=zdoDK5wz(Ffm@dz!hWW`&wxQZ3iJ+b&SD^8}x)vUM| zi{zY#*e)=)Q%=eRb84rwu9lQs^8`zuwG5!W24UBcUK+8RsusrDjc=st8OSD?-@^l> zwN?yLt>?9jWj~-LpN}o@ELp1+$HE5s)tlPaFxW_9y|KL7ON>-)f*iYPjf3ZCPqw%j zxRqnZ{1tC8PTT@chPG)SogdB*tCzA3-n@udOw^8YMY|SD!q6>T?ZHCsx3e~qB`K5@&;ohn(+o>B%1Ih z;zf7)ZeC*PZbmfBYlCTAUT%=T1~arAc1kkC^WxB4H%A(PV%C?VnCT`s@)ft4ZjO=c zAThZ2tM0Pgyw*sPo8wN)J96Bdz!Wzj@H$U%^G3Z4Hz!>a+?=YP-{!PydYgL5Y;eP8 zU6R|dJT^)i8=l+dE$4JMNj$a9xyCZvyxkzN%{wrJ3X**CUN?v_s`D_ApNM#rdXoOZ zDDD4*k-P~HR6SWwo8CpYNjd8yR2N_rui?4c99`X<5OI$9K9F-)xWDB~0z$<^_<-~b z6-TK)#1(dhi=C+x7N)w0GqGB1m>9152nO?7DSBzDk6|dUy%Huy!zbiYm^ejs2{L)( zl5jB!J|#QD#j&c-U<_^SLV8Dt32C2$FOn{sX1uawmsXTtMqB7y{{jrOUry6y#pGWM z^h;73A%?X63Ih1(T=ZhG2VKjeuOWcX;MDdF$&3_@%Ob^4xI)4r#WeVqwxq^h1y9VT glv7%L4K!FNDygzu2W@tCQDtF;wbKo Result> #[cfg(test)] mod tests { + use anyhow::Context; use wasmtime_wasi::preview2::{wasi::command::Command, Table, WasiView}; use { anyhow::{anyhow, Result}, spin_abi_conformance::{ - InvocationStyle, KeyValueReport, MysqlReport, PostgresReport, RedisReport, Report, - TestConfig, WasiReport, + InvocationStyle, KeyValueReport, LlmReport, MysqlReport, PostgresReport, RedisReport, + Report, TestConfig, WasiReport, }, std::io::Cursor, tokio::fs, @@ -251,7 +252,11 @@ mod tests { let engine = Engine::new(&config)?; - let component = Component::new(&engine, crate::componentize(module)?)?; + let component = Component::new( + &engine, + crate::componentize(module).context("could not componentize")?, + ) + .context("failed to instantiate componentized bytes")?; let report = spin_abi_conformance::test( &component, @@ -260,7 +265,8 @@ mod tests { invocation_style: InvocationStyle::InboundHttp, }, ) - .await?; + .await + .context("abi conformance test failed")?; let expected = Report { inbound_http: Ok(()), @@ -295,6 +301,7 @@ mod tests { get_keys: Ok(()), close: Ok(()), }, + llm: LlmReport { infer: Ok(()) }, wasi: WasiReport { env: Ok(()), epoch: Ok(()), diff --git a/tests/case-helper/src/lib.rs b/tests/case-helper/src/lib.rs index ed9cbdb..b5f7100 100644 --- a/tests/case-helper/src/lib.rs +++ b/tests/case-helper/src/lib.rs @@ -109,6 +109,10 @@ pub enum Command { KeyValueClose { store: u32, }, + LlmInfer { + model: String, + prompt: String, + }, WasiEnv { key: String, }, diff --git a/tests/rust-case-0.2/src/lib.rs b/tests/rust-case-0.2/src/lib.rs index 9ec8a5c..944f3ab 100644 --- a/tests/rust-case-0.2/src/lib.rs +++ b/tests/rust-case-0.2/src/lib.rs @@ -152,6 +152,8 @@ impl fmt::Display for key_value::Error { impl error::Error for key_value::Error {} +wit_bindgen_rust::import!("../wit/llm.wit"); + fn dispatch(body: Option>) -> Response { match execute(body) { Ok(()) => { @@ -352,6 +354,9 @@ fn execute(body: Option>) -> Result<()> { Command::KeyValueClose { store } => { key_value::close(*store); } + Command::LlmInfer { model, prompt } => { + llm::infer(model, prompt, None); + } Command::WasiEnv { key } => Command::env(key.clone())?, Command::WasiEpoch => Command::epoch()?, diff --git a/tests/rust-case-0.8/src/lib.rs b/tests/rust-case-0.8/src/lib.rs index 65cc457..26aff63 100644 --- a/tests/rust-case-0.8/src/lib.rs +++ b/tests/rust-case-0.8/src/lib.rs @@ -260,6 +260,9 @@ fn execute(body: Option>) -> Result<()> { Command::KeyValueClose { store } => { spin::key_value::close(store); } + Command::LlmInfer { model, prompt } => { + spin::llm::infer(&model, &prompt, None); + } Command::WasiEnv { key } => Command::env(key)?, Command::WasiEpoch => Command::epoch()?, diff --git a/tests/wit-0.8/llm.wit b/tests/wit-0.8/llm.wit new file mode 100644 index 0000000..2b1533c --- /dev/null +++ b/tests/wit-0.8/llm.wit @@ -0,0 +1,70 @@ +// A WASI interface dedicated to performing inferencing for Large Language Models. +interface llm { + /// A Large Language Model. + type inferencing-model = string + + /// Inference request parameters + record inferencing-params { + /// The maximum tokens that should be inferred. + /// + /// Note: the backing implementation may return less tokens. + max-tokens: u32, + /// The amount the model should avoid repeating tokens. + repeat-penalty: float32, + /// The number of tokens the model should apply the repeat penalty to. + repeat-penalty-last-n-token-count: u32, + /// The randomness with which the next token is selected. + temperature: float32, + /// The number of possible next tokens the model will choose from. + top-k: u32, + /// The probability total of next tokens the model will choose from. + top-p: float32 + } + + /// The set of errors which may be raised by functions in this interface + variant error { + model-not-supported, + runtime-error(string), + invalid-input(string) + } + + /// An inferencing result + record inferencing-result { + /// The text generated by the model + // TODO: this should be a stream + text: string, + /// Usage information about the inferencing request + usage: inferencing-usage + } + + /// Usage information related to the inferencing result + record inferencing-usage { + /// Number of tokens in the prompt + prompt-token-count: u32, + /// Number of tokens generated by the inferencing operation + generated-token-count: u32 + } + + /// Perform inferencing using the provided model and prompt with the given optional params + infer: func(model: inferencing-model, prompt: string, params: option) -> result + + /// The model used for generating embeddings + type embedding-model = string + + /// Generate embeddings for the supplied list of text + generate-embeddings: func(model: embedding-model, text: list) -> result + + /// Result of generating embeddings + record embeddings-result { + /// The embeddings generated by the request + embeddings: list>, + /// Usage related to the embeddings generation request + usage: embeddings-usage + } + + /// Usage related to an embeddings generation request + record embeddings-usage { + /// Number of tokens in the prompt + prompt-token-count: u32, + } +} diff --git a/tests/wit-0.8/reactor.wit b/tests/wit-0.8/reactor.wit index c06a4bd..c5898ac 100644 --- a/tests/wit-0.8/reactor.wit +++ b/tests/wit-0.8/reactor.wit @@ -8,6 +8,7 @@ world reactor { import redis import key-value import http + import llm export inbound-http export inbound-redis } diff --git a/tests/wit/llm.wit b/tests/wit/llm.wit new file mode 100644 index 0000000..66ca408 --- /dev/null +++ b/tests/wit/llm.wit @@ -0,0 +1,67 @@ +/// A Large Language Model. +type inferencing-model = string + +/// Inference request parameters +record inferencing-params { + /// The maximum tokens that should be inferred. + /// + /// Note: the backing implementation may return less tokens. + max-tokens: u32, + /// The amount the model should avoid repeating tokens. + repeat-penalty: float32, + /// The number of tokens the model should apply the repeat penalty to. + repeat-penalty-last-n-token-count: u32, + /// The randomness with which the next token is selected. + temperature: float32, + /// The number of possible next tokens the model will choose from. + top-k: u32, + /// The probability total of next tokens the model will choose from. + top-p: float32 +} + +/// The set of errors which may be raised by functions in this interface +variant error { + model-not-supported, + runtime-error(string), + invalid-input(string) +} + +/// An inferencing result +record inferencing-result { + /// The text generated by the model + // TODO: this should be a stream + text: string, + /// Usage information about the inferencing request + usage: inferencing-usage +} + +/// Usage information related to the inferencing result +record inferencing-usage { + /// Number of tokens in the prompt + prompt-token-count: u32, + /// Number of tokens generated by the inferencing operation + generated-token-count: u32 +} + +/// Perform inferencing using the provided model and prompt with the given optional params +infer: func(model: inferencing-model, prompt: string, params: option) -> expected + +/// The model used for generating embeddings +type embedding-model = string + +/// Generate embeddings for the supplied list of text +generate-embeddings: func(model: embedding-model, text: list) -> expected + +/// Result of generating embeddings +record embeddings-result { + /// The embeddings generated by the request + embeddings: list>, + /// Usage related to the embeddings generation request + usage: embeddings-usage +} + +/// Usage related to an embeddings generation request +record embeddings-usage { + /// Number of tokens in the prompt + prompt-token-count: u32, +} \ No newline at end of file diff --git a/wit/llm.wit b/wit/llm.wit new file mode 100644 index 0000000..2b1533c --- /dev/null +++ b/wit/llm.wit @@ -0,0 +1,70 @@ +// A WASI interface dedicated to performing inferencing for Large Language Models. +interface llm { + /// A Large Language Model. + type inferencing-model = string + + /// Inference request parameters + record inferencing-params { + /// The maximum tokens that should be inferred. + /// + /// Note: the backing implementation may return less tokens. + max-tokens: u32, + /// The amount the model should avoid repeating tokens. + repeat-penalty: float32, + /// The number of tokens the model should apply the repeat penalty to. + repeat-penalty-last-n-token-count: u32, + /// The randomness with which the next token is selected. + temperature: float32, + /// The number of possible next tokens the model will choose from. + top-k: u32, + /// The probability total of next tokens the model will choose from. + top-p: float32 + } + + /// The set of errors which may be raised by functions in this interface + variant error { + model-not-supported, + runtime-error(string), + invalid-input(string) + } + + /// An inferencing result + record inferencing-result { + /// The text generated by the model + // TODO: this should be a stream + text: string, + /// Usage information about the inferencing request + usage: inferencing-usage + } + + /// Usage information related to the inferencing result + record inferencing-usage { + /// Number of tokens in the prompt + prompt-token-count: u32, + /// Number of tokens generated by the inferencing operation + generated-token-count: u32 + } + + /// Perform inferencing using the provided model and prompt with the given optional params + infer: func(model: inferencing-model, prompt: string, params: option) -> result + + /// The model used for generating embeddings + type embedding-model = string + + /// Generate embeddings for the supplied list of text + generate-embeddings: func(model: embedding-model, text: list) -> result + + /// Result of generating embeddings + record embeddings-result { + /// The embeddings generated by the request + embeddings: list>, + /// Usage related to the embeddings generation request + usage: embeddings-usage + } + + /// Usage related to an embeddings generation request + record embeddings-usage { + /// Number of tokens in the prompt + prompt-token-count: u32, + } +} diff --git a/wit/spin.wit b/wit/spin.wit index 0b2a509..d48de89 100644 --- a/wit/spin.wit +++ b/wit/spin.wit @@ -31,6 +31,7 @@ world reactor { import redis import key-value import http + import llm export inbound-http export inbound-redis