diff --git a/packages/core/package.json b/packages/core/package.json index 846dc907950..eefe38b1674 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -82,7 +82,6 @@ "handlebars": "^4.7.8", "js-sha1": "0.7.0", "js-tiktoken": "1.0.15", - "langchain": "0.3.6", "ollama-ai-provider": "0.16.1", "openai": "4.82.0", "pino": "^9.6.0", diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts index f15f5e26a33..19b82e5d7df 100644 --- a/packages/core/src/generation.ts +++ b/packages/core/src/generation.ts @@ -4,7 +4,6 @@ import { createMistral } from "@ai-sdk/mistral"; import { createGroq } from "@ai-sdk/groq"; import { createOpenAI } from "@ai-sdk/openai"; import { bedrock } from "@ai-sdk/amazon-bedrock"; -import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; import { generateObject as aiGenerateObject, generateText as aiGenerateText, @@ -1380,12 +1379,8 @@ export async function splitChunks( ): Promise { elizaLogger.debug(`[splitChunks] Starting text split`); - const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: Number(chunkSize), - chunkOverlap: Number(bleed), - }); + const chunks = splitText(content, chunkSize, bleed); - const chunks = await textSplitter.splitText(content); elizaLogger.debug(`[splitChunks] Split complete:`, { numberOfChunks: chunks.length, averageChunkSize: @@ -1396,6 +1391,19 @@ export async function splitChunks( return chunks; } +export function splitText(content: string, chunkSize: number, bleed: number): string[] { + const chunks: string[] = []; + let start = 0; + + while (start < content.length) { + const end = Math.min(start + chunkSize, content.length); + chunks.push(content.substring(start, end)); + start = end - bleed; // Apply overlap + } + + return chunks; +} + /** * Sends a message to the model and parses the response as a boolean value * @param opts - The options for the generateText request