From 30499b5266f808c067cca66396d60ea6a5760c23 Mon Sep 17 00:00:00 2001 From: Mikyo King Date: Sun, 2 Mar 2025 13:27:49 -0700 Subject: [PATCH 1/4] fix(langchainjs): deprecate v0.1 support --- .../package.json | 4 +- .../src/instrumentation.ts | 23 +- .../src/instrumentationUtils.ts | 15 +- .../test/langchainV1.test.ts | 772 ------------------ 4 files changed, 4 insertions(+), 810 deletions(-) delete mode 100644 js/packages/openinference-instrumentation-langchain/test/langchainV1.test.ts diff --git a/js/packages/openinference-instrumentation-langchain/package.json b/js/packages/openinference-instrumentation-langchain/package.json index 59931c3f1..0a12cf3da 100644 --- a/js/packages/openinference-instrumentation-langchain/package.json +++ b/js/packages/openinference-instrumentation-langchain/package.json @@ -40,15 +40,13 @@ "@opentelemetry/instrumentation": "^0.46.0" }, "peerDependencies": { - "@langchain/core": "^0.1.0 || ^0.2.0 || ^0.3.0" + "@langchain/core": "^0.2.0 || ^0.3.0" }, "devDependencies": { "@langchain/core": "^0.3.13", "@langchain/coreV0.2": "npm:@langchain/core@^0.2.0", - "@langchain/coreV0.1": "npm:@langchain/core@^0.1.0", "@langchain/openai": "^0.3.11", "@langchain/openaiV0.2": "npm:@langchain/openai@^0.2.0", - "@langchain/openaiV0.1": "npm:@langchain/openai@^0.1.0", "@opentelemetry/exporter-trace-otlp-proto": "^0.50.0", "@opentelemetry/resources": "^1.25.1", "@opentelemetry/sdk-trace-base": "^1.25.1", diff --git a/js/packages/openinference-instrumentation-langchain/src/instrumentation.ts b/js/packages/openinference-instrumentation-langchain/src/instrumentation.ts index e86ac7a31..f4f697e76 100644 --- a/js/packages/openinference-instrumentation-langchain/src/instrumentation.ts +++ b/js/packages/openinference-instrumentation-langchain/src/instrumentation.ts @@ -1,5 +1,4 @@ import type * as CallbackManagerModuleV02 from "@langchain/core/callbacks/manager"; -import type * as CallbackManagerModuleV01 from "@langchain/coreV0.1/callbacks/manager"; import { InstrumentationBase, InstrumentationConfig, @@ -27,9 +26,7 @@ export function isPatched() { return _isOpenInferencePatched; } -type CallbackManagerModule = - | typeof CallbackManagerModuleV01 - | typeof CallbackManagerModuleV02; +type CallbackManagerModule = typeof CallbackManagerModuleV02; /** * An auto instrumentation class for LangChain that creates {@link https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md|OpenInference} Compliant spans for LangChain @@ -121,24 +118,6 @@ export class LangChainInstrumentation extends InstrumentationBase { - return function ( - this: typeof CallbackManagerModuleV01, - ...args: Parameters< - (typeof CallbackManagerModuleV01.CallbackManager)["configure"] - > - ) { - const handlers = args[0]; - const newHandlers = addTracerToHandlers( - instrumentation.oiTracer, - handlers, - ); - args[0] = newHandlers; - return original.apply(this, args); }; }); diff --git a/js/packages/openinference-instrumentation-langchain/src/instrumentationUtils.ts b/js/packages/openinference-instrumentation-langchain/src/instrumentationUtils.ts index 5f20a3b24..fcb972b00 100644 --- a/js/packages/openinference-instrumentation-langchain/src/instrumentationUtils.ts +++ b/js/packages/openinference-instrumentation-langchain/src/instrumentationUtils.ts @@ -1,5 +1,4 @@ import type * as CallbackManagerModuleV02 from "@langchain/core/callbacks/manager"; -import type * as CallbackManagerModuleV01 from "@langchain/coreV0.1/callbacks/manager"; import { LangChainTracer } from "./tracer"; import { OITracer } from "@arizeai/openinference-core"; @@ -10,25 +9,15 @@ import { OITracer } from "@arizeai/openinference-core"; * @returns the callback handlers with the {@link LangChainTracer} added * * If the handlers are an array, we add the tracer to the array if it is not already present - * - * There are some slight differences in the CallbackHandler interface between V0.1 and v0.2 - * So we have to cast our tracer to any to avoid type errors - * We support both versions and our tracer is compatible with either as it will extend the BaseTracer from the installed version which will be the same as the version of handlers passed in here */ -export function addTracerToHandlers( - tracer: OITracer, - handlers?: CallbackManagerModuleV01.Callbacks, -): CallbackManagerModuleV01.Callbacks; export function addTracerToHandlers( tracer: OITracer, handlers?: CallbackManagerModuleV02.Callbacks, ): CallbackManagerModuleV02.Callbacks; export function addTracerToHandlers( tracer: OITracer, - handlers?: - | CallbackManagerModuleV01.Callbacks - | CallbackManagerModuleV02.Callbacks, -): CallbackManagerModuleV01.Callbacks | CallbackManagerModuleV02.Callbacks { + handlers?: CallbackManagerModuleV02.Callbacks, +): CallbackManagerModuleV02.Callbacks { if (handlers == null) { return [new LangChainTracer(tracer)]; } diff --git a/js/packages/openinference-instrumentation-langchain/test/langchainV1.test.ts b/js/packages/openinference-instrumentation-langchain/test/langchainV1.test.ts deleted file mode 100644 index f03b577eb..000000000 --- a/js/packages/openinference-instrumentation-langchain/test/langchainV1.test.ts +++ /dev/null @@ -1,772 +0,0 @@ -import { - InMemorySpanExporter, - SimpleSpanProcessor, -} from "@opentelemetry/sdk-trace-base"; -import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; -import { LangChainInstrumentation } from "../src"; -import * as CallbackManager from "@langchain/coreV0.1/callbacks/manager"; -import { ChatPromptTemplate } from "@langchain/coreV0.1/prompts"; -import { MemoryVectorStore } from "langchainV0.1/vectorstores/memory"; -import { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openaiV0.1"; -import { RecursiveCharacterTextSplitter } from "langchainV0.1/text_splitter"; -import "dotenv/config"; -import { Stream } from "openai/streaming"; -import { - MESSAGE_FUNCTION_CALL_NAME, - OpenInferenceSpanKind, - SemanticConventions, -} from "@arizeai/openinference-semantic-conventions"; -import { LangChainTracer } from "../src/tracer"; -import { context, trace } from "@opentelemetry/api"; -import { completionsResponse, functionCallResponse } from "./fixtures"; -import { DynamicTool } from "@langchain/coreV0.1/tools"; -import { createRetrievalChain } from "langchainV0.1/chains/retrieval"; -import { createStuffDocumentsChain } from "langchainV0.1/chains/combine_documents"; -import { - OITracer, - setAttributes, - setSession, -} from "@arizeai/openinference-core"; - -const memoryExporter = new InMemorySpanExporter(); - -const { - INPUT_VALUE, - LLM_INPUT_MESSAGES, - OUTPUT_VALUE, - LLM_OUTPUT_MESSAGES, - INPUT_MIME_TYPE, - OUTPUT_MIME_TYPE, - MESSAGE_ROLE, - MESSAGE_CONTENT, - DOCUMENT_CONTENT, - DOCUMENT_METADATA, - OPENINFERENCE_SPAN_KIND, - LLM_MODEL_NAME, - LLM_INVOCATION_PARAMETERS, - LLM_TOKEN_COUNT_COMPLETION, - LLM_TOKEN_COUNT_PROMPT, - LLM_TOKEN_COUNT_TOTAL, - TOOL_NAME, - LLM_FUNCTION_CALL, - PROMPT_TEMPLATE_TEMPLATE, - PROMPT_TEMPLATE_VARIABLES, - RETRIEVAL_DOCUMENTS, - METADATA, -} = SemanticConventions; - -jest.mock("@langchain/openaiV0.1", () => { - const originalModule = jest.requireActual("@langchain/openaiV0.1"); - class MockChatOpenAI extends originalModule.ChatOpenAI { - constructor(...args: Parameters) { - super(...args); - this.client = { - chat: { - completions: { - create: jest.fn().mockResolvedValue(completionsResponse), - }, - }, - }; - } - } - return { - ...originalModule, - ChatOpenAI: MockChatOpenAI, - OpenAIEmbeddings: class extends originalModule.OpenAIEmbeddings { - embedDocuments = async () => { - return Promise.resolve([ - [1, 2, 3], - [4, 5, 6], - [7, 8, 9], - ]); - }; - embedQuery = async () => { - return Promise.resolve([1, 2, 4]); - }; - }, - }; -}); - -const expectedSpanAttributes = { - [OPENINFERENCE_SPAN_KIND]: OpenInferenceSpanKind.LLM, - [INPUT_VALUE]: JSON.stringify({ - messages: [ - [ - { - lc: 1, - type: "constructor", - id: ["langchain_core", "messages", "HumanMessage"], - kwargs: { - content: "hello, this is a test", - additional_kwargs: {}, - response_metadata: {}, - }, - }, - ], - ], - }), - [INPUT_MIME_TYPE]: "application/json", - [OUTPUT_VALUE]: JSON.stringify({ - generations: [ - [ - { - text: "This is a test.", - message: { - lc: 1, - type: "constructor", - id: ["langchain_core", "messages", "AIMessage"], - kwargs: { - content: "This is a test.", - tool_calls: [], - invalid_tool_calls: [], - additional_kwargs: {}, - response_metadata: { - tokenUsage: { - completionTokens: 5, - promptTokens: 12, - totalTokens: 17, - }, - finish_reason: "stop", - }, - id: "chatcmpl-8adq9JloOzNZ9TyuzrKyLpGXexh6p", - }, - }, - generationInfo: { finish_reason: "stop" }, - }, - ], - ], - llmOutput: { - tokenUsage: { completionTokens: 5, promptTokens: 12, totalTokens: 17 }, - }, - }), - [LLM_TOKEN_COUNT_COMPLETION]: 5, - [LLM_TOKEN_COUNT_PROMPT]: 12, - [LLM_TOKEN_COUNT_TOTAL]: 17, - [OUTPUT_MIME_TYPE]: "application/json", - [`${LLM_INPUT_MESSAGES}.0.${MESSAGE_ROLE}`]: "user", - [`${LLM_INPUT_MESSAGES}.0.${MESSAGE_CONTENT}`]: "hello, this is a test", - [`${LLM_OUTPUT_MESSAGES}.0.${MESSAGE_ROLE}`]: "assistant", - [`${LLM_OUTPUT_MESSAGES}.0.${MESSAGE_CONTENT}`]: "This is a test.", - [LLM_MODEL_NAME]: "gpt-3.5-turbo", - [LLM_INVOCATION_PARAMETERS]: - '{"model":"gpt-3.5-turbo","temperature":0,"top_p":1,"frequency_penalty":0,"presence_penalty":0,"n":1,"stream":true,"stream_options":{"include_usage":true}}', - metadata: - '{"ls_provider":"openai","ls_model_name":"gpt-3.5-turbo","ls_model_type":"chat","ls_temperature":0}', -}; - -describe("LangChainInstrumentation", () => { - const tracerProvider = new NodeTracerProvider(); - tracerProvider.register(); - const instrumentation = new LangChainInstrumentation(); - instrumentation.disable(); - const provider = new NodeTracerProvider(); - provider.getTracer("default"); - - instrumentation.setTracerProvider(tracerProvider); - tracerProvider.addSpanProcessor(new SimpleSpanProcessor(memoryExporter)); - - const PROMPT_TEMPLATE = `Use the context below to answer the question. - ---------------- - {context} - - Question: - {input} - `; - const prompt = ChatPromptTemplate.fromTemplate(PROMPT_TEMPLATE); - - // @ts-expect-error the moduleExports property is private. This is needed to make the test work with auto-mocking - instrumentation._modules[0].moduleExports = CallbackManager; - beforeAll(() => { - instrumentation.enable(); - }); - afterAll(() => { - instrumentation.disable(); - }); - beforeEach(() => { - memoryExporter.reset(); - }); - afterEach(() => { - jest.resetAllMocks(); - jest.clearAllMocks(); - }); - it("should patch the callback manager module", async () => { - expect( - (CallbackManager as { openInferencePatched?: boolean }) - .openInferencePatched, - ).toBe(true); - }); - - const testDocuments = [ - "dogs are cute", - "rainbows are colorful", - "water is wet", - ]; - - it("should add attributes to llm spans", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - temperature: 0, - }); - - await chatModel.invoke("hello, this is a test"); - - const span = memoryExporter.getFinishedSpans()[0]; - expect(span).toBeDefined(); - }); - - it("should add attributes to llm spans when streaming", async () => { - // Do this to update the mock to return a streaming response - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { ChatOpenAI } = jest.requireMock("@langchain/openaiV0.1"); - - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - streaming: true, - }); - - chatModel.client.chat.completions.create.mockResolvedValue( - new Stream(async function* iterator() { - yield { choices: [{ delta: { content: "This is " } }] }; - yield { choices: [{ delta: { content: "a test stream." } }] }; - yield { choices: [{ delta: { finish_reason: "stop" } }] }; - }, new AbortController()), - ); - - await chatModel.invoke("hello, this is a test"); - - const span = memoryExporter.getFinishedSpans()[0]; - expect(span).toBeDefined(); - - const expectedStreamingAttributes = { - ...expectedSpanAttributes, - [`${LLM_OUTPUT_MESSAGES}.0.${MESSAGE_CONTENT}`]: "This is a test stream.", - [LLM_INVOCATION_PARAMETERS]: - '{"model":"gpt-3.5-turbo","temperature":1,"top_p":1,"frequency_penalty":0,"presence_penalty":0,"n":1,"stream":true}', - [LLM_TOKEN_COUNT_PROMPT]: 13, - [LLM_TOKEN_COUNT_COMPLETION]: 6, - [LLM_TOKEN_COUNT_TOTAL]: 19, - [OUTPUT_VALUE]: - '{"generations":[[{"text":"This is a test stream.","generationInfo":{"prompt":0,"completion":0},"message":{"lc":1,"type":"constructor","id":["langchain_core","messages","ChatMessageChunk"],"kwargs":{"content":"This is a test stream.","additional_kwargs":{},"response_metadata":{"estimatedTokenUsage":{"promptTokens":13,"completionTokens":6,"totalTokens":19},"prompt":0,"completion":0}}}}]],"llmOutput":{"estimatedTokenUsage":{"promptTokens":13,"completionTokens":6,"totalTokens":19}}}', - [METADATA]: "{}", - }; - delete expectedStreamingAttributes[ - `${LLM_OUTPUT_MESSAGES}.0.${MESSAGE_ROLE}` - ]; - - expect(span.attributes).toStrictEqual(expectedStreamingAttributes); - }); - - it("should properly nest spans", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - }); - const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - }); - const docs = await textSplitter.createDocuments(testDocuments); - const vectorStore = await MemoryVectorStore.fromDocuments( - docs, - new OpenAIEmbeddings({ - openAIApiKey: "my-api-key", - }), - ); - const combineDocsChain = await createStuffDocumentsChain({ - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore there is a slight mismatch in types due to version differences, this still works - llm: chatModel, - prompt, - }); - const chain = await createRetrievalChain({ - combineDocsChain, - retriever: vectorStore.asRetriever(), - }); - await chain.invoke({ - input: "What are cats?", - }); - - const spans = memoryExporter.getFinishedSpans(); - - const rootSpan = spans.find((span) => span.parentSpanId == null); - const llmSpan = spans.find( - (span) => - span.attributes[SemanticConventions.OPENINFERENCE_SPAN_KIND] === - OpenInferenceSpanKind.LLM, - ); - const retrieverSpan = spans.find( - (span) => - span.attributes[SemanticConventions.OPENINFERENCE_SPAN_KIND] === - OpenInferenceSpanKind.RETRIEVER, - ); - - const retrievalChainSpan = spans.find( - (span) => span.name === "retrieval_chain", - ); - - const retrieveDocumentsSpan = spans.find( - (span) => span.name === "retrieve_documents", - ); - - // Langchain creates a ton of generic spans that are deeply nested. This is a simple test to ensure we have the spans we care about and they are at least nested under something. It is not possible to test the exact nesting structure because it is too complex and generic. - expect(rootSpan).toBe(retrievalChainSpan); - expect(retrieverSpan).toBeDefined(); - expect(llmSpan).toBeDefined(); - - expect(retrieverSpan?.parentSpanId).toBe( - retrieveDocumentsSpan?.spanContext().spanId, - ); - expect(llmSpan?.parentSpanId).toBeDefined(); - }); - - it("should add documents to retriever spans", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - }); - - const textSplitter = new RecursiveCharacterTextSplitter({ - chunkSize: 1000, - }); - const docs = await textSplitter.createDocuments(testDocuments); - const vectorStore = await MemoryVectorStore.fromDocuments( - docs, - new OpenAIEmbeddings({ - openAIApiKey: "my-api-key", - }), - ); - const combineDocsChain = await createStuffDocumentsChain({ - llm: chatModel, - prompt, - }); - const chain = await createRetrievalChain({ - combineDocsChain, - retriever: vectorStore.asRetriever(), - }); - await chain.invoke({ - input: "What are cats?", - }); - - const spans = memoryExporter.getFinishedSpans(); - const retrieverSpan = spans.find( - (span) => - span.attributes[SemanticConventions.OPENINFERENCE_SPAN_KIND] === - OpenInferenceSpanKind.RETRIEVER, - ); - - expect(retrieverSpan).toBeDefined(); - expect(retrieverSpan?.attributes).toStrictEqual({ - [OPENINFERENCE_SPAN_KIND]: OpenInferenceSpanKind.RETRIEVER, - [OUTPUT_MIME_TYPE]: "application/json", - [OUTPUT_VALUE]: - '{"documents":[{"pageContent":"dogs are cute","metadata":{"loc":{"lines":{"from":1,"to":1}}}},{"pageContent":"rainbows are colorful","metadata":{"loc":{"lines":{"from":1,"to":1}}}},{"pageContent":"water is wet","metadata":{"loc":{"lines":{"from":1,"to":1}}}}]}', - [INPUT_MIME_TYPE]: "text/plain", - [INPUT_VALUE]: "What are cats?", - [`${RETRIEVAL_DOCUMENTS}.0.${DOCUMENT_CONTENT}`]: "dogs are cute", - [`${RETRIEVAL_DOCUMENTS}.0.${DOCUMENT_METADATA}`]: JSON.stringify({ - loc: { - lines: { - from: 1, - to: 1, - }, - }, - }), - [`${RETRIEVAL_DOCUMENTS}.1.${DOCUMENT_CONTENT}`]: "rainbows are colorful", - [`${RETRIEVAL_DOCUMENTS}.1.${DOCUMENT_METADATA}`]: JSON.stringify({ - loc: { - lines: { - from: 1, - to: 1, - }, - }, - }), - [`${RETRIEVAL_DOCUMENTS}.2.${DOCUMENT_CONTENT}`]: "water is wet", - [`${RETRIEVAL_DOCUMENTS}.2.${DOCUMENT_METADATA}`]: JSON.stringify({ - loc: { - lines: { - from: 1, - to: 1, - }, - }, - }), - metadata: "{}", - }); - }); - - it("should add function calls to spans", async () => { - // Do this to update the mock to return a function call response - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { ChatOpenAI } = jest.requireMock("@langchain/openaiV0.1"); - - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - temperature: 1, - }); - - chatModel.client.chat.completions.create.mockResolvedValue( - functionCallResponse, - ); - - const weatherFunction = { - name: "get_current_weather", - description: "Get the current weather in a given location", - parameters: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state, e.g. San Francisco, CA", - }, - unit: { type: "string", enum: ["celsius", "fahrenheit"] }, - }, - required: ["location"], - }, - }; - - await chatModel.invoke( - "whats the weather like in seattle, wa in fahrenheit?", - { - functions: [weatherFunction], - }, - ); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans).toBeDefined(); - - const llmSpan = spans.find( - (span) => - span.attributes[OPENINFERENCE_SPAN_KIND] === OpenInferenceSpanKind.LLM, - ); - expect(llmSpan).toBeDefined(); - expect(llmSpan?.attributes).toStrictEqual({ - [OPENINFERENCE_SPAN_KIND]: OpenInferenceSpanKind.LLM, - [LLM_MODEL_NAME]: "gpt-3.5-turbo", - [LLM_FUNCTION_CALL]: - '{"name":"get_current_weather","arguments":"{\\"location\\":\\"Seattle, WA\\",\\"unit\\":\\"fahrenheit\\"}"}', - [`${LLM_INPUT_MESSAGES}.0.${MESSAGE_ROLE}`]: "user", - [`${LLM_INPUT_MESSAGES}.0.${MESSAGE_CONTENT}`]: - "whats the weather like in seattle, wa in fahrenheit?", - [`${LLM_OUTPUT_MESSAGES}.0.${MESSAGE_FUNCTION_CALL_NAME}`]: - "get_current_weather", - [`${LLM_OUTPUT_MESSAGES}.0.${MESSAGE_CONTENT}`]: "", - [`${LLM_OUTPUT_MESSAGES}.0.${MESSAGE_ROLE}`]: "assistant", - [LLM_TOKEN_COUNT_COMPLETION]: 22, - [LLM_TOKEN_COUNT_PROMPT]: 88, - [LLM_TOKEN_COUNT_TOTAL]: 110, - [LLM_INVOCATION_PARAMETERS]: - '{"model":"gpt-3.5-turbo","temperature":1,"top_p":1,"frequency_penalty":0,"presence_penalty":0,"n":1,"stream":false,"functions":[{"name":"get_current_weather","description":"Get the current weather in a given location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}]}', - [INPUT_VALUE]: - '{"messages":[[{"lc":1,"type":"constructor","id":["langchain_core","messages","HumanMessage"],"kwargs":{"content":"whats the weather like in seattle, wa in fahrenheit?","additional_kwargs":{},"response_metadata":{}}}]]}', - [INPUT_MIME_TYPE]: "application/json", - [OUTPUT_VALUE]: - '{"generations":[[{"text":"","message":{"lc":1,"type":"constructor","id":["langchain_core","messages","AIMessage"],"kwargs":{"content":"","tool_calls":[],"invalid_tool_calls":[],"additional_kwargs":{"function_call":{"name":"get_current_weather","arguments":"{\\"location\\":\\"Seattle, WA\\",\\"unit\\":\\"fahrenheit\\"}"}},"response_metadata":{"tokenUsage":{"completionTokens":22,"promptTokens":88,"totalTokens":110},"finish_reason":"function_call"}}},"generationInfo":{"finish_reason":"function_call"}}]],"llmOutput":{"tokenUsage":{"completionTokens":22,"promptTokens":88,"totalTokens":110}}}', - [OUTPUT_MIME_TYPE]: "application/json", - metadata: "{}", - }); - }); - - it("should add a prompt template to a span if found ", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - }); - const chain = prompt.pipe(chatModel); - await chain.invoke({ - context: "This is a test.", - input: "What is this?", - }); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans).toBeDefined(); - - const promptSpan = spans.find((span) => span.name === "ChatPromptTemplate"); - - expect(promptSpan).toBeDefined(); - expect(promptSpan?.attributes).toStrictEqual({ - [OPENINFERENCE_SPAN_KIND]: OpenInferenceSpanKind.CHAIN, - [PROMPT_TEMPLATE_TEMPLATE]: PROMPT_TEMPLATE, - [PROMPT_TEMPLATE_VARIABLES]: JSON.stringify({ - context: "This is a test.", - input: "What is this?", - }), - [INPUT_VALUE]: '{"context":"This is a test.","input":"What is this?"}', - [INPUT_MIME_TYPE]: "application/json", - [OUTPUT_VALUE]: - '{"lc":1,"type":"constructor","id":["langchain_core","prompt_values","ChatPromptValue"],"kwargs":{"messages":[{"lc":1,"type":"constructor","id":["langchain_core","messages","HumanMessage"],"kwargs":{"content":"Use the context below to answer the question.\\n ----------------\\n This is a test.\\n \\n Question:\\n What is this?\\n ","additional_kwargs":{},"response_metadata":{}}}]}}', - [OUTPUT_MIME_TYPE]: "application/json", - metadata: "{}", - }); - }); - - it("should add tool information to tool spans", async () => { - const simpleTool = new DynamicTool({ - name: "test_tool", - description: - "call this to get the value of a test, input should be an empty string", - func: async () => Promise.resolve("this is a test tool"), - }); - - await simpleTool.call(""); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans).toBeDefined(); - - const toolSpan = spans.find( - (span) => - span.attributes[OPENINFERENCE_SPAN_KIND] === OpenInferenceSpanKind.TOOL, - ); - expect(toolSpan).toBeDefined(); - expect(toolSpan?.attributes).toStrictEqual({ - [OPENINFERENCE_SPAN_KIND]: OpenInferenceSpanKind.TOOL, - [TOOL_NAME]: "test_tool", - [INPUT_VALUE]: "", - [INPUT_MIME_TYPE]: "text/plain", - [OUTPUT_VALUE]: "this is a test tool", - [OUTPUT_MIME_TYPE]: "text/plain", - metadata: "{}", - }); - }); - - it("should capture context attributes and add them to spans", async () => { - await context.with( - setSession( - setAttributes(context.active(), { - "test-attribute": "test-value", - }), - { sessionId: "session-id" }, - ), - async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - temperature: 0, - }); - await chatModel.invoke("hello, this is a test"); - }, - ); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans.length).toBe(1); - const span = spans[0]; - expect(span.attributes).toMatchInlineSnapshot(` -{ - "input.mime_type": "application/json", - "input.value": "{"messages":[[{"lc":1,"type":"constructor","id":["langchain_core","messages","HumanMessage"],"kwargs":{"content":"hello, this is a test","additional_kwargs":{},"response_metadata":{}}}]]}", - "llm.input_messages.0.message.content": "hello, this is a test", - "llm.input_messages.0.message.role": "user", - "llm.invocation_parameters": "{"model":"gpt-3.5-turbo","temperature":0,"top_p":1,"frequency_penalty":0,"presence_penalty":0,"n":1,"stream":false}", - "llm.model_name": "gpt-3.5-turbo", - "llm.output_messages.0.message.content": "This is a test.", - "llm.output_messages.0.message.role": "assistant", - "llm.token_count.completion": 5, - "llm.token_count.prompt": 12, - "llm.token_count.total": 17, - "metadata": "{}", - "openinference.span.kind": "LLM", - "output.mime_type": "application/json", - "output.value": "{"generations":[[{"text":"This is a test.","message":{"lc":1,"type":"constructor","id":["langchain_core","messages","AIMessage"],"kwargs":{"content":"This is a test.","tool_calls":[],"invalid_tool_calls":[],"additional_kwargs":{},"response_metadata":{"tokenUsage":{"completionTokens":5,"promptTokens":12,"totalTokens":17},"finish_reason":"stop"}}},"generationInfo":{"finish_reason":"stop"}}]],"llmOutput":{"tokenUsage":{"completionTokens":5,"promptTokens":12,"totalTokens":17}}}", - "session.id": "session-id", - "test-attribute": "test-value", -} -`); - }); - - it("should extract session ID from run metadata with session_id", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - }); - - await chatModel.invoke("test message", { - metadata: { - session_id: "test-session-123", - }, - }); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans[0].attributes[SemanticConventions.SESSION_ID]).toBe( - "test-session-123", - ); - }); - - it("should extract session ID from run metadata with thread_id", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - }); - - await chatModel.invoke("test message", { - metadata: { - thread_id: "thread-456", - }, - }); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans[0].attributes[SemanticConventions.SESSION_ID]).toBe( - "thread-456", - ); - }); - - it("should extract session ID from run metadata with conversation_id", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - }); - - await chatModel.invoke("test message", { - metadata: { - conversation_id: "conv-789", - }, - }); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans[0].attributes[SemanticConventions.SESSION_ID]).toBe( - "conv-789", - ); - }); - - it("should prioritize session_id over thread_id and conversation_id", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - }); - - await chatModel.invoke("test message", { - metadata: { - session_id: "session-123", - thread_id: "thread-456", - conversation_id: "conv-789", - }, - }); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans[0].attributes[SemanticConventions.SESSION_ID]).toBe( - "session-123", - ); - }); - - it("should handle missing session identifiers in metadata", async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - }); - - await chatModel.invoke("test message", { - metadata: { - other_field: "some-value", - }, - }); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans[0].attributes[SemanticConventions.SESSION_ID]).toBeUndefined(); - }); -}); - -describe("LangChainInstrumentation with TraceConfigOptions", () => { - const tracerProvider = new NodeTracerProvider(); - tracerProvider.register(); - const instrumentation = new LangChainInstrumentation({ - traceConfig: { - hideInputs: true, - }, - }); - instrumentation.disable(); - const provider = new NodeTracerProvider(); - provider.getTracer("default"); - - instrumentation.setTracerProvider(tracerProvider); - tracerProvider.addSpanProcessor(new SimpleSpanProcessor(memoryExporter)); - - // @ts-expect-error the moduleExports property is private. This is needed to make the test work with auto-mocking - instrumentation._modules[0].moduleExports = CallbackManager; - beforeAll(() => { - instrumentation.enable(); - }); - afterAll(() => { - instrumentation.disable(); - }); - beforeEach(() => { - memoryExporter.reset(); - }); - afterEach(() => { - jest.resetAllMocks(); - jest.clearAllMocks(); - }); - it("should patch the callback manager module", async () => { - expect( - (CallbackManager as { openInferencePatched?: boolean }) - .openInferencePatched, - ).toBe(true); - }); - - it("should respect trace config options", async () => { - await context.with( - setSession( - setAttributes(context.active(), { - "test-attribute": "test-value", - }), - { sessionId: "session-id" }, - ), - async () => { - const chatModel = new ChatOpenAI({ - openAIApiKey: "my-api-key", - modelName: "gpt-3.5-turbo", - temperature: 0, - }); - await chatModel.invoke("hello, this is a test"); - }, - ); - - const spans = memoryExporter.getFinishedSpans(); - expect(spans.length).toBe(1); - const span = spans[0]; - expect(span.attributes).toMatchInlineSnapshot(` -{ - "input.value": "__REDACTED__", - "llm.invocation_parameters": "{"model":"gpt-3.5-turbo","temperature":0,"top_p":1,"frequency_penalty":0,"presence_penalty":0,"n":1,"stream":false}", - "llm.model_name": "gpt-3.5-turbo", - "llm.output_messages.0.message.content": "This is a test.", - "llm.output_messages.0.message.role": "assistant", - "llm.token_count.completion": 5, - "llm.token_count.prompt": 12, - "llm.token_count.total": 17, - "metadata": "{}", - "openinference.span.kind": "LLM", - "output.mime_type": "application/json", - "output.value": "{"generations":[[{"text":"This is a test.","message":{"lc":1,"type":"constructor","id":["langchain_core","messages","AIMessage"],"kwargs":{"content":"This is a test.","tool_calls":[],"invalid_tool_calls":[],"additional_kwargs":{},"response_metadata":{"tokenUsage":{"completionTokens":5,"promptTokens":12,"totalTokens":17},"finish_reason":"stop"}}},"generationInfo":{"finish_reason":"stop"}}]],"llmOutput":{"tokenUsage":{"completionTokens":5,"promptTokens":12,"totalTokens":17}}}", - "session.id": "session-id", - "test-attribute": "test-value", -} -`); - }); -}); - -describe("LangChainTracer", () => { - const testSerialized = { - lc: 1, - type: "not_implemented" as const, - id: [], - }; - it("should delete runs after they are ended", async () => { - const oiTracer = new OITracer({ tracer: trace.getTracer("default") }); - const langChainTracer = new LangChainTracer(oiTracer); - for (let i = 0; i < 10; i++) { - await langChainTracer.handleLLMStart(testSerialized, [], "runId"); - expect(Object.keys(langChainTracer["runs"]).length).toBe(1); - - await langChainTracer.handleRetrieverStart(testSerialized, "", "runId2"); - expect(Object.keys(langChainTracer["runs"]).length).toBe(2); - - await langChainTracer.handleLLMEnd({ generations: [] }, "runId"); - expect(Object.keys(langChainTracer["runs"]).length).toBe(1); - - await langChainTracer.handleRetrieverEnd([], "runId2"); - expect(Object.keys(langChainTracer["runs"]).length).toBe(0); - } - - expect(langChainTracer["runs"]).toBeDefined(); - expect(Object.keys(langChainTracer["runs"]).length).toBe(0); - }); -}); From 903c240b07848f74d711680ff1c041d522cbd230 Mon Sep 17 00:00:00 2001 From: Mikyo King Date: Sun, 2 Mar 2025 13:29:48 -0700 Subject: [PATCH 2/4] cleanp --- js/packages/openinference-instrumentation-langchain/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/js/packages/openinference-instrumentation-langchain/README.md b/js/packages/openinference-instrumentation-langchain/README.md index ce89fa3f0..62fe0fd97 100644 --- a/js/packages/openinference-instrumentation-langchain/README.md +++ b/js/packages/openinference-instrumentation-langchain/README.md @@ -26,3 +26,7 @@ lcInstrumentation.manuallyInstrument(CallbackManagerModule); ``` For more information on OpenTelemetry Node.js SDK, see the [OpenTelemetry Node.js SDK documentation](https://opentelemetry.io/docs/instrumentation/js/getting-started/nodejs/). + +## Deprecations + +LangChain v0.1 was deprecated on 2025-03-02 due to security vulerabilities in the core package. From 2a574a92344aa704a0c98c4cd5f7dddbd6d885d8 Mon Sep 17 00:00:00 2001 From: Mikyo King Date: Sun, 2 Mar 2025 13:41:47 -0700 Subject: [PATCH 3/4] cleanup --- js/pnpm-lock.yaml | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index ffc19e1f7..a844cf8a9 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -42,7 +42,7 @@ importers: version: 5.0.10 ts-jest: specifier: ^29.2.2 - version: 29.2.4(@babel/core@7.24.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.9))(jest@29.7.0(@types/node@20.14.11))(typescript@5.5.4) + version: 29.2.4(@babel/core@7.24.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.9))(jest@29.7.0)(typescript@5.5.4) tsc-alias: specifier: ^1.8.10 version: 1.8.10 @@ -108,18 +108,12 @@ importers: '@langchain/core': specifier: ^0.3.13 version: 0.3.13(openai@4.56.0(zod@3.23.8)) - '@langchain/coreV0.1': - specifier: npm:@langchain/core@^0.1.0 - version: '@langchain/core@0.1.63(langchain@0.3.3(@langchain/core@0.3.13(openai@4.56.0(zod@3.23.8)))(openai@4.56.0(zod@3.23.8)))(openai@4.56.0(zod@3.23.8))' '@langchain/coreV0.2': specifier: npm:@langchain/core@^0.2.0 version: '@langchain/core@0.2.36(openai@4.56.0(zod@3.23.8))' '@langchain/openai': specifier: ^0.3.11 version: 0.3.11(@langchain/core@0.3.13(openai@4.56.0(zod@3.23.8))) - '@langchain/openaiV0.1': - specifier: npm:@langchain/openai@^0.1.0 - version: '@langchain/openai@0.1.3(langchain@0.3.3(@langchain/core@0.3.13(openai@4.56.0(zod@3.23.8)))(openai@4.56.0(zod@3.23.8)))' '@langchain/openaiV0.2': specifier: npm:@langchain/openai@^0.2.0 version: '@langchain/openai@0.2.8' @@ -897,10 +891,6 @@ packages: resolution: {integrity: sha512-M+CW4oXle5fdoz2T2SwdOef8pl3/1XmUx1vjn2mXUVM/128aO0l23FMF0SNBsAbRV6P+p/TuzjodchJbi0Ht/A==} engines: {node: '>=18'} - '@langchain/openai@0.1.3': - resolution: {integrity: sha512-riv/JC9x2A8b7GcHu8sx+mlZJ8KAwSSi231IPTlcciYnKozmrQ5H0vrtiD31fxiDbaRsk7tyCpkSBIOQEo7CyQ==} - engines: {node: '>=18'} - '@langchain/openai@0.2.8': resolution: {integrity: sha512-p5fxEAKuR8UV9jWIxkZ6AY/vAPSYxJI0Pf/UM4T3FKk/dn99G/mAEDLhfI4pBf7B8o8TudSVyBW2hRjZqlQu7g==} engines: {node: '>=18'} @@ -3927,17 +3917,6 @@ snapshots: - encoding - langchain - '@langchain/openai@0.1.3(langchain@0.3.3(@langchain/core@0.3.13(openai@4.56.0(zod@3.23.8)))(openai@4.56.0(zod@3.23.8)))': - dependencies: - '@langchain/core': 0.1.63(langchain@0.3.3(@langchain/core@0.3.13(openai@4.56.0(zod@3.23.8)))(openai@4.56.0(zod@3.23.8)))(openai@4.56.0(zod@3.23.8)) - js-tiktoken: 1.0.14 - openai: 4.56.0(zod@3.23.8) - zod: 3.23.8 - zod-to-json-schema: 3.23.2(zod@3.23.8) - transitivePeerDependencies: - - encoding - - langchain - '@langchain/openai@0.2.8': dependencies: '@langchain/core': 0.2.36(openai@4.56.0(zod@3.23.8)) @@ -6081,7 +6060,7 @@ snapshots: dependencies: typescript: 5.5.4 - ts-jest@29.2.4(@babel/core@7.24.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.9))(jest@29.7.0(@types/node@20.14.11))(typescript@5.5.4): + ts-jest@29.2.4(@babel/core@7.24.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.9))(jest@29.7.0)(typescript@5.5.4): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 From 30b2a945829af65fc59cdfed01489612a0b1f0ba Mon Sep 17 00:00:00 2001 From: Mikyo King Date: Mon, 3 Mar 2025 12:07:25 -0700 Subject: [PATCH 4/4] add a changeset --- js/.changeset/cute-clouds-tell.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 js/.changeset/cute-clouds-tell.md diff --git a/js/.changeset/cute-clouds-tell.md b/js/.changeset/cute-clouds-tell.md new file mode 100644 index 000000000..b5abc897c --- /dev/null +++ b/js/.changeset/cute-clouds-tell.md @@ -0,0 +1,5 @@ +--- +"@arizeai/openinference-instrumentation-langchain": major +--- + +deprecate support for v0.1