diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs index bf016dfc2e4..d624ec63abb 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs @@ -15,7 +15,6 @@ using Microsoft.Shared.Diagnostics; #pragma warning disable CA2213 // Disposable fields should be disposed -#pragma warning disable CA1002 // Do not expose generic lists #pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test #pragma warning disable SA1202 // 'protected' members should come before 'private' members #pragma warning disable S107 // Methods should not have too many parameters @@ -214,7 +213,7 @@ public override async Task GetResponseAsync( // A single request into this GetResponseAsync may result in multiple requests to the inner client. // Create an activity to group them together for better observability. - using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient)); + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}"); // Copy the original messages in order to avoid enumerating the original messages multiple times. // The IEnumerable can represent an arbitrary amount of work. @@ -309,7 +308,7 @@ public override async IAsyncEnumerable GetStreamingResponseA // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client. // Create an activity to group them together for better observability. - using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient)); + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}"); UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes // Copy the original messages in order to avoid enumerating the original messages multiple times. @@ -795,7 +794,16 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul { _ = Throw.IfNull(context); - using Activity? activity = _activitySource?.StartActivity(context.Function.Name); + using Activity? activity = _activitySource?.StartActivity( + $"execute_tool {context.Function.Name}", + ActivityKind.Internal, + default(ActivityContext), + [ + new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"), + new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId), + new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name), + new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description), + ]); long startingTimestamp = 0; if (_logger.IsEnabled(LogLevel.Debug)) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs index c74bd3aa3c1..c22dc292c8a 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs @@ -25,7 +25,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating chat client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.32, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.33, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// public sealed partial class OpenTelemetryChatClient : DelegatingChatClient diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs index 14332d1253f..99a3ed684af 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs @@ -19,7 +19,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating embedding generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.32, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.33, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// /// The type of input used to produce embeddings. @@ -86,6 +86,20 @@ public OpenTelemetryEmbeddingGenerator(IEmbeddingGenerator i ); } + /// + /// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry. + /// + /// + /// if potentially sensitive information should be included in telemetry; + /// if telemetry shouldn't include raw inputs and outputs. + /// The default value is . + /// + /// + /// By default, telemetry includes metadata, such as token counts, but not raw inputs + /// and outputs or additional options data. + /// + public bool EnableSensitiveData { get; set; } + /// public override object? GetService(Type serviceType, object? serviceKey = null) => serviceType == typeof(ActivitySource) ? _activitySource : @@ -163,20 +177,19 @@ protected override void Dispose(bool disposing) _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.EmbeddingDimensions, dimensionsValue); } - if (options is not null && - _system is not null) + // Log all additional request options as per-provider tags. This is non-normative, but it covers cases where + // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.request.service_tier), + // and more generally cases where there's additional useful information to be logged. + // Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data. + if (EnableSensitiveData && + _system is not null && + options?.AdditionalProperties is { } props) { - // Log all additional request options as per-provider tags. This is non-normative, but it covers cases where - // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.request.service_tier), - // and more generally cases where there's additional useful information to be logged. - if (options.AdditionalProperties is { } props) + foreach (KeyValuePair prop in props) { - foreach (KeyValuePair prop in props) - { - _ = activity.AddTag( - OpenTelemetryConsts.GenAI.Request.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)), - prop.Value); - } + _ = activity.AddTag( + OpenTelemetryConsts.GenAI.Request.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)), + prop.Value); } } } @@ -247,7 +260,8 @@ private void TraceResponse( // Log all additional response properties as per-provider tags. This is non-normative, but it covers cases where // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.response.system_fingerprint), // and more generally cases where there's additional useful information to be logged. - if (_system is not null && + if (EnableSensitiveData && + _system is not null && embeddings?.AdditionalProperties is { } props) { foreach (KeyValuePair prop in props) diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index b585ebf3086..c1a22066227 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -103,7 +103,14 @@ public static class Token public static class Tool { + public const string Name = "gen_ai.tool.name"; + public const string Description = "gen_ai.tool.description"; public const string Message = "gen_ai.tool.message"; + + public static class Call + { + public const string Id = "gen_ai.tool.call.id"; + } } public static class User diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs index 4302348cc56..26554946dca 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs @@ -532,11 +532,11 @@ public async Task FunctionInvocationTrackedWithActivity(bool enableTelemetry) Func configure = b => b.Use(c => new FunctionInvokingChatClient(new OpenTelemetryChatClient(c, sourceName: sourceName))); - await InvokeAsync(() => InvokeAndAssertAsync(options, plan, configurePipeline: configure)); + await InvokeAsync(() => InvokeAndAssertAsync(options, plan, configurePipeline: configure), streaming: false); - await InvokeAsync(() => InvokeAndAssertStreamingAsync(options, plan, configurePipeline: configure)); + await InvokeAsync(() => InvokeAndAssertStreamingAsync(options, plan, configurePipeline: configure), streaming: true); - async Task InvokeAsync(Func work) + async Task InvokeAsync(Func work, bool streaming) { var activities = new List(); using TracerProvider? tracerProvider = enableTelemetry ? @@ -552,9 +552,9 @@ async Task InvokeAsync(Func work) { Assert.Collection(activities, activity => Assert.Equal("chat", activity.DisplayName), - activity => Assert.Equal("Func1", activity.DisplayName), + activity => Assert.Equal("execute_tool Func1", activity.DisplayName), activity => Assert.Equal("chat", activity.DisplayName), - activity => Assert.Equal(nameof(FunctionInvokingChatClient), activity.DisplayName)); + activity => Assert.Equal(streaming ? "FunctionInvokingChatClient.GetStreamingResponseAsync" : "FunctionInvokingChatClient.GetResponseAsync", activity.DisplayName)); for (int i = 0; i < activities.Count - 1; i++) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs index c6a4adb1e97..25e01afb1df 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs @@ -16,9 +16,10 @@ namespace Microsoft.Extensions.AI; public class OpenTelemetryEmbeddingGeneratorTests { [Theory] - [InlineData(null)] - [InlineData("replacementmodel")] - public async Task ExpectedInformationLogged_Async(string? perRequestModelId) + [InlineData(null, false)] + [InlineData("replacementmodel", false)] + [InlineData("replacementmodel", true)] + public async Task ExpectedInformationLogged_Async(string? perRequestModelId, bool enableSensitiveData) { var sourceName = Guid.NewGuid().ToString(); var activities = new List(); @@ -45,7 +46,7 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId) AdditionalProperties = new() { ["system_fingerprint"] = "abcdefgh", - ["AndSomethingElse"] = "value2", + ["AndSomethingElse"] = "value3", } }; }, @@ -56,7 +57,7 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId) using var generator = innerGenerator .AsBuilder() - .UseOpenTelemetry(loggerFactory, sourceName) + .UseOpenTelemetry(loggerFactory, sourceName, configure: g => g.EnableSensitiveData = enableSensitiveData) .Build(); var options = new EmbeddingGenerationOptions @@ -85,12 +86,12 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId) Assert.Equal(expectedModelName, activity.GetTagItem("gen_ai.request.model")); Assert.Equal(1234, activity.GetTagItem("gen_ai.request.embedding.dimensions")); - Assert.Equal("value1", activity.GetTagItem("gen_ai.testservice.request.service_tier")); - Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.request.something_else")); + Assert.Equal(enableSensitiveData ? "value1" : null, activity.GetTagItem("gen_ai.testservice.request.service_tier")); + Assert.Equal(enableSensitiveData ? "value2" : null, activity.GetTagItem("gen_ai.testservice.request.something_else")); Assert.Equal(10, activity.GetTagItem("gen_ai.response.input_tokens")); - Assert.Equal("abcdefgh", activity.GetTagItem("gen_ai.testservice.response.system_fingerprint")); - Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.response.and_something_else")); + Assert.Equal(enableSensitiveData ? "abcdefgh" : null, activity.GetTagItem("gen_ai.testservice.response.system_fingerprint")); + Assert.Equal(enableSensitiveData ? "value3" : null, activity.GetTagItem("gen_ai.testservice.response.and_something_else")); Assert.True(activity.Duration.TotalMilliseconds > 0); }