From d45810d7583c6901d3c46449600a2d86e8d6db1b Mon Sep 17 00:00:00 2001 From: Paras Date: Wed, 9 Apr 2025 14:51:23 +0200 Subject: [PATCH 1/2] feat: change to emit STE event for inline chat action --- .../agenticChat/agenticChatController.ts | 4 +- .../agenticChat/qAgenticChatServer.ts | 8 +- .../language-server/chat/chatController.ts | 4 +- .../src/language-server/chat/qChatServer.ts | 4 + .../shared/telemetry/telemetryService.test.ts | 130 ++++++++++++++++++ .../src/shared/telemetry/telemetryService.ts | 34 +++++ 6 files changed, 180 insertions(+), 4 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts index a1653f3bf..4ea94e915 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/agenticChatController.ts @@ -1190,7 +1190,9 @@ export class AgenticChatController implements ChatHandlers { } } - async onInlineChatResult(handler: InlineChatResultParams) {} + async onInlineChatResult(params: InlineChatResultParams) { + await this.#telemetryService.emitInlineChatResultLog(params) + } async onCodeInsertToCursorPosition(params: InsertToCursorPositionParams) { // Implementation based on https://github.com/aws/aws-toolkit-vscode/blob/1814cc84228d4bf20270574c5980b91b227f31cf/packages/core/src/amazonq/commons/controllers/contentController.ts#L38 diff --git a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/qAgenticChatServer.ts b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/qAgenticChatServer.ts index dcc881fd7..0e92ac8cf 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/agenticChat/qAgenticChatServer.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/agenticChat/qAgenticChatServer.ts @@ -75,9 +75,9 @@ export const QAgenticChatServer = amazonQServiceManager ) - /* + /* Calling handleDidChangeConfiguration once to ensure we get configuration atleast once at start up - + TODO: TODO: consider refactoring such responsibilities to common service manager config/initialisation server */ await amazonQServiceManager.handleDidChangeConfiguration() @@ -161,6 +161,10 @@ export const QAgenticChatServer = return chatController.onButtonClick(params) }) + chat.onInlineChatResult(params => { + return chatController.onInlineChatResult(params) + }) + logging.log('Q Chat server has been initialized') return () => { diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts index c701f2f04..75b926cf7 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts @@ -303,7 +303,9 @@ export class ChatController implements ChatHandlers { } } - async onInlineChatResult(handler: InlineChatResultParams) {} + async onInlineChatResult(params: InlineChatResultParams) { + await this.#telemetryService.emitInlineChatResultLog(params) + } async onCodeInsertToCursorPosition(params: InsertToCursorPositionParams) { // Implementation based on https://github.com/aws/aws-toolkit-vscode/blob/1814cc84228d4bf20270574c5980b91b227f31cf/packages/core/src/amazonq/commons/controllers/contentController.ts#L38 diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/qChatServer.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/qChatServer.ts index 9e412d37c..aecb615fe 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/qChatServer.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/qChatServer.ts @@ -150,6 +150,10 @@ export const QChatServerFactory = return chatController.onCodeInsertToCursorPosition(params) }) + chat.onLogInlineChatResult(params => { + return chatController.onLogInlineChatResult(params) + }) + logging.log('Q Chat server has been initialized') return () => { diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts index 70ac2db85..83d1d6dd2 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.test.ts @@ -982,4 +982,134 @@ describe('TelemetryService', () => { sinon.assert.notCalled(codeWhisperServiceStub.sendTelemetryEvent) }) }) + + describe('Inline chat result notification', () => { + let telemetryService: TelemetryService + let mockCredentialsProvider: MockCredentialsProvider + + beforeEach(() => { + mockCredentialsProvider = new MockCredentialsProvider() + mockCredentialsProvider.setConnectionMetadata({ + sso: { + startUrl: 'idc-start-url', + }, + }) + + codeWhisperServiceStub.getCredentialsType.returns('bearer') + telemetryService = new TelemetryService( + baseAmazonQServiceManagerStub, + mockCredentialsProvider, + telemetry, + logging + ) + }) + + afterEach(() => { + sinon.restore() + }) + + it('should send InlineChatEvent with correct parameters', () => { + const timestamp = new Date() + telemetryService.emitInlineChatResultLog({ + requestId: 'mock-request-id', + inputLength: 10, + selectedLines: 2, + suggestionAddedChars: 20, + suggestionAddedLines: 3, + suggestionDeletedChars: 10, + suggestionDeletedLines: 2, + codeIntent: true, + userDecision: 'ACCEPT', + responseStartLatency: 1250, + responseEndLatency: 1500, + programmingLanguage: { + languageName: 'typescript', + }, + }) + + const expectedEvent = { + telemetryEvent: { + inlineChatEvent: { + requestId: 'mock-request-id', + timestamp: timestamp, + inputLength: 10, + numSelectedLines: 2, + numSuggestionAddChars: 20, + numSuggestionAddLines: 3, + numSuggestionDelChars: 10, + numSuggestionDelLines: 2, + codeIntent: true, + userDecision: 'ACCEPT', + responseStartLatency: 1250, + responseEndLatency: 1500, + programmingLanguage: { + languageName: 'typescript', + }, + }, + }, + } + sinon.assert.calledOnceWithExactly(codeWhisperServiceStub.sendTelemetryEvent, expectedEvent) + }) + + it('should not send InlineChatEvent when credentialsType is IAM', () => { + codeWhisperServiceStub.getCredentialsType.returns('iam') + telemetryService = new TelemetryService( + baseAmazonQServiceManagerStub, + mockCredentialsProvider, + telemetry, + logging + ) + const timestamp = new Date() + telemetryService.emitInlineChatResultLog({ + requestId: 'mock-request-id', + inputLength: 10, + selectedLines: 2, + suggestionAddedChars: 20, + suggestionAddedLines: 3, + suggestionDeletedChars: 10, + suggestionDeletedLines: 2, + codeIntent: true, + userDecision: 'ACCEPT', + responseStartLatency: 1250, + responseEndLatency: 1500, + programmingLanguage: { + languageName: 'typescript', + }, + }) + sinon.assert.notCalled(codeWhisperServiceStub.sendTelemetryEvent) + }) + + it('should not send InlineChatEvent when login is BuilderID, but user chose OPTOUT option', () => { + mockCredentialsProvider.setConnectionMetadata({ + sso: { + startUrl: BUILDER_ID_START_URL, + }, + }) + telemetryService = new TelemetryService( + baseAmazonQServiceManagerStub, + mockCredentialsProvider, + telemetry, + logging + ) + telemetryService.updateOptOutPreference('OPTOUT') + const timestamp = new Date() + telemetryService.emitInlineChatResultLog({ + requestId: 'mock-request-id', + inputLength: 10, + selectedLines: 2, + suggestionAddedChars: 20, + suggestionAddedLines: 3, + suggestionDeletedChars: 10, + suggestionDeletedLines: 2, + codeIntent: true, + userDecision: 'ACCEPT', + responseStartLatency: 1250, + responseEndLatency: 1500, + programmingLanguage: { + languageName: 'typescript', + }, + }) + sinon.assert.notCalled(codeWhisperServiceStub.sendTelemetryEvent) + }) + }) }) diff --git a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts index f661aa65f..37ef7573c 100644 --- a/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/telemetry/telemetryService.ts @@ -18,6 +18,14 @@ import { TelemetryEvent, ChatAddMessageEvent, UserIntent, + InlineChatEvent, + UUID, + Timestamp, + PrimitiveInteger, + Boolean, + InlineChatUserDecision, + Double, + ProgrammingLanguage, } from '../../client/token/codewhispererbearertokenclient' import { getCompletionType, getSsoConnectionType, isAwsError } from '../utils' import { @@ -30,6 +38,7 @@ import { import { CodewhispererLanguage, getRuntimeLanguage } from '../languageDetection' import { CONVERSATION_ID_METRIC_KEY } from '../../language-server/chat/telemetry/chatTelemetryController' import { AmazonQBaseServiceManager } from '../amazonQServiceManager/BaseAmazonQServiceManager' +import { InlineChatResultParams } from '@aws/language-server-runtimes/protocol' export class TelemetryService { // Using Base service manager here to support fallback cases such as in codeWhispererServer @@ -453,4 +462,29 @@ export class TelemetryService { chatAddMessageEvent: event, }) } + + public emitInlineChatResultLog(params: InlineChatResultParams) { + const event: InlineChatEvent = { + requestId: params.requestId, + timestamp: new Date(), + inputLength: params.inputLength, + numSelectedLines: params.selectedLines, + numSuggestionAddChars: params.suggestionAddedChars, + numSuggestionAddLines: params.suggestionAddedLines, + numSuggestionDelChars: params.suggestionDeletedChars, + numSuggestionDelLines: params.suggestionDeletedLines, + codeIntent: params.codeIntent, + userDecision: params.userDecision, + responseStartLatency: params.responseStartLatency, + responseEndLatency: params.responseEndLatency, + } + if (params.programmingLanguage) { + event.programmingLanguage = { + languageName: getRuntimeLanguage(params.programmingLanguage.languageName as CodewhispererLanguage), + } + } + return this.invokeSendTelemetryEvent({ + inlineChatEvent: event, + }) + } } From 4d342d67efbf4cfd4ca76ae75dadcd676e0b3391 Mon Sep 17 00:00:00 2001 From: Paras Date: Fri, 25 Apr 2025 15:32:25 +0200 Subject: [PATCH 2/2] chore: add support for emitting telemetry related to inlineChatResult --- .../src/language-server/chat/qChatServer.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/qChatServer.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/qChatServer.ts index aecb615fe..5718e17c2 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/qChatServer.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/qChatServer.ts @@ -96,9 +96,9 @@ export const QChatServerFactory = amazonQServiceManager ) - /* + /* Calling handleDidChangeConfiguration once to ensure we get configuration atleast once at start up - + TODO: TODO: consider refactoring such responsibilities to common service manager config/initialisation server */ await amazonQServiceManager.handleDidChangeConfiguration() @@ -150,8 +150,8 @@ export const QChatServerFactory = return chatController.onCodeInsertToCursorPosition(params) }) - chat.onLogInlineChatResult(params => { - return chatController.onLogInlineChatResult(params) + chat.onInlineChatResult(params => { + return chatController.onInlineChatResult(params) }) logging.log('Q Chat server has been initialized')