Add changelog

Author: colin-sentry

CHANGELOG.md

@@ -22,6 +22,7 @@
 - Do not PII-scrub code locations by default. ([#3116](https://github.com/getsentry/relay/pull/3116))
 - Accept transactions with unfinished spans. ([#3162](https://github.com/getsentry/relay/pull/3162))
 - Don't run validation on renormalization, and don't normalize spans from librelay calls. ([#3214](https://github.com/getsentry/relay/pull/3214))
+- Do not consider strings with the word "tokens" plural to be passwords ([#3106](https://github.com/getsentry/relay/pull/3106))
 
 **Internal**:
 

relay-event-schema/src/protocol/span.rs

@@ -260,6 +260,26 @@ pub struct SpanData {
     #[metastructure(field = "http.response.status_code", legacy_alias = "status_code")]
     pub http_response_status_code: Annotated<Value>,
 
+    /// The input messages to an AI model call
+    #[metastructure(field = "ai.input_messages")]
+    pub ai_input_messages: Annotated<Value>,
+
+    /// The number of tokens used to generate the response to an AI call
+    #[metastructure(field = "ai.completion_tokens.used")]
+    pub ai_completion_tokens_used: Annotated<Value>,
+
+    /// The number of tokens used to process a request for an AI call
+    #[metastructure(field = "ai.prompt_tokens.used")]
+    pub ai_prompt_tokens_used: Annotated<Value>,
+
+    /// The total number of tokens used to for an AI call
+    #[metastructure(field = "ai.total_tokens.used")]
+    pub ai_total_tokens_used: Annotated<Value>,
+
+    /// The responses to an AI model call
+    #[metastructure(field = "ai.responses")]
+    pub ai_responses: Annotated<Value>,
+
     /// Label identifying a thread from where the span originated.
     #[metastructure(field = "thread.name")]
     pub thread_name: Annotated<Value>,

relay-pii/src/processor.rs

@@ -473,7 +473,7 @@ mod tests {
         NativeDebugImage, Request, Span, TagEntry, Tags, TraceContext,
     };
     use relay_protocol::{
-        assert_annotated_snapshot, get_value, Annotated, FromValue, Object, Value,
+        assert_annotated_snapshot, get_value, Annotated, FromValue, Object, Val, Value,
     };
     use serde_json::json;
 
@@ -1339,6 +1339,33 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_span_data_tokens() {
+        let mut span = Span::from_value(
+            json!({
+                "data": {
+                    "ai.total_tokens.used": 10,
+                }
+            })
+            .into(),
+        );
+
+        let ds_config = DataScrubbingConfig {
+            scrub_data: true,
+            scrub_defaults: true,
+            ..Default::default()
+        };
+        let pii_config = ds_config.pii_config().unwrap().as_ref().unwrap();
+
+        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
+        process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
+
+        assert_eq!(
+            Val::from(get_value!(span.data.ai_total_tokens_used!)).as_u64(),
+            Some(10),
+        );
+    }
+
     #[test]
     fn test_scrub_breadcrumb_data_http_not_scrubbed() {
         let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(

relay-pii/src/regexes.rs

@@ -286,6 +286,6 @@ static US_SSN_REGEX: Lazy<Regex> = Lazy::new(|| {
 
 static PASSWORD_KEY_REGEX: Lazy<Regex> = Lazy::new(|| {
     Regex::new(
-        r"(?i)(password|secret|passwd|api_key|apikey|auth|credentials|mysql_pwd|privatekey|private_key|token([a-r,t-z]|\b))"
+        r"(?i)(password|secret|passwd|api_key|apikey|auth|credentials|mysql_pwd|privatekey|private_key|token([^s]|\b))"
     ).unwrap()
 });