From 4e8dcfa3abbb4b25ef2bac6916e32fdcfe5f610b Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 20 Jun 2025 15:58:53 +0100 Subject: [PATCH 1/2] Increase otel attribute limits and make them configurable --- apps/webapp/app/env.server.ts | 10 ++ .../environmentVariablesRepository.server.ts | 91 ++++++++++++++++++- apps/webapp/app/v3/otlpExporter.server.ts | 51 ++++++++--- .../migration.sql | 2 + .../database/prisma/schema.prisma | 3 + packages/core/src/v3/limits.ts | 49 ++++++++-- 6 files changed, 183 insertions(+), 23 deletions(-) create mode 100644 internal-packages/database/prisma/migrations/20250620144525_add_built_in_overrides_to_runtime_environment/migration.sql diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index ffdfacf164..ea22f8afcb 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { isValidDatabaseUrl } from "./utils/db"; import { isValidRegex } from "./utils/regex"; import { BoolEnv } from "./utils/boolEnv"; +import { OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT, OTEL_LINK_COUNT_LIMIT } from "@trigger.dev/core/v3"; const EnvironmentSchema = z.object({ NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]), @@ -276,6 +277,15 @@ const EnvironmentSchema = z.object({ PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"), + TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"), + TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"), + TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), + TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), + TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT: z.string().default("10"), + TRIGGER_OTEL_LINK_COUNT_LIMIT: z.string().default("2"), + TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT: z.string().default("10"), + TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT: z.string().default("10"), + CHECKPOINT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000), // Internal OTEL environment variables diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index 66c1ddd9d1..ea6243f4a5 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -810,6 +810,7 @@ export const RuntimeEnvironmentForEnvRepoPayload = { apiKey: true, organizationId: true, branchName: true, + builtInEnvironmentVariableOverrides: true, }, } as const; @@ -1025,5 +1026,93 @@ async function resolveBuiltInProdVariables( async function resolveCommonBuiltInVariables( runtimeEnvironment: RuntimeEnvironmentForEnvRepo ): Promise> { - return []; + return [ + { + key: "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_LINK_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_LINK_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_LINK_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT) + ), + }, + { + key: "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT", + value: resolveBuiltInEnvironmentVariableOverrides( + "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT", + runtimeEnvironment, + String(env.TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT) + ), + }, + ]; +} + +function resolveBuiltInEnvironmentVariableOverrides( + key: string, + runtimeEnvironment: RuntimeEnvironmentForEnvRepo, + defaultValue: string +) { + const overrides = runtimeEnvironment.builtInEnvironmentVariableOverrides; + + if (!overrides) { + return defaultValue; + } + + if ( + !Array.isArray(overrides) && + typeof overrides === "object" && + key in overrides && + typeof overrides[key] === "string" + ) { + return overrides[key]; + } + + return defaultValue; } diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index b87718f86e..888a6f46bb 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -39,7 +39,8 @@ class OTLPExporter { constructor( private readonly _eventRepository: EventRepository, - private readonly _verbose: boolean + private readonly _verbose: boolean, + private readonly _spanAttributeValueLengthLimit: number ) { this._tracer = trace.getTracer("otlp-exporter"); } @@ -52,7 +53,7 @@ class OTLPExporter { this.#logExportTracesVerbose(request); const events = this.#filterResourceSpans(request.resourceSpans).flatMap((resourceSpan) => { - return convertSpansToCreateableEvents(resourceSpan); + return convertSpansToCreateableEvents(resourceSpan, this._spanAttributeValueLengthLimit); }); const enrichedEvents = enrichCreatableEvents(events); @@ -79,7 +80,7 @@ class OTLPExporter { this.#logExportLogsVerbose(request); const events = this.#filterResourceLogs(request.resourceLogs).flatMap((resourceLog) => { - return convertLogsToCreateableEvents(resourceLog); + return convertLogsToCreateableEvents(resourceLog, this._spanAttributeValueLengthLimit); }); const enrichedEvents = enrichCreatableEvents(events); @@ -180,7 +181,10 @@ class OTLPExporter { } } -function convertLogsToCreateableEvents(resourceLog: ResourceLogs): Array { +function convertLogsToCreateableEvents( + resourceLog: ResourceLogs, + spanAttributeValueLengthLimit: number +): Array { const resourceAttributes = resourceLog.resource?.attributes ?? []; const resourceProperties = extractEventProperties(resourceAttributes); @@ -213,10 +217,10 @@ function convertLogsToCreateableEvents(resourceLog: ResourceLogs): Array { +function convertSpansToCreateableEvents( + resourceSpan: ResourceSpans, + spanAttributeValueLengthLimit: number +): Array { const resourceAttributes = resourceSpan.resource?.attributes ?? []; const resourceProperties = extractEventProperties(resourceAttributes); @@ -323,10 +330,10 @@ function convertSpansToCreateableEvents(resourceSpan: ResourceSpans): Array { + return isStringValue(attribute.value) + ? { + key: attribute.key, + value: { + stringValue: attribute.value.stringValue.slice(0, maximumLength), + }, + } + : attribute; + }); +} + export const otlpExporter = new OTLPExporter( eventRepository, - process.env.OTLP_EXPORTER_VERBOSE === "1" + process.env.OTLP_EXPORTER_VERBOSE === "1", + process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT + ? parseInt(process.env.SERVER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, 10) + : 8192 ); diff --git a/internal-packages/database/prisma/migrations/20250620144525_add_built_in_overrides_to_runtime_environment/migration.sql b/internal-packages/database/prisma/migrations/20250620144525_add_built_in_overrides_to_runtime_environment/migration.sql new file mode 100644 index 0000000000..04ee6a77a8 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20250620144525_add_built_in_overrides_to_runtime_environment/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "RuntimeEnvironment" ADD COLUMN "builtInEnvironmentVariableOverrides" JSONB; \ No newline at end of file diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index c83bf6947b..3143b089f2 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -241,6 +241,9 @@ model RuntimeEnvironment { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + /// Allows us to customize the built-in environment variables for a specific environment, like TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT + builtInEnvironmentVariableOverrides Json? + tunnelId String? backgroundWorkers BackgroundWorker[] diff --git a/packages/core/src/v3/limits.ts b/packages/core/src/v3/limits.ts index 212446faad..4cead21454 100644 --- a/packages/core/src/v3/limits.ts +++ b/packages/core/src/v3/limits.ts @@ -1,13 +1,46 @@ import { AttributeValue, Attributes } from "@opentelemetry/api"; +import { getEnvVar } from "./utils/getEnv.js"; + +function getOtelEnvVarLimit(key: string, defaultValue: number) { + const value = getEnvVar(key); + + if (!value) { + return defaultValue; + } + + return parseInt(value, 10); +} + +export const OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT", + 256 +); +export const OTEL_LOG_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT", + 256 +); +export const OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", + 131072 +); +export const OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT", + 131072 +); +export const OTEL_SPAN_EVENT_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT", + 10 +); +export const OTEL_LINK_COUNT_LIMIT = getOtelEnvVarLimit("TRIGGER_OTEL_LINK_COUNT_LIMIT", 2); +export const OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT", + 10 +); +export const OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT = getOtelEnvVarLimit( + "TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT", + 10 +); -export const OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 256; -export const OTEL_LOG_ATTRIBUTE_COUNT_LIMIT = 256; -export const OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = 1028; -export const OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT = 1028; -export const OTEL_SPAN_EVENT_COUNT_LIMIT = 10; -export const OTEL_LINK_COUNT_LIMIT = 2; -export const OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT = 10; -export const OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT = 10; export const OFFLOAD_IO_PACKET_LENGTH_LIMIT = 128 * 1024; export function imposeAttributeLimits(attributes: Attributes): Attributes { From 302f552df424cd89946e98ba2ca2531447561506 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 20 Jun 2025 16:07:45 +0100 Subject: [PATCH 2/2] Fix broken webapp test --- apps/webapp/test/timelineSpanEvents.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/webapp/test/timelineSpanEvents.test.ts b/apps/webapp/test/timelineSpanEvents.test.ts index adc02b6f57..8299ca83f3 100644 --- a/apps/webapp/test/timelineSpanEvents.test.ts +++ b/apps/webapp/test/timelineSpanEvents.test.ts @@ -76,7 +76,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => { expect(result.some((event) => event.name === "Dequeued")).toBe(true); expect(result.some((event) => event.name === "Launched")).toBe(true); expect(result.some((event) => event.name === "Attempt created")).toBe(true); - expect(result.some((event) => event.name === "Importing src/trigger/chat.ts")).toBe(true); + expect(result.some((event) => event.name === "Importing task file")).toBe(true); }); test("should sort events by timestamp", () => { @@ -86,7 +86,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => { expect(result[0].name).toBe("Dequeued"); expect(result[1].name).toBe("Attempt created"); expect(result[2].name).toBe("Launched"); - expect(result[3].name).toBe("Importing src/trigger/chat.ts"); + expect(result[3].name).toBe("Importing Importing task file"); }); test("should calculate offsets correctly from the first event", () => { @@ -176,7 +176,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => { expect(result.find((e) => e.name === "Attempt created")?.helpText).toBe( "An attempt was created for the run" ); - expect(result.find((e) => e.name === "Importing src/trigger/chat.ts")?.helpText).toBe( + expect(result.find((e) => e.name === "Importing task file")?.helpText).toBe( "A task file was imported" ); }); @@ -187,7 +187,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => { expect(result.find((e) => e.name === "Dequeued")?.duration).toBe(0); expect(result.find((e) => e.name === "Launched")?.duration).toBe(127); expect(result.find((e) => e.name === "Attempt created")?.duration).toBe(56); - expect(result.find((e) => e.name === "Importing src/trigger/chat.ts")?.duration).toBe(67); + expect(result.find((e) => e.name === "Importing task file")?.duration).toBe(67); }); test("should use fallback name for import event without file property", () => { @@ -214,7 +214,7 @@ describe("createTimelineSpanEventsFromSpanEvents", () => { // Without fork event, import should also be visible for non-admins expect(result.length).toBe(2); expect(result.some((event) => event.name === "Dequeued")).toBe(true); - expect(result.some((event) => event.name === "Importing src/trigger/chat.ts")).toBe(true); + expect(result.some((event) => event.name === "Importing task file")).toBe(true); // create_attempt should still be admin-only expect(result.some((event) => event.name === "Attempt created")).toBe(false);