Skip to content

Commit

Permalink
Better sampling override behavior (#4014)
Browse files Browse the repository at this point in the history
  • Loading branch information
trask authored Feb 24, 2025
1 parent ff2d17d commit aa2d6f3
Show file tree
Hide file tree
Showing 15 changed files with 684 additions and 151 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@
package com.microsoft.applicationinsights.agent.internal.exporter;

import static com.azure.monitor.opentelemetry.autoconfigure.implementation.utils.AzureMonitorMsgId.EXPORTER_MAPPING_ERROR;
import static com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils.shouldSample;

import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes;
import com.azure.monitor.opentelemetry.autoconfigure.implementation.LogDataMapper;
import com.azure.monitor.opentelemetry.autoconfigure.implementation.logging.OperationLogger;
import com.azure.monitor.opentelemetry.autoconfigure.implementation.models.TelemetryItem;
import com.azure.monitor.opentelemetry.autoconfigure.implementation.quickpulse.QuickPulse;
import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride;
import com.microsoft.applicationinsights.agent.internal.sampling.AiFixedPercentageSampler;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
Expand All @@ -21,6 +22,8 @@
import io.opentelemetry.sdk.common.CompletableResultCode;
import io.opentelemetry.sdk.logs.data.LogRecordData;
import io.opentelemetry.sdk.logs.export.LogRecordExporter;
import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
import io.opentelemetry.sdk.trace.samplers.SamplingResult;
import io.opentelemetry.semconv.ExceptionAttributes;
import java.util.Collection;
import java.util.List;
Expand Down Expand Up @@ -96,45 +99,62 @@ private CompletableResultCode internalExport(Collection<LogRecordData> logs) {
return CompletableResultCode.ofFailure();
}
for (LogRecordData log : logs) {
try {
int severityNumber = log.getSeverity().getSeverityNumber();
if (severityNumber < severityThreshold) {
continue;
}
internalExport(log);
}
// always returning success, because all error handling is performed internally
return CompletableResultCode.ofSuccess();
}

String stack = log.getAttributes().get(ExceptionAttributes.EXCEPTION_STACKTRACE);
private void internalExport(LogRecordData log) {
try {
int severityNumber = log.getSeverity().getSeverityNumber();
if (severityNumber < severityThreshold) {
return;
}

SamplingOverrides samplingOverrides =
stack != null ? exceptionSamplingOverrides : logSamplingOverrides;
String stack = log.getAttributes().get(ExceptionAttributes.EXCEPTION_STACKTRACE);

SpanContext spanContext = log.getSpanContext();
SamplingOverrides samplingOverrides =
stack != null ? exceptionSamplingOverrides : logSamplingOverrides;

Double samplingPercentage = samplingOverrides.getOverridePercentage(log.getAttributes());
SpanContext spanContext = log.getSpanContext();
Double parentSpanSampleRate = log.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE);

if (samplingPercentage != null && !shouldSample(spanContext, samplingPercentage)) {
continue;
}
AiFixedPercentageSampler sampler = samplingOverrides.getOverride(log.getAttributes());

if (samplingPercentage == null
&& spanContext.isValid()
&& !spanContext.getTraceFlags().isSampled()) {
// if there is no sampling override, and the log is part of an unsampled trace, then don't
// capture it
continue;
}
boolean hasSamplingOverride = sampler != null;

logger.debug("exporting log: {}", log);
if (!hasSamplingOverride
&& spanContext.isValid()
&& !spanContext.getTraceFlags().isSampled()) {
// if there is no sampling override, and the log is part of an unsampled trace,
// then don't capture it
return;
}

TelemetryItem telemetryItem = mapper.map(log, stack, samplingPercentage);
telemetryItemConsumer.accept(telemetryItem);
Double sampleRate = null;
if (hasSamplingOverride) {
SamplingResult samplingResult = sampler.shouldSampleLog(spanContext, parentSpanSampleRate);
if (samplingResult.getDecision() != SamplingDecision.RECORD_AND_SAMPLE) {
return;
}
sampleRate = samplingResult.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE);
}

exportingLogLogger.recordSuccess();
} catch (Throwable t) {
exportingLogLogger.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR);
if (sampleRate == null) {
sampleRate = parentSpanSampleRate;
}

logger.debug("exporting log: {}", log);

// TODO (trask) no longer need to check AiSemanticAttributes.SAMPLE_RATE in map() method
TelemetryItem telemetryItem = mapper.map(log, stack, sampleRate);
telemetryItemConsumer.accept(telemetryItem);

exportingLogLogger.recordSuccess();
} catch (Throwable t) {
exportingLogLogger.recordFailure(t.getMessage(), t, EXPORTER_MAPPING_ERROR);
}
// always returning success, because all error handling is performed internally
return CompletableResultCode.ofSuccess();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

package com.microsoft.applicationinsights.agent.internal.exporter;

import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplerUtil;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import io.opentelemetry.api.trace.SpanContext;
import java.util.concurrent.ThreadLocalRandom;
Expand All @@ -23,7 +23,7 @@ public static boolean shouldSample(SpanContext spanContext, double percentage) {
return false;
}
if (spanContext.isValid()) {
return AiSampler.shouldRecordAndSample(spanContext.getTraceId(), percentage);
return SamplerUtil.shouldRecordAndSample(spanContext.getTraceId(), percentage);
}
// this is a standalone log (not part of a trace), so randomly sample at the given percentage
return ThreadLocalRandom.current().nextDouble() < percentage / 100;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import static java.util.concurrent.TimeUnit.MINUTES;

import com.azure.core.util.logging.ClientLogger;
import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes;
import com.azure.monitor.opentelemetry.autoconfigure.implementation.AzureMonitorExporterProviderKeys;
import com.azure.monitor.opentelemetry.autoconfigure.implementation.AzureMonitorLogRecordExporterProvider;
import com.azure.monitor.opentelemetry.autoconfigure.implementation.AzureMonitorMetricExporterProvider;
Expand Down Expand Up @@ -37,14 +38,14 @@
import com.microsoft.applicationinsights.agent.internal.exporter.AgentLogExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.AgentMetricExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.AgentSpanExporter;
import com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils;
import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient;
import com.microsoft.applicationinsights.agent.internal.legacyheaders.AiLegacyHeaderSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithLogProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.LogExporterWithAttributeProcessor;
import com.microsoft.applicationinsights.agent.internal.processors.SpanExporterWithAttributeProcessor;
import com.microsoft.applicationinsights.agent.internal.profiler.triggers.AlertTriggerSpanProcessor;
import com.microsoft.applicationinsights.agent.internal.sampling.AiFixedPercentageSampler;
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
import com.microsoft.applicationinsights.agent.internal.telemetry.MetricFilter;
Expand All @@ -66,6 +67,7 @@
import io.opentelemetry.sdk.metrics.internal.view.AiViewRegistry;
import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
import io.opentelemetry.sdk.trace.export.SpanExporter;
import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
Expand Down Expand Up @@ -579,10 +581,15 @@ private static SpanExporter createSpanExporter(
return false;
},
(span, event) -> {
Double samplingPercentage =
exceptionSamplingOverrides.getOverridePercentage(event.getAttributes());
return samplingPercentage != null
&& !ExporterUtils.shouldSample(span.getSpanContext(), samplingPercentage);
AiFixedPercentageSampler sampler =
exceptionSamplingOverrides.getOverride(event.getAttributes());
return sampler != null
&& sampler
.shouldSampleLog(
span.getSpanContext(),
span.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE))
.getDecision()
== SamplingDecision.DROP;
});

BatchItemProcessor batchItemProcessor = telemetryClient.getGeneralBatchItemProcessor();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.microsoft.applicationinsights.agent.internal.sampling;

import com.azure.monitor.opentelemetry.autoconfigure.implementation.AiSemanticAttributes;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.SpanContext;
import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.context.Context;
import io.opentelemetry.sdk.trace.ReadableSpan;
import io.opentelemetry.sdk.trace.data.LinkData;
import io.opentelemetry.sdk.trace.samplers.Sampler;
import io.opentelemetry.sdk.trace.samplers.SamplingResult;
import java.util.List;
import javax.annotation.Nullable;

public class AiFixedPercentageSampler implements Sampler {

private final double percentage;

public static AiFixedPercentageSampler create(double percentage) {
return new AiFixedPercentageSampler(percentage);
}

private AiFixedPercentageSampler(double percentage) {
this.percentage = percentage;
}

@Override
public SamplingResult shouldSample(
Context parentContext,
String traceId,
String name,
SpanKind spanKind,
Attributes attributes,
List<LinkData> parentLinks) {

Span parentSpan = Span.fromContext(parentContext);
SpanContext parentSpanContext = parentSpan.getSpanContext();
Double parentSpanSampleRate = null;
if (parentSpan instanceof ReadableSpan) {
parentSpanSampleRate =
((ReadableSpan) parentSpan).getAttribute(AiSemanticAttributes.SAMPLE_RATE);
}

return internalShouldSample(parentSpanContext, parentSpanSampleRate, traceId);
}

public SamplingResult shouldSampleLog(SpanContext spanContext, @Nullable Double spanSampleRate) {
return internalShouldSample(spanContext, spanSampleRate, spanContext.getTraceId());
}

private SamplingResult internalShouldSample(
SpanContext parentSpanContext, @Nullable Double parentSpanSampleRate, String traceId) {

SamplingResult samplingResult =
useLocalParentDecisionIfPossible(parentSpanContext, parentSpanSampleRate);
if (samplingResult != null) {
return samplingResult;
}

return SamplerUtil.shouldSample(traceId, percentage);
}

@Nullable
private SamplingResult useLocalParentDecisionIfPossible(
SpanContext parentSpanContext, @Nullable Double parentSpanSampleRate) {

// remote parent-based sampling messes up item counts since item count is not propagated in
// tracestate (yet), but local parent-based sampling doesn't have this issue since we are
// propagating item count locally

if (!parentSpanContext.isValid() || parentSpanContext.isRemote()) {
return null;
}

if (!parentSpanContext.isSampled()) {
if (percentage < 100) {
// only 100% sampling override will override an unsampled parent!!
return SamplingResult.drop();
} else {
// falls back in this case to percentage
return null;
}
}

if (parentSpanSampleRate == null) {
return null;
}

if (percentage < parentSpanSampleRate || percentage == 100) {
// falls back in this case to percentage
return null;
}
// don't sample more dependencies than parent in this case
return SamplerUtil.createSamplingResultWithSampleRateAndItemCount(parentSpanSampleRate);
}

@Override
public String getDescription() {
return "FixedPercentageSampler";
}
}
Loading

0 comments on commit aa2d6f3

Please # to comment.