Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Linux Consumption metrics publisher for Legion #10750

Open
wants to merge 6 commits into
base: dev
Choose a base branch
from
Open

Linux Consumption metrics publisher for Legion #10750

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ef86f05
Linux Consumption metrics publisher for Legion
vivekjilla Jan 16, 2025
d8c9b85
nit
vivekjilla Jan 16, 2025
20d43c1
Removing cyclic dependency between metricslogger and metricspublisher
vivekjilla Jan 16, 2025
bdfe28f
Remove Cyclic Dependency between metrics logger and publisher
jainharsh98 Feb 3, 2025
16aa650
Fixing Unit test case failures
jainharsh98 Feb 3, 2025
42a846e
Disabling events not used warnings
jainharsh98 Feb 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NuGet.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
<add key="AzureFunctionsRelease" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsRelease/nuget/v3/index.json" />
<add key="AzureFunctionsPreRelease" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsPreRelease/nuget/v3/index.json" />
<add key="AzureFunctionsTempStaging" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsTempStaging/nuget/v3/index.json" />
<add key="AzureFunctions" value="https://pkgs.dev.azure.com/azfunc/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctions/nuget/v3/index.json" />
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What package required adding this feed? I would prefer we use an existing feed if possible.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This feed was added to test it with the package Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption for internal consumption. Please refer to #10750 (comment). Is it okay to push it in the tempStaging or PreRelease feed?

</packageSources>
</configuration>
92 changes: 10 additions & 82 deletions src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class FlexConsumptionMetricsPublisher : IMetricsPublisher, IDisposable
private readonly IHostMetricsProvider _metricsProvider;
private readonly object _lock = new object();
private readonly IFileSystem _fileSystem;
private readonly LegionMetricsFileManager _metricsFileManager;

private Timer _metricsPublisherTimer;
private bool _started = false;
Expand All @@ -44,6 +45,7 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_fileSystem = fileSystem ?? new FileSystem();
_metricsFileManager = new LegionMetricsFileManager(_options.MetricsFilePath, _fileSystem, _logger, _options.MaxFileCount);
_metricsProvider = metricsProvider ?? throw new ArgumentNullException(nameof(metricsProvider));

if (_standbyOptions.CurrentValue.InStandbyMode)
Expand All @@ -66,13 +68,13 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor

internal bool IsAlwaysReady { get; set; }

internal string MetricsFilePath { get; set; }
internal LegionMetricsFileManager MetricsFileManager => _metricsFileManager;

public void Start()
{
Initialize();

_logger.LogInformation($"Starting metrics publisher (AlwaysReady={IsAlwaysReady}, MetricsPath='{MetricsFilePath}').");
_logger.LogInformation($"Starting metrics publisher (AlwaysReady={IsAlwaysReady}, MetricsPath='{_metricsFileManager.MetricsFilePath}').");

_metricsPublisherTimer = new Timer(OnFunctionMetricsPublishTimer, null, _initialPublishDelay, _metricPublishInterval);
_started = true;
Expand All @@ -86,7 +88,6 @@ internal void Initialize()
_metricPublishInterval = TimeSpan.FromMilliseconds(_options.MetricsPublishIntervalMS);
_initialPublishDelay = TimeSpan.FromMilliseconds(_options.InitialPublishDelayMS);
_intervalStopwatch = ValueStopwatch.StartNew();
MetricsFilePath = _options.MetricsFilePath;

IsAlwaysReady = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsAlwaysReadyInstance) == "1";
}
Expand Down Expand Up @@ -136,7 +137,12 @@ internal async Task OnPublishMetrics(DateTime now)
FunctionExecutionTimeMS = FunctionExecutionCount = 0;
}

await PublishMetricsAsync(metrics);
await _metricsFileManager.PublishMetricsAsync(metrics);
}
catch (Exception ex) when (!ex.IsFatal())
{
// ensure no background exceptions escape
_logger.LogError(ex, $"Error publishing metrics.");
}
finally
{
Expand All @@ -149,84 +155,6 @@ private async void OnFunctionMetricsPublishTimer(object state)
await OnPublishMetrics(DateTime.UtcNow);
}

private async Task PublishMetricsAsync(Metrics metrics)
{
string fileName = string.Empty;

try
{
bool metricsPublishEnabled = !string.IsNullOrEmpty(MetricsFilePath);
if (metricsPublishEnabled && !PrepareDirectoryForFile())
{
return;
}

string metricsContent = JsonConvert.SerializeObject(metrics);
_logger.PublishingMetrics(metricsContent);

if (metricsPublishEnabled)
{
fileName = $"{Guid.NewGuid().ToString().ToLower()}.json";
string filePath = Path.Combine(MetricsFilePath, fileName);

using (var streamWriter = _fileSystem.File.CreateText(filePath))
{
await streamWriter.WriteAsync(metricsContent);
}
}
}
catch (Exception ex) when (!ex.IsFatal())
{
// TODO: consider using a retry strategy here
_logger.LogError(ex, $"Error writing metrics file '{fileName}'.");
}
}

private bool PrepareDirectoryForFile()
{
if (string.IsNullOrEmpty(MetricsFilePath))
{
return false;
}

// ensure the directory exists
_fileSystem.Directory.CreateDirectory(MetricsFilePath);

var metricsDirectoryInfo = _fileSystem.DirectoryInfo.FromDirectoryName(MetricsFilePath);
var files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();

// ensure we're under the max file count
if (files.Count < _options.MaxFileCount)
{
return true;
}

// we're at or over limit
// delete enough files that we have space to write a new one
int numToDelete = files.Count - _options.MaxFileCount + 1;
var filesToDelete = files.Take(numToDelete).ToArray();

_logger.LogDebug($"Deleting {filesToDelete.Length} metrics file(s).");

foreach (var file in filesToDelete)
{
try
{
file.Delete();
}
catch (Exception ex) when (!ex.IsFatal())
{
// best effort
_logger.LogError(ex, $"Error deleting metrics file '{file.FullName}'.");
}
}

files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();

// return true if we have space for a new file
return files.Count < _options.MaxFileCount;
}

private void OnStandbyOptionsChange()
{
if (!_standbyOptions.CurrentValue.InStandbyMode)
Expand Down
109 changes: 109 additions & 0 deletions src/WebJobs.Script.WebHost/Metrics/LegionMetricsFileManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;

namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics
{
public class LegionMetricsFileManager
{
private readonly IFileSystem _fileSystem;
private readonly int _maxFileCount;
private readonly ILogger _logger;

public LegionMetricsFileManager(string metricsFilePath, IFileSystem fileSystem, ILogger logger, int maxFileCount)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
MetricsFilePath = metricsFilePath;
_maxFileCount = maxFileCount;
}

internal string MetricsFilePath { get; set; }

private bool PrepareDirectoryForFile()
{
if (string.IsNullOrEmpty(MetricsFilePath))
{
return false;
}

// ensure the directory exists
_fileSystem.Directory.CreateDirectory(MetricsFilePath);

var metricsDirectoryInfo = _fileSystem.DirectoryInfo.FromDirectoryName(MetricsFilePath);
var files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();

// ensure we're under the max file count
if (files.Count < _maxFileCount)
{
return true;
}

// we're at or over limit
// delete enough files that we have space to write a new one
int numToDelete = files.Count - _maxFileCount + 1;
var filesToDelete = files.Take(numToDelete).ToArray();

_logger.LogDebug($"Deleting {filesToDelete.Length} metrics file(s).");

foreach (var file in filesToDelete)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could benefit from a Parrallel.ForEach - but this is just a code move so no worries.

{
try
{
file.Delete();
}
catch (Exception ex) when (!ex.IsFatal())
{
// best effort
_logger.LogError(ex, $"Error deleting metrics file '{file.FullName}'.");
}
}

files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();

// return true if we have space for a new file
return files.Count < _maxFileCount;
}

public async Task PublishMetricsAsync(object metrics)
{
string fileName = string.Empty;

try
{
bool metricsPublishEnabled = !string.IsNullOrEmpty(MetricsFilePath);
if (metricsPublishEnabled && !PrepareDirectoryForFile())
{
return;
}

string metricsContent = JsonConvert.SerializeObject(metrics);
_logger.PublishingMetrics(metricsContent);

if (metricsPublishEnabled)
{
fileName = $"{Guid.NewGuid().ToString().ToLower()}.json";
string filePath = Path.Combine(MetricsFilePath, fileName);

using (var streamWriter = _fileSystem.File.CreateText(filePath))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: can use concise using statement:

using var streamWriter = _fileSystem.File.CreateText(filePath);
await streamWriter.WriteAsync(metricsContext);

{
await streamWriter.WriteAsync(metricsContent);
}
}
}
catch (Exception ex) when (!ex.IsFatal())
{
// TODO: consider using a retry strategy here
_logger.LogError(ex, $"Error writing metrics file '{fileName}'.");
}
}
}
}
Loading
Loading