From b5d5812d46d6e3bc8dcc814586b7c7e7be783395 Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Fri, 18 Dec 2020 17:28:41 -0500 Subject: [PATCH] add warning when running out of disk. --- src/Runner.Common/Constants.cs | 6 +++ src/Runner.Worker/JobExtension.cs | 51 ++++++++++++++++++- .../WellKnownDistributedTaskVariables.cs | 1 + 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 82401bb110f..9ccb28f6b6f 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -140,8 +140,14 @@ public static class ReturnCode public const int RunOnceRunnerUpdating = 4; } + public static class Features + { + public static readonly string DiskSpaceWarning = "runner.diskspace.warning"; + } + public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry"; public static readonly string WorkerCrash = "WORKER_CRASH"; + public static readonly string LowDiskSpace = "LOW_DISK_SPACE"; public static readonly string UnsupportedCommand = "UNSUPPORTED_COMMAND"; public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/"; } diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index d4f5d21b4ed..d37b72dfd5b 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -1,10 +1,11 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; using System.Runtime.Serialization; +using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating.Tokens; @@ -41,6 +42,8 @@ public sealed class JobExtension : RunnerService, IJobExtension private readonly HashSet _existingProcesses = new HashSet(StringComparer.OrdinalIgnoreCase); private bool _processCleanup; private string _processLookupId = $"github_{Guid.NewGuid()}"; + private CancellationTokenSource _diskSpaceCheckToken = new CancellationTokenSource(); + private Task _diskSpaceCheckTask = null; // Download all required actions. // Make sure all condition inputs are valid. @@ -325,6 +328,12 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } + jobContext.Global.EnvironmentVariables.TryGetValue(Constants.Runner.Features.DiskSpaceWarning, out var enableWarning); + if (StringUtil.ConvertToBoolean(enableWarning, defaultValue: true)) + { + _diskSpaceCheckTask = CheckDiskSpaceAsync(context, _diskSpaceCheckToken.Token); + } + return steps; } catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested) @@ -335,7 +344,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.Result = TaskResult.Canceled; throw; } - catch (FailedToResolveActionDownloadInfoException ex) + catch (FailedToResolveActionDownloadInfoException ex) { // Log the error and fail the JobExtension Initialization. Trace.Error($"Caught exception from JobExtenion Initialization: {ex}"); @@ -529,6 +538,11 @@ public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestM } } } + + if (_diskSpaceCheckTask != null) + { + _diskSpaceCheckToken.Cancel(); + } } catch (Exception ex) { @@ -544,6 +558,39 @@ public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestM } } + private async Task CheckDiskSpaceAsync(IExecutionContext context, CancellationToken token) + { + while (!token.IsCancellationRequested) + { + // Add warning when disk is lower than system.runner.lowdiskspacethreshold from service (default to 100 MB on service side) + var lowDiskSpaceThreshold = context.Global.Variables.GetInt(WellKnownDistributedTaskVariables.RunnerLowDiskspaceThreshold); + if (lowDiskSpaceThreshold == null) + { + Trace.Info($"Low diskspace warning is not enabled."); + return; + } + var workDirRoot = Directory.GetDirectoryRoot(HostContext.GetDirectory(WellKnownDirectory.Work)); + var driveInfo = new DriveInfo(workDirRoot); + var freeSpaceInMB = driveInfo.AvailableFreeSpace / 1024 / 1024; + if (freeSpaceInMB < lowDiskSpaceThreshold) + { + var issue = new Issue() { Type = IssueType.Warning, Message = $"You are running out of disk space. The runner will stop working when the machine runs out of disk space. Free space left: {freeSpaceInMB} MB" }; + issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.LowDiskSpace; + context.AddIssue(issue); + return; + } + + try + { + await Task.Delay(10 * 1000, token); + } + catch (TaskCanceledException) + { + // ignore + } + } + } + private Dictionary SnapshotProcesses() { Dictionary snapshot = new Dictionary(); diff --git a/src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs b/src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs index a5495bc036e..2b487e78250 100644 --- a/src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs +++ b/src/Sdk/DTWebApi/WebApi/WellKnownDistributedTaskVariables.cs @@ -5,5 +5,6 @@ namespace GitHub.DistributedTask.WebApi public static class WellKnownDistributedTaskVariables { public static readonly String JobId = "system.jobId"; + public static readonly String RunnerLowDiskspaceThreshold = "system.runner.lowdiskspacethreshold"; } }