From fbae6f23e04441aae2c8de05d8fcffbdf307fedc Mon Sep 17 00:00:00 2001 From: Julio Barba Date: Thu, 5 Dec 2019 11:08:41 -0500 Subject: [PATCH] Remove job message PII variable masking code --- src/Runner.Worker/Variables.cs | 28 --------- src/Runner.Worker/Worker.cs | 2 +- src/Runner.Worker/WorkerUtilties.cs | 92 ----------------------------- src/Test/L0/Worker/WorkerL0.cs | 56 ------------------ 4 files changed, 1 insertion(+), 177 deletions(-) delete mode 100644 src/Runner.Worker/WorkerUtilties.cs diff --git a/src/Runner.Worker/Variables.cs b/src/Runner.Worker/Variables.cs index cee65f5bedf..a0281ff69e9 100644 --- a/src/Runner.Worker/Variables.cs +++ b/src/Runner.Worker/Variables.cs @@ -77,34 +77,6 @@ public Variables(IHostContext hostContext, IDictionary co public string System_TFCollectionUrl => Get(WellKnownDistributedTaskVariables.TFCollectionUrl); - public static readonly HashSet PiiVariables = new HashSet(StringComparer.OrdinalIgnoreCase) - { - "Build.AuthorizeAs", - "Build.QueuedBy", - "Build.RequestedFor", - "Build.RequestedForEmail", - "Build.SourceBranch", - "Build.SourceBranchName", - "Build.SourceTfvcShelveset", - "Build.SourceVersion", - "Build.SourceVersionAuthor", - "Job.AuthorizeAs", - "Release.Deployment.RequestedFor", - "Release.Deployment.RequestedForEmail", - "Release.RequestedFor", - "Release.RequestedForEmail", - }; - - public static readonly string PiiArtifactVariablePrefix = "Release.Artifacts"; - - public static readonly List PiiArtifactVariableSuffixes = new List() - { - "SourceBranch", - "SourceBranchName", - "SourceVersion", - "RequestedFor" - }; - public string Get(string name) { Variable variable; diff --git a/src/Runner.Worker/Worker.cs b/src/Runner.Worker/Worker.cs index 26d1a895aa6..f96bef6108a 100644 --- a/src/Runner.Worker/Worker.cs +++ b/src/Runner.Worker/Worker.cs @@ -74,7 +74,7 @@ public async Task RunAsync(string pipeIn, string pipeOut) SetCulture(jobMessage); // Start the job. - Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(WorkerUtilities.ScrubPiiData(jobMessage))}"); + Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(jobMessage)}"); Task jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token); // Start listening for a cancel message from the channel. diff --git a/src/Runner.Worker/WorkerUtilties.cs b/src/Runner.Worker/WorkerUtilties.cs deleted file mode 100644 index ac147046943..00000000000 --- a/src/Runner.Worker/WorkerUtilties.cs +++ /dev/null @@ -1,92 +0,0 @@ -using GitHub.DistributedTask.Pipelines.ContextData; -using GitHub.DistributedTask.WebApi; -using Pipelines = GitHub.DistributedTask.Pipelines; -using System; -using System.Collections.Generic; -using System.Linq; -using GitHub.Runner.Sdk; - -namespace GitHub.Runner.Worker -{ - public class WorkerUtilities - { - public static Pipelines.AgentJobRequestMessage ScrubPiiData(Pipelines.AgentJobRequestMessage message) - { - ArgUtil.NotNull(message, nameof(message)); - - var scrubbedVariables = new Dictionary(); - - // Scrub the known PII variables - foreach (var variable in message.Variables) - { - if (Variables.PiiVariables.Contains(variable.Key) || - (variable.Key.StartsWith(Variables.PiiArtifactVariablePrefix, StringComparison.OrdinalIgnoreCase) - && Variables.PiiArtifactVariableSuffixes.Any(varSuffix => variable.Key.EndsWith(varSuffix, StringComparison.OrdinalIgnoreCase)))) - { - scrubbedVariables[variable.Key] = "[PII]"; - } - else - { - scrubbedVariables[variable.Key] = variable.Value; - } - } - - var scrubbedRepositories = new List(); - - // Scrub the repository resources - foreach (var repository in message.Resources.Repositories) - { - Pipelines.RepositoryResource scrubbedRepository = repository.Clone(); - - var versionInfo = repository.Properties.Get(Pipelines.RepositoryPropertyNames.VersionInfo); - - if (versionInfo != null) - { - scrubbedRepository.Properties.Set( - Pipelines.RepositoryPropertyNames.VersionInfo, - new Pipelines.VersionInfo() - { - Author = "[PII]", - Message = versionInfo.Message - }); - } - - scrubbedRepositories.Add(scrubbedRepository); - } - - var scrubbedJobResources = new Pipelines.JobResources(); - - scrubbedJobResources.Containers.AddRange(message.Resources.Containers); - scrubbedJobResources.Endpoints.AddRange(message.Resources.Endpoints); - scrubbedJobResources.Repositories.AddRange(scrubbedRepositories); - scrubbedJobResources.SecureFiles.AddRange(message.Resources.SecureFiles); - - var contextData = new DictionaryContextData(); - if (message.ContextData?.Count > 0) - { - foreach (var pair in message.ContextData) - { - contextData[pair.Key] = pair.Value; - } - } - - // Reconstitute a new agent job request message from the scrubbed parts - return new Pipelines.AgentJobRequestMessage( - plan: message.Plan, - timeline: message.Timeline, - jobId: message.JobId, - jobDisplayName: message.JobDisplayName, - jobName: message.JobName, - jobContainer: message.JobContainer, - jobServiceContainers: message.JobServiceContainers, - environmentVariables: message.EnvironmentVariables, - variables: scrubbedVariables, - maskHints: message.MaskHints, - jobResources: scrubbedJobResources, - contextData: contextData, - workspaceOptions: message.Workspace, - steps: message.Steps, - scopes: message.Scopes); - } - } -} diff --git a/src/Test/L0/Worker/WorkerL0.cs b/src/Test/L0/Worker/WorkerL0.cs index f0d75ac5c69..4b03a327792 100644 --- a/src/Test/L0/Worker/WorkerL0.cs +++ b/src/Test/L0/Worker/WorkerL0.cs @@ -187,62 +187,6 @@ await Assert.ThrowsAsync( } } - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public void VerifyJobRequestMessagePiiDataIsScrubbed() - { - // Arrange - Pipelines.AgentJobRequestMessage message = CreateJobRequestMessage("jobwithpiidata"); - - // Populate PII variables - foreach (string piiVariable in Variables.PiiVariables) - { - message.Variables.Add(piiVariable, "MyPiiVariable"); - } - - foreach (string piiVariableSuffix in Variables.PiiArtifactVariableSuffixes) - { - message.Variables.Add($"{Variables.PiiArtifactVariablePrefix}.MyArtifact.{piiVariableSuffix}", "MyPiiVariable"); - } - - // Populate the repository PII data - Pipelines.RepositoryResource repository = new Pipelines.RepositoryResource(); - - repository.Properties.Set( - Pipelines.RepositoryPropertyNames.VersionInfo, - new Pipelines.VersionInfo() - { - Author = "MyAuthor", - Message = "MyMessage" - }); - - message.Resources.Repositories.Add(repository); - - // Act - Pipelines.AgentJobRequestMessage scrubbedMessage = WorkerUtilities.ScrubPiiData(message); - - // Assert - foreach (string piiVariable in Variables.PiiVariables) - { - scrubbedMessage.Variables.TryGetValue(piiVariable, out VariableValue value); - - Assert.Equal("[PII]", value.Value); - } - - foreach (string piiVariableSuffix in Variables.PiiArtifactVariableSuffixes) - { - scrubbedMessage.Variables.TryGetValue($"{Variables.PiiArtifactVariablePrefix}.MyArtifact.{piiVariableSuffix}", out VariableValue value); - - Assert.Equal("[PII]", value.Value); - } - - Pipelines.RepositoryResource scrubbedRepo = scrubbedMessage.Resources.Repositories[0]; - Pipelines.VersionInfo scrubbedInfo = scrubbedRepo.Properties.Get(Pipelines.RepositoryPropertyNames.VersionInfo); - - Assert.Equal("[PII]", scrubbedInfo.Author); - } - private bool IsMessageIdentical(Pipelines.AgentJobRequestMessage source, Pipelines.AgentJobRequestMessage target) { if (source == null && target == null)