From e51dc04b292da900551b4061fe6bab64b9493961 Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Thu, 10 Oct 2024 11:22:52 -0400 Subject: [PATCH 1/6] Standard SecNetPerf Scenarios --- scripts/secnetperf-helpers.psm1 | 41 ++++++++++++++--------------- scripts/secnetperf.ps1 | 23 +++++++---------- src/perf/lib/PerfClient.cpp | 46 +++++++++++++++++++++++++++++++++ src/perf/lib/SecNetPerfMain.cpp | 24 ++++++++++++++++- 4 files changed, 98 insertions(+), 36 deletions(-) diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index 0ecd7c94b0..0a768477ee 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -524,7 +524,7 @@ function Get-LatencyOutput { # Invokes secnetperf with the given arguments for both TCP and QUIC. function Invoke-Secnetperf { - param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter, $Environment, $RunId, $SyncerSecret) + param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $Scenario, $io, $Filter, $Environment, $RunId, $SyncerSecret) $values = @(@(), @()) $latency = $null @@ -536,26 +536,25 @@ function Invoke-Secnetperf { $tcpSupported = 0 } $metric = "throughput" - if ($exeArgs.Contains("conns:16cpu")) { # TODO: figure out a better way to detect max RPS tests + if ($Scenario.Contains("rps")) { $metric = "rps" - } elseif ($exeArgs.Contains("plat:1")) { + } elseif ($Scenario.Contains("latency")) { $metric = "latency" $latency = @(@(), @()) $extraOutput = Repo-Path "latency.txt" if (!$isWindows) { chmod +rw "$extraOutput" } - } elseif ($exeArgs.Contains("prate:1")) { + } elseif ($Scenario.Contains("hps")) { $metric = "hps" } for ($tcp = 0; $tcp -le $tcpSupported; $tcp++) { # Set up all the parameters and paths for running the test. - $execMode = $ExeArgs.Substring(0, $ExeArgs.IndexOf(" ")) # First arg is the exec mode $clientPath = Repo-Path $SecNetPerfPath - $serverArgs = "$execMode -io:$io" - $clientArgs = "-target:$RemoteName $ExeArgs -tcp:$tcp -trimout -watchdog:25000" + $serverArgs = "-scenario:$Scenario -io:$io" + $clientArgs = "-target:$RemoteName -scenario:$Scenario -io:$io -tcp:$tcp -trimout -watchdog:25000" if ($io -eq "xdp" -or $io -eq "qtip") { $serverArgs += " -pollidle:10000" $clientArgs += " -pollidle:10000" @@ -584,9 +583,9 @@ function Invoke-Secnetperf { $useSudo = (!$isWindows -and $io -eq "xdp") if ($tcp -eq 0) { - $artifactName = "$TestId-quic" + $artifactName = "$Scenario-quic" } else { - $artifactName = "$TestId-tcp" + $artifactName = "$Scenario-tcp" } New-Item -ItemType Directory "artifacts/logs/$artifactName" -ErrorAction Ignore | Out-Null $artifactDir = Repo-Path "artifacts/logs/$artifactName" @@ -715,14 +714,14 @@ function Invoke-Secnetperf { } } -function CheckRegressionResult($values, $testid, $transport, $regressionJson, $envStr) { +function CheckRegressionResult($values, $scenario, $transport, $regressionJson, $envStr) { $sum = 0 foreach ($item in $values) { $sum += $item } $avg = $sum / $values.Length - $Testid = "$testid-$transport" + $Scenario = "$scenario-$transport" $res = @{ Baseline = "N/A" @@ -734,14 +733,14 @@ function CheckRegressionResult($values, $testid, $transport, $regressionJson, $e } try { - $res.Baseline = $regressionJson.$Testid.$envStr.baseline - $res.BestResult = $regressionJson.$Testid.$envStr.BestResult - $res.BestResultCommit = $regressionJson.$Testid.$envStr.BestResultCommit + $res.Baseline = $regressionJson.$Scenario.$envStr.baseline + $res.BestResult = $regressionJson.$Scenario.$envStr.BestResult + $res.BestResultCommit = $regressionJson.$Scenario.$envStr.BestResultCommit $res.CumulativeResult = $avg $res.AggregateFunction = "AVG" if ($avg -lt $res.Baseline) { - Write-GHError "Regression detected in $Testid for $envStr. See summary table for details." + Write-GHError "Regression detected in $Scenario for $envStr. See summary table for details." $res.HasRegression = $true } } catch { @@ -751,7 +750,7 @@ function CheckRegressionResult($values, $testid, $transport, $regressionJson, $e return $res } -function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envStr) { +function CheckRegressionLat($values, $regressionJson, $scenario, $transport, $envStr) { # TODO: Right now, we are not using a watermark based method for regression detection of latency percentile values because we don't know how to determine a "Best Ever" distribution. # (we are just looking at P0, P50, P99 columns, and computing the baseline for each percentile as the mean - 2 * std of the last 20 runs. ) @@ -764,7 +763,7 @@ function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envS } $RpsAvg /= $NumRuns - $Testid = "$testid-$transport" + $Scenario = "$scenario-$transport" $res = @{ Baseline = "N/A" @@ -776,14 +775,14 @@ function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envS } try { - $res.Baseline = $regressionJson.$Testid.$envStr.baseline - $res.BestResult = $regressionJson.$Testid.$envStr.BestResult - $res.BestResultCommit = $regressionJson.$Testid.$envStr.BestResultCommit + $res.Baseline = $regressionJson.$Scenario.$envStr.baseline + $res.BestResult = $regressionJson.$Scenario.$envStr.BestResult + $res.BestResultCommit = $regressionJson.$Scenario.$envStr.BestResultCommit $res.CumulativeResult = $RpsAvg $res.AggregateFunction = "AVG" if ($RpsAvg -lt $res.Baseline) { - Write-GHError "RPS Regression detected in $Testid for $envStr. See summary table for details." + Write-GHError "RPS Regression detected in $Scenario for $envStr. See summary table for details." $res.HasRegression = $true } } catch { diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index 332e2fe3c4..ee626b76b8 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -254,11 +254,7 @@ if ($isWindows) { $allTests = [System.Collections.Specialized.OrderedDictionary]::new() # > All tests: -$allTests["tput-up"] = "-exec:maxtput -up:12s -ptput:1" -$allTests["tput-down"] = "-exec:maxtput -down:12s -ptput:1" -$allTests["hps-conns-100"] = "-exec:maxtput -rconn:1 -share:1 -conns:100 -run:12s -prate:1" -$allTests["rps-up-512-down-4000"] = "-exec:lowlat -rstream:1 -up:512 -down:4000 -run:20s -plat:1" -$allTests["max-rps-up-512-down-4000"] = "-exec:lowlat -conns:16cpu -streams:10 -rstream:1 -up:512 -down:4000 -run:20s -plat:1" +$allTests = @("upload", "download", "hps", "rps-single", "rps-multi", "latency") $hasFailures = $false $json["run_args"] = $allTests @@ -336,9 +332,8 @@ $regressionJson = Get-Content -Raw -Path "watermark_regression.json" | ConvertFr # Run all the test cases. Write-Host "Setup complete! Running all tests" -foreach ($testId in $allTests.Keys) { - $ExeArgs = $allTests[$testId] + " -io:$io" - $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $testId $ExeArgs $io $filter $environment $RunId $SyncerSecret +foreach ($scenario in $allTests) { + $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $scenario $io $filter $environment $RunId $SyncerSecret $Test = $Output[-1] if ($Test.HasFailures) { $hasFailures = $true } @@ -349,15 +344,15 @@ foreach ($testId in $allTests.Keys) { } else { $transport = "quic" } - $json["$testId-$transport"] = $Test.Values[$tcp] + $json["$scenario-$transport"] = $Test.Values[$tcp] if ($Test.Metric -eq "latency") { - $json["$testId-$transport-lat"] = $Test.Latency[$tcp] - $LatencyRegression = CheckRegressionLat $Test.Values[$tcp] $regressionJson $testId $transport "$os-$arch-$environment-$io-$tls" - $json["$testId-$transport-regression"] = $LatencyRegression + $json["$scenario-$transport-lat"] = $Test.Latency[$tcp] + $LatencyRegression = CheckRegressionLat $Test.Values[$tcp] $regressionJson $scenario $transport "$os-$arch-$environment-$io-$tls" + $json["$scenario-$transport-regression"] = $LatencyRegression } else { - $ResultRegression = CheckRegressionResult $Test.Values[$tcp] $testId $transport $regressionJson "$os-$arch-$environment-$io-$tls" - $json["$testId-$transport-regression"] = $ResultRegression + $ResultRegression = CheckRegressionResult $Test.Values[$tcp] $scenario $transport $regressionJson "$os-$arch-$environment-$io-$tls" + $json["$scenario-$transport-regression"] = $ResultRegression } } } diff --git a/src/perf/lib/PerfClient.cpp b/src/perf/lib/PerfClient.cpp index 6cd808fe3a..cf4142ea27 100644 --- a/src/perf/lib/PerfClient.cpp +++ b/src/perf/lib/PerfClient.cpp @@ -108,6 +108,52 @@ PerfClient::Init( CountMult[0] = CxPlatProcCount(); + // + // Scenario profile sets new defauls for values below, that may then be + // further overridden by command line arguments. + // + const char* ScenarioStr = GetValue(argc, argv, "scenario"); + if (ScenarioStr != nullptr) { + if (IsValue(ScenarioStr, "upload")) { + Upload = S_TO_US(12); // 12 seconds + Timed = TRUE; + PrintThroughput = TRUE; + } else if (IsValue(ScenarioStr, "download")) { + Download = S_TO_US(12); // 12 seconds + Timed = TRUE; + PrintThroughput = TRUE; + } else if (IsValue(ScenarioStr, "hps")) { + ConnectionCount = 16 * CxPlatProcCount(); + RunTime = S_TO_US(12); // 12 seconds + RepeatConnections = TRUE; + PrintIoRate = TRUE; + } else if (IsValue(ScenarioStr, "rps-single")) { + Upload = 512; + Download = 4000; + StreamCount = 100; + RunTime = S_TO_US(20); // 20 seconds + RepeatStreams = TRUE; + PrintLatency = TRUE; + } else if (IsValue(ScenarioStr, "rps-multi")) { + Upload = 512; + Download = 4000; + ConnectionCount = 16 * CxPlatProcCount(); + StreamCount = 10; + RunTime = S_TO_US(20); // 20 seconds + RepeatStreams = TRUE; + PrintLatency = TRUE; + } else if (IsValue(ScenarioStr, "latency")) { + Upload = 512; + Download = 4000; + RunTime = S_TO_US(20); // 20 seconds + RepeatStreams = TRUE; + PrintLatency = TRUE; + } else { + WriteOutput("Failed to parse scenario profile[%s]!\n"); + return QUIC_STATUS_INVALID_PARAMETER; + } + } + // // Remote target/server options // diff --git a/src/perf/lib/SecNetPerfMain.cpp b/src/perf/lib/SecNetPerfMain.cpp index 3ba2f5486f..3c58c1627e 100644 --- a/src/perf/lib/SecNetPerfMain.cpp +++ b/src/perf/lib/SecNetPerfMain.cpp @@ -77,6 +77,8 @@ PrintHelp( " -platency<0/1> Print latency statistics. (def:0)\n" "\n" " Scenario options:\n" + " -scenario: Scenario profile to use.\n" + " - {upload, download, hps, rps-single, rps-multi, latency}.\n" " -conns:<####> The number of connections to use. (def:1)\n" " -streams:<####> The number of streams to send on at a time. (def:0)\n" " -upload:<####>[unit] The length of bytes to send on each stream, with an optional (time or length) unit. (def:0)\n" @@ -199,6 +201,25 @@ QuicMainStart( return Status; } + const char* ScenarioStr = GetValue(argc, argv, "scenario"); + if (ScenarioStr != nullptr) { + if (IsValue(ScenarioStr, "upload") || + IsValue(ScenarioStr, "download") || + IsValue(ScenarioStr, "hps")) { + PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_MAX_THROUGHPUT; + TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_MAX_THROUGHPUT; + } else if ( + IsValue(ScenarioStr, "rps-single") || + IsValue(ScenarioStr, "rps-multi") || + IsValue(ScenarioStr, "latency")) { + PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_LOW_LATENCY; + TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_LOW_LATENCY; + } else { + WriteOutput("Failed to parse scenario profile[%s]!\n"); + return QUIC_STATUS_INVALID_PARAMETER; + } + } + const char* ExecStr = GetValue(argc, argv, "exec"); if (ExecStr != nullptr) { if (IsValue(ExecStr, "lowlat")) { @@ -212,7 +233,8 @@ QuicMainStart( } else if (IsValue(ExecStr, "realtime")) { PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_REAL_TIME; } else { - WriteOutput("Failed to parse execution profile[%s], use lowlat as default for QUIC, lowlat as default for TCP.\n", ExecStr); + WriteOutput("Failed to parse execution profile[%s]!\n"); + return QUIC_STATUS_INVALID_PARAMETER; } } From fadaad25c068fbba3a8786d0e25ce9e47c2d7ca9 Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Thu, 10 Oct 2024 11:43:42 -0400 Subject: [PATCH 2/6] Fix compiler errors --- src/perf/lib/PerfClient.cpp | 2 +- src/perf/lib/SecNetPerfMain.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/perf/lib/PerfClient.cpp b/src/perf/lib/PerfClient.cpp index cf4142ea27..b70f008422 100644 --- a/src/perf/lib/PerfClient.cpp +++ b/src/perf/lib/PerfClient.cpp @@ -149,7 +149,7 @@ PerfClient::Init( RepeatStreams = TRUE; PrintLatency = TRUE; } else { - WriteOutput("Failed to parse scenario profile[%s]!\n"); + WriteOutput("Failed to parse scenario profile[%s]!\n", ScenarioStr); return QUIC_STATUS_INVALID_PARAMETER; } } diff --git a/src/perf/lib/SecNetPerfMain.cpp b/src/perf/lib/SecNetPerfMain.cpp index 3c58c1627e..c5727d9f01 100644 --- a/src/perf/lib/SecNetPerfMain.cpp +++ b/src/perf/lib/SecNetPerfMain.cpp @@ -215,7 +215,7 @@ QuicMainStart( PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_LOW_LATENCY; TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_LOW_LATENCY; } else { - WriteOutput("Failed to parse scenario profile[%s]!\n"); + WriteOutput("Failed to parse scenario profile[%s]!\n", ScenarioStr); return QUIC_STATUS_INVALID_PARAMETER; } } @@ -233,7 +233,7 @@ QuicMainStart( } else if (IsValue(ExecStr, "realtime")) { PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_REAL_TIME; } else { - WriteOutput("Failed to parse execution profile[%s]!\n"); + WriteOutput("Failed to parse execution profile[%s]!\n", ExecStr); return QUIC_STATUS_INVALID_PARAMETER; } } From 10a156e1d98fc04da3e4932cc48a8eabadfc24cb Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Fri, 11 Oct 2024 08:08:50 -0400 Subject: [PATCH 3/6] nits --- scripts/secnetperf.ps1 | 8 +++----- src/perf/lib/PerfClient.cpp | 2 +- src/perf/lib/SecNetPerfMain.cpp | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index ee626b76b8..5abff3b504 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -251,13 +251,11 @@ if ($isWindows) { $kernelVersion = bash -c "uname -r" $json["os_version"] = "$osName $kernelVersion" } -$allTests = [System.Collections.Specialized.OrderedDictionary]::new() -# > All tests: -$allTests = @("upload", "download", "hps", "rps-single", "rps-multi", "latency") +# Test all supported scenarios. +$allScenarios = @("upload", "download", "hps", "rps", "rps-multi", "latency") $hasFailures = $false -$json["run_args"] = $allTests try { @@ -332,7 +330,7 @@ $regressionJson = Get-Content -Raw -Path "watermark_regression.json" | ConvertFr # Run all the test cases. Write-Host "Setup complete! Running all tests" -foreach ($scenario in $allTests) { +foreach ($scenario in $allScenarios) { $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $scenario $io $filter $environment $RunId $SyncerSecret $Test = $Output[-1] if ($Test.HasFailures) { $hasFailures = $true } diff --git a/src/perf/lib/PerfClient.cpp b/src/perf/lib/PerfClient.cpp index b70f008422..01dacf3fe3 100644 --- a/src/perf/lib/PerfClient.cpp +++ b/src/perf/lib/PerfClient.cpp @@ -127,7 +127,7 @@ PerfClient::Init( RunTime = S_TO_US(12); // 12 seconds RepeatConnections = TRUE; PrintIoRate = TRUE; - } else if (IsValue(ScenarioStr, "rps-single")) { + } else if (IsValue(ScenarioStr, "rps")) { Upload = 512; Download = 4000; StreamCount = 100; diff --git a/src/perf/lib/SecNetPerfMain.cpp b/src/perf/lib/SecNetPerfMain.cpp index c5727d9f01..d4103fb13a 100644 --- a/src/perf/lib/SecNetPerfMain.cpp +++ b/src/perf/lib/SecNetPerfMain.cpp @@ -78,7 +78,7 @@ PrintHelp( "\n" " Scenario options:\n" " -scenario: Scenario profile to use.\n" - " - {upload, download, hps, rps-single, rps-multi, latency}.\n" + " - {upload, download, hps, rps, rps-multi, latency}.\n" " -conns:<####> The number of connections to use. (def:1)\n" " -streams:<####> The number of streams to send on at a time. (def:0)\n" " -upload:<####>[unit] The length of bytes to send on each stream, with an optional (time or length) unit. (def:0)\n" @@ -209,7 +209,7 @@ QuicMainStart( PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_MAX_THROUGHPUT; TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_MAX_THROUGHPUT; } else if ( - IsValue(ScenarioStr, "rps-single") || + IsValue(ScenarioStr, "rps") || IsValue(ScenarioStr, "rps-multi") || IsValue(ScenarioStr, "latency")) { PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_LOW_LATENCY; From 3548145fd910e483437c9c28dbfd68ee7ec277fb Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Fri, 11 Oct 2024 08:40:04 -0400 Subject: [PATCH 4/6] reorder IsValue checks --- src/perf/lib/PerfClient.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/perf/lib/PerfClient.cpp b/src/perf/lib/PerfClient.cpp index 01dacf3fe3..9f00687115 100644 --- a/src/perf/lib/PerfClient.cpp +++ b/src/perf/lib/PerfClient.cpp @@ -127,18 +127,18 @@ PerfClient::Init( RunTime = S_TO_US(12); // 12 seconds RepeatConnections = TRUE; PrintIoRate = TRUE; - } else if (IsValue(ScenarioStr, "rps")) { + } else if (IsValue(ScenarioStr, "rps-multi")) { Upload = 512; Download = 4000; - StreamCount = 100; + ConnectionCount = 16 * CxPlatProcCount(); + StreamCount = 10; RunTime = S_TO_US(20); // 20 seconds RepeatStreams = TRUE; PrintLatency = TRUE; - } else if (IsValue(ScenarioStr, "rps-multi")) { + } else if (IsValue(ScenarioStr, "rps")) { Upload = 512; Download = 4000; - ConnectionCount = 16 * CxPlatProcCount(); - StreamCount = 10; + StreamCount = 100; RunTime = S_TO_US(20); // 20 seconds RepeatStreams = TRUE; PrintLatency = TRUE; From 3afdfbfec39c54ecc159d049102e8be9fbe2992a Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Fri, 11 Oct 2024 09:22:17 -0400 Subject: [PATCH 5/6] Fix string compare helper --- src/inc/msquichelper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/msquichelper.h b/src/inc/msquichelper.h index 4572ddbd3e..529c68413d 100644 --- a/src/inc/msquichelper.h +++ b/src/inc/msquichelper.h @@ -304,7 +304,7 @@ IsValue( _In_z_ const char* toTestAgainst ) { - return _strnicmp(name, toTestAgainst, CXPLAT_MIN(strlen(name), strlen(toTestAgainst))) == 0; + return _strnicmp(name, toTestAgainst, strlen(toTestAgainst)) == 0; } inline From 410e49227065e7c1f52b0cfae97d7883a74aea92 Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Fri, 11 Oct 2024 10:27:05 -0400 Subject: [PATCH 6/6] More streams --- src/perf/lib/PerfClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/perf/lib/PerfClient.cpp b/src/perf/lib/PerfClient.cpp index 9f00687115..bdde1efdb5 100644 --- a/src/perf/lib/PerfClient.cpp +++ b/src/perf/lib/PerfClient.cpp @@ -131,7 +131,7 @@ PerfClient::Init( Upload = 512; Download = 4000; ConnectionCount = 16 * CxPlatProcCount(); - StreamCount = 10; + StreamCount = 100; RunTime = S_TO_US(20); // 20 seconds RepeatStreams = TRUE; PrintLatency = TRUE;