From 6ead5312d2c4d13347acf83945e03a70d1354d5b Mon Sep 17 00:00:00 2001 From: Daiki AMINAKA <1991.daiki@gmail.com> Date: Tue, 4 Jun 2024 11:23:01 -0700 Subject: [PATCH] Fix HPS test for Linux xdp (#4298) * increase nofile for sudo. default was 1024 * enable linux XDP HPS test * use bash instead of pwsh * use internally pwsh * try full path for bash * print stderr to see pipeline specific issue * use file * print output * rollback and ccommentout throw * print * set LD_LIBRARY_PATH * print more * accelerate * export * rollback * 'Args' should not be used. it become empty * print parameters * test * cleanup * use limits.conf for ulimit setting --- docs/InteropServerSetup.md | 11 ++++++++++- scripts/prepare-machine.ps1 | 5 +++++ scripts/secnetperf-helpers.psm1 | 33 ++++++++++++++++++++++----------- scripts/secnetperf.ps1 | 5 +++++ scripts/spin.ps1 | 3 ++- scripts/test.ps1 | 3 ++- 6 files changed, 46 insertions(+), 14 deletions(-) diff --git a/docs/InteropServerSetup.md b/docs/InteropServerSetup.md index 8af46ac98..b5f2b5754 100644 --- a/docs/InteropServerSetup.md +++ b/docs/InteropServerSetup.md @@ -91,7 +91,7 @@ Feel free to update `$OutputDir` to whatever local directory you wish. The following commands (run as root) should configure core dumps to be created in the local directory: -``` +```sh # Enable core dumps for the system. sudo sh -c "echo 'root soft core unlimited' >> /etc/security/limits.conf" sudo sh -c "echo 'root hard core unlimited' >> /etc/security/limits.conf" @@ -101,3 +101,12 @@ sudo sh -c "echo '* hard core unlimited' >> /etc/security/limits.conf" # Set the core dump pattern. sudo sh -c "echo -n '%e.%p.%t.core' > /proc/sys/kernel/core_pattern" ``` + +The following are sudo basd run especially for using XDP +```sh +# Increase the number of file descriptors. +sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf" +sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf" +sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf" +sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf" +``` \ No newline at end of file diff --git a/scripts/prepare-machine.ps1 b/scripts/prepare-machine.ps1 index e357954f1..1b5aa32ef 100644 --- a/scripts/prepare-machine.ps1 +++ b/scripts/prepare-machine.ps1 @@ -558,6 +558,11 @@ if ($IsLinux) { sudo sh -c "echo 'root hard core unlimited' >> /etc/security/limits.conf" sudo sh -c "echo '* soft core unlimited' >> /etc/security/limits.conf" sudo sh -c "echo '* hard core unlimited' >> /etc/security/limits.conf" + # Increase the number of file descriptors. + sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf" + sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf" + sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf" + sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf" #sudo cat /etc/security/limits.conf # Set the core dump pattern. diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index 5f0045e42..b8782a806 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -254,9 +254,14 @@ function Cleanup-State { # Waits for a remote job to be ready based on looking for a particular string in # the output. function Start-RemoteServer { - param ($Session, $Command) + param ($Session, $Command, $ServerArgs, $UseSudo) # Start the server on the remote in an async job. - $job = Invoke-Command -Session $Session -ScriptBlock { iex $Using:Command } -AsJob + + if ($UseSudo) { + $job = Invoke-Command -Session $Session -ScriptBlock { iex "sudo LD_LIBRARY_PATH=$(Split-Path $Using:Command -Parent) $Using:Command $Using:ServerArgs" } -AsJob + } else { + $job = Invoke-Command -Session $Session -ScriptBlock { iex "$Using:Command $Using:ServerArgs"} -AsJob + } # Poll the job for 10 seconds to see if it started. $stopWatch = [system.diagnostics.stopwatch]::StartNew() while ($stopWatch.ElapsedMilliseconds -lt 10000) { @@ -303,15 +308,22 @@ function Stop-RemoteServer { # Creates a new local process to asynchronously run the test. function Start-LocalTest { - param ($FullPath, $FullArgs, $OutputDir) + param ($FullPath, $FullArgs, $OutputDir, $UseSudo) $pinfo = New-Object System.Diagnostics.ProcessStartInfo if ($IsWindows) { $pinfo.FileName = $FullPath $pinfo.Arguments = $FullArgs } else { # We use bash to execute the test so we can collect core dumps. - $pinfo.FileName = "bash" - $pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''`"" + $NOFILE = Invoke-Expression "bash -c 'ulimit -n'" + $CommonCommand = "ulimit -n $NOFILE && ulimit -c unlimited && LD_LIBRARY_PATH=$(Split-Path $FullPath -Parent) LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''" + if ($UseSudo) { + $pinfo.FileName = "/usr/bin/sudo" + $pinfo.Arguments = "/usr/bin/bash -c `"$CommonCommand`"" + } else { + $pinfo.FileName = "bash" + $pinfo.Arguments = "-c `"$CommonCommand`"" + } $pinfo.WorkingDirectory = $OutputDir } $pinfo.RedirectStandardOutput = $true @@ -499,14 +511,13 @@ function Invoke-Secnetperf { } # These scenarios are currently broken! TODO - Figure out why and fix them. - if (($io -eq "wsk" -and $metric -eq "hps") -or - (!$isWindows -and $io -eq "xdp" -and $metric -eq "hps")) { + if ($io -eq "wsk" -and $metric -eq "hps") { Write-Host "> secnetperf $clientArgs BROKEN!" continue } # Linux XDP requires sudo for now - $sudo = (!$IsWindows -and $io -eq "xdp") ? "sudo -E LD_LIBRARY_PATH=$RemoteDir/$(Split-Path $SecNetPerfPath -Parent) " : "" + $useSudo = (!$IsWindows -and $io -eq "xdp") $artifactName = $tcp -eq 0 ? "$TestId-quic" : "$TestId-tcp" New-Item -ItemType Directory "artifacts/logs/$artifactName" -ErrorAction Ignore | Out-Null @@ -536,7 +547,7 @@ function Invoke-Secnetperf { # Start the server running. "> secnetperf $serverArgs" | Add-Content $serverOut - $job = Start-RemoteServer $Session "$sudo$RemoteDir/$SecNetPerfPath $serverArgs" + $job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo # Run the test multiple times, failing (for now) only if all tries fail. # TODO: Once all failures have been fixed, consider all errors fatal. @@ -546,12 +557,12 @@ function Invoke-Secnetperf { Write-Host "==============================`nRUN $($try+1):" "> secnetperf $clientArgs" | Add-Content $clientOut try { - $process = Start-LocalTest "$sudo$clientPath" $clientArgs $artifactDir + $process = Start-LocalTest "$clientPath" $clientArgs $artifactDir $useSudo $rawOutput = Wait-LocalTest $process $artifactDir ($io -eq "wsk") 30000 Write-Host $rawOutput $values[$tcp] += Get-TestOutput $rawOutput $metric if ($extraOutput) { - if ($sudo -ne "") { + if ($useSudo) { sudo chown $UserName $extraOutput } $latency[$tcp] += Get-LatencyOutput $extraOutput diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index 445d4ce05..592e96041 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -273,6 +273,11 @@ if (!$isWindows) { sudo sh -c "echo "root hard core unlimited" >> /etc/security/limits.conf" sudo sh -c "echo "* soft core unlimited" >> /etc/security/limits.conf" sudo sh -c "echo "* hard core unlimited" >> /etc/security/limits.conf" + # Increase the number of file descriptors. + sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf" + sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf" + sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf" + sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf" } # Set the core dump pattern. diff --git a/scripts/spin.ps1 b/scripts/spin.ps1 index f8995e797..7eaeb58f2 100644 --- a/scripts/spin.ps1 +++ b/scripts/spin.ps1 @@ -188,7 +188,8 @@ if (![string]::IsNullOrWhiteSpace($ExtraArtifactDir)) { # Run the script. if ($IsLinux -and $UseXdp) { - Invoke-Expression ('sudo pwsh -c "$RunExecutable $Arguments"') + $NOFILE = Invoke-Expression "bash -c 'ulimit -n'" + Invoke-Expression ('/usr/bin/sudo bash -c "ulimit -n $NOFILE && pwsh $RunExecutable $Arguments"') } else { Invoke-Expression ($RunExecutable + " " + $Arguments) } diff --git a/scripts/test.ps1 b/scripts/test.ps1 index 71018c02c..efcd1f589 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -366,7 +366,8 @@ for ($iteration = 1; $iteration -le $NumIterations; $iteration++) { # Run the script. foreach ($TestPath in $TestPaths) { if ($IsLinux -and $UseXdp) { - Invoke-Expression ("sudo pwsh -c " + $RunTest + " -Path $TestPath " + $TestArguments) + $NOFILE = Invoke-Expression "bash -c 'ulimit -n'" + Invoke-Expression ('/usr/bin/sudo bash -c "ulimit -n $NOFILE && pwsh $RunTest -Path $TestPath $TestArguments"') } else { Invoke-Expression ($RunTest + " -Path $TestPath " + $TestArguments) }