Fix HPS test for Linux xdp (#4298)

* increase nofile for sudo. default was 1024

* enable linux XDP HPS test

* use bash instead of pwsh

* use internally pwsh

* try full path for bash

* print stderr to see pipeline specific issue

* use file

* print output

* rollback and ccommentout throw

* print

* set LD_LIBRARY_PATH

* print more

* accelerate

* export

* rollback

* 'Args' should not be used. it become empty

* print parameters

* test

* cleanup

* use limits.conf for ulimit setting

docs/InteropServerSetup.md

@@ -91,7 +91,7 @@ Feel free to update `$OutputDir` to whatever local directory you wish.
 
 The following commands (run as root) should configure core dumps to be created in the local directory:
 
-```
+```sh
 # Enable core dumps for the system.
 sudo sh -c "echo 'root soft core unlimited' >> /etc/security/limits.conf"
 sudo sh -c "echo 'root hard core unlimited' >> /etc/security/limits.conf"
@@ -101,3 +101,12 @@ sudo sh -c "echo '* hard core unlimited' >> /etc/security/limits.conf"
 # Set the core dump pattern.
 sudo sh -c "echo -n '%e.%p.%t.core' > /proc/sys/kernel/core_pattern"
 ```
+
+The following are sudo basd run especially for using XDP
+```sh
+# Increase the number of file descriptors.
+sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf"
+sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf"
+sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf"
+sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf"
+```

scripts/prepare-machine.ps1

@@ -558,6 +558,11 @@ if ($IsLinux) {
         sudo sh -c "echo 'root hard core unlimited' >> /etc/security/limits.conf"
         sudo sh -c "echo '* soft core unlimited' >> /etc/security/limits.conf"
         sudo sh -c "echo '* hard core unlimited' >> /etc/security/limits.conf"
+        # Increase the number of file descriptors.
+        sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf"
         #sudo cat /etc/security/limits.conf
 
         # Set the core dump pattern.

scripts/secnetperf-helpers.psm1

@@ -254,9 +254,14 @@ function Cleanup-State {
 # Waits for a remote job to be ready based on looking for a particular string in
 # the output.
 function Start-RemoteServer {
-    param ($Session, $Command)
+    param ($Session, $Command, $ServerArgs, $UseSudo)
     # Start the server on the remote in an async job.
-    $job = Invoke-Command -Session $Session -ScriptBlock { iex $Using:Command } -AsJob
+
+    if ($UseSudo) {
+        $job = Invoke-Command -Session $Session -ScriptBlock { iex "sudo LD_LIBRARY_PATH=$(Split-Path $Using:Command -Parent)  $Using:Command $Using:ServerArgs" } -AsJob
+    } else {
+        $job = Invoke-Command -Session $Session -ScriptBlock { iex "$Using:Command $Using:ServerArgs"} -AsJob
+    }
     # Poll the job for 10 seconds to see if it started.
     $stopWatch = [system.diagnostics.stopwatch]::StartNew()
     while ($stopWatch.ElapsedMilliseconds -lt 10000) {
@@ -303,15 +308,22 @@ function Stop-RemoteServer {
 
 # Creates a new local process to asynchronously run the test.
 function Start-LocalTest {
-    param ($FullPath, $FullArgs, $OutputDir)
+    param ($FullPath, $FullArgs, $OutputDir, $UseSudo)
     $pinfo = New-Object System.Diagnostics.ProcessStartInfo
     if ($IsWindows) {
         $pinfo.FileName = $FullPath
         $pinfo.Arguments = $FullArgs
     } else {
         # We use bash to execute the test so we can collect core dumps.
-        $pinfo.FileName = "bash"
+        $NOFILE = Invoke-Expression "bash -c 'ulimit -n'"
-        $pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''`""
+        $CommonCommand = "ulimit -n $NOFILE && ulimit -c unlimited && LD_LIBRARY_PATH=$(Split-Path $FullPath -Parent) LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''"
+        if ($UseSudo) {
+            $pinfo.FileName = "/usr/bin/sudo"
+            $pinfo.Arguments = "/usr/bin/bash -c `"$CommonCommand`""
+        } else {
+            $pinfo.FileName = "bash"
+            $pinfo.Arguments = "-c `"$CommonCommand`""
+        }
         $pinfo.WorkingDirectory = $OutputDir
     }
     $pinfo.RedirectStandardOutput = $true
@@ -499,14 +511,13 @@ function Invoke-Secnetperf {
     }
 
     # These scenarios are currently broken! TODO - Figure out why and fix them.
-    if (($io -eq "wsk" -and $metric -eq "hps") -or
+    if ($io -eq "wsk" -and $metric -eq "hps") {
-        (!$isWindows -and $io -eq "xdp" -and $metric -eq "hps")) {
         Write-Host "> secnetperf $clientArgs BROKEN!"
         continue
     }
 
      # Linux XDP requires sudo for now
-    $sudo = (!$IsWindows -and $io -eq "xdp") ? "sudo -E LD_LIBRARY_PATH=$RemoteDir/$(Split-Path $SecNetPerfPath -Parent) " : ""
+    $useSudo = (!$IsWindows -and $io -eq "xdp")
 
     $artifactName = $tcp -eq 0 ? "$TestId-quic" : "$TestId-tcp"
     New-Item -ItemType Directory "artifacts/logs/$artifactName" -ErrorAction Ignore | Out-Null
@@ -536,7 +547,7 @@ function Invoke-Secnetperf {
 
     # Start the server running.
     "> secnetperf $serverArgs" | Add-Content $serverOut
-    $job = Start-RemoteServer $Session "$sudo$RemoteDir/$SecNetPerfPath $serverArgs"
+    $job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo
 
     # Run the test multiple times, failing (for now) only if all tries fail.
     # TODO: Once all failures have been fixed, consider all errors fatal.
@@ -546,12 +557,12 @@ function Invoke-Secnetperf {
         Write-Host "==============================`nRUN $($try+1):"
         "> secnetperf $clientArgs" | Add-Content $clientOut
         try {
-            $process = Start-LocalTest "$sudo$clientPath" $clientArgs $artifactDir
+            $process = Start-LocalTest "$clientPath" $clientArgs $artifactDir $useSudo
             $rawOutput = Wait-LocalTest $process $artifactDir ($io -eq "wsk") 30000
             Write-Host $rawOutput
             $values[$tcp] += Get-TestOutput $rawOutput $metric
             if ($extraOutput) {
-                if ($sudo -ne "") {
+                if ($useSudo) {
                     sudo chown $UserName $extraOutput
                 }
                 $latency[$tcp] += Get-LatencyOutput $extraOutput

scripts/secnetperf.ps1

@@ -273,6 +273,11 @@ if (!$isWindows) {
         sudo sh -c "echo "root hard core unlimited" >> /etc/security/limits.conf"
         sudo sh -c "echo "* soft core unlimited" >> /etc/security/limits.conf"
         sudo sh -c "echo "* hard core unlimited" >> /etc/security/limits.conf"
+        # Increase the number of file descriptors.
+        sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf"
     }
 
     # Set the core dump pattern.

scripts/spin.ps1

@@ -188,7 +188,8 @@ if (![string]::IsNullOrWhiteSpace($ExtraArtifactDir)) {
 
 # Run the script.
 if ($IsLinux -and $UseXdp) {
-    Invoke-Expression ('sudo pwsh -c "$RunExecutable $Arguments"')
+    $NOFILE = Invoke-Expression "bash -c 'ulimit -n'"
+    Invoke-Expression ('/usr/bin/sudo bash -c "ulimit -n $NOFILE && pwsh $RunExecutable $Arguments"')
 } else {
     Invoke-Expression ($RunExecutable + " " + $Arguments)
 }

scripts/test.ps1

@@ -366,7 +366,8 @@ for ($iteration = 1; $iteration -le $NumIterations; $iteration++) {
     # Run the script.
     foreach ($TestPath in $TestPaths) {
         if ($IsLinux -and $UseXdp) {
-            Invoke-Expression ("sudo pwsh -c " + $RunTest + " -Path $TestPath " + $TestArguments)
+            $NOFILE = Invoke-Expression "bash -c 'ulimit -n'"
+            Invoke-Expression ('/usr/bin/sudo bash -c "ulimit -n $NOFILE && pwsh $RunTest -Path $TestPath $TestArguments"')
         } else {
             Invoke-Expression ($RunTest + " -Path $TestPath " + $TestArguments)
         }