Run E2E tests on Linux in build pipeline and add Backward/Forward E2E tests (#737)

2020-11-03 11:54:28 -08:00 · 2020-11-03 11:54:28 -08:00 · 27e67ac447
--- a/azure-pipelines-e2e-tests-template.yml
+++ b/azure-pipelines-e2e-tests-template.yml
@ -1,57 +1,145 @@
 parameters:
- name: 'versions'
+- name: tests
  type: object
  default: {}
- name: 'testOptions'
+- name: backwardCompatibleRelease
+  type: string
+  default: ''
+- name: forwardCompatibleRelease
  type: string
  default: ''

+
 stages:
- ${{ each version in parameters.versions }}:
-  - stage: E2E_Tests_${{ replace(version, '.', '_') }}
-    displayName: E2E tests for Spark ${{ version }}
+- ${{ each test in parameters.tests }}:
+  - stage: E2E_Tests_${{ replace(test.version, '.', '_') }}
+    displayName: E2E tests for Spark ${{ test.version }}
    dependsOn: Build
    jobs:
-    - job: Run
-      pool: Hosted VS2017
-  
-      variables:
-        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-          _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
-        HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
-        DOTNET_WORKER_DIR: $(CurrentDotnetWorkerDir)
+    - ${{ each option in test.jobOptions }}:
+      - job: Run_${{ replace(option.pool, ' ', '_') }}
+        pool: ${{ option.pool }}

-      steps:
-      - task: DownloadBuildArtifacts@0
-        displayName: Download Build Artifacts
-        inputs:
-          artifactName: Microsoft.Spark.Binaries
-          downloadPath: $(Build.ArtifactStagingDirectory)
+        variables:
+          ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+            _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)

-      - task: CopyFiles@2
-        displayName: Copy jars
-        inputs:
-          sourceFolder: $(ArtifactPath)/Jars
-          contents: '**/*.jar'
-          targetFolder: $(Build.SourcesDirectory)/src/scala
+        steps:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Build Artifacts
+          inputs:
+            artifactName: Microsoft.Spark.Binaries
+            downloadPath: $(Build.ArtifactStagingDirectory)

-      - task: BatchScript@1
-        displayName: Download Winutils.exe
-        inputs:
-          filename: script\download-hadoop-utils.cmd
-          arguments: $(Build.BinariesDirectory)
+        - pwsh: |
+            $framework = "netcoreapp3.1"

-      - task: BatchScript@1
-        displayName: 'Download Spark Distro ${{ version }}'
-        inputs:
-          filename: script\download-spark-distros.cmd
-          arguments: $(Build.BinariesDirectory) ${{ version }}
+            if ($env:AGENT_OS -eq 'Windows_NT') {
+              $runtimeIdentifier = "win-x64"
+            } else {
+              $runtimeIdentifier = "linux-x64"
+            }

-      - task: DotNetCoreCLI@2
-        displayName: 'E2E tests'
-        inputs:
-          command: test
-          projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
-          arguments: '--configuration $(buildConfiguration) ${{ parameters.testOptions }}'
-        env:
-          SPARK_HOME: $(Build.BinariesDirectory)\spark-${{ version }}-bin-hadoop2.7
+            $pathSeparator = [IO.Path]::DirectorySeparatorChar
+            $artifactPath = "$(Build.ArtifactStagingDirectory)${pathSeparator}Microsoft.Spark.Binaries"
+            echo "##vso[task.setvariable variable=PATH_SEPARATOR]$pathSeparator"
+            echo "##vso[task.setvariable variable=ArtifactPath]$artifactPath"
+
+            $backwardCompatibleRelease = "${{ parameters.backwardCompatibleRelease }}"
+            echo "##vso[task.setvariable variable=BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR]$(Build.BinariesDirectory)${pathSeparator}Microsoft.Spark.Worker-${backwardCompatibleRelease}"
+            echo "##vso[task.setvariable variable=BACKWARD_COMPATIBLE_WORKER_URL]https://github.com/dotnet/spark/releases/download/v${backwardCompatibleRelease}/Microsoft.Spark.Worker.${framework}.${runtimeIdentifier}-${backwardCompatibleRelease}.zip"
+
+            $dotnetWorkerDir = "${artifactPath}${pathSeparator}Microsoft.Spark.Worker${pathSeparator}${framework}${pathSeparator}${runtimeIdentifier}"
+            echo "##vso[task.setvariable variable=CURRENT_DOTNET_WORKER_DIR]$dotnetWorkerDir"
+            if ($env:AGENT_OS -eq 'Linux') {
+              chmod +x "${dotnetWorkerDir}${pathSeparator}Microsoft.Spark.Worker"
+            }
+          displayName: 'Setup Variables and Permissions'
+
+        - checkout: self
+          path: s$(PATH_SEPARATOR)dotnet-spark
+
+        - task: CopyFiles@2
+          displayName: Copy jars
+          inputs:
+            sourceFolder: $(ArtifactPath)$(PATH_SEPARATOR)Jars
+            contents: '**$(PATH_SEPARATOR)*.jar'
+            targetFolder: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark$(PATH_SEPARATOR)src$(PATH_SEPARATOR)scala
+
+        - task: PowerShell@2
+          condition: eq( variables['Agent.OS'], 'Windows_NT' )
+          displayName: Download Winutils.exe
+          inputs:
+            workingDirectory: $(Build.BinariesDirectory)
+            pwsh: true
+            targetType: inline
+            script: |
+              echo "Download Hadoop utils for Windows."
+              curl -k -L -o hadoop.zip https://github.com/steveloughran/winutils/releases/download/tag_2017-08-29-hadoop-2.8.1-native/hadoop-2.8.1.zip
+              unzip hadoop.zip
+              New-Item -ItemType Directory -Force -Path hadoop\bin
+              cp hadoop-2.8.1\winutils.exe hadoop\bin
+
+        - pwsh: |
+            echo "Downloading Spark ${{ test.version }}"
+            curl -k -L -o spark-${{ test.version }}.tgz https://archive.apache.org/dist/spark/spark-${{ test.version }}/spark-${{ test.version }}-bin-hadoop2.7.tgz
+            tar xzvf spark-${{ test.version }}.tgz
+          displayName: 'Download Spark Distro ${{ test.version }}'
+          workingDirectory: $(Build.BinariesDirectory)
+
+        - task: DotNetCoreCLI@2
+          displayName: 'E2E tests'
+          inputs:
+            command: test
+            projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
+            arguments: '--configuration $(buildConfiguration) ${{ option.testOptions }}'
+            workingDirectory: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark
+          env:
+            HADOOP_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)hadoop
+            SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
+            DOTNET_WORKER_DIR: $(CURRENT_DOTNET_WORKER_DIR)
+
+        - pwsh: |
+            echo "Downloading ${env:BACKWARD_COMPATIBLE_WORKER_URL}"
+            curl -k -L -o Microsoft.Spark.Worker-${{ parameters.backwardCompatibleRelease }}.zip ${env:BACKWARD_COMPATIBLE_WORKER_URL}
+            unzip Microsoft.Spark.Worker-${{ parameters.backwardCompatibleRelease }}.zip -d $([System.IO.Directory]::GetParent($env:BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR).FullName)
+
+            if ($env:AGENT_OS -eq 'Linux') {
+              chmod +x "${env:BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR}${env:PATH_SEPARATOR}Microsoft.Spark.Worker"
+            }
+          displayName: 'Setup Backward Compatible Microsoft Spark Worker ${{ parameters.backwardCompatibleRelease }}'
+          workingDirectory: $(Build.BinariesDirectory)
+          env:
+            SPARK_VERSION: ${{ test.version }}
+
+        - task: DotNetCoreCLI@2
+          displayName: 'E2E Backward Compatibility Tests'
+          inputs:
+            command: test
+            projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
+            arguments: '--configuration $(buildConfiguration) ${{ option.backwardCompatibleTestOptions }}'
+            workingDirectory: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark
+          env:
+            HADOOP_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)hadoop
+            SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
+            DOTNET_WORKER_DIR: $(BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR)
+
+        - checkout: forwardCompatibleRelease
+          path: s$(PATH_SEPARATOR)dotnet-spark-${{ parameters.forwardCompatibleRelease }}
+
+        - task: Maven@3
+          displayName: 'Maven build src for forward compatible release v${{ parameters.forwardCompatibleRelease }}'
+          inputs:
+            mavenPomFile: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark-${{ parameters.forwardCompatibleRelease }}$(PATH_SEPARATOR)src$(PATH_SEPARATOR)scala$(PATH_SEPARATOR)pom.xml
+
+        - task: DotNetCoreCLI@2
+          displayName: 'E2E Forward Compatibility Tests'
+          inputs:
+            command: test
+            projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
+            arguments: '--configuration $(buildConfiguration) ${{ option.forwardCompatibleTestOptions }}'
+            workingDirectory: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark-${{ parameters.forwardCompatibleRelease }}
+          env:
+            HADOOP_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)hadoop
+            SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
+            DOTNET_WORKER_DIR: $(CURRENT_DOTNET_WORKER_DIR)
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -12,13 +12,45 @@ variables:
  _TeamName: DotNetSpark
  MSBUILDSINGLELOADCONTEXT: 1
  ArtifactPath: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries'
-  CurrentDotnetWorkerDir: '$(ArtifactPath)\Microsoft.Spark.Worker\netcoreapp3.1\win-x64'
+
+  backwardCompatibleRelease: '1.0.0'
+  forwardCompatibleRelease: '1.0.0'
+
+  backwardCompatibleTestOptions_Windows_2_3: ""
+  forwardCompatibleTestOptions_Windows_2_3: ""
+  backwardCompatibleTestOptions_Linux_2_3: ""
+  forwardCompatibleTestOptions_Linux_2_3: ""
+
+  backwardCompatibleTestOptions_Windows_2_4: ""
+  forwardCompatibleTestOptions_Windows_2_4: ""
+  backwardCompatibleTestOptions_Linux_2_4: ""
+  # Filter HyperspaceTests not due to functionality changes, but to incompatible tests running on Linux.
+  # Please see https://github.com/dotnet/spark/pull/737 for the fix.
+  forwardCompatibleTestOptions_Linux_2_4: "--filter \
+  (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestExplainAPI)&\
+  (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestIndexCreateAndDelete)&\
+  (FullyQualifiedName!=Microsoft.Spark.Extensions.Hyperspace.E2ETest.HyperspaceTests.TestSignatures)"
+
+  backwardCompatibleTestOptions_Windows_3: "--filter \
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\
+  (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf)"
+  forwardCompatibleTestOptions_Windows_3: ""
+  backwardCompatibleTestOptions_Linux_3: $(backwardCompatibleTestOptions_Windows_3)
+  forwardCompatibleTestOptions_Linux_3: $(forwardCompatibleTestOptions_Linux_2_4)

  # Azure DevOps variables are transformed into environment variables, with these variables we
  # avoid the first time experience and telemetry to speed up the build.
  DOTNET_CLI_TELEMETRY_OPTOUT: 1
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1

+resources:
+  repositories:
+  - repository: forwardCompatibleRelease
+    type: github
+    endpoint: dotnet
+    name: dotnet/spark
+    ref: refs/tags/v$(forwardCompatibleRelease)
+
 stages:
 - stage: Build
  displayName: Build Sources
@ -103,7 +135,7 @@ stages:
        inputs:
          artifactName: Microsoft.Spark.Binaries
          downloadPath: $(Build.ArtifactStagingDirectory)
-    
+
      - task: MicroBuildSigningPlugin@2
        displayName: Install MicroBuild plugin
        inputs:
@ -173,18 +205,143 @@ stages:

 - template: azure-pipelines-e2e-tests-template.yml
  parameters:
-    versions:
-    - '2.3.0'
-    - '2.3.1'
-    - '2.3.2'
-    - '2.3.3'
-    - '2.3.4'
-    - '2.4.0'
-    - '2.4.1'
-    - '2.4.3'
-    - '2.4.4'
-    - '2.4.5'
-    - '2.4.6'
-    - '2.4.7'
-    - '3.0.0'
-    - '3.0.1'
+    backwardCompatibleRelease: $(backwardCompatibleRelease)
+    forwardCompatibleRelease: $(forwardCompatibleRelease)
+    tests:
+    - version: '2.3.0'
+      jobOptions:
+      # 'Hosted Ubuntu 1604' test is disabled due to https://github.com/dotnet/spark/issues/753
+      - pool: 'Hosted VS2017'
+        testOptions: ''
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_3)
+    - version: '2.3.1'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_3)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_3)
+    - version: '2.3.2'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_3)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_3)
+    - version: '2.3.3'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_3)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_3)
+    - version: '2.3.4'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_3)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_3)
+    - version: '2.4.0'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
+    - version: '2.4.1'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
+    - version: '2.4.3'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
+    - version: '2.4.4'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
+    - version: '2.4.5'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
+    - version: '2.4.6'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
+    - version: '2.4.7'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
+    - version: '3.0.0'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3)
+    - version: '3.0.1'
+      jobOptions:
+      - pool: 'Hosted VS2017'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3)
+      - pool: 'Hosted Ubuntu 1604'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3)
--- a/script/download-hadoop-utils.cmd
+++ b/script/download-hadoop-utils.cmd
@ -1,14 +0,0 @@
-@echo off
-
-setlocal
-
-set OutputDir=%1
-cd %OutputDir%
-
-echo "Download Hadoop utils for Windows."
-curl -k -L -o hadoop.zip https://github.com/steveloughran/winutils/releases/download/tag_2017-08-29-hadoop-2.8.1-native/hadoop-2.8.1.zip
-unzip hadoop.zip
-mkdir -p hadoop\bin
-cp hadoop-2.8.1\winutils.exe hadoop\bin
-
-endlocal
--- a/script/download-spark-distros.cmd
+++ b/script/download-spark-distros.cmd
@ -1,14 +0,0 @@
-@echo off
-
-setlocal
-
-set OutputDir=%1
-set SparkVersion=%2
-
-cd %OutputDir%
-
-echo "Downloading Spark distros."
-
-curl -k -L -o spark-%SparkVersion%.tgz https://archive.apache.org/dist/spark/spark-%SparkVersion%/spark-%SparkVersion%-bin-hadoop2.7.tgz && tar xzvf spark-%SparkVersion%.tgz
-
-endlocal
--- a/script/download-worker-release.cmd
+++ b/script/download-worker-release.cmd
@ -1,13 +0,0 @@
-@echo off
-
-setlocal
-
-set OutputDir=%1
-set OldestCompatibleWorkerVersion=%2
-cd %OutputDir%
-
-echo "Download oldest backwards compatible worker release - %OldestCompatibleWorkerVersion%"
-curl -k -L -o Microsoft.Spark.Worker.zip https://github.com/dotnet/spark/releases/download/v%OldestCompatibleWorkerVersion%/Microsoft.Spark.Worker.netcoreapp3.1.win-x64-%OldestCompatibleWorkerVersion%.zip
-unzip Microsoft.Spark.Worker.zip
-
-endlocal
--- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs
+++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Hyperspace.E2ETest/HyperspaceTests.cs
@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.

 using System;
+using Microsoft.Spark.E2ETest;
 using Microsoft.Spark.E2ETest.Utils;
 using Microsoft.Spark.Extensions.Hyperspace.Index;
 using Microsoft.Spark.Sql;
@ -36,7 +37,7 @@ namespace Microsoft.Spark.Extensions.Hyperspace.E2ETest
            _sampleDataFrame = _spark.Read()
                .Option("header", true)
                .Option("delimiter", ";")
-                .Csv("Resources\\people.csv");
+                .Csv($"{TestEnvironment.ResourceDirectory}people.csv");
            _sampleIndexName = "sample_dataframe";
            _sampleIndexConfig = new IndexConfig(_sampleIndexName, new[] { "job" }, new[] { "name" });
            _hyperspace.CreateIndex(_sampleDataFrame, _sampleIndexConfig);