[evaluation] ci: Re-enable Windows tests (#37411)

* ci: Revert back to repo's platform matrix * fix: Wrap async clients in async with blocks * tests,fix: Ensure that promptflow service is stopped
2024-09-18 19:42:40 -04:00 · 2024-09-18 19:42:40 -04:00 · 5337072502
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py
@ -65,11 +65,10 @@ async def ensure_service_availability(rai_svc_url: str, token: str, capability:
    headers = get_common_headers(token)
    svc_liveness_url = rai_svc_url + "/checkannotation"

-    client = get_async_http_client()
-
-    response = await client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
-        svc_liveness_url, headers=headers, timeout=CommonConstants.DEFAULT_HTTP_TIMEOUT
-    )
+    async with get_async_http_client() as client:
+        response = await client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
+            svc_liveness_url, headers=headers, timeout=CommonConstants.DEFAULT_HTTP_TIMEOUT
+        )

    if response.status_code != 200:
        raise Exception(  # pylint: disable=broad-exception-raised
@ -143,11 +142,10 @@ async def submit_request(question: str, answer: str, metric: str, rai_svc_url: s
    url = rai_svc_url + "/submitannotation"
    headers = get_common_headers(token)

-    client = get_async_http_client()
-
-    response = await client.post(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
-        url, json=payload, headers=headers, timeout=CommonConstants.DEFAULT_HTTP_TIMEOUT
-    )
+    async with get_async_http_client() as client:
+        response = await client.post(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
+            url, json=payload, headers=headers, timeout=CommonConstants.DEFAULT_HTTP_TIMEOUT
+        )

    if response.status_code != 202:
        print("Fail evaluating '%s' with error message: %s" % (payload["UserTextList"], response.text))
@ -180,11 +178,10 @@ async def fetch_result(operation_id: str, rai_svc_url: str, credential: TokenCre
        token = await fetch_or_reuse_token(credential, token)
        headers = get_common_headers(token)

-        client = get_async_http_client()
-
-        response = await client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
-            url, headers=headers, timeout=CommonConstants.DEFAULT_HTTP_TIMEOUT
-        )
+        async with get_async_http_client() as client:
+            response = await client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
+                url, headers=headers, timeout=CommonConstants.DEFAULT_HTTP_TIMEOUT
+            )

        if response.status_code == 200:
            return response.json()
@ -342,16 +339,15 @@ async def _get_service_discovery_url(azure_ai_project: dict, token: str) -> str:
    """
    headers = get_common_headers(token)

-    client = get_async_http_client()
-
-    response = await client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
-        f"https://management.azure.com/subscriptions/{azure_ai_project['subscription_id']}/"
-        f"resourceGroups/{azure_ai_project['resource_group_name']}/"
-        f"providers/Microsoft.MachineLearningServices/workspaces/{azure_ai_project['project_name']}?"
-        f"api-version=2023-08-01-preview",
-        headers=headers,
-        timeout=CommonConstants.DEFAULT_HTTP_TIMEOUT,
-    )
+    async with get_async_http_client() as client:
+        response = await client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
+            f"https://management.azure.com/subscriptions/{azure_ai_project['subscription_id']}/"
+            f"resourceGroups/{azure_ai_project['resource_group_name']}/"
+            f"providers/Microsoft.MachineLearningServices/workspaces/{azure_ai_project['project_name']}?"
+            f"api-version=2023-08-01-preview",
+            headers=headers,
+            timeout=CommonConstants.DEFAULT_HTTP_TIMEOUT,
+        )

    if response.status_code != 200:
        raise Exception("Failed to retrieve the discovery service URL")  # pylint: disable=broad-exception-raised
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py
@ -321,7 +321,7 @@ class AdversarialSimulator:
            )
        )

-        async with semaphore:
+        async with semaphore, session:
            _, conversation_history = await simulate_conversation(
                bots=bots,
                session=session,
--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_model_tools/_proxy_completion_model.py
@ -196,17 +196,16 @@ class ProxyChatCompletionsModel(OpenAIChatCompletionsModel):
            retry_mode=RetryMode.Exponential,
        )

-        exp_retry_client = get_async_http_client().with_policies(retry_policy=retry_policy)
-
        # initial 15 seconds wait before attempting to fetch result
        # Need to wait both in this thread and in the async thread for some reason?
        # Someone not under a crunch and with better async understandings should dig into this more.
        await asyncio.sleep(15)
        time.sleep(15)

-        response = await exp_retry_client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
-            self.result_url, headers=proxy_headers
-        )
+        async with get_async_http_client().with_policies(retry_policy=retry_policy) as exp_retry_client:
+            response = await exp_retry_client.get(  # pylint: disable=too-many-function-args,unexpected-keyword-arg
+                self.result_url, headers=proxy_headers
+            )

        response.raise_for_status()

--- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_model_tools/_rai_client.py
+++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_model_tools/_rai_client.py
@ -122,7 +122,9 @@ class RAIClient:
        }

        session = self._create_async_client()
-        response = await session.get(url=url, headers=headers)  # pylint: disable=unexpected-keyword-arg
+
+        async with session:
+            response = await session.get(url=url, headers=headers)  # pylint: disable=unexpected-keyword-arg

        if response.status_code == 200:
            return response.json()
--- a/sdk/evaluation/azure-ai-evaluation/tests/conftest.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/conftest.py
@ -471,3 +471,28 @@ def pytest_collection_modifyitems(items):
                # If item's parent was marked as 'localtest', mark the child as such, but not if
                # it was marked as 'azuretest'.
                item.add_marker(pytest.mark.localtest)
+
+
+def pytest_sessionfinish() -> None:
+
+    def stop_promptflow_service() -> None:
+        """Ensure that the promptflow service is stopped when pytest exits.
+
+        .. note::
+
+            The azure-sdk-for-python CI performs a cleanup step that deletes
+            the python environment that the tests run in.
+
+            At time of writing, at least one test starts the promptflow service
+            (served from `waitress-serve`). The promptflow service is a separate
+            process that gets orphaned by pytest.
+
+            Crucially, that process has a handles on files in the python environment.
+            On Windows, this causes the cleanup step to fail with a permission issue
+            since the OS disallows deletion of files in use by a process.
+        """
+        from promptflow._cli._pf._service import stop_service
+
+        stop_service()
+
+    stop_promptflow_service()
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_synthetic_conversation_bot.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_synthetic_conversation_bot.py
@ -84,9 +84,11 @@ class TestConversationBot:
            )
        )

-        parsed_response, req, time_taken, full_response = await bot.generate_response(
-            session=client, conversation_history=[], max_history=0, turn_number=0
-        )
+        async with client:
+            parsed_response, req, time_taken, full_response = await bot.generate_response(
+                session=client, conversation_history=[], max_history=0, turn_number=0
+            )
+
        assert (
            parsed_response["samples"][0]
            == bot_invalid_jinja_params["instantiation_parameters"]["conversation_starter"]
--- a/sdk/evaluation/ci.yml
+++ b/sdk/evaluation/ci.yml
@ -28,13 +28,6 @@ extends:
    ServiceDirectory: evaluation
    ValidateFormatting: true
    TestProxy: true
-    # This custom matrix config should be dropped once:
-    #  * Resolve the issue of windows runners crashing because a file isn't deletable
-    MatrixConfigs: 
-      - Name: ai_ci_matrix
-        Path: sdk/evaluation/platform-matrix.json
-        Selection: sparse
-        GenerateVMJobs: true
    Artifacts:
    - name: azure-ai-evaluation
      safeName: azureaievaluation
--- a/sdk/evaluation/platform-matrix.json
+++ b/sdk/evaluation/platform-matrix.json
@ -1,51 +0,0 @@
-{
-  "displayNames": {
-    "--disablecov": "",
-    "false": "",
-    "true": ""
-  },
-  "matrix": {
-    "Agent": {
-      "macos-latest": { "OSVmImage": "env:MACVMIMAGE", "Pool": "env:MACPOOL" },
-      "ubuntu-20.04": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }
-    },
-    "PythonVersion": [ "3.8", "3.11", "3.10" ],
-    "CoverageArg": "--disablecov",
-    "TestSamples": "false"
-  },
-  "include": [
-    {
-      "CoverageConfig": {
-        "ubuntu2004_39_coverage": {
-          "OSVmImage": "env:LINUXVMIMAGE",
-          "Pool": "env:LINUXPOOL",
-          "PythonVersion": "3.9",
-          "CoverageArg": "",
-          "TestSamples": "false"
-        }
-      }
-    },
-    {
-      "CoverageConfig": {
-        "ubuntu2004_pypy39": {
-          "OSVmImage": "env:LINUXVMIMAGE",
-          "Pool": "env:LINUXPOOL",
-          "PythonVersion": "pypy3.9",
-          "CoverageArg": "",
-          "TestSamples": "false"
-        }
-      }
-    },
-    {
-      "Config": {
-        "Ubuntu2004_312": {
-          "OSVmImage": "env:LINUXVMIMAGE",
-          "Pool": "env:LINUXPOOL",
-          "PythonVersion": "3.12",
-          "CoverageArg": "--disablecov",
-          "TestSamples": "false"
-        }
-      }
-    }
-  ]
-}