Update promptflow-ci to use check-directory-file-changes to allow for required status check (#3420)

* Update promptflow-ci to use check-directory-file-changes to allow for required status check

* update echo message

* update logging for promptflow_ci.py

* fix syntax issue

* update logging again

* update skipped flows

* update skipped-flows

* remove whitespace

* remove backslash

* update list of flows

* update errors count

---------

Co-authored-by: lykelly19 <kellyly@microsoft.com>

.github/workflows/check-changed-files.yaml

@@ -49,7 +49,7 @@ jobs:
           if ${{ steps.changed-scripts.outputs.all_changed_and_modified_files == '' }}; then
             echo "No changed files under ${{ inputs.folder_path }}"
           else
-            echo "Updated files under ${{ inputs.folder_path }}: ${{ steps.changed-scripts.outputs.all_changed_and_modified_files }}"
+            echo "Updated file(s) found under ${{ inputs.folder_path }}: ${{ steps.changed-scripts.outputs.all_changed_and_modified_files }}"
           fi
           echo "files-changed=${{ steps.changed-scripts.outputs.all_changed_and_modified_files }}" >> "$GITHUB_OUTPUT"
 

.github/workflows/promptflow-ci.yml

@@ -5,17 +5,9 @@ on:
   pull_request:
     branches:
       - main
-    paths:
-      - assets/promptflow/models/**
-      - .github/workflows/promptflow-ci.yml
-      - scripts/promptflow-ci/**
   pull_request_target:
     branches:
       - main
-    paths:
-      - assets/promptflow/models/**
-      - .github/workflows/promptflow-ci.yml
-      - scripts/promptflow-ci/**
     types:
       - opened
       - labeled
@@ -35,10 +27,18 @@ jobs:
   check-execution-context:
     uses: Azure/azureml-assets/.github/workflows/check-execution-context.yaml@main
 
+  check-directory-file-changes:
+    needs: check-execution-context
+    if: fromJSON(needs.check-execution-context.outputs.continue)
+    uses: Azure/azureml-assets/.github/workflows/check-changed-files.yaml@main
+    with:
+      folder_path: "assets/promptflow/models/**,.github/workflows/promptflow-ci.yml,scripts/promptflow-ci/**"
+      forked_pr_input: ${{ needs.check-execution-context.outputs.forked_pr }}
+
   check_spec_yaml:
     runs-on: ubuntu-latest
-    if: fromJSON(needs.check-execution-context.outputs.continue)
-    needs: check-execution-context
+    if: needs.check-directory-file-changes.outputs.files-changed != ''
+    needs: [check-execution-context, check-directory-file-changes]
     name: Check spec.yaml fields
     timeout-minutes: 45
     steps:
@@ -60,8 +60,8 @@ jobs:
 
   run_promptflow_ci_job:
     runs-on: ubuntu-latest
-    if: fromJSON(needs.check-execution-context.outputs.continue)
-    needs: check-execution-context
+    if: needs.check-directory-file-changes.outputs.files-changed != ''
+    needs: [check-execution-context, check-directory-file-changes]
     name: Flow tests
     environment: Testing
     timeout-minutes: 60

scripts/promptflow-ci/promptflow_ci.py

@@ -165,7 +165,15 @@ if __name__ == "__main__":
     parser.add_argument('--flow_submit_mode', type=str, default="sync")
     parser.add_argument('--run_time', type=str, default="default-mir")
     parser.add_argument('--skipped_flows', type=str,
-                        default="bring_your_own_data_qna,template_chat_flow,template_eval_flow")
+                        default="bring_your_own_data_qna,template_chat_flow,template_eval_flow,playground-ayod-rag,"
+                        "chat-quality-safety-eval,qna-non-rag-metrics-eval,qna-quality-safety-eval,"
+                        "qna-rag-metrics-eval,rai-eval-ui-dag-flow,rai-qna-quality-safety-eval,"
+                        "qna-ada-similarity-eval,qna-relevance-eval,ask-wikipedia,classification-accuracy-eval,"
+                        "qna-f1-score-eval,count-cars,analyze-conversations,rerank-qna,multi-index-rerank-qna,"
+                        "detect-defects,analyze-documents,qna-gpt-similarity-eval,use-functions-with-chat-models,"
+                        "qna-groundedness-eval,bring-your-own-data-chat-qna,qna-coherence-eval,template-standard-flow,"
+                        "qna-with-your-own-data-using-faiss-index,chat-with-wikipedia,web-classification,"
+                        "qna-fluency-eval")
     # Skip bring_your_own_data_qna test, the flow has a bug.
     # Bug 2773738: Add retry when ClientAuthenticationError
     # https://msdata.visualstudio.com/Vienna/_workitems/edit/2773738
@@ -196,17 +204,16 @@ if __name__ == "__main__":
             flow_dir).name not in skipped_flows]
     # Check download models
     log_debug(f"Flows to validate: {flows_dirs}.")
-    errors = []
+    errors = 0
     for model_dir in flows_dirs:
         try:
             validate_downlaod(model_dir)
         except Exception as e:
-            errors.append(e)
+            log_error(f"Error found for {os.path.join(model_dir, MODEL_FILE)}: {e}")
+            errors += 1
 
-    if len(errors) > 0:
-        log_error(f"Found {len(errors)} errors when downloading models.")
-        for error in errors:
-            log_error(error)
+    if errors > 0:
+        log_error(f"Found {errors} errors when downloading models.")
         exit(1)
 
     # Check run flows