From 144920a4d946ca88e973125d1a613ffa668c51ec Mon Sep 17 00:00:00 2001 From: Kelly <40868256+lykelly19@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:48:30 -0700 Subject: [PATCH] Update promptflow-ci to use check-directory-file-changes to allow for required status check (#3420) * Update promptflow-ci to use check-directory-file-changes to allow for required status check * update echo message * update logging for promptflow_ci.py * fix syntax issue * update logging again * update skipped flows * update skipped-flows * remove whitespace * remove backslash * update list of flows * update errors count --------- Co-authored-by: lykelly19 --- .github/workflows/check-changed-files.yaml | 2 +- .github/workflows/promptflow-ci.yml | 24 +++++++++++----------- scripts/promptflow-ci/promptflow_ci.py | 21 ++++++++++++------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check-changed-files.yaml b/.github/workflows/check-changed-files.yaml index a95d54af0e..0267338c17 100644 --- a/.github/workflows/check-changed-files.yaml +++ b/.github/workflows/check-changed-files.yaml @@ -49,7 +49,7 @@ jobs: if ${{ steps.changed-scripts.outputs.all_changed_and_modified_files == '' }}; then echo "No changed files under ${{ inputs.folder_path }}" else - echo "Updated files under ${{ inputs.folder_path }}: ${{ steps.changed-scripts.outputs.all_changed_and_modified_files }}" + echo "Updated file(s) found under ${{ inputs.folder_path }}: ${{ steps.changed-scripts.outputs.all_changed_and_modified_files }}" fi echo "files-changed=${{ steps.changed-scripts.outputs.all_changed_and_modified_files }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/promptflow-ci.yml b/.github/workflows/promptflow-ci.yml index d385a1a304..ea77888613 100644 --- a/.github/workflows/promptflow-ci.yml +++ b/.github/workflows/promptflow-ci.yml @@ -5,17 +5,9 @@ on: pull_request: branches: - main - paths: - - assets/promptflow/models/** - - .github/workflows/promptflow-ci.yml - - scripts/promptflow-ci/** pull_request_target: branches: - main - paths: - - assets/promptflow/models/** - - .github/workflows/promptflow-ci.yml - - scripts/promptflow-ci/** types: - opened - labeled @@ -35,10 +27,18 @@ jobs: check-execution-context: uses: Azure/azureml-assets/.github/workflows/check-execution-context.yaml@main + check-directory-file-changes: + needs: check-execution-context + if: fromJSON(needs.check-execution-context.outputs.continue) + uses: Azure/azureml-assets/.github/workflows/check-changed-files.yaml@main + with: + folder_path: "assets/promptflow/models/**,.github/workflows/promptflow-ci.yml,scripts/promptflow-ci/**" + forked_pr_input: ${{ needs.check-execution-context.outputs.forked_pr }} + check_spec_yaml: runs-on: ubuntu-latest - if: fromJSON(needs.check-execution-context.outputs.continue) - needs: check-execution-context + if: needs.check-directory-file-changes.outputs.files-changed != '' + needs: [check-execution-context, check-directory-file-changes] name: Check spec.yaml fields timeout-minutes: 45 steps: @@ -60,8 +60,8 @@ jobs: run_promptflow_ci_job: runs-on: ubuntu-latest - if: fromJSON(needs.check-execution-context.outputs.continue) - needs: check-execution-context + if: needs.check-directory-file-changes.outputs.files-changed != '' + needs: [check-execution-context, check-directory-file-changes] name: Flow tests environment: Testing timeout-minutes: 60 diff --git a/scripts/promptflow-ci/promptflow_ci.py b/scripts/promptflow-ci/promptflow_ci.py index 8b3311356c..90c790a2d1 100644 --- a/scripts/promptflow-ci/promptflow_ci.py +++ b/scripts/promptflow-ci/promptflow_ci.py @@ -165,7 +165,15 @@ if __name__ == "__main__": parser.add_argument('--flow_submit_mode', type=str, default="sync") parser.add_argument('--run_time', type=str, default="default-mir") parser.add_argument('--skipped_flows', type=str, - default="bring_your_own_data_qna,template_chat_flow,template_eval_flow") + default="bring_your_own_data_qna,template_chat_flow,template_eval_flow,playground-ayod-rag," + "chat-quality-safety-eval,qna-non-rag-metrics-eval,qna-quality-safety-eval," + "qna-rag-metrics-eval,rai-eval-ui-dag-flow,rai-qna-quality-safety-eval," + "qna-ada-similarity-eval,qna-relevance-eval,ask-wikipedia,classification-accuracy-eval," + "qna-f1-score-eval,count-cars,analyze-conversations,rerank-qna,multi-index-rerank-qna," + "detect-defects,analyze-documents,qna-gpt-similarity-eval,use-functions-with-chat-models," + "qna-groundedness-eval,bring-your-own-data-chat-qna,qna-coherence-eval,template-standard-flow," + "qna-with-your-own-data-using-faiss-index,chat-with-wikipedia,web-classification," + "qna-fluency-eval") # Skip bring_your_own_data_qna test, the flow has a bug. # Bug 2773738: Add retry when ClientAuthenticationError # https://msdata.visualstudio.com/Vienna/_workitems/edit/2773738 @@ -196,17 +204,16 @@ if __name__ == "__main__": flow_dir).name not in skipped_flows] # Check download models log_debug(f"Flows to validate: {flows_dirs}.") - errors = [] + errors = 0 for model_dir in flows_dirs: try: validate_downlaod(model_dir) except Exception as e: - errors.append(e) + log_error(f"Error found for {os.path.join(model_dir, MODEL_FILE)}: {e}") + errors += 1 - if len(errors) > 0: - log_error(f"Found {len(errors)} errors when downloading models.") - for error in errors: - log_error(error) + if errors > 0: + log_error(f"Found {errors} errors when downloading models.") exit(1) # Check run flows