From 83df1490519508d63e4eb033c12d8d4a4b3ebf08 Mon Sep 17 00:00:00 2001 From: Anton Schwaighofer Date: Tue, 7 Nov 2023 09:26:29 +0000 Subject: [PATCH] ENH: Apply security fixes (#912) --- hi-ml-azure/src/health_azure/traverse.py | 13 +++++++-- hi-ml-azure/testazure/testazure/test_himl.py | 2 +- hi-ml-cpath/requirements_run.txt | 2 +- .../testSSL/testSSL/test_ssl_containers.py | 2 +- hi-ml-multimodal/requirements_test.txt | 2 +- hi-ml/src/health_ml/utils/reports.py | 5 ++-- hi-ml/testhiml/testhiml/utils/test_reports.py | 29 +++++++++---------- 7 files changed, 30 insertions(+), 25 deletions(-) diff --git a/hi-ml-azure/src/health_azure/traverse.py b/hi-ml-azure/src/health_azure/traverse.py index ddc045e1..bdd82749 100644 --- a/hi-ml-azure/src/health_azure/traverse.py +++ b/hi-ml-azure/src/health_azure/traverse.py @@ -8,7 +8,7 @@ from io import StringIO from typing import Any, Dict, Iterable, Union, List, Optional import param -from ruamel import yaml +from ruamel.yaml import YAML def is_basic_type(o: Any) -> bool: @@ -138,7 +138,13 @@ def object_to_yaml(o: Any) -> str: :param o: The object to inspect. :return: A string in YAML format. """ - return yaml.safe_dump(object_to_dict(o), default_flow_style=False) # type: ignore + yaml = YAML(typ='safe', pure=True) + stream = StringIO() + yaml.default_flow_style = False + yaml.dump(object_to_dict(o), stream) + result = stream.getvalue() + stream.close() + return result def yaml_to_dict(s: str) -> Dict[str, Any]: @@ -150,7 +156,8 @@ def yaml_to_dict(s: str) -> Dict[str, Any]: or dictionaries again. """ stream = StringIO(s) - return yaml.safe_load(stream=stream) + yaml = YAML(typ='safe', pure=True) + return yaml.load(stream=stream) def _write_dict_to_object(o: Any, d: Dict[str, Any], traversed_fields: Optional[List] = None) -> List[str]: diff --git a/hi-ml-azure/testazure/testazure/test_himl.py b/hi-ml-azure/testazure/testazure/test_himl.py index e54b4ebb..e24fe66e 100644 --- a/hi-ml-azure/testazure/testazure/test_himl.py +++ b/hi-ml-azure/testazure/testazure/test_himl.py @@ -77,7 +77,7 @@ from testazure.utils_testazure import ( ) INEXPENSIVE_TESTING_CLUSTER_NAME = "lite-testing-ds2" -EXPECTED_QUEUED = "This command will be run in AzureML:" +EXPECTED_QUEUED = "Successfully queued run" GITHUB_SHIBBOLETH = "GITHUB_RUN_ID" # https://docs.github.com/en/actions/reference/environment-variables AZUREML_FLAG = himl.AZUREML_FLAG diff --git a/hi-ml-cpath/requirements_run.txt b/hi-ml-cpath/requirements_run.txt index 34e4ae6f..dd68e8af 100644 --- a/hi-ml-cpath/requirements_run.txt +++ b/hi-ml-cpath/requirements_run.txt @@ -10,7 +10,7 @@ numpy==1.22.0 pillow==9.3.0 pydicom==2.3.0 scikit-image==0.19.3 -scipy==1.7.3 +scipy==1.10.0 seaborn==0.10.1 simpleitk==2.1.1.2 tifffile==2022.10.10 diff --git a/hi-ml-cpath/testSSL/testSSL/test_ssl_containers.py b/hi-ml-cpath/testSSL/testSSL/test_ssl_containers.py index ae7459b3..1269269e 100644 --- a/hi-ml-cpath/testSSL/testSSL/test_ssl_containers.py +++ b/hi-ml-cpath/testSSL/testSSL/test_ssl_containers.py @@ -143,7 +143,7 @@ def test_ssl_container_cifar10_resnet_simclr() -> None: # Note: It is possible that after the PyTorch 1.10 upgrade, we can't get parity between local runs and runs on # the hosted build agents. If that suspicion is confirmed, we need to add branching for local and cloud results. expected_metrics = { - 'simclr/val/loss': 2.8596301078796387, + 'simclr/val/loss': 2.859630584716797, 'ssl_online_evaluator/val/loss': 2.2664988040924072, 'ssl_online_evaluator/val/AccuracyAtThreshold05': 0.20000000298023224, 'simclr/train/loss': 3.6261773109436035, diff --git a/hi-ml-multimodal/requirements_test.txt b/hi-ml-multimodal/requirements_test.txt index 7b29988d..8ad4332b 100644 --- a/hi-ml-multimodal/requirements_test.txt +++ b/hi-ml-multimodal/requirements_test.txt @@ -2,7 +2,7 @@ bump2version==1.0.1 coverage==6.3.2 flake8==5.0.2 ipykernel==6.15.0 -ipython==7.34.0 +ipython==8.11.0 mypy==0.931 papermill==2.3.4 pycobertura==2.0.1 diff --git a/hi-ml/src/health_ml/utils/reports.py b/hi-ml/src/health_ml/utils/reports.py index 5fa98b9e..e513870c 100644 --- a/hi-ml/src/health_ml/utils/reports.py +++ b/hi-ml/src/health_ml/utils/reports.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, OrderedDict, Tuple import jinja2 -import ruamel.yaml +from ruamel.yaml import YAML import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -407,7 +407,8 @@ class HTMLReport: # TODO: add option to overwrite report title with entry here assert report_config_path.suffix == ".yml", f"Expected a .yml file but found {report_config_path.suffix}" with open(report_config_path, "r") as f_path: - yaml_contents = ruamel.yaml.load(f_path) + yaml = YAML(typ='safe', pure=True) + yaml_contents = yaml.load(f_path) return yaml_contents diff --git a/hi-ml/testhiml/testhiml/utils/test_reports.py b/hi-ml/testhiml/testhiml/utils/test_reports.py index aa63c425..b1c27036 100644 --- a/hi-ml/testhiml/testhiml/utils/test_reports.py +++ b/hi-ml/testhiml/testhiml/utils/test_reports.py @@ -11,7 +11,7 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd import pytest -import ruamel.yaml +from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap as OrderedDict, CommentedSeq as OrderedList from health_ml.utils.reports import HTMLReport, IMAGE_KEY_HTML, TABLE_KEY_HTML, REPORT_CONTENTS_KEY, ReportComponentKey @@ -226,28 +226,25 @@ def test_html_report_read_config(html_report: HTMLReport, dummy_df: pd.DataFrame plt.plot(dummy_df[[dummy_df_cols[0]]], dummy_df[[dummy_df_cols[1]]]) - report_config_contents = OrderedDict( - { - REPORT_CONTENTS_KEY: OrderedList( - [ - { - ReportComponentKey.TYPE.value: ReportComponentKey.TABLE.value, - ReportComponentKey.VALUE.value: table_path, - } - ] - ) - } - ) + report_config_contents = { + REPORT_CONTENTS_KEY: [ + { + ReportComponentKey.TYPE.value: ReportComponentKey.TABLE.value, + ReportComponentKey.VALUE.value: str(table_path), + } + ] + } report_config_path = tmp_path / "report_config.yml" - with open(report_config_path, "w+") as f_path: - ruamel.yaml.dump(report_config_contents, f_path) + with open(report_config_path, "w+", encoding="utf-8") as f_path: + yaml = YAML(typ='safe', pure=True) + yaml.dump(report_config_contents, f_path) report_config = html_report.read_config_yaml(report_config_path) assert list(report_config.keys()) == [REPORT_CONTENTS_KEY] assert len(report_config[REPORT_CONTENTS_KEY]) == 1 report_contents_first_entry = report_config[REPORT_CONTENTS_KEY][0] assert report_contents_first_entry[ReportComponentKey.TYPE.value] == ReportComponentKey.TABLE.value - assert report_contents_first_entry[ReportComponentKey.VALUE.value] == table_path + assert report_contents_first_entry[ReportComponentKey.VALUE.value] == str(table_path) html_report.add_yaml_contents_to_report(report_config) html_template_difference = html_report.template.replace(html_template_before, "")