зеркало из https://github.com/microsoft/hi-ml.git
ENH: Rename histopathology folder to match overall naming scheme (#512)
Top-level folder has been renamed from `hi-ml-histopathology` to hi-ml-cpath, the Python namespace from `histopathology` to `health_cpath`.
This commit is contained in:
Родитель
65ba65d16f
Коммит
2407f93f6c
|
@ -20,7 +20,7 @@ node_modules
|
|||
cifar-10-python.tar.gz
|
||||
cifar-10-batches-py/
|
||||
None/
|
||||
hi-ml-histopathology/testSSL/test_outputs
|
||||
hi-ml-cpath/testSSL/test_outputs
|
||||
test_outputs/
|
||||
lightning_logs/
|
||||
# Downloaded datasets
|
||||
|
|
|
@ -10,7 +10,7 @@ flags:
|
|||
paths:
|
||||
- hi-ml-azure
|
||||
carryforward: true
|
||||
hi-ml-histopathology:
|
||||
hi-ml-cpath:
|
||||
paths:
|
||||
- hi-ml-histopathology
|
||||
- hi-ml-cpath
|
||||
carryforward: true
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
name: Histopathology PR
|
||||
name: Pathology PR
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
|
@ -7,16 +7,16 @@ on:
|
|||
pull_request:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "hi-ml-histopathology/**"
|
||||
- ".github/workflows/histopathology-pr.yml"
|
||||
- "hi-ml-cpath/**"
|
||||
- ".github/workflows/cpath-pr.yml"
|
||||
# When the core hi-ml libraries change, test the histo test too, to find problems with backwards compatibility
|
||||
- "hi-ml-azure/**"
|
||||
- "hi-ml/**"
|
||||
|
||||
env:
|
||||
pythonVersion: 3.7
|
||||
folder: hi-ml-histopathology
|
||||
module_for_coverage_reporting: histopathology
|
||||
folder: hi-ml-cpath
|
||||
module_for_coverage_reporting: health_cpath
|
||||
HIML_TENANT_ID: ${{ secrets.HIML_TENANT_ID }}
|
||||
HIML_RESOURCE_GROUP: ${{ secrets.HIML_RESOURCE_GROUP }}
|
||||
HIML_SUBSCRIPTION_ID: ${{ secrets.HIML_SUBSCRIPTION_ID }}
|
|
@ -164,7 +164,7 @@ temp_requirements.txt
|
|||
cifar-10-python.tar.gz
|
||||
cifar-10-batches-py/
|
||||
None/
|
||||
hi-ml-histopathology/testSSL/test_outputs
|
||||
hi-ml-cpath/testSSL/test_outputs
|
||||
test_outputs/
|
||||
lightning_logs/
|
||||
# Downloaded datasets
|
||||
|
|
|
@ -40,10 +40,10 @@ repos:
|
|||
files: ^hi-ml-azure/
|
||||
args: [--config, hi-ml-azure/.flake8]
|
||||
- id: flake8
|
||||
name: flake8 ./hi-ml-histopathology/
|
||||
alias: flake8-hi-ml-histopathology
|
||||
files: ^hi-ml-histopathology/
|
||||
args: [--config, hi-ml-histopathology/.flake8]
|
||||
name: flake8 ./hi-ml-cpath/
|
||||
alias: flake8-hi-ml-cpath
|
||||
files: ^hi-ml-cpath/
|
||||
args: [--config, hi-ml-cpath/.flake8]
|
||||
- id: flake8
|
||||
name: flake8 ./hi-ml-multimodal/
|
||||
alias: flake8-hi-ml-multimodal
|
||||
|
|
6
Makefile
6
Makefile
|
@ -42,7 +42,7 @@ conda_update:
|
|||
conda env update -n $(CONDA_DEFAULT_ENV) --file hi-ml/environment.yml
|
||||
|
||||
# Set the conda environment for local development work, that contains all packages need for hi-ml, hi-ml-azure
|
||||
# and hi-ml-histopathology with hi-ml and hi-ml-azure installed in editable mode
|
||||
# and hi-ml-cpath with hi-ml and hi-ml-azure installed in editable mode
|
||||
conda: conda_update call_pip_local
|
||||
|
||||
## Actions
|
||||
|
@ -120,10 +120,10 @@ combine: pip_test
|
|||
mkdir -p coverage
|
||||
cp hi-ml/.coverage coverage/hi-ml-coverage
|
||||
cp hi-ml-azure/.coverage coverage/hi-ml-azure-coverage
|
||||
cp hi-ml-histopathology/.coverage coverage/hi-ml-histopathology-coverage
|
||||
cp hi-ml-cpath/.coverage coverage/hi-ml-cpath-coverage
|
||||
cp .coveragerc coverage/
|
||||
cd coverage && \
|
||||
coverage combine hi-ml-coverage hi-ml-azure-coverage hi-ml-histopathology-coverage && \
|
||||
coverage combine hi-ml-coverage hi-ml-azure-coverage hi-ml-cpath-coverage && \
|
||||
coverage html && \
|
||||
coverage xml && \
|
||||
pycobertura show --format text --output coverage.txt coverage.xml
|
||||
|
|
|
@ -12,7 +12,7 @@ This repository consists of two Python packages, as well as project-specific cod
|
|||
|
||||
* PyPi package [hi-ml-azure](https://pypi.org/project/hi-ml-azure/) - providing helper functions for running in AzureML.
|
||||
* PyPi package [hi-ml](https://pypi.org/project/hi-ml/) - providing ML components.
|
||||
* hi-ml-histopathology: Models and workflows for working with histopathology images
|
||||
* hi-ml-cpath: Models and workflows for working with histopathology images
|
||||
|
||||
## Getting started
|
||||
|
||||
|
@ -24,7 +24,7 @@ For just the AzureML helper functions:
|
|||
|
||||
* Install from `pypi` via `pip`, by running `pip install hi-ml-azure`
|
||||
|
||||
For the histopathology workflows, please follow the instructions [here](hi-ml-histopathology/README.md).
|
||||
For the histopathology workflows, please follow the instructions [here](hi-ml-cpath/README.md).
|
||||
|
||||
If you would like to contribute to the code, please check the [developer guide](docs/source/developers.md).
|
||||
|
||||
|
|
|
@ -21,7 +21,7 @@ import sys
|
|||
# Make the source code for both packages available here
|
||||
sys.path.insert(0, os.path.abspath('../../hi-ml/src'))
|
||||
sys.path.insert(0, os.path.abspath('../../hi-ml-azure/src'))
|
||||
sys.path.insert(0, os.path.abspath('../../hi-ml-histopathology/src'))
|
||||
sys.path.insert(0, os.path.abspath('../../hi-ml-cpath/src'))
|
||||
sys.path.insert(0, os.path.abspath('../../hi-ml-multimodal/src'))
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
|
@ -72,7 +72,7 @@ html_theme = 'sphinx_rtd_theme'
|
|||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = [] # type: ignore
|
||||
|
||||
modindex_common_prefix = ["health_azure.", "health_ml.", "histopathology."]
|
||||
modindex_common_prefix = ["health_azure.", "health_ml.", "health_cpath."]
|
||||
|
||||
source_suffix = {
|
||||
'.rst': 'restructuredtext',
|
||||
|
|
|
@ -20,7 +20,7 @@ We provide a set of example debugging configs for each of hi-ml module:
|
|||
|
||||
* [launch.json in hi-ml](https://github.com/microsoft/hi-ml/tree/main/hi-ml/.vscode/launch.json)
|
||||
* [launch.json in hi-ml-azure](https://github.com/microsoft/hi-ml/tree/main/hi-ml-azure/.vscode/launch.json)
|
||||
* [launch.json in hi-ml-histopathology](https://github.com/microsoft/hi-ml/tree/main/hi-ml-histopathology/.vscode/launch.json)
|
||||
* [launch.json in hi-ml-cpath](https://github.com/microsoft/hi-ml/tree/main/hi-ml-cpath/.vscode/launch.json)
|
||||
|
||||
VS Code restricts debugging to user-written code only by default. If you want to step through external code and
|
||||
standard libraries functions, set `"justMyCode": false` inside the debugging config block in the `launch.json` file.
|
||||
|
|
|
@ -15,14 +15,14 @@ at once or individual projects separately in VSCode.
|
|||
* To open all projects at once, use VSCode's "Open Workspace from File", and
|
||||
select `himl-projects.code-workspace`.
|
||||
* To open individual projects, use VSCode's "Open Folder", and select one of the
|
||||
folders `hi-ml-azure`, `hi-ml`, or `hi-ml-histopathology`
|
||||
folders `hi-ml-azure`, `hi-ml`, or `hi-ml-cpath`
|
||||
|
||||
## Creating a Conda environment
|
||||
|
||||
Different projects in this repository use different Conda environments:
|
||||
|
||||
* The `himl` Conda environment should be used for work on the `hi-ml` and `hi-ml-azure` projects.
|
||||
* The `HimlHisto` Conda environment should be used for work on `hi-ml-histopathology`.
|
||||
* The `HimlHisto` Conda environment should be used for work on `hi-ml-cpath`.
|
||||
|
||||
Please select the right Python interpreter for your project (or all projects if using the `himl-projects` workspace)
|
||||
inside VSCode, by choosing "Python: Select Interpreter" from the command palette (Ctrl-Shift-P on VSCode for Windows)
|
||||
|
@ -138,7 +138,7 @@ When running the tests locally, they can either be run against the source direct
|
|||
|
||||
### Test discovery in VSCode
|
||||
|
||||
All tests in the repository should be picked up automatically by VSCode. In particular, this includes the tests in the `hi-ml-histopathology` folder, which
|
||||
All tests in the repository should be picked up automatically by VSCode. In particular, this includes the tests in the `hi-ml-cpath` folder, which
|
||||
are not always necessary when working on the core `hi-ml` projects.
|
||||
|
||||
## Creating a New Release
|
||||
|
|
|
@ -52,7 +52,7 @@ upload annotations to a deployed DSA.
|
|||
For example:
|
||||
|
||||
```shell
|
||||
SCRIPT="hi-ml-histopathology/src/histopathology/utils/girder.py"
|
||||
SCRIPT="hi-ml-cpath/src/histopathology/utils/girder.py"
|
||||
python $SCRIPT \
|
||||
--run-id "Experiment_transformer_Gleason_run_0" \
|
||||
--dsa-url "https://my-deployed-dsa.azure.com/" \
|
||||
|
|
|
@ -1,12 +1,12 @@
|
|||
# HI-ML Tools for Computational Pathology
|
||||
|
||||
The directory [`hi-ml-histopathology`](https://github.com/microsoft/hi-ml/tree/main/hi-ml-histopathology) contains code
|
||||
The directory [`hi-ml-cpath`](https://github.com/microsoft/hi-ml/tree/main/hi-ml-cpath) contains code
|
||||
for runnning experiments in Computational Pathology.
|
||||
|
||||
## Setting up your computer
|
||||
|
||||
The tools for computational pathology cannot be as a Python package, but rather directly from the Git repository. Please
|
||||
follow the instructions in [README](https://github.com/microsoft/hi-ml/blob/main/hi-ml-histopathology/README.md) to set
|
||||
follow the instructions in [README](https://github.com/microsoft/hi-ml/blob/main/hi-ml-cpath/README.md) to set
|
||||
up your local Python environment.
|
||||
|
||||
## Onboarding to Azure
|
||||
|
@ -34,13 +34,13 @@ Please follow the [detailed instructions](public_datasets.md) to download and pr
|
|||
|
||||
## New Model configurations
|
||||
|
||||
To define your own model configuration, place a class definition in the directory `histopathology.configs`. The class should
|
||||
To define your own model configuration, place a class definition in the directory `health_cpath.configs`. The class should
|
||||
inherit from a
|
||||
[LightningContainer](https://github.com/microsoft/hi-ml/blob/39911d217c919d8213ad36c9c776f69369d98509/hi-ml/src/health_ml/lightning_container.py#L24).
|
||||
As an example, please check the [HelloWorld
|
||||
model](https://github.com/microsoft/hi-ml/blob/0793cbd1a874920d04b0a8f1298a7a112cfd712c/hi-ml/src/health_ml/configs/hello_world.py#L232)
|
||||
or the [base class for the MIL
|
||||
models](https://github.com/microsoft/hi-ml/blob/1d96c9bcdb326ad4d145ab082f45a2116d776a76/hi-ml-histopathology/src/histopathology/configs/classification/BaseMIL.py#L39).
|
||||
models](https://github.com/microsoft/hi-ml/blob/1d96c9bcdb326ad4d145ab082f45a2116d776a76/hi-ml-cpath/src/histopathology/configs/classification/BaseMIL.py#L39).
|
||||
|
||||
## Mount datasets
|
||||
|
||||
|
@ -49,7 +49,7 @@ or mount them. "Mounting" here means that the dataset will be loaded on-demand o
|
|||
docs](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-train-with-datasets#mount-vs-download)). This is ideal if you expect that
|
||||
you will only need a small number of files, or if the disk of your machine is too small to download the full dataset.
|
||||
|
||||
You can mount the dataset by executing this script in `<root>/hi-ml-histopathology`:
|
||||
You can mount the dataset by executing this script in `<root>/hi-ml-cpath`:
|
||||
|
||||
```shell
|
||||
python src/histopathology/scripts/mount_azure_dataset.py --dataset_id PANDA
|
||||
|
|
|
@ -22,11 +22,11 @@ You will also need to run the dataset preparations for the PANDA dataset, as des
|
|||
|
||||
## Running the model as-is
|
||||
|
||||
If you have a GPU available, you can run training on that machine, by executing in `<root>/hi-ml-histopathology`:
|
||||
If you have a GPU available, you can run training on that machine, by executing in `<root>/hi-ml-cpath`:
|
||||
|
||||
```shell
|
||||
conda activate HimlHisto
|
||||
python ../hi-ml/src/health_ml/runner.py --model histopathology.SlidesPandaImageNetMILBenchmark
|
||||
python ../hi-ml/src/health_ml/runner.py --model health_cpath.idesPandaImageNetMILBenchmark
|
||||
```
|
||||
|
||||
Running the model will automatically mount (download on-the-fly) the PANDA dataset from Azure. To enable that, you will
|
||||
|
@ -44,7 +44,7 @@ addition, you can turn on fine-tuning of the encoder, which will improve the res
|
|||
|
||||
```shell
|
||||
conda activate HimlHisto
|
||||
python ../hi-ml/src/health_ml/runner.py --model histopathology.SlidesPandaImageNetMILBenchmark --is_finetune --cluster=<your_cluster_name>
|
||||
python ../hi-ml/src/health_ml/runner.py --model health_cpath.idesPandaImageNetMILBenchmark --is_finetune --cluster=<your_cluster_name>
|
||||
```
|
||||
|
||||
Then the script will output "Successfully queued run number ..." and a line prefixed "Run URL: ...". Open that
|
||||
|
@ -87,16 +87,16 @@ This will mean that the job starts faster, but may not run at maximum speed beca
|
|||
To use cross-validation, supply the additional commandline flag `--crossval_count=5` for 5-fold cross-validation, like:
|
||||
|
||||
```shell
|
||||
python ../hi-ml/src/health_ml/runner.py --model histopathology.SlidesPandaImageNetMILBenchmark --crossval_count=5 --cluster=<your_cluster_name>
|
||||
python ../hi-ml/src/health_ml/runner.py --model health_cpath.idesPandaImageNetMILBenchmark --crossval_count=5 --cluster=<your_cluster_name>
|
||||
```
|
||||
|
||||
Cross-validation will start 5 training runs in parallel. For this reason, cross-validation can only be used in AzureML.
|
||||
|
||||
To compute aggregated metrics of the hyperdrive run in Azure ML, replace the `run_id` in
|
||||
`hi-ml-histopathology/src/histopathology/scripts/aggregate_metrics_crossvalidation.py` with the Run ID of the hyperdrive
|
||||
`hi-ml-cpath/src/histopathology/scripts/aggregate_metrics_crossvalidation.py` with the Run ID of the hyperdrive
|
||||
run, and run the script as follows:
|
||||
|
||||
```shell
|
||||
conda activate HimlHisto
|
||||
python hi-ml-histopathology/src/histopathology/scripts/aggregate_metrics_crossvalidation.py
|
||||
python hi-ml-cpath/src/histopathology/scripts/aggregate_metrics_crossvalidation.py
|
||||
```
|
||||
|
|
|
@ -39,7 +39,7 @@ repository. To prepare:
|
|||
|
||||
- In the last statement, where we upload the full dataset to Azure, replace `<your_storage_account>` with the
|
||||
name of your Azure storage account.
|
||||
- For Python to pick up the paths in `hi-ml-histopathology/src/histopathology/scripts/tcga_dataset_prep.py`, you need to
|
||||
- For Python to pick up the paths in `hi-ml-cpath/src/histopathology/scripts/tcga_dataset_prep.py`, you need to
|
||||
add the `hi
|
||||
|
||||
Note: Depending on the speed of your internet connection, this script can run for several hours because it downloads
|
||||
|
@ -64,8 +64,8 @@ do
|
|||
done
|
||||
# Create a summary file dataset.csv with all file paths and class labels
|
||||
cd ..
|
||||
export PYTHONPATH=`pwd`/hi-ml-histopathology/src
|
||||
python hi-ml-histopathology/src/histopathology/scripts/tcga_dataset_prep.py
|
||||
export PYTHONPATH=`pwd`/hi-ml-cpath/src
|
||||
python hi-ml-cpath/src/histopathology/scripts/tcga_dataset_prep.py
|
||||
# Upload
|
||||
azcopy copy TCGA-Crck https://<your_storage_account>.blob.core.windows.net/datasets/ --recursive
|
||||
```
|
||||
|
|
|
@ -25,9 +25,9 @@ that class:
|
|||
`health_ml.configs` namespace. It can be in any module/folder inside of that namespace.
|
||||
- If the class is outside of the `health_ml.configs` (as would be normal if using the `himl-runner` from a package),
|
||||
you need to provide some "hints" where to start searching. It is enough to provide the start of the namespace string:
|
||||
for example, `--model histopathology.PandaImageNetMIL` is effectively telling the runner to search for the
|
||||
for example, `--model health_cpath.PandaImageNetMIL` is effectively telling the runner to search for the
|
||||
`PandaImageNetMIL` class _anywhere_ in the `histopathology` namespace. You can think of this as
|
||||
`histopathology.*.PandaImageNetMIL`
|
||||
`health_cpath.*.PandaImageNetMIL`
|
||||
|
||||
## Running ML experiments in Azure ML
|
||||
|
||||
|
|
|
@ -16,11 +16,11 @@ from the Cancer Genome Atlas (TCGA) with accompanying binarized MicroSatellite I
|
|||
TCGA-CRCk, the dataset is already tiled, i.e., the WSI are not available. In [public_datasets.md](./public_datasets.md)
|
||||
you will find instructions on how to download and setup the TCGA-CRCk dataset.
|
||||
|
||||
To train an image encoder using SSL locally run this in the `hi-ml-histopathology` folder, with the `HimlHisto` conda
|
||||
To train an image encoder using SSL locally run this in the `hi-ml-cpath` folder, with the `HimlHisto` conda
|
||||
enviroment activated:
|
||||
|
||||
```bash
|
||||
python ../hi-ml/src/health_ml/runner.py --model histopathology.CRCK_SimCLR
|
||||
python ../hi-ml/src/health_ml/runner.py --model health_cpath.CRCK_SimCLR
|
||||
```
|
||||
|
||||
The model class
|
||||
|
@ -76,7 +76,7 @@ the TCGA-CRCk dataset. Assuming you are using a total of 8 GPUs (e.g. 1 node wi
|
|||
Azure you can start training with the following command in the repository root folder:
|
||||
|
||||
```bash
|
||||
python hi-ml/src/health_ml/runner.py --model histopathology.CRCK_SimCLR --cluster CLUSTER_NAME --conda_env hi-ml-histopathology/environment.yml
|
||||
python hi-ml/src/health_ml/runner.py --model health_cpath.CRCK_SimCLR --cluster CLUSTER_NAME --conda_env hi-ml-cpath/environment.yml
|
||||
```
|
||||
|
||||
A SimCLR run with 200 epochs, 8 GPUs, and a batch size of 48 (per GPU) takes about 6 hours. On Azure we use
|
||||
|
|
|
@ -37,7 +37,7 @@ The suitable AzureML base Docker images are therefore the ones containing `cuda1
|
|||
|
||||
## Performance
|
||||
|
||||
An exploratory set of scripts for comparing loading images with OpenSlide or cuCIM, and performing tiling using both libraries can be found at [`slide_image_loading`](https://github.com/microsoft/hi-ml/tree/main/hi-ml-histopathology/other/slide_image_loading).
|
||||
An exploratory set of scripts for comparing loading images with OpenSlide or cuCIM, and performing tiling using both libraries can be found at [`slide_image_loading`](https://github.com/microsoft/hi-ml/tree/main/hi-ml-cpath/other/slide_image_loading).
|
||||
|
||||
### Loading and saving at lowest resolution
|
||||
|
||||
|
|
|
@ -50,7 +50,7 @@ class RunPytestConfig(param.Parameterized):
|
|||
coverage_module: str = param.String(
|
||||
default="",
|
||||
doc="This value is used as an argument to --cov of pytest to collect code coverage for the specified pyhton "
|
||||
"module. For example, in the subfolder hi-ml-histopathology, one can collect code coverage for the "
|
||||
"module. For example, in the subfolder hi-ml-cpath, one can collect code coverage for the "
|
||||
"histopathology module by setting `module=histopathology`. If set to '' (default), no coverage is collected."
|
||||
)
|
||||
cluster: str = param.String(default="", doc="The name of the AzureML compute cluster where the script should run.")
|
||||
|
|
|
@ -22,7 +22,7 @@
|
|||
"request": "launch",
|
||||
"program": "${workspaceFolder}/../hi-ml/src/health_ml/runner.py",
|
||||
"args": [
|
||||
"--model=histopathology.configs.classification.DeepSMILECrck",
|
||||
"--model=health_cpath.configs.classification.DeepSMILECrck",
|
||||
"--cluster=lite-testing-ds2",
|
||||
],
|
||||
"console": "integratedTerminal"
|
||||
|
@ -33,7 +33,7 @@
|
|||
"request": "launch",
|
||||
"program": "${workspaceFolder}/../hi-ml/src/health_ml/runner.py",
|
||||
"args": [
|
||||
"--model=histopathology.DeepSMILECrck",
|
||||
"--model=health_cpath.DeepSMILECrck",
|
||||
],
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
|
@ -43,7 +43,7 @@
|
|||
"request": "launch",
|
||||
"program": "${workspaceFolder}/../hi-ml/src/health_ml/runner.py",
|
||||
"args": [
|
||||
"--model=histopathology.SlidesPandaImageNetMIL",
|
||||
"--model=health_cpath.SlidesPandaImageNetMIL",
|
||||
"--pl_fast_dev_run=10",
|
||||
"--crossval_count=0",
|
||||
"--batch_size=2",
|
||||
|
@ -60,7 +60,7 @@
|
|||
"request": "launch",
|
||||
"program": "${workspaceFolder}/../hi-ml/src/health_ml/runner.py",
|
||||
"args": [
|
||||
"--model=histopathology.TilesPandaImageNetMIL",
|
||||
"--model=health_cpath.TilesPandaImageNetMIL",
|
||||
"--pl_fast_dev_run=10",
|
||||
"--crossval_count=0",
|
||||
"--batch_size=2",
|
|
@ -67,5 +67,10 @@
|
|||
"${workspaceFolder}/src",
|
||||
"${workspaceFolder}/testhisto",
|
||||
"${workspaceFolder}/testSSL"
|
||||
]
|
||||
],
|
||||
"workbench.colorCustomizations": {
|
||||
"activityBar.background": "#4B0769",
|
||||
"titleBar.activeBackground": "#690A93",
|
||||
"titleBar.activeForeground": "#FDF9FF"
|
||||
}
|
||||
}
|
|
@ -46,7 +46,7 @@ flake8:
|
|||
|
||||
# run mypy, assuming test requirements already installed
|
||||
mypy:
|
||||
mypy --install-types --show-error-codes --non-interactive --package histopathology
|
||||
mypy --install-types --show-error-codes --non-interactive --package health_cpath
|
||||
mypy --install-types --show-error-codes --non-interactive --package SSL
|
||||
mypy --install-types --show-error-codes --non-interactive --package testhisto
|
||||
mypy --install-types --show-error-codes --non-interactive --package testSSL
|
||||
|
@ -60,14 +60,14 @@ pytest:
|
|||
|
||||
# run pytest with coverage on package
|
||||
pytest_coverage:
|
||||
pytest --cov=histopathology --cov SSL --cov-branch --cov-report=html --cov-report=xml --cov-report=term-missing --cov-config=.coveragerc
|
||||
pytest --cov=health_cpath --cov SSL --cov-branch --cov-report=html --cov-report=xml --cov-report=term-missing --cov-config=.coveragerc
|
||||
|
||||
# Run regression tests and compare performance
|
||||
regression_test_tilespandaimagenetmil:
|
||||
{ \
|
||||
cd ../ ;\
|
||||
python hi-ml/src/health_ml/runner.py --model=histopathology.TilesPandaImageNetMIL --batch_size=2 \
|
||||
--cluster pr-gpu --mount_in_azureml --conda_env hi-ml-histopathology/environment.yml \
|
||||
python hi-ml/src/health_ml/runner.py --model=health_cpath.TilesPandaImageNetMIL --batch_size=2 \
|
||||
--cluster pr-gpu --mount_in_azureml --conda_env hi-ml-health_cpath/environment.yml \
|
||||
--regression_test_folder=testhisto/RegressionTestResults/TilesPANDAImageNetMIL/\
|
||||
HD_4ab0d833-fe55-44e8-aa04-cbaadbcc2733_0 --regression_test_csv_tolerance=0.5 --is_finetune \
|
||||
--regression_metrics='test/accuracy,test/macro_accuracy,test/weighted_accuracy,test/auroc,test/ISUP 0,test/ISUP 1,\
|
||||
|
@ -77,8 +77,8 @@ regression_test_tilespandaimagenetmil:
|
|||
regression_test_slidespandaimagenetmil:
|
||||
{ \
|
||||
cd ../ ;\
|
||||
python hi-ml/src/health_ml/runner.py --model=histopathology.SlidesPandaImageNetMILBenchmark \
|
||||
--cluster pr-gpu --mount_in_azureml --conda_env hi-ml-histopathology/environment.yml \
|
||||
python hi-ml/src/health_ml/runner.py --model=health_cpath.SlidesPandaImageNetMILBenchmark \
|
||||
--cluster pr-gpu --mount_in_azureml --conda_env hi-ml-cpath/environment.yml \
|
||||
--regression_test_folder=testhisto/RegressionTestResults/SlidesPANDAImageNetMIL/\
|
||||
HD_0e805b91-319d-4fde-8bc3-1cea3a6d08dd_0 --regression_test_csv_tolerance=0.5 --is_finetune \
|
||||
--regression_metrics='test/accuracy,test/macro_accuracy,test/weighted_accuracy,test/auroc,test/ISUP 0,test/ISUP 1,\
|
||||
|
@ -88,9 +88,9 @@ regression_test_slidespandaimagenetmil:
|
|||
regression_test_tcgacrcksslmil:
|
||||
{ \
|
||||
cd ../ ;\
|
||||
python hi-ml/src/health_ml/runner.py --model=histopathology.TcgaCrckSSLMIL \
|
||||
--cluster pr-gpu --conda_env hi-ml-histopathology/environment.yml --max_epochs=50 \
|
||||
--regression_test_folder=hi-ml-histopathology/testhisto/RegressionTestResults/\
|
||||
python hi-ml/src/health_ml/runner.py --model=health_cpath.TcgaCrckSSLMIL \
|
||||
--cluster pr-gpu --conda_env hi-ml-cpath/environment.yml --max_epochs=50 \
|
||||
--regression_test_folder=hi-ml-cpath/testhisto/RegressionTestResults/\
|
||||
TcgaCrckSSLMIL/HD_d76ef6cd-0403-4923-b8fa-dfd2827c5d74 --regression_test_csv_tolerance=0.5 \
|
||||
--regression_metrics=test/accuracy,test/auroc,test/f1score,test/precision,test/recall;\
|
||||
}
|
||||
|
@ -98,9 +98,9 @@ regression_test_tcgacrcksslmil:
|
|||
regression_test_crck_simclr:
|
||||
{ \
|
||||
cd ../; \
|
||||
python hi-ml/src/health_ml/runner.py --model=histopathology.CRCK_SimCLR \
|
||||
--cluster pr-gpu --conda_env hi-ml-histopathology/environment.yml \
|
||||
--regression_test_folder=hi-ml-histopathology/testhisto/RegressionTestResults/CRCK_SimCLR/\
|
||||
python hi-ml/src/health_ml/runner.py --model=health_cpath.CRCK_SimCLR \
|
||||
--cluster pr-gpu --conda_env hi-ml-cpath/environment.yml \
|
||||
--regression_test_folder=hi-ml-cpath/testhisto/RegressionTestResults/CRCK_SimCLR/\
|
||||
CRCK_SimCLR_1653673515_42d53d78 --regression_test_csv_tolerance=0.5 \
|
||||
--regression_metrics=ssl_online_evaluator/val/AreaUnderRocCurve,\
|
||||
ssl_online_evaluator/val/AreaUnderPRCurve,ssl_online_evaluator/val/AccuracyAtThreshold05 --max_epochs=200;\
|
||||
|
@ -113,9 +113,9 @@ regression tests: regression_test_tilespandaimagenetmil regression_test_slidespa
|
|||
smoke_test_slidespandaimagenetmil:
|
||||
{ \
|
||||
cd ../ ;\
|
||||
python hi-ml/src/health_ml/runner.py --model=histopathology.SlidesPandaImageNetMILBenchmark \
|
||||
--mount_in_azureml --conda_env hi-ml-histopathology/environment.yml --is_finetune \
|
||||
--conda_env hi-ml-histopathology/environment.yml --crossval_count=0 --num_top_slides=2 --num_top_tiles=2 \
|
||||
python hi-ml/src/health_ml/runner.py --model=health_cpath.SlidesPandaImageNetMILBenchmark \
|
||||
--mount_in_azureml --conda_env hi-ml-cpath/environment.yml --is_finetune \
|
||||
--conda_env hi-ml-cpath/environment.yml --crossval_count=0 --num_top_slides=2 --num_top_tiles=2 \
|
||||
--pl_limit_train_batches=2 --pl_limit_val_batches=2 --pl_limit_test_batches=2 \
|
||||
--max_bag_size=3 --max_bag_size_inf=3 --max_epochs=2;\
|
||||
}
|
||||
|
@ -124,8 +124,8 @@ smoke_test_slidespandaimagenetmil:
|
|||
smoke_test_tilespandaimagenetmil:
|
||||
{ \
|
||||
cd ../ ;\
|
||||
python hi-ml/src/health_ml/runner.py --model=histopathology.TilesPandaImageNetMIL \
|
||||
--mount_in_azureml --conda_env hi-ml-histopathology/environment.yml \
|
||||
python hi-ml/src/health_ml/runner.py --model=health_cpath.TilesPandaImageNetMIL \
|
||||
--mount_in_azureml --conda_env hi-ml-cpath/environment.yml \
|
||||
--is_finetune --batch_size=2 --crossval_count=0 --num_top_slides=2 --num_top_tiles=2 \
|
||||
--pl_limit_train_batches=2 --pl_limit_val_batches=2 --pl_limit_test_batches=2 \
|
||||
--max_bag_size=3 --max_bag_size_inf=3 --max_epochs=2;\
|
||||
|
@ -135,8 +135,8 @@ smoke_test_tilespandaimagenetmil:
|
|||
smoke_test_tcgacrcksslmil:
|
||||
{ \
|
||||
cd ../ ;\
|
||||
python hi-ml/src/health_ml/runner.py --model=histopathology.TcgaCrckSSLMIL \
|
||||
--conda_env hi-ml-histopathology/environment.yml --crossval_count=1 \
|
||||
python hi-ml/src/health_ml/runner.py --model=health_cpath.TcgaCrckSSLMIL \
|
||||
--conda_env hi-ml-cpath/environment.yml --crossval_count=1 \
|
||||
--pl_limit_train_batches=2 --pl_limit_val_batches=2 --pl_limit_test_batches=2 \
|
||||
--max_bag_size=3 --max_bag_size_inf=3 --max_epochs=2;\
|
||||
}
|
||||
|
@ -145,8 +145,8 @@ smoke_test_tcgacrcksslmil:
|
|||
smoke_test_crck_simclr:
|
||||
{ \
|
||||
cd ../; \
|
||||
python hi-ml/src/health_ml/runner.py --model=histopathology.CRCK_SimCLR \
|
||||
--conda_env hi-ml-histopathology/environment.yml \
|
||||
python hi-ml/src/health_ml/runner.py --model=health_cpath.CRCK_SimCLR \
|
||||
--conda_env hi-ml-cpath/environment.yml \
|
||||
--pl_limit_train_batches=2 --pl_limit_val_batches=2 --pl_limit_test_batches=2 \
|
||||
--is_debug_model=True --num_workers=0 --max_epochs=2; \
|
||||
}
|
|
@ -13,7 +13,7 @@ readthedocs](https://hi-ml.readthedocs.io/en/latest/histopathology.html).
|
|||
For working on the histopathology folder, please create a separate Conda environment.
|
||||
|
||||
```shell
|
||||
cd hi-ml-histopathology
|
||||
cd hi-ml-cpath
|
||||
make env
|
||||
```
|
||||
|
||||
|
@ -23,7 +23,7 @@ from the command palette.
|
|||
In the event of dependencies update, you can update your conda environment by running the following:
|
||||
|
||||
```shell
|
||||
cd hi-ml-histopathology
|
||||
cd hi-ml-cpath
|
||||
make conda_update
|
||||
```
|
||||
|
||||
|
@ -36,15 +36,15 @@ In addition, please download an AzureML workspace configuration file for the wor
|
|||
* Select "Download config file".
|
||||
* Save that file into the the repository root.
|
||||
|
||||
Once that config file is in place, all Python runs that you start inside the `hi-ml-histopathology` folder will automatically use this config file.
|
||||
Once that config file is in place, all Python runs that you start inside the `hi-ml-cpath` folder will automatically use this config file.
|
||||
|
||||
## Running histopathology models
|
||||
|
||||
To test your setup, please execute in the `hi-ml-histopathology` folder:
|
||||
To test your setup, please execute in the `hi-ml-cpath` folder:
|
||||
|
||||
```shell
|
||||
conda activate HimlHisto
|
||||
python ../hi-ml/src/health_ml/runner.py --model histopathology.TcgaCrckImageNetMIL --cluster=training-nd24
|
||||
python ../hi-ml/src/health_ml/runner.py --model health_cpath.TcgaCrckImageNetMIL --cluster=training-nd24
|
||||
```
|
||||
|
||||
This should start an AzureML job in the AzureML workspace that you configured above via `config.json`. You may need to adjust the name of
|
||||
|
@ -52,12 +52,12 @@ the compute cluster (`training-nd24` in the above example).
|
|||
|
||||
### Conda environment
|
||||
|
||||
If you start your jobs in the `hi-ml-histopathology` folder, they will automatically pick up the Conda environment file that is present in that folder.
|
||||
If you start your jobs in a different folder, you need to add the `--conda_env` option to point to the file `<repo_root>/hi-ml-histopathology/environment.yml`.
|
||||
If you start your jobs in the `hi-ml-cpath` folder, they will automatically pick up the Conda environment file that is present in that folder.
|
||||
If you start your jobs in a different folder, you need to add the `--conda_env` option to point to the file `<repo_root>/hi-ml-cpath/environment.yml`.
|
||||
|
||||
## Running histopathology tests
|
||||
|
||||
In the `hi-ml-histopathology` folder, run
|
||||
In the `hi-ml-cpath` folder, run
|
||||
|
||||
```shell
|
||||
make call_pytest
|
|
@ -13,8 +13,8 @@ from monai.data.dataset import Dataset
|
|||
from monai.data.image_reader import WSIReader
|
||||
from tqdm import tqdm
|
||||
|
||||
from histopathology.datasets.panda_dataset import LoadPandaROId, PandaDataset
|
||||
from histopathology.preprocessing import tiling
|
||||
from health_cpath.datasets.panda_dataset import LoadPandaROId, PandaDataset
|
||||
from health_cpath.preprocessing import tiling
|
||||
|
||||
CSV_COLUMNS = ['slide_id', 'tile_id', 'image', 'mask', 'tile_x', 'tile_y', 'occupancy',
|
||||
'data_provider', 'slide_isup_grade', 'slide_gleason_score']
|
|
@ -14,7 +14,7 @@ import numpy as np
|
|||
|
||||
from health_azure import DatasetConfig, submit_to_azure_if_needed
|
||||
|
||||
from histopathology.preprocessing.create_tiles_dataset import process_slide, save_tile, generate_tiles
|
||||
from health_cpath.preprocessing.create_tiles_dataset import process_slide, save_tile, generate_tiles
|
||||
|
||||
|
||||
def profile_cucim(input_file: Path,
|
||||
|
@ -137,7 +137,7 @@ def profile_main(mount_point: Path,
|
|||
label: str,
|
||||
process: Callable) -> None:
|
||||
def wrap_main() -> None:
|
||||
from histopathology.preprocessing.create_tiles_dataset import main
|
||||
from health_cpath.preprocessing.create_tiles_dataset import main
|
||||
main(process,
|
||||
panda_dir=mount_point,
|
||||
root_output_dir=output_folder / label,
|
|
@ -7,9 +7,9 @@ from typing import Any
|
|||
|
||||
from SSL.lightning_containers.ssl_container import EncoderName, SSLContainer, SSLDatasetName
|
||||
from SSL.utils import SSLTrainingType
|
||||
from histopathology.datasets.default_paths import TCGA_CRCK_DATASET_ID
|
||||
from histopathology.datasets.tcga_crck_tiles_dataset import TcgaCrck_TilesDatasetWithReturnIndex
|
||||
from histopathology.configs.SSL.HistoSimCLRContainer import HistoSSLContainer
|
||||
from health_cpath.datasets.default_paths import TCGA_CRCK_DATASET_ID
|
||||
from health_cpath.datasets.tcga_crck_tiles_dataset import TcgaCrck_TilesDatasetWithReturnIndex
|
||||
from health_cpath.configs.SSL.HistoSimCLRContainer import HistoSSLContainer
|
||||
|
||||
|
||||
class SSLDatasetNameHiml(SSLDatasetName, Enum): # type: ignore
|
|
@ -8,9 +8,9 @@ from typing import Any
|
|||
from SSL.lightning_containers.ssl_container import EncoderName, SSLContainer, SSLDatasetName
|
||||
from SSL.utils import SSLTrainingType
|
||||
from health_azure.utils import is_running_in_azure_ml
|
||||
from histopathology.datasets.panda_tiles_dataset import PandaTilesDatasetWithReturnIndex
|
||||
from histopathology.configs.SSL.HistoSimCLRContainer import HistoSSLContainer
|
||||
from histopathology.datasets.default_paths import PANDA_TILES_DATASET_ID
|
||||
from health_cpath.datasets.panda_tiles_dataset import PandaTilesDatasetWithReturnIndex
|
||||
from health_cpath.configs.SSL.HistoSimCLRContainer import HistoSSLContainer
|
||||
from health_cpath.datasets.default_paths import PANDA_TILES_DATASET_ID
|
||||
|
||||
|
||||
class SSLDatasetNameHiml(SSLDatasetName, Enum): # type: ignore
|
|
@ -21,14 +21,14 @@ from health_ml.lightning_container import LightningContainer
|
|||
from health_ml.utils.checkpoint_utils import get_best_checkpoint_path
|
||||
from health_ml.utils.common_utils import CHECKPOINT_FOLDER, DEFAULT_AML_UPLOAD_DIR
|
||||
|
||||
from histopathology.datamodules.base_module import CacheLocation, CacheMode, HistoDataModule
|
||||
from histopathology.datasets.base_dataset import SlidesDataset
|
||||
from histopathology.models.deepmil import TilesDeepMILModule, SlidesDeepMILModule, BaseDeepMILModule
|
||||
from histopathology.models.transforms import EncodeTilesBatchd, LoadTilesBatchd
|
||||
from histopathology.utils.deepmil_utils import EncoderParams, PoolingParams
|
||||
from histopathology.utils.output_utils import DeepMILOutputsHandler
|
||||
from histopathology.utils.naming import MetricsKey, PlotOption, SlideKey, ModelKey
|
||||
from histopathology.utils.tiles_selection_utils import TilesSelector
|
||||
from health_cpath.datamodules.base_module import CacheLocation, CacheMode, HistoDataModule
|
||||
from health_cpath.datasets.base_dataset import SlidesDataset
|
||||
from health_cpath.models.deepmil import TilesDeepMILModule, SlidesDeepMILModule, BaseDeepMILModule
|
||||
from health_cpath.models.transforms import EncodeTilesBatchd, LoadTilesBatchd
|
||||
from health_cpath.utils.deepmil_utils import EncoderParams, PoolingParams
|
||||
from health_cpath.utils.output_utils import DeepMILOutputsHandler
|
||||
from health_cpath.utils.naming import MetricsKey, PlotOption, SlideKey, ModelKey
|
||||
from health_cpath.utils.tiles_selection_utils import TilesSelector
|
||||
|
||||
|
||||
class BaseMIL(LightningContainer, EncoderParams, PoolingParams):
|
|
@ -14,18 +14,18 @@ damage response defect classification directly from H&E whole-slide images. arXi
|
|||
from typing import Any
|
||||
|
||||
from health_ml.networks.layers.attention_layers import AttentionLayer
|
||||
from histopathology.configs.run_ids import innereye_ssl_checkpoint_crck_4ws
|
||||
from histopathology.datamodules.base_module import TilesDataModule
|
||||
from histopathology.datamodules.tcga_crck_module import TcgaCrckTilesDataModule
|
||||
from histopathology.datasets.default_paths import TCGA_CRCK_DATASET_ID
|
||||
from histopathology.models.encoders import (
|
||||
from health_cpath.configs.run_ids import innereye_ssl_checkpoint_crck_4ws
|
||||
from health_cpath.datamodules.base_module import TilesDataModule
|
||||
from health_cpath.datamodules.tcga_crck_module import TcgaCrckTilesDataModule
|
||||
from health_cpath.datasets.default_paths import TCGA_CRCK_DATASET_ID
|
||||
from health_cpath.models.encoders import (
|
||||
HistoSSLEncoder,
|
||||
ImageNetEncoder,
|
||||
ImageNetSimCLREncoder,
|
||||
SSLEncoder,
|
||||
)
|
||||
from histopathology.configs.classification.BaseMIL import BaseMILTiles
|
||||
from histopathology.datasets.tcga_crck_tiles_dataset import TcgaCrck_TilesDataset
|
||||
from health_cpath.configs.classification.BaseMIL import BaseMILTiles
|
||||
from health_cpath.datasets.tcga_crck_tiles_dataset import TcgaCrck_TilesDataset
|
||||
|
||||
|
||||
class DeepSMILECrck(BaseMILTiles):
|
|
@ -6,22 +6,22 @@ from typing import Any, Optional, Set
|
|||
|
||||
from health_azure.utils import is_running_in_azure_ml
|
||||
from health_ml.networks.layers.attention_layers import AttentionLayer
|
||||
from histopathology.configs.run_ids import innereye_ssl_checkpoint_binary
|
||||
from histopathology.datamodules.panda_module import (
|
||||
from health_cpath.configs.run_ids import innereye_ssl_checkpoint_binary
|
||||
from health_cpath.datamodules.panda_module import (
|
||||
PandaSlidesDataModule,
|
||||
PandaTilesDataModule)
|
||||
from histopathology.datasets.panda_tiles_dataset import PandaTilesDataset
|
||||
from histopathology.models.encoders import (
|
||||
from health_cpath.datasets.panda_tiles_dataset import PandaTilesDataset
|
||||
from health_cpath.models.encoders import (
|
||||
HistoSSLEncoder,
|
||||
ImageNetEncoder,
|
||||
ImageNetSimCLREncoder,
|
||||
SSLEncoder)
|
||||
from histopathology.configs.classification.BaseMIL import BaseMILSlides, BaseMILTiles, BaseMIL
|
||||
from histopathology.datasets.panda_dataset import PandaDataset
|
||||
from histopathology.datasets.default_paths import (
|
||||
from health_cpath.configs.classification.BaseMIL import BaseMILSlides, BaseMILTiles, BaseMIL
|
||||
from health_cpath.datasets.panda_dataset import PandaDataset
|
||||
from health_cpath.datasets.default_paths import (
|
||||
PANDA_DATASET_ID,
|
||||
PANDA_TILES_DATASET_ID)
|
||||
from histopathology.utils.naming import PlotOption
|
||||
from health_cpath.utils.naming import PlotOption
|
||||
|
||||
|
||||
class BaseDeepSMILEPanda(BaseMIL):
|
|
@ -14,18 +14,18 @@ from health_ml.networks.layers.attention_layers import (
|
|||
TransformerPoolingBenchmark
|
||||
)
|
||||
from health_ml.deep_learning_config import OptimizerParams
|
||||
from histopathology.datasets.panda_dataset import PandaDataset
|
||||
from histopathology.datamodules.panda_module_benchmark import PandaSlidesDataModuleBenchmark
|
||||
from histopathology.models.encoders import (
|
||||
from health_cpath.datasets.panda_dataset import PandaDataset
|
||||
from health_cpath.datamodules.panda_module_benchmark import PandaSlidesDataModuleBenchmark
|
||||
from health_cpath.models.encoders import (
|
||||
HistoSSLEncoder,
|
||||
ImageNetEncoder_Resnet50,
|
||||
ImageNetSimCLREncoder,
|
||||
SSLEncoder,
|
||||
)
|
||||
from histopathology.configs.classification.DeepSMILEPanda import DeepSMILESlidesPanda
|
||||
from histopathology.models.deepmil import SlidesDeepMILModule
|
||||
from histopathology.utils.deepmil_utils import EncoderParams, PoolingParams
|
||||
from histopathology.utils.naming import MetricsKey, ModelKey, SlideKey
|
||||
from health_cpath.configs.classification.DeepSMILEPanda import DeepSMILESlidesPanda
|
||||
from health_cpath.models.deepmil import SlidesDeepMILModule
|
||||
from health_cpath.utils.deepmil_utils import EncoderParams, PoolingParams
|
||||
from health_cpath.utils.naming import MetricsKey, ModelKey, SlideKey
|
||||
|
||||
|
||||
class PandaSlidesDeepMILModuleBenchmark(SlidesDeepMILModule):
|
|
@ -15,10 +15,10 @@ from torch.utils.data import DataLoader
|
|||
from health_ml.utils.bag_utils import BagDataset, multibag_collate
|
||||
from health_ml.utils.common_utils import _create_generator
|
||||
|
||||
from histopathology.utils.wsi_utils import image_collate
|
||||
from histopathology.models.transforms import LoadTilesBatchd
|
||||
from histopathology.datasets.base_dataset import SlidesDataset, TilesDataset
|
||||
from histopathology.utils.naming import ModelKey
|
||||
from health_cpath.utils.wsi_utils import image_collate
|
||||
from health_cpath.models.transforms import LoadTilesBatchd
|
||||
from health_cpath.datasets.base_dataset import SlidesDataset, TilesDataset
|
||||
from health_cpath.utils.naming import ModelKey
|
||||
|
||||
from monai.transforms.compose import Compose
|
||||
from monai.transforms.io.dictionary import LoadImaged
|
|
@ -6,9 +6,9 @@ from typing import Tuple
|
|||
|
||||
from health_ml.utils.split_dataset import DatasetSplits
|
||||
|
||||
from histopathology.datamodules.base_module import SlidesDataModule, TilesDataModule
|
||||
from histopathology.datasets.panda_dataset import PandaDataset
|
||||
from histopathology.datasets.panda_tiles_dataset import PandaTilesDataset
|
||||
from health_cpath.datamodules.base_module import SlidesDataModule, TilesDataModule
|
||||
from health_cpath.datasets.panda_dataset import PandaDataset
|
||||
from health_cpath.datasets.panda_tiles_dataset import PandaTilesDataset
|
||||
|
||||
|
||||
class PandaTilesDataModule(TilesDataModule):
|
|
@ -12,8 +12,8 @@ from typing import Tuple
|
|||
|
||||
from health_ml.utils.split_dataset import DatasetSplits
|
||||
|
||||
from histopathology.datamodules.base_module import SlidesDataModule
|
||||
from histopathology.datasets.panda_dataset import PandaDataset
|
||||
from health_cpath.datamodules.base_module import SlidesDataModule
|
||||
from health_cpath.datasets.panda_dataset import PandaDataset
|
||||
|
||||
|
||||
class PandaSlidesDataModuleBenchmark(SlidesDataModule):
|
|
@ -7,8 +7,8 @@ from typing import Tuple
|
|||
|
||||
from health_ml.utils.split_dataset import DatasetSplits
|
||||
|
||||
from histopathology.datamodules.base_module import TilesDataModule
|
||||
from histopathology.datasets.tcga_crck_tiles_dataset import TcgaCrck_TilesDataset
|
||||
from health_cpath.datamodules.base_module import TilesDataModule
|
||||
from health_cpath.datasets.tcga_crck_tiles_dataset import TcgaCrck_TilesDataset
|
||||
|
||||
|
||||
class TcgaCrckTilesDataModule(TilesDataModule):
|
|
@ -12,7 +12,7 @@ import torch
|
|||
from sklearn.utils.class_weight import compute_class_weight
|
||||
from torch.utils.data import Dataset
|
||||
|
||||
from histopathology.utils.naming import SlideKey
|
||||
from health_cpath.utils.naming import SlideKey
|
||||
|
||||
|
||||
class TilesDataset(Dataset):
|
|
@ -13,7 +13,7 @@ from monai.transforms import MapTransform
|
|||
|
||||
from health_ml.utils import box_utils
|
||||
|
||||
from histopathology.datasets.base_dataset import SlidesDataset
|
||||
from health_cpath.datasets.base_dataset import SlidesDataset
|
||||
|
||||
try:
|
||||
from cucim import CuImage
|
|
@ -9,9 +9,9 @@ from typing import Any, Callable, Optional, Tuple, Union
|
|||
import pandas as pd
|
||||
from torchvision.datasets.vision import VisionDataset
|
||||
|
||||
from histopathology.datasets.base_dataset import TilesDataset
|
||||
from histopathology.models.transforms import load_pil_image
|
||||
from histopathology.utils.naming import TileKey
|
||||
from health_cpath.datasets.base_dataset import TilesDataset
|
||||
from health_cpath.models.transforms import load_pil_image
|
||||
from health_cpath.utils.naming import TileKey
|
||||
|
||||
from SSL.data.dataset_cls_utils import DataClassBaseWithReturnIndex
|
||||
|
|
@ -10,8 +10,8 @@ import pandas as pd
|
|||
|
||||
from torchvision.datasets.vision import VisionDataset
|
||||
|
||||
from histopathology.datasets.base_dataset import TilesDataset
|
||||
from histopathology.models.transforms import load_pil_image
|
||||
from health_cpath.datasets.base_dataset import TilesDataset
|
||||
from health_cpath.models.transforms import load_pil_image
|
||||
from SSL.data.dataset_cls_utils import DataClassBaseWithReturnIndex
|
||||
|
||||
|
|
@ -8,7 +8,7 @@ from typing import Optional, Union
|
|||
|
||||
import pandas as pd
|
||||
|
||||
from histopathology.datasets.base_dataset import SlidesDataset
|
||||
from health_cpath.datasets.base_dataset import SlidesDataset
|
||||
|
||||
|
||||
class TcgaPradDataset(SlidesDataset):
|
|
@ -13,13 +13,13 @@ from torchmetrics import AUROC, F1, Accuracy, ConfusionMatrix, Precision, Recall
|
|||
|
||||
from health_ml.utils import log_on_epoch
|
||||
from health_ml.deep_learning_config import OptimizerParams
|
||||
from histopathology.models.encoders import IdentityEncoder
|
||||
from histopathology.utils.deepmil_utils import EncoderParams, PoolingParams
|
||||
from health_cpath.models.encoders import IdentityEncoder
|
||||
from health_cpath.utils.deepmil_utils import EncoderParams, PoolingParams
|
||||
|
||||
from histopathology.datasets.base_dataset import TilesDataset
|
||||
from histopathology.utils.naming import MetricsKey, ResultsKey, SlideKey, ModelKey, TileKey
|
||||
from histopathology.utils.output_utils import (BatchResultsType, DeepMILOutputsHandler, EpochResultsType,
|
||||
validate_class_names)
|
||||
from health_cpath.datasets.base_dataset import TilesDataset
|
||||
from health_cpath.utils.naming import MetricsKey, ResultsKey, SlideKey, ModelKey, TileKey
|
||||
from health_cpath.utils.output_utils import (BatchResultsType, DeepMILOutputsHandler, EpochResultsType,
|
||||
validate_class_names)
|
||||
|
||||
RESULTS_COLS = [ResultsKey.SLIDE_ID, ResultsKey.TILE_ID, ResultsKey.IMAGE_PATH, ResultsKey.PROB,
|
||||
ResultsKey.CLASS_PROBS, ResultsKey.PRED_LABEL, ResultsKey.TRUE_LABEL, ResultsKey.BAG_ATTN]
|
|
@ -13,9 +13,9 @@ from torch import Tensor as T, nn
|
|||
from torchvision.models import resnet18
|
||||
from torchvision.transforms import Compose
|
||||
|
||||
from histopathology.utils.layer_utils import (get_imagenet_preprocessing,
|
||||
load_weights_to_model,
|
||||
setup_feature_extractor)
|
||||
from health_cpath.utils.layer_utils import (get_imagenet_preprocessing,
|
||||
load_weights_to_model,
|
||||
setup_feature_extractor)
|
||||
from SSL.lightning_modules.ssl_classifier_module import SSLClassifier
|
||||
from SSL.utils import create_ssl_image_classifier
|
||||
from SSL import encoders
|
|
@ -13,7 +13,7 @@ from monai.config.type_definitions import KeysCollection
|
|||
from monai.transforms.transform import MapTransform, Randomizable
|
||||
from torchvision.transforms.functional import to_tensor
|
||||
|
||||
from histopathology.models.encoders import TileEncoder
|
||||
from health_cpath.models.encoders import TileEncoder
|
||||
|
||||
PathOrString = Union[Path, str]
|
||||
|
|
@ -23,9 +23,9 @@ from monai.data.image_reader import WSIReader
|
|||
from tqdm import tqdm
|
||||
from health_ml.utils.box_utils import Box
|
||||
|
||||
from histopathology.preprocessing import tiling
|
||||
from histopathology.utils.naming import SlideKey, TileKey
|
||||
from histopathology.datasets.panda_dataset import PandaDataset, LoadPandaROId
|
||||
from health_cpath.preprocessing import tiling
|
||||
from health_cpath.utils.naming import SlideKey, TileKey
|
||||
from health_cpath.datasets.panda_dataset import PandaDataset, LoadPandaROId
|
||||
|
||||
CSV_COLUMNS = (
|
||||
'slide_id',
|
|
@ -17,10 +17,10 @@ from monai.data import Dataset
|
|||
from monai.data.image_reader import WSIReader
|
||||
from tqdm import tqdm
|
||||
|
||||
from histopathology.datasets.base_dataset import SlidesDataset
|
||||
from histopathology.preprocessing import tiling
|
||||
from histopathology.preprocessing.loading import LoadROId, segment_foreground
|
||||
from histopathology.utils.naming import SlideKey, TileKey
|
||||
from health_cpath.datasets.base_dataset import SlidesDataset
|
||||
from health_cpath.preprocessing import tiling
|
||||
from health_cpath.preprocessing.loading import LoadROId, segment_foreground
|
||||
from health_cpath.utils.naming import SlideKey, TileKey
|
||||
|
||||
logging.basicConfig(format='%(asctime)s %(message)s', filemode='w')
|
||||
logger = logging.getLogger()
|
||||
|
@ -291,7 +291,7 @@ def main(slides_dataset: SlidesDataset, root_output_dir: Union[str, Path],
|
|||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from histopathology.datasets.tcga_prad_dataset import TcgaPradDataset
|
||||
from health_cpath.datasets.tcga_prad_dataset import TcgaPradDataset
|
||||
|
||||
# Example set up for an existing slides dataset:
|
||||
main(slides_dataset=TcgaPradDataset("/tmp/datasets/TCGA-PRAD"),
|
|
@ -12,7 +12,7 @@ from health_ml.utils import box_utils
|
|||
from monai.data.image_reader import WSIReader
|
||||
from monai.transforms import MapTransform
|
||||
|
||||
from histopathology.utils.naming import SlideKey
|
||||
from health_cpath.utils.naming import SlideKey
|
||||
|
||||
|
||||
def get_luminance(slide: np.ndarray) -> np.ndarray:
|
|
@ -11,16 +11,16 @@ from matplotlib import pyplot as plt
|
|||
|
||||
from health_azure.utils import get_aml_run_from_run_id, get_workspace
|
||||
from health_ml.utils.reports import HTMLReport
|
||||
from histopathology.utils.analysis_plot_utils import (add_training_curves_legend, plot_confusion_matrices,
|
||||
plot_crossval_roc_and_pr_curves,
|
||||
plot_crossval_training_curves)
|
||||
from histopathology.utils.output_utils import (AML_LEGACY_TEST_OUTPUTS_CSV, AML_TEST_OUTPUTS_CSV,
|
||||
AML_VAL_OUTPUTS_CSV)
|
||||
from histopathology.utils.report_utils import (collect_crossval_metrics, collect_crossval_outputs,
|
||||
crossval_runs_have_val_and_test_outputs, get_best_epoch_metrics,
|
||||
get_best_epochs, get_crossval_metrics_table, get_formatted_run_info,
|
||||
collect_class_info)
|
||||
from histopathology.utils.naming import MetricsKey, ModelKey
|
||||
from health_cpath.utils.analysis_plot_utils import (add_training_curves_legend, plot_confusion_matrices,
|
||||
plot_crossval_roc_and_pr_curves,
|
||||
plot_crossval_training_curves)
|
||||
from health_cpath.utils.output_utils import (AML_LEGACY_TEST_OUTPUTS_CSV, AML_TEST_OUTPUTS_CSV,
|
||||
AML_VAL_OUTPUTS_CSV)
|
||||
from health_cpath.utils.report_utils import (collect_crossval_metrics, collect_crossval_outputs,
|
||||
crossval_runs_have_val_and_test_outputs, get_best_epoch_metrics,
|
||||
get_best_epochs, get_crossval_metrics_table, get_formatted_run_info,
|
||||
collect_class_info)
|
||||
from health_cpath.utils.naming import MetricsKey, ModelKey
|
||||
|
||||
|
||||
def generate_html_report(parent_run_id: str, output_dir: Path,
|
|
@ -5,10 +5,10 @@
|
|||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
from histopathology.datasets.default_paths import TCGA_CRCK_DATASET_ID
|
||||
from health_cpath.datasets.default_paths import TCGA_CRCK_DATASET_ID
|
||||
|
||||
from histopathology.utils.tcga_utils import extract_fields
|
||||
from histopathology.datasets.tcga_prad_dataset import TcgaPradDataset
|
||||
from health_cpath.utils.tcga_utils import extract_fields
|
||||
from health_cpath.datasets.tcga_prad_dataset import TcgaPradDataset
|
||||
|
||||
|
||||
def check_dataset_csv_paths(dataset_dir: Path) -> None:
|
|
@ -17,7 +17,7 @@ import seaborn as sns
|
|||
from sklearn.manifold import TSNE
|
||||
from sklearn.metrics import auc, precision_recall_curve, roc_curve, confusion_matrix
|
||||
|
||||
from histopathology.utils.naming import ResultsKey
|
||||
from health_cpath.utils.naming import ResultsKey
|
||||
|
||||
TRAIN_STYLE = dict(ls='-')
|
||||
VAL_STYLE = dict(ls='--')
|
|
@ -8,7 +8,7 @@ from torch import nn
|
|||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
from torchvision.models.resnet import resnet18, resnet50
|
||||
from histopathology.models.encoders import (
|
||||
from health_cpath.models.encoders import (
|
||||
HistoSSLEncoder,
|
||||
ImageNetEncoder,
|
||||
ImageNetEncoder_Resnet50,
|
||||
|
@ -16,7 +16,7 @@ from histopathology.models.encoders import (
|
|||
SSLEncoder,
|
||||
TileEncoder,
|
||||
)
|
||||
from histopathology.utils.download_utils import get_checkpoint_downloader
|
||||
from health_cpath.utils.download_utils import get_checkpoint_downloader
|
||||
from health_ml.networks.layers.attention_layers import (
|
||||
AttentionLayer,
|
||||
GatedAttentionLayer,
|
|
@ -35,8 +35,8 @@ from girder_client import GirderClient
|
|||
from health_azure.logging import logging_to_stdout
|
||||
from health_azure.utils import get_aml_run_from_run_id
|
||||
|
||||
from histopathology.utils.naming import ResultsKey
|
||||
from histopathology.utils.output_utils import AML_TEST_OUTPUTS_CSV
|
||||
from health_cpath.utils.naming import ResultsKey
|
||||
from health_cpath.utils.output_utils import AML_TEST_OUTPUTS_CSV
|
||||
|
||||
|
||||
TypeRectangleJSON = Dict[str, Union[str, float, Dict[str, str]]]
|
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче