зеркало из https://github.com/microsoft/MLOS.git
Introduce basic experiment visualization module mlos_viz via dabl (#624)
Adds a basic `mlos_viz.plot(exp)` style API for simple visualizations of `ExperimentData` results relative to the experiment's objectives (building off of #628 and https://github.com/dabl/dabl/pull/335). Note: this PR currently omits unit tests for the new module due to the complexity of testing visualizations. We intend to add this in future PRs. There is however, a working example of its use here right now: https://github.com/Microsoft-CISL/sqlite-autotuning/pull/41 --------- Co-authored-by: Sergiy Matusevych <sergiy.matusevych@gmail.com>
This commit is contained in:
Brian Kroth
GitHub
Родитель
221cee31ee
Коммит
1f3b26d9f0
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 0.2.6
|
||||
current_version = 0.3.0
|
||||
commit = True
|
||||
tag = True
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
"conda",
|
||||
"configspace",
|
||||
"coro",
|
||||
"dabl",
|
||||
"DABL",
|
||||
"dataframe",
|
||||
"devcontainer",
|
||||
"discretization",
|
||||
|
|
|
|||
|
|
@ -30,11 +30,11 @@ get_python_deps() {
|
|||
python3 /tmp/conda-tmp/$pkg/setup.py egg_info --egg-base "$tmpdir/"
|
||||
cat "$tmpdir/$pkg.egg-info/requires.txt" \
|
||||
| grep -v -e '^\[' -e '^\s*$' \
|
||||
| grep -v '^mlos-core' \
|
||||
| grep -v '^mlos-' \
|
||||
| sort -u \
|
||||
> /tmp/conda-tmp/$pkg.requirements.txt
|
||||
}
|
||||
for pkg in mlos_core mlos_bench; do
|
||||
for pkg in mlos_core mlos_bench mlos_viz; do
|
||||
get_python_deps "$pkg"
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ if [ -d .devcontainer/tmp ]; then
|
|||
fi
|
||||
mkdir -p .devcontainer/tmp/
|
||||
cp -v conda-envs/mlos.yml .devcontainer/tmp/mlos.yml
|
||||
for pkg in mlos_core mlos_bench; do
|
||||
for pkg in mlos_core mlos_bench mlos_viz; do
|
||||
mkdir -p .devcontainer/tmp/$pkg
|
||||
cp -v $pkg/setup.py .devcontainer/tmp/$pkg/setup.py
|
||||
cp -v $pkg/_version.py .devcontainer/tmp/$pkg/_version.py
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ if (Test-Path .devcontainer/tmp) {
|
|||
New-Item -Type Directory .devcontainer/tmp
|
||||
|
||||
Copy-Item conda-envs/mlos.yml .devcontainer/tmp/mlos.yml
|
||||
foreach ($pkg in @('mlos_core', 'mlos_bench')) {
|
||||
foreach ($pkg in @('mlos_core', 'mlos_bench', 'mlos_viz')) {
|
||||
New-Item -Type Directory ".devcontainer/tmp/${pkg}"
|
||||
Copy-Item "$pkg/setup.py" ".devcontainer/tmp/${pkg}/setup.py"
|
||||
Copy-Item "$pkg/_version.py" ".devcontainer/tmp/${pkg}/_version.py"
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ docker run -i --rm \
|
|||
-v "$reporoot":/src:ro \
|
||||
--workdir /src \
|
||||
markdown-link-check:latest \
|
||||
find ./doc ./mlos_core ./mlos_bench ./.devcontainer -name '*.md' -not -path './node_modules/*' \
|
||||
find ./doc ./mlos_core ./mlos_bench ./mlos_viz ./.devcontainer -name '*.md' -not -path './node_modules/*' \
|
||||
-exec markdown-link-check '{}' --config ./.github/workflows/markdown-link-check-config.json -q -v ';'
|
||||
|
||||
docker run -i --rm \
|
||||
|
|
|
|||
|
|
@ -25,6 +25,10 @@ updates:
|
|||
directory: "/mlos_bench/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/mlos_viz/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/doc/"
|
||||
schedule:
|
||||
|
|
|
|||
|
|
@ -41,6 +41,20 @@ if (!($mlos_bench_whl)) {
|
|||
exit 1
|
||||
}
|
||||
|
||||
# Build the mlos_viz wheel.
|
||||
Set-Location mlos_viz
|
||||
if (Test-Path dist) {
|
||||
Remove-Item -Recurse -Force dist
|
||||
}
|
||||
conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
|
||||
Set-Location ..
|
||||
$mlos_viz_whl = (Resolve-Path mlos_viz/dist/mlos_viz-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
|
||||
Write-Host "mlos_viz_whl: $mlos_viz_whl"
|
||||
if (!($mlos_viz_whl)) {
|
||||
Write-Error "Failed to find mlos_viz wheel."
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Setup a clean environment to test installing/using them.
|
||||
$PythonVersReq = (conda list -n $env:CONDA_ENV_NAME | Select-String -AllMatches -Pattern '^python\s+([0-9]+\.[0-9]+)[0-9.]*\s+').Matches.Groups[1].Value
|
||||
#$PythonVersReq = conda run -n $env:CONDA_ENV_NAME python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
|
||||
|
|
@ -77,6 +91,12 @@ if ($LASTEXITCODE -ne 0) {
|
|||
Write-Error "Failed to install mlos_bench wheels."
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
# Install mlos_viz wheel.
|
||||
conda run -n mlos-dist-test pip install "${mlos_viz_whl}[full-tests]"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Error "Failed to install mlos_viz wheels."
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
|
||||
# Just pick one simple test to run for now.
|
||||
# The rest should have been handled in a separate step.
|
||||
|
|
@ -92,3 +112,10 @@ if ($LASTEXITCODE -ne 0) {
|
|||
Write-Error "Failed to run mlos_bench tests."
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
|
||||
# Run a simple mlos_viz test.
|
||||
conda run -n mlos-dist-test python -m pytest mlos_viz/mlos_viz/tests/test_dabl_plot.py
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Error "Failed to run mlos_viz tests."
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,10 +73,10 @@ jobs:
|
|||
with:
|
||||
#path: ${{ env.CONDA_DIR }}/envs/${{ env.CONDA_ENV_NAME }}
|
||||
path: ${{ env.CONDA_DIR }}/pkgs
|
||||
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
|
||||
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
|
||||
restore-keys: |
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
|
||||
|
||||
- name: Restore cached pip packages
|
||||
id: restore-pip-cache
|
||||
|
|
@ -84,10 +84,10 @@ jobs:
|
|||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ${{ env.PIP_CACHE_DIR }}
|
||||
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
|
||||
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
|
||||
restore-keys: |
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
|
||||
|
||||
- name: Log some environment variables for debugging
|
||||
run: |
|
||||
|
|
|
|||
|
|
@ -23,6 +23,6 @@ jobs:
|
|||
with:
|
||||
use-quiet-mode: yes
|
||||
use-verbose-mode: yes
|
||||
folder-path: './doc, ./mlos_core, ./mlos_bench, ./.devcontainer'
|
||||
folder-path: './doc, ./mlos_core, ./mlos_bench, ./mlos_viz, ./.devcontainer'
|
||||
file-path: './CODE_OF_CONDUCT.md, ./CONTRIBUTING.md, ./README.md, ./SECURITY.md'
|
||||
config-file: './.github/workflows/markdown-link-check-config.json'
|
||||
|
|
|
|||
|
|
@ -61,10 +61,10 @@ jobs:
|
|||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ${{ env.CONDA }}/envs/mlos
|
||||
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
|
||||
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
|
||||
restore-keys: |
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_*/setup.py') }}-${{ env.cache_cur_date }}
|
||||
|
||||
- name: Log some environment variables for debugging
|
||||
run: |
|
||||
|
|
@ -109,7 +109,7 @@ jobs:
|
|||
# Only run the coverage checks on the devcontainer job.
|
||||
- name: Run tests
|
||||
run: |
|
||||
conda run -n $env:CONDA_ENV_NAME pytest --junitxml=junit/test-results.xml mlos_core/ mlos_bench/
|
||||
conda run -n $env:CONDA_ENV_NAME pytest --junitxml=junit/test-results.xml mlos_core/ mlos_bench/ mlos_viz/
|
||||
|
||||
# Note: unlike the Makefile version, the pwsh version of these rules are all run within a single shell context, so we can
|
||||
# split commands across lines with CWD maintained (hence we also require the "cd .." here).
|
||||
|
|
|
|||
128
Makefile
128
Makefile
|
|
@ -6,9 +6,10 @@ PYTHON_VERSION := $(shell echo "${CONDA_ENV_NAME}" | sed -r -e 's/^mlos[-]?//')
|
|||
ENV_YML := conda-envs/${CONDA_ENV_NAME}.yml
|
||||
|
||||
# Find the non-build python files we should consider as rule dependencies.
|
||||
PYTHON_FILES := $(shell find ./ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./((mlos_core|mlos_bench)/)?build/' -e '^./doc/source/')
|
||||
PYTHON_FILES := $(shell find ./ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./(mlos_(core|bench|viz)/)?build/' -e '^./doc/source/')
|
||||
MLOS_CORE_PYTHON_FILES := $(shell find ./mlos_core/ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./mlos_core/build/')
|
||||
MLOS_BENCH_PYTHON_FILES := $(shell find ./mlos_bench/ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./mlos_bench/build/')
|
||||
MLOS_VIZ_PYTHON_FILES := $(shell find ./mlos_viz/ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./mlos_viz/build/')
|
||||
SCRIPT_FILES := $(shell find ./ -name '*.sh' -or -name '*.ps1' -or -name '*.cmd')
|
||||
SQL_FILES := $(shell find ./ -name '*.sql')
|
||||
MD_FILES := $(shell find ./ -name '*.md' | grep -v '^./doc/')
|
||||
|
|
@ -30,7 +31,7 @@ all: check test dist dist-test doc licenseheaders
|
|||
.PHONY: conda-env
|
||||
conda-env: build/conda-env.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
build/conda-env.${CONDA_ENV_NAME}.build-stamp: ${ENV_YML} mlos_core/setup.py mlos_bench/setup.py
|
||||
build/conda-env.${CONDA_ENV_NAME}.build-stamp: ${ENV_YML} mlos_core/setup.py mlos_bench/setup.py mlos_viz/setup.py
|
||||
@echo "CONDA_SOLVER: ${CONDA_SOLVER}"
|
||||
@echo "CONDA_EXPERIMENTAL_SOLVER: ${CONDA_EXPERIMENTAL_SOLVER}"
|
||||
@echo "CONDA_INFO_LEVEL: ${CONDA_INFO_LEVEL}"
|
||||
|
|
@ -48,10 +49,15 @@ clean-conda-env:
|
|||
check: pycodestyle pydocstyle pylint mypy # cspell licenseheaders markdown-link-check
|
||||
|
||||
.PHONY: pycodestyle
|
||||
pycodestyle: conda-env build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp build/pycodestyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
pycodestyle: conda-env
|
||||
pycodestyle: build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
pycodestyle: build/pycodestyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
pycodestyle: build/pycodestyle.mlos_viz.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
|
||||
build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
|
||||
build/pycodestyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
|
||||
build/pycodestyle.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
|
||||
|
||||
build/pycodestyle.%.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp setup.cfg
|
||||
# Check for decent pep8 code style with pycodestyle.
|
||||
|
|
@ -60,10 +66,15 @@ build/pycodestyle.%.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_N
|
|||
touch $@
|
||||
|
||||
.PHONY: pydocstyle
|
||||
pydocstyle: conda-env build/pydocstyle.mlos_core.${CONDA_ENV_NAME}.build-stamp build/pydocstyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
pydocstyle: conda-env
|
||||
pydocstyle: build/pydocstyle.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
pydocstyle: build/pydocstyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
pydocstyle: build/pydocstyle.mlos_viz.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
|
||||
build/pydocstyle.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
|
||||
build/pydocstyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
|
||||
build/pydocstyle.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
|
||||
|
||||
build/pydocstyle.%.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp setup.cfg
|
||||
# Check for decent pep8 doc style with pydocstyle.
|
||||
|
|
@ -74,8 +85,11 @@ build/pydocstyle.%.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NA
|
|||
licenseheaders: build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
build/licenseheaders.${CONDA_ENV_NAME}.build-stamp: $(PYTHON_FILES) $(SCRIPT_FILES) $(SQL_FILES) doc/mit-license.tmpl
|
||||
# Note: to avoid makefile dependency loops, we don't touch the setup.py files as that would force the conda-env to be rebuilt.
|
||||
conda run -n ${CONDA_ENV_NAME} licenseheaders -t doc/mit-license.tmpl -E .py .sh .ps1 .sql .cmd -x mlos_bench/setup.py mlos_core/setup.py
|
||||
# Note: to avoid makefile dependency loops, we don't touch the setup.py
|
||||
# files as that would force the conda-env to be rebuilt.
|
||||
conda run -n ${CONDA_ENV_NAME} licenseheaders -t doc/mit-license.tmpl \
|
||||
-E .py .sh .ps1 .sql .cmd \
|
||||
-x mlos_bench/setup.py mlos_core/setup.py mlos_viz/setup.py
|
||||
touch $@
|
||||
|
||||
.PHONY: cspell
|
||||
|
|
@ -107,32 +121,49 @@ build/markdown-link-check-container.build-stamp:
|
|||
touch $@
|
||||
|
||||
.PHONY: pylint
|
||||
pylint: conda-env build/pylint.mlos_core.${CONDA_ENV_NAME}.build-stamp build/pylint.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
pylint: conda-env
|
||||
pylint: build/pylint.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
pylint: build/pylint.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
pylint: build/pylint.mlos_viz.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
|
||||
build/pylint.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
|
||||
build/pylint.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
|
||||
build/pylint.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
|
||||
|
||||
build/pylint.%.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp .pylintrc
|
||||
conda run -n ${CONDA_ENV_NAME} pylint -j0 $(filter-out .pylintrc,$+)
|
||||
touch $@
|
||||
|
||||
.PHONY: flake8
|
||||
flake8: conda-env build/flake8.mlos_core.${CONDA_ENV_NAME}.build-stamp build/flake8.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
flake8: conda-env
|
||||
flake8: build/flake8.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
flake8: build/flake8.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
flake8: build/flake8.mlos_viz.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
build/flake8.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
|
||||
build/flake8.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
|
||||
build/flake8.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
|
||||
|
||||
build/flake8.%.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp setup.cfg
|
||||
conda run -n ${CONDA_ENV_NAME} flake8 -j0 $(filter-out setup.cfg,$+)
|
||||
touch $@
|
||||
|
||||
.PHONY: mypy
|
||||
mypy: conda-env build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp build/mypy.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
mypy: conda-env
|
||||
mypy: build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
mypy: build/mypy.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
mypy: build/mypy.mlos_viz.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
|
||||
build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
|
||||
build/mypy.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES) build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
build/mypy.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES) build/mypy.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
NON_MYPY_FILES := scripts/dmypy-wrapper.sh build/conda-env.${CONDA_ENV_NAME}.build-stamp build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp setup.cfg
|
||||
NON_MYPY_FILES := scripts/dmypy-wrapper.sh setup.cfg
|
||||
NON_MYPY_FILES += build/conda-env.${CONDA_ENV_NAME}.build-stamp
|
||||
NON_MYPY_FILES += build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
NON_MYPY_FILES += build/mypy.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
build/mypy.%.${CONDA_ENV_NAME}.build-stamp: scripts/dmypy-wrapper.sh build/conda-env.${CONDA_ENV_NAME}.build-stamp setup.cfg
|
||||
conda run -n ${CONDA_ENV_NAME} scripts/dmypy-wrapper.sh \
|
||||
$(filter-out $(NON_MYPY_FILES),$+)
|
||||
|
|
@ -165,6 +196,16 @@ build/pytest.mlos_bench.${CONDA_ENV_NAME}.needs-build-stamp:
|
|||
echo "PYTEST_MODULES: $(PYTEST_MODULES)"
|
||||
touch $@
|
||||
|
||||
# Run the mlos_viz target update after mlos_bench target update.
|
||||
build/pytest.mlos_viz.${CONDA_ENV_NAME}.needs-build-stamp: build/pytest.mlos_bench.${CONDA_ENV_NAME}.needs-build-stamp
|
||||
build/pytest.mlos_viz.${CONDA_ENV_NAME}.needs-build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp
|
||||
build/pytest.mlos_viz.${CONDA_ENV_NAME}.needs-build-stamp: $(MLOS_VIZ_PYTHON_FILES) conftest.py setup.cfg
|
||||
build/pytest.mlos_viz.${CONDA_ENV_NAME}.needs-build-stamp:
|
||||
# Update the PYTEST_MODULES list to include mlos_viz.
|
||||
$(eval PYTEST_MODULES += mlos_viz)
|
||||
echo "PYTEST_MODULES: $(PYTEST_MODULES)"
|
||||
touch $@
|
||||
|
||||
PYTEST_OPTIONS :=
|
||||
|
||||
# Allow optionally skipping coverage calculations during local testing to skip up inner dev loop.
|
||||
|
|
@ -179,7 +220,10 @@ endif
|
|||
# NOTE: When run locally, the junit/test-results.xml will only include the
|
||||
# tests from the latest run, but this file is only used for upstream reporting,
|
||||
# so probably shouldn't matter.
|
||||
build/pytest.${CONDA_ENV_NAME}.build-stamp: build/pytest.mlos_core.${CONDA_ENV_NAME}.needs-build-stamp build/pytest.mlos_bench.${CONDA_ENV_NAME}.needs-build-stamp
|
||||
build/pytest.${CONDA_ENV_NAME}.build-stamp: build/pytest.mlos_core.${CONDA_ENV_NAME}.needs-build-stamp
|
||||
build/pytest.${CONDA_ENV_NAME}.build-stamp: build/pytest.mlos_bench.${CONDA_ENV_NAME}.needs-build-stamp
|
||||
build/pytest.${CONDA_ENV_NAME}.build-stamp: build/pytest.mlos_viz.${CONDA_ENV_NAME}.needs-build-stamp
|
||||
build/pytest.${CONDA_ENV_NAME}.build-stamp:
|
||||
# Make sure to update the list of modules needed everytime in case the test fails and we need to rerun it.
|
||||
for pytest_module in $(PYTEST_MODULES); do rm -f build/pytest.$${pytest_module}.${CONDA_ENV_NAME}.needs-build-stamp; done
|
||||
# Run pytest for the modules: $(PYTEST_MODULES)
|
||||
|
|
@ -194,7 +238,10 @@ build/pytest.${CONDA_ENV_NAME}.build-stamp: build/pytest.mlos_core.${CONDA_ENV_N
|
|||
dist: bdist_wheel
|
||||
|
||||
.PHONY: bdist_wheel
|
||||
bdist_wheel: conda-env mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl
|
||||
bdist_wheel: conda-env
|
||||
bdist_wheel: mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl
|
||||
bdist_wheel: mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl
|
||||
bdist_wheel: mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl
|
||||
|
||||
mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl: mlos_core/dist/tmp/mlos-core-latest.tar
|
||||
mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl: MODULE_NAME := mlos_core
|
||||
|
|
@ -210,6 +257,13 @@ mlos_bench/dist/tmp/mlos-bench-latest.tar: mlos_bench/setup.py mlos_bench/MANIFE
|
|||
mlos_bench/dist/tmp/mlos-bench-latest.tar: MODULE_NAME := mlos_bench
|
||||
mlos_bench/dist/tmp/mlos-bench-latest.tar: PACKAGE_NAME := mlos-bench
|
||||
|
||||
mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl: mlos_viz/dist/tmp/mlos-viz-latest.tar
|
||||
mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl: MODULE_NAME := mlos_viz
|
||||
mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl: PACKAGE_NAME := mlos-viz
|
||||
mlos_viz/dist/tmp/mlos-bench-latest.tar: mlos_viz/setup.py mlos_viz/MANIFEST.in $(mlos_viz_PYTHON_FILES)
|
||||
mlos_viz/dist/tmp/mlos-bench-latest.tar: MODULE_NAME := mlos_viz
|
||||
mlos_viz/dist/tmp/mlos-bench-latest.tar: PACKAGE_NAME := mlos-viz
|
||||
|
||||
%-latest.tar: build/conda-env.${CONDA_ENV_NAME}.build-stamp
|
||||
%-latest.tar:
|
||||
mkdir -p $(MODULE_NAME)/dist/tmp
|
||||
|
|
@ -247,7 +301,9 @@ dist-test-env: dist build/dist-test-env.$(PYTHON_VERSION).build-stamp
|
|||
build/dist-test-env.$(PYTHON_VERSION).build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp
|
||||
# Use the same version of python as the one we used to build the wheels.
|
||||
build/dist-test-env.$(PYTHON_VERSION).build-stamp: PYTHON_VERS_REQ=$(shell conda list -n ${CONDA_ENV_NAME} | egrep '^python\s+' | sed -r -e 's/^python\s+//' | cut -d' ' -f1 | cut -d. -f1-2)
|
||||
build/dist-test-env.$(PYTHON_VERSION).build-stamp: mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl
|
||||
build/dist-test-env.$(PYTHON_VERSION).build-stamp: mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl
|
||||
build/dist-test-env.$(PYTHON_VERSION).build-stamp: mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl
|
||||
build/dist-test-env.$(PYTHON_VERSION).build-stamp: mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl
|
||||
# Create a clean test environment for checking the wheel files.
|
||||
$(MAKE) dist-test-env-clean
|
||||
conda create -y ${CONDA_INFO_LEVEL} -n mlos-dist-test-$(PYTHON_VERSION) python=$(PYTHON_VERS_REQ)
|
||||
|
|
@ -259,6 +315,8 @@ build/dist-test-env.$(PYTHON_VERSION).build-stamp: mlos_core/dist/tmp/mlos_core-
|
|||
conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install "mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl[full-tests]"
|
||||
# Test that the config dir for mlos_bench got distributed.
|
||||
test -e `conda env list | grep "mlos-dist-test-$(PYTHON_VERSION) " | awk '{ print $$2 }'`/lib/python$(PYTHON_VERS_REQ)/site-packages/mlos_bench/config/README.md
|
||||
# Test a clean install of the mlos_viz wheel.
|
||||
conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install "mlos_viz/dist/tmp/mlos_viz-latest-py3-none-any.whl[full-tests]"
|
||||
touch $@
|
||||
|
||||
.PHONY: dist-test
|
||||
|
|
@ -271,10 +329,13 @@ build/dist-test.$(PYTHON_VERSION).build-stamp: $(PYTHON_FILES) build/dist-test-e
|
|||
# Note: this will pick up the local directory and change the output if we're using PYTHONPATH=.
|
||||
conda run -n mlos-dist-test-$(PYTHON_VERSION) pip list --verbose | grep mlos-core | grep ' pip'
|
||||
conda run -n mlos-dist-test-$(PYTHON_VERSION) pip list --verbose | grep mlos-bench | grep ' pip'
|
||||
conda run -n mlos-dist-test-$(PYTHON_VERSION) pip list --verbose | grep mlos-viz | grep ' pip'
|
||||
# Run a simple test that uses the mlos_core wheel (full tests can be checked with `make test`).
|
||||
conda run -n mlos-dist-test-$(PYTHON_VERSION) python3 -m pytest mlos_core/mlos_core/tests/spaces/spaces_test.py
|
||||
# Run a simple test that uses the mlos_bench wheel (full tests can be checked with `make test`).
|
||||
conda run -n mlos-dist-test-$(PYTHON_VERSION) python3 -m pytest mlos_bench/mlos_bench/tests/environments/mock_env_test.py
|
||||
# Run a simple test that uses the mlos_viz wheel (full tests can be checked with `make test`).
|
||||
conda run -n mlos-dist-test-$(PYTHON_VERSION) python3 -m pytest mlos_viz/mlos_viz/tests/test_dabl_plot.py
|
||||
touch $@
|
||||
|
||||
dist-test-clean: dist-test-env-clean
|
||||
|
|
@ -291,7 +352,12 @@ build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_E
|
|||
conda run -n ${CONDA_ENV_NAME} pip install -U twine
|
||||
touch $@
|
||||
|
||||
build/publish.%.py.build-stamp: build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp build/pytest.${CONDA_ENV_NAME}.build-stamp build/dist-test.$(PYTHON_VERSION).build-stamp build/check-doc.build-stamp build/linklint-doc.build-stamp
|
||||
build/publish.%.py.build-stamp: build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp
|
||||
build/publish.%.py.build-stamp: build/pytest.${CONDA_ENV_NAME}.build-stamp
|
||||
build/publish.%.py.build-stamp: build/dist-test.$(PYTHON_VERSION).build-stamp
|
||||
build/publish.%.py.build-stamp: build/check-doc.build-stamp
|
||||
build/publish.%.py.build-stamp: build/linklint-doc.build-stamp
|
||||
build/publish.%.py.build-stamp:
|
||||
rm -f mlos_*/dist/*.tar.gz
|
||||
ls mlos_*/dist/*.tar | xargs -I% gzip -k %
|
||||
repo_name=`echo "$@" | sed -e 's|build/publish\.||' -e 's|\.py\.build-stamp||'` \
|
||||
|
|
@ -326,12 +392,18 @@ doc/source/api/mlos_bench/modules.rst: $(MLOS_BENCH_PYTHON_FILES) $(COMMON_DOC_F
|
|||
cd doc/ && conda run -n ${CONDA_ENV_NAME} sphinx-apidoc -f -e -M -o source/api/mlos_bench/ ../mlos_bench/ ../mlos_*/setup.py
|
||||
# Save the help output of the mlos_bench scripts to include in the documentation.
|
||||
# First make sure that the latest version of mlos_bench is installed (since it uses git based tagging).
|
||||
conda run -n ${CONDA_ENV_NAME} pip install -e mlos_core -e mlos_bench
|
||||
conda run -n ${CONDA_ENV_NAME} pip install -e mlos_core -e mlos_bench -e mlos_viz
|
||||
conda run -n ${CONDA_ENV_NAME} mlos_bench --help > doc/source/api/mlos_bench/mlos_bench.run.usage.txt
|
||||
echo ".. literalinclude:: mlos_bench.run.usage.txt" >> doc/source/api/mlos_bench/mlos_bench.run.rst
|
||||
echo " :language: none" >> doc/source/api/mlos_bench/mlos_bench.run.rst
|
||||
|
||||
SPHINX_API_RST_FILES := doc/source/api/mlos_core/modules.rst doc/source/api/mlos_bench/modules.rst
|
||||
doc/source/api/mlos_viz/modules.rst: $(MLOS_VIZ_PYTHON_FILES) $(COMMON_DOC_FILES)
|
||||
rm -rf doc/source/api/mlos_viz
|
||||
cd doc/ && conda run -n ${CONDA_ENV_NAME} sphinx-apidoc -f -e -M -o source/api/mlos_viz/ ../mlos_viz/ ../mlos_*/setup.py
|
||||
|
||||
SPHINX_API_RST_FILES := doc/source/api/mlos_core/modules.rst
|
||||
SPHINX_API_RST_FILES += doc/source/api/mlos_bench/modules.rst
|
||||
SPHINX_API_RST_FILES += doc/source/api/mlos_viz/modules.rst
|
||||
|
||||
.PHONY: sphinx-apidoc
|
||||
sphinx-apidoc: $(SPHINX_API_RST_FILES)
|
||||
|
|
@ -386,8 +458,11 @@ build/check-doc.build-stamp: doc/build/html/index.html doc/build/html/htmlcov/in
|
|||
test -s doc/build/html/index.html
|
||||
test -s doc/build/html/generated/mlos_core.optimizers.optimizer.BaseOptimizer.html
|
||||
test -s doc/build/html/generated/mlos_bench.environments.Environment.html
|
||||
test -s doc/build/html/generated/mlos_viz.plot.html
|
||||
test -s doc/build/html/api/mlos_core/mlos_core.html
|
||||
test -s doc/build/html/api/mlos_bench/mlos_bench.html
|
||||
test -s doc/build/html/api/mlos_viz/mlos_viz.html
|
||||
test -s doc/build/html/api/mlos_viz/mlos_viz.dabl.html
|
||||
grep -q -e '--config CONFIG' doc/build/html/api/mlos_bench/mlos_bench.run.html
|
||||
# Check doc logs for errors (but skip over some known ones) ...
|
||||
@cat doc/build/log.txt \
|
||||
|
|
@ -440,18 +515,14 @@ clean-doc:
|
|||
clean-check:
|
||||
rm -f build/pylint.build-stamp
|
||||
rm -f build/pylint.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pylint.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pylint.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/mypy.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/mypy.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pylint.mlos_*.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/mypy.mlos_*.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pycodestyle.build-stamp
|
||||
rm -f build/pycodestyle.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pycodestyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pycodestyle.mlos_*.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pydocstyle.build-stamp
|
||||
rm -f build/pydocstyle.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pydocstyle.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pydocstyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pydocstyle.mlos_*.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/licenseheaders-prereqs.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
|
|
@ -459,10 +530,8 @@ clean-check:
|
|||
clean-test:
|
||||
rm -f build/pytest.build-stamp
|
||||
rm -f build/pytest.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pytest.mlos_core.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pytest.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pytest.mlos_core.${CONDA_ENV_NAME}.needs-build-stamp
|
||||
rm -f build/pytest.mlos_bench.${CONDA_ENV_NAME}.needs-build-stamp
|
||||
rm -f build/pytest.mlos_*.${CONDA_ENV_NAME}.build-stamp
|
||||
rm -f build/pytest.mlos_*.${CONDA_ENV_NAME}.needs-build-stamp
|
||||
rm -rf .pytest_cache/
|
||||
rm -f coverage.xml .coverage* build/.converage*
|
||||
rm -rf htmlcov/
|
||||
|
|
@ -474,6 +543,7 @@ dist-clean:
|
|||
rm -rf build dist
|
||||
rm -rf mlos_core/build mlos_core/dist
|
||||
rm -rf mlos_bench/build mlos_bench/dist
|
||||
rm -rf mlos_viz/build mlos_viz/dist
|
||||
|
||||
.PHONY: clean
|
||||
clean: clean-check clean-test dist-clean clean-doc clean-doc-env dist-test-clean
|
||||
|
|
@ -483,6 +553,8 @@ clean: clean-check clean-test dist-clean clean-doc clean-doc-env dist-test-clean
|
|||
rm -rf mlos_core/mlos_core.egg-info
|
||||
rm -rf mlos_bench.egg-info
|
||||
rm -rf mlos_bench/mlos_bench.egg-info
|
||||
rm -rf mlos_viz.egg-info
|
||||
rm -rf mlos_viz/mlos_viz.egg-info
|
||||
rm -rf __pycache__
|
||||
find . -type d -name __pycache__ -print0 | xargs -t -r -0 rm -rf
|
||||
find . -type f -name '*.pyc' -print0 | xargs -t -r -0 rm -f
|
||||
|
|
|
|||
|
|
@ -37,3 +37,4 @@ dependencies:
|
|||
- types-setuptools
|
||||
- "--editable ../mlos_core[full-tests]"
|
||||
- "--editable ../mlos_bench[full-tests]"
|
||||
- "--editable ../mlos_viz[full-tests]"
|
||||
|
|
|
|||
|
|
@ -37,3 +37,4 @@ dependencies:
|
|||
- types-setuptools
|
||||
- "--editable ../mlos_core[full-tests]"
|
||||
- "--editable ../mlos_bench[full-tests]"
|
||||
- "--editable ../mlos_viz[full-tests]"
|
||||
|
|
|
|||
|
|
@ -37,3 +37,4 @@ dependencies:
|
|||
- types-setuptools
|
||||
- "--editable ../mlos_core[full-tests]"
|
||||
- "--editable ../mlos_bench[full-tests]"
|
||||
- "--editable ../mlos_viz[full-tests]"
|
||||
|
|
|
|||
|
|
@ -37,3 +37,4 @@ dependencies:
|
|||
- types-setuptools
|
||||
- "--editable ../mlos_core[full-tests]"
|
||||
- "--editable ../mlos_bench[full-tests]"
|
||||
- "--editable ../mlos_viz[full-tests]"
|
||||
|
|
|
|||
|
|
@ -41,3 +41,4 @@ dependencies:
|
|||
- types-setuptools
|
||||
- "--editable ../mlos_core[full-tests]"
|
||||
- "--editable ../mlos_bench[full-tests]"
|
||||
- "--editable ../mlos_viz[full-tests]"
|
||||
|
|
|
|||
|
|
@ -35,3 +35,4 @@ dependencies:
|
|||
- types-setuptools
|
||||
- "--editable ../mlos_core[full-tests]"
|
||||
- "--editable ../mlos_bench[full-tests]"
|
||||
- "--editable ../mlos_viz[full-tests]"
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ cd "$scriptdir/.."
|
|||
rm -rf doc/source/source_tree_docs/
|
||||
mkdir -p doc/source/source_tree_docs/
|
||||
|
||||
for readme_file_path in README.md mlos_core/README.md mlos_bench/README.md; do
|
||||
for readme_file_path in README.md mlos_core/README.md mlos_bench/README.md mlos_viz/README.md; do
|
||||
file_dir=$(dirname "$readme_file_path")
|
||||
mkdir -p "doc/source/source_tree_docs/$file_dir"
|
||||
|
||||
|
|
|
|||
|
|
@ -26,16 +26,17 @@ import sphinx_rtd_theme
|
|||
|
||||
sys.path.insert(0, os.path.abspath('../../mlos_core/mlos_core'))
|
||||
sys.path.insert(1, os.path.abspath('../../mlos_bench/mlos_bench'))
|
||||
sys.path.insert(1, os.path.abspath('../../mlos_viz/mlos_viz'))
|
||||
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
|
||||
project = 'MlosCore'
|
||||
copyright = '2022, GSL'
|
||||
copyright = '2024, GSL'
|
||||
author = 'GSL'
|
||||
|
||||
# The full version, including alpha/beta/rc tags
|
||||
release = '0.2.6'
|
||||
release = '0.3.0'
|
||||
|
||||
try:
|
||||
from setuptools_scm import get_version
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Welcome to the MLOS documentation!
|
|||
.. image:: badges/coverage.svg
|
||||
:target: htmlcov/index.html
|
||||
|
||||
`MLOS <source_tree_docs/index.html>`_ is a project to enable autotuning for systems.
|
||||
`MLOS <source_tree_docs/index.html>`_ is a project to enable autotuning for systems via automated benchmarking including managing the storage and visualization of the results.
|
||||
|
||||
See below for additional documentation sections.
|
||||
|
||||
|
|
@ -15,8 +15,9 @@ See below for additional documentation sections.
|
|||
:caption: Source Tree Documentation
|
||||
|
||||
source_tree_docs/index
|
||||
source_tree_docs/mlos_bench/index
|
||||
source_tree_docs/mlos_core/index
|
||||
source_tree_docs/mlos_bench/index
|
||||
source_tree_docs/mlos_viz/index
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
|
@ -30,6 +31,7 @@ See below for additional documentation sections.
|
|||
|
||||
api/mlos_core/modules
|
||||
api/mlos_bench/modules
|
||||
api/mlos_viz/modules
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
|
@ -41,4 +43,4 @@ See below for additional documentation sections.
|
|||
:maxdepth: 1
|
||||
:caption: References
|
||||
|
||||
Github Source Tree <https://aka.ms/mlos-core/src>
|
||||
Github Source Tree <https://github.com/microsoft/MLOS>
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
##########################
|
||||
MLOS Package APIs Overview
|
||||
##########################
|
||||
|
||||
This is a list of major functions and classes provided by the MLOS packages.
|
||||
|
||||
#############################
|
||||
mlos-core API
|
||||
#############################
|
||||
|
|
@ -221,6 +227,20 @@ Storage
|
|||
|
||||
Storage
|
||||
|
||||
.. currentmodule:: mlos_bench.storage.base_experiment_data
|
||||
.. autosummary::
|
||||
:toctree: generated/
|
||||
:template: class.rst
|
||||
|
||||
ExperimentData
|
||||
|
||||
.. currentmodule:: mlos_bench.storage.base_trial_data
|
||||
.. autosummary::
|
||||
:toctree: generated/
|
||||
:template: class.rst
|
||||
|
||||
TrialData
|
||||
|
||||
SQL DB Storage
|
||||
--------------
|
||||
.. currentmodule:: mlos_bench.storage.sql.storage
|
||||
|
|
@ -229,3 +249,25 @@ SQL DB Storage
|
|||
:template: class.rst
|
||||
|
||||
SqlStorage
|
||||
|
||||
#############################
|
||||
mlos-viz API
|
||||
#############################
|
||||
|
||||
This is a list of major functions and classes provided by `mlos_viz`.
|
||||
|
||||
.. currentmodule:: mlos_viz
|
||||
|
||||
.. currentmodule:: mlos_viz
|
||||
.. autosummary::
|
||||
:toctree: generated/
|
||||
:template: class.rst
|
||||
|
||||
MlosVizMethod
|
||||
|
||||
.. currentmodule:: mlos_viz
|
||||
.. autosummary::
|
||||
:toctree: generated/
|
||||
:template: function.rst
|
||||
|
||||
plot
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"folders": [
|
||||
{
|
||||
"path": "."
|
||||
},
|
||||
{
|
||||
"path": "mlos_core"
|
||||
},
|
||||
{
|
||||
"path": "mlos_bench"
|
||||
},
|
||||
{
|
||||
"path": "mlos_viz"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -3,8 +3,8 @@
|
|||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
Version number for the mlos_core package.
|
||||
Version number for the mlos_bench package.
|
||||
"""
|
||||
|
||||
# NOTE: This should be managed by bumpversion.
|
||||
_VERSION = '0.2.6'
|
||||
_VERSION = '0.3.0'
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ class ExperimentData(metaclass=ABCMeta):
|
|||
Base interface for accessing the stored benchmark data.
|
||||
"""
|
||||
|
||||
RESULT_COLUMN_PREFIX = "result."
|
||||
CONFIG_COLUMN_PREFIX = "config."
|
||||
|
||||
def __init__(self, *, exp_id: str, description: str,
|
||||
root_env_config: str, git_repo: str, git_commit: str):
|
||||
self._exp_id = exp_id
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ class ExperimentSqlData(ExperimentData):
|
|||
)
|
||||
)
|
||||
configs_df = pandas.DataFrame(
|
||||
[(row.trial_id, row.config_id, "config." + row.param_id, row.param_value)
|
||||
[(row.trial_id, row.config_id, self.CONFIG_COLUMN_PREFIX + row.param_id, row.param_value)
|
||||
for row in cur_configs.fetchall()],
|
||||
columns=['trial_id', 'config_id', 'param', 'value']
|
||||
).pivot(
|
||||
|
|
@ -168,7 +168,7 @@ class ExperimentSqlData(ExperimentData):
|
|||
)
|
||||
)
|
||||
results_df = pandas.DataFrame(
|
||||
[(row.trial_id, "result." + row.metric_id, row.metric_value)
|
||||
[(row.trial_id, self.RESULT_COLUMN_PREFIX + row.metric_id, row.metric_value)
|
||||
for row in cur_results.fetchall()],
|
||||
columns=['trial_id', 'metric', 'value']
|
||||
).pivot(
|
||||
|
|
|
|||
|
|
@ -7,4 +7,4 @@ Version number for the mlos_core package.
|
|||
"""
|
||||
|
||||
# NOTE: This should be managed by bumpversion.
|
||||
_VERSION = '0.2.6'
|
||||
_VERSION = '0.3.0'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
# setuptools-scm has an issue that causes it to include all files in git in the sdist produced.
|
||||
# Here we exclude test files.
|
||||
prune **/tests
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
# mlos_viz
|
||||
|
||||
The [`mlos_viz`](./) module is an aid to visualizing experiment benchmarking and optimization results generated and stored by [`mlos_bench`](../mlos_bench/).
|
||||
|
||||
Its core API is `mlos_viz.plot(experiment)`, initially implemented as a wrapper around [`dabl`](https://github.com/dabl/dabl) to provide a basic visual overview of the results, where `experiment` is an [`ExperimentData`](../mlos_bench/mlos_bench/storage/base_experiment_data.py) objected returned from the [`mlos_bench.storage`](../mlos_bench/mlos_bench/storage/) layer.
|
||||
|
||||
In the future, we plan to add more automatic visualizations, interactive visualizations, feedback to the `mlos_bench` experiment trial scheduler, etc.
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
Version number for the mlos_viz package.
|
||||
"""
|
||||
|
||||
# NOTE: This should be managed by bumpversion.
|
||||
_VERSION = '0.1.0'
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
mlos_viz is a framework to help visualizing, explain, and gain insights from results
|
||||
from the mlos_bench framework for benchmarking and optimization automation.
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
|
||||
import warnings
|
||||
|
||||
from matplotlib import pyplot as plt
|
||||
import seaborn as sns
|
||||
|
||||
from mlos_bench.storage.base_experiment_data import ExperimentData
|
||||
|
||||
|
||||
class MlosVizMethod(Enum):
|
||||
"""
|
||||
What method to use for visualizing the experiment results.
|
||||
"""
|
||||
|
||||
AUTO = "dabl" # use dabl as the current default
|
||||
DABL = "dabl"
|
||||
|
||||
|
||||
def _plot_optimizer_trends(exp_data: ExperimentData) -> None:
|
||||
"""
|
||||
Plots the optimizer trends for the Experiment.
|
||||
|
||||
Intended to be used from a Jupyter notebook.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exp_data: ExperimentData
|
||||
The experiment data to plot.
|
||||
"""
|
||||
for objective in exp_data.objectives:
|
||||
objective_column = ExperimentData.RESULT_COLUMN_PREFIX + objective
|
||||
results_df = exp_data.results
|
||||
plt.rcParams["figure.figsize"] = (10, 4)
|
||||
|
||||
sns.scatterplot(x=results_df.trial_id, y=results_df[objective_column], alpha=0.7) # Result of each trial
|
||||
sns.lineplot(x=results_df.trial_id, y=results_df[objective_column].cummin()) # the best result so far (cummin)
|
||||
|
||||
plt.yscale('log')
|
||||
|
||||
plt.xlabel("Trial number")
|
||||
plt.ylabel(objective)
|
||||
|
||||
plt.title("Optimizer Trends for Experiment: " + exp_data.exp_id)
|
||||
plt.grid()
|
||||
plt.show() # type: ignore[no-untyped-call]
|
||||
|
||||
|
||||
def ignore_plotter_warnings(plotter_method: MlosVizMethod = MlosVizMethod.AUTO) -> None:
|
||||
"""
|
||||
Suppress some annoying warnings from third-party data visualization packages by
|
||||
adding them to the warnings filter.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
plotter_method: MlosVizMethod
|
||||
The method to use for visualizing the experiment results.
|
||||
"""
|
||||
warnings.filterwarnings("ignore", category=FutureWarning)
|
||||
|
||||
if plotter_method == MlosVizMethod.DABL:
|
||||
import mlos_viz.dabl # pylint: disable=import-outside-toplevel
|
||||
mlos_viz.dabl.ignore_plotter_warnings()
|
||||
else:
|
||||
raise NotImplementedError(f"Unhandled method: {plotter_method}")
|
||||
|
||||
|
||||
def plot(exp_data: ExperimentData,
|
||||
plotter_method: MlosVizMethod = MlosVizMethod.AUTO,
|
||||
filter_warnings: bool = True) -> None:
|
||||
"""
|
||||
Plots the results of the experiment.
|
||||
|
||||
Intended to be used from a Jupyter notebook.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exp_data: ExperimentData
|
||||
The experiment data to plot.
|
||||
plotter_method: MlosVizMethod
|
||||
The method to use for visualizing the experiment results.
|
||||
filter_warnings: bool
|
||||
Whether or not to filter some warnings from the plotter.
|
||||
"""
|
||||
_plot_optimizer_trends(exp_data)
|
||||
|
||||
if filter_warnings:
|
||||
ignore_plotter_warnings(plotter_method)
|
||||
|
||||
if MlosVizMethod.DABL:
|
||||
import mlos_viz.dabl # pylint: disable=import-outside-toplevel
|
||||
mlos_viz.dabl.plot(exp_data)
|
||||
else:
|
||||
raise NotImplementedError(f"Unhandled method: {plotter_method}")
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
Small wrapper functions for dabl plotting functions via mlos_bench data.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
import dabl
|
||||
|
||||
from mlos_bench.storage.base_experiment_data import ExperimentData
|
||||
|
||||
|
||||
def plot(exp_data: ExperimentData) -> None:
|
||||
"""
|
||||
Plots the Experiment results data using dabl.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exp_data : ExperimentData
|
||||
The ExperimentData (e.g., obtained from the storage layer) to plot.
|
||||
"""
|
||||
for objective in exp_data.objectives:
|
||||
objective_column = ExperimentData.RESULT_COLUMN_PREFIX + objective
|
||||
dabl.plot(exp_data.results, objective_column)
|
||||
|
||||
|
||||
def ignore_plotter_warnings() -> None:
|
||||
"""
|
||||
Add some filters to ignore warnings from the plotter.
|
||||
"""
|
||||
warnings.filterwarnings("ignore", module="dabl", category=UserWarning, message="Could not infer format")
|
||||
warnings.filterwarnings("ignore", module="dabl", category=UserWarning, message="(Dropped|Discarding) .* outliers")
|
||||
warnings.filterwarnings("ignore", module="dabl", category=UserWarning, message="Not plotting highly correlated")
|
||||
warnings.filterwarnings("ignore", module="dabl", category=UserWarning,
|
||||
message="Missing values in target_col have been removed for regression")
|
||||
from sklearn.exceptions import UndefinedMetricWarning # pylint: disable=import-outside-toplevel
|
||||
warnings.filterwarnings("ignore", module="sklearn", category=UndefinedMetricWarning, message="Recall is ill-defined")
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
Unit tests for mlos_viz.dabl.plot.
|
||||
"""
|
||||
|
||||
|
||||
def test_placeholder() -> None:
|
||||
"""Placeholder test."""
|
||||
# TODO: Remove this and implement real tests for mlos_viz.plot()
|
||||
# See Also: https://stackoverflow.com/questions/27948126/how-can-i-write-unit-tests-against-code-that-uses-matplotlib
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
Setup instructions for the mlos_viz package.
|
||||
"""
|
||||
|
||||
# pylint: disable=duplicate-code
|
||||
|
||||
from logging import warning
|
||||
from itertools import chain
|
||||
from typing import Dict, List
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
from _version import _VERSION # pylint: disable=import-private-name
|
||||
|
||||
|
||||
# A simple routine to read and adjust the README.md for this module into a format
|
||||
# suitable for packaging.
|
||||
# See Also: copy-source-tree-docs.sh
|
||||
# Unfortunately we can't use that directly due to the way packaging happens inside a
|
||||
# temp directory.
|
||||
# Similarly, we can't use a utility script outside this module, so this code has to
|
||||
# be duplicated for now.
|
||||
def _get_long_desc_from_readme(base_url: str) -> dict:
|
||||
pkg_dir = os.path.dirname(__file__)
|
||||
readme_path = os.path.join(pkg_dir, 'README.md')
|
||||
if not os.path.isfile(readme_path):
|
||||
return {}
|
||||
jsonc_re = re.compile(r'```jsonc')
|
||||
link_re = re.compile(r'\]\(([^:#)]+)(#[a-zA-Z0-9_-]+)?\)')
|
||||
with open(readme_path, mode='r', encoding='utf-8') as readme_fh:
|
||||
lines = readme_fh.readlines()
|
||||
# Tweak the lexers for local expansion by pygments instead of github's.
|
||||
lines = [link_re.sub(f"]({base_url}" + r'/\1\2)', line) for line in lines]
|
||||
# Tweak source source code links.
|
||||
lines = [jsonc_re.sub(r'```json', line) for line in lines]
|
||||
return {
|
||||
'long_description': ''.join(lines),
|
||||
'long_description_content_type': 'text/markdown',
|
||||
}
|
||||
|
||||
|
||||
try:
|
||||
from setuptools_scm import get_version
|
||||
version = get_version(root='..', relative_to=__file__)
|
||||
if version is not None:
|
||||
_VERSION = version # noqa: F811
|
||||
except ImportError:
|
||||
warning("setuptools_scm not found, using version from _version.py")
|
||||
except LookupError as e:
|
||||
warning(f"setuptools_scm failed to find git version, using version from _version.py: {e}")
|
||||
|
||||
|
||||
extra_requires: Dict[str, List[str]] = {}
|
||||
|
||||
# construct special 'full' extra that adds requirements for all built-in
|
||||
# backend integrations and additional extra features.
|
||||
extra_requires['full'] = list(set(chain(*extra_requires.values())))
|
||||
|
||||
extra_requires['full-tests'] = extra_requires['full'] + [
|
||||
'pytest',
|
||||
'pytest-forked',
|
||||
'pytest-xdist',
|
||||
'pytest-cov',
|
||||
'pytest-local-badge',
|
||||
]
|
||||
|
||||
# pylint: disable=duplicate-code
|
||||
MODULE_BASE_NAME = 'mlos_viz'
|
||||
setup(
|
||||
name='mlos-viz',
|
||||
version=_VERSION,
|
||||
packages=find_packages(exclude=[f"{MODULE_BASE_NAME}.tests", f"{MODULE_BASE_NAME}.tests.*"]),
|
||||
package_data={
|
||||
'': ['py.typed', '**/*.pyi'],
|
||||
},
|
||||
install_requires=[
|
||||
'mlos-bench==' + _VERSION,
|
||||
'dabl>=0.2.6',
|
||||
],
|
||||
extras_require=extra_requires,
|
||||
author='Microsoft',
|
||||
license='MIT',
|
||||
**_get_long_desc_from_readme('https://github.com/microsoft/MLOS/tree/main/mlos_viz'),
|
||||
author_email='mlos-maintainers@service.microsoft.com',
|
||||
description=('MLOS Visualization Python interface for benchmark automation and optimization results.'),
|
||||
url='https://github.com/microsoft/MLOS',
|
||||
project_urls={
|
||||
'Documentation': 'https://microsoft.github.io/MLOS',
|
||||
'Package Source': 'https://github.com/microsoft/MLOS/tree/main/mlos_viz/',
|
||||
},
|
||||
python_requires='>=3.8',
|
||||
keywords=[
|
||||
'autotuning',
|
||||
'benchmarking',
|
||||
'optimization',
|
||||
'systems',
|
||||
],
|
||||
classifiers=[
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: Science/Research",
|
||||
"Intended Audience :: System Administrators",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
],
|
||||
)
|
||||
|
|
@ -99,6 +99,12 @@ ignore_missing_imports = True
|
|||
[mypy-matplotlib.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-seaborn.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
[mypy-dabl.*]
|
||||
ignore_missing_imports = True
|
||||
|
||||
# https://github.com/pytest-dev/pytest/issues/10435
|
||||
[mypy-pytest]
|
||||
ignore_missing_imports = True
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче