зеркало из https://github.com/microsoft/MLOS.git
Switch to Github Actions for CI/CD runs (#274)
This commit is contained in:
Brian Kroth
GitHub
Родитель
dbac3f4ddd
Коммит
a0a76cfcf6
|
|
@ -1,236 +0,0 @@
|
|||
name: 'MlosCore Linux'
|
||||
|
||||
trigger:
|
||||
- main
|
||||
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
- '*'
|
||||
|
||||
schedules:
|
||||
- cron: "0 0 * * *"
|
||||
displayName: Nightly build
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
always: "true"
|
||||
|
||||
jobs:
|
||||
- job: 'Linux'
|
||||
|
||||
pool:
|
||||
vmImage: ubuntu-latest
|
||||
|
||||
# Test multiple versions of python.
|
||||
strategy:
|
||||
matrix:
|
||||
DefaultPython:
|
||||
# Floating most recent version of python (useful to catch new compatibility issues in nightly builds).
|
||||
conda_env: 'mlos_core'
|
||||
# Python 3.6 is currently incompatible.
|
||||
# Python36:
|
||||
# conda_env: 'mlos_core-3.6'
|
||||
Python38:
|
||||
conda_env: 'mlos_core-3.8'
|
||||
Python39:
|
||||
conda_env: 'mlos_core-3.9'
|
||||
Python310:
|
||||
conda_env: 'mlos_core-3.10'
|
||||
# FIXME: There are currently several missing/broken dependencies for python 3.11.
|
||||
#Python311:
|
||||
# conda_env: 'mlos_core-3.11'
|
||||
|
||||
variables:
|
||||
- name: CONDA_CACHE_DIR
|
||||
value: "/usr/share/miniconda/envs/$(conda_env)"
|
||||
#value: "/usr/share/miniconda/envs"
|
||||
- name: cache_cur_date
|
||||
value: "0000-00-00"
|
||||
- name: cache_cur_hour
|
||||
value: "00"
|
||||
- name: cache_prev_hour
|
||||
value: "23"
|
||||
|
||||
steps:
|
||||
- bash: echo "##vso[task.prependpath]$CONDA/bin"
|
||||
displayName: 'Add conda to PATH'
|
||||
- bash: |
|
||||
echo "##vso[task.setvariable variable=cache_cur_date;]`date +%Y-%m-%d`"
|
||||
echo "##vso[task.setvariable variable=cache_cur_hour;]`date +%H`"
|
||||
echo "##vso[task.setvariable variable=cache_prev_hour;]`date -d'1 hour ago' +%H`"
|
||||
displayName: 'Set cache timestamp variables'
|
||||
- bash: |
|
||||
echo "cache_cur_date: $(cache_cur_date)"
|
||||
echo "cache_cur_hour: $(cache_cur_hour)"
|
||||
echo "cache_prev_hour: $(cache_prev_hour)"
|
||||
echo "CONDA_CACHE_DIR: $(CONDA_CACHE_DIR)"
|
||||
displayName: 'Print some pipeline variables'
|
||||
- task: Cache@2
|
||||
displayName: Restore cached conda environment
|
||||
inputs:
|
||||
key: 'conda | "$(Agent.OS)" | "$(conda_env)" | conda-envs/$(conda_env).yml | mlos_core/setup.py | mlos_bench/setup.py | "$(cache_cur_date)" | "$(cache_cur_hour)" | .azure-pipelines/linux.yml'
|
||||
path: $(CONDA_CACHE_DIR)
|
||||
# Prefer cached environments from the last hour or day.
|
||||
restoreKeys: |
|
||||
conda | "$(Agent.OS)" | "$(conda_env)" | conda-envs/$(conda_env).yml | mlos_core/setup.py | mlos_bench/setup.py | "$(cache_cur_date)" | "$(cache_cur_hour)"
|
||||
conda | "$(Agent.OS)" | "$(conda_env)" | conda-envs/$(conda_env).yml | mlos_core/setup.py | mlos_bench/setup.py | "$(cache_cur_date)" | "$(cache_prev_hour)"
|
||||
# conda | "$(Agent.OS)" | "$(conda_env)" | conda-envs/$(conda_env).yml | mlos_core/setup.py | mlos_bench/setup.py | "$(cache_cur_date)"
|
||||
cacheHitVar: CONDA_CACHE_RESTORED
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
printenv
|
||||
displayName: 'Log some environment variables for debugging'
|
||||
|
||||
- bash: |
|
||||
conda config --set channel_priority strict
|
||||
#conda update -v -y -n base conda
|
||||
displayName: Update and configure conda
|
||||
|
||||
# Try and speed up the pipeline by using a faster solver:
|
||||
- bash: |
|
||||
conda install -v -y -n base conda-libmamba-solver
|
||||
# Try to set either of the configs for the solver.
|
||||
conda config --set experimental_solver libmamba || true
|
||||
conda config --set solver libmamba || true
|
||||
echo "##vso[task.setvariable variable=CONDA_EXPERIMENTAL_SOLVER;]libmamba"
|
||||
echo "##vso[task.setvariable variable=EXPERIMENTAL_SOLVER;]libmamba"
|
||||
displayName: Install and default to mamba solver
|
||||
- bash: rm -f /usr/share/miniconda/pkgs/cache/*.json
|
||||
displayName: Workaround a mamba cache bug in ADO
|
||||
# https://github.com/mamba-org/mamba/issues/488
|
||||
|
||||
- bash: make CONDA_ENV_NAME=$(conda_env) CONDA_INFO_LEVEL=-v conda-env
|
||||
displayName: 'Create/update mlos_core conda environment'
|
||||
|
||||
- bash: conda run -n $(conda_env) pip install pytest-azurepipelines
|
||||
displayName: 'Install pytest-azurepipelines'
|
||||
|
||||
# This is moreso about code cleanliness, which is a dev thing, not a
|
||||
# functionality thing, and the rules for that change between python versions,
|
||||
# so only do this for the default in the devcontainer.
|
||||
#- bash: make CONDA_ENV_NAME=$(conda_env) check
|
||||
# displayName: 'Run lint checks'
|
||||
- bash: make CONDA_ENV_NAME=$(conda_env) test
|
||||
displayName: 'Run tests'
|
||||
|
||||
- bash: make CONDA_ENV_NAME=$(conda_env) CONDA_INFO_LEVEL=-v dist dist-test
|
||||
displayName: 'Generate and test binary distribution files'
|
||||
|
||||
- job: DevContainer
|
||||
pool:
|
||||
vmImage: 'ubuntu-latest'
|
||||
variables:
|
||||
DOCKER_BUILDKIT: 1
|
||||
BUILDKIT_INLINE_CACHE: 1
|
||||
steps:
|
||||
- bash: |
|
||||
set -x
|
||||
printenv
|
||||
displayName: 'Log some environment variables for debugging'
|
||||
- bash: |
|
||||
if [ "$(Build.Reason)" == "Schedule" ]; then
|
||||
echo "##vso[task.setvariable variable=NO_CACHE;]true"
|
||||
fi
|
||||
displayName: 'Set NO_CACHE variable for nightly builds'
|
||||
- bash: |
|
||||
set -x
|
||||
echo "NO_CACHE: $NO_CACHE"
|
||||
.devcontainer/build/build-devcontainer.sh
|
||||
displayName: 'Build the devcontainer image'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker run -d --rm --user root \
|
||||
--env SYSTEM_JOBDISPLAYNAME=$(System.JobDisplayName) \
|
||||
--volume /var/run/docker.sock:/var/run/docker.sock \
|
||||
--env DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) \
|
||||
--volume $(pwd):/workspaces/MlosCore \
|
||||
--env CONTAINER_WORKSPACE_FOLDER=/workspaces/MlosCore \
|
||||
--env LOCAL_WORKSPACE_FOLDER=$(pwd) \
|
||||
--workdir /workspaces/MlosCore \
|
||||
--name mlos-core-devcontainer mlos-core-devcontainer sleep infinity
|
||||
displayName: 'Start the devcontainer in the background'
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user root mlos-core-devcontainer groupmod -g `id -g` vscode
|
||||
docker exec --user root mlos-core-devcontainer usermod -u `id -u` -g `id -g` vscode
|
||||
docker exec --user root mlos-core-devcontainer chown -R vscode:vscode /home/vscode
|
||||
docker exec --user root mlos-core-devcontainer mkdir -p /opt/conda/pkgs/cache /var/cache/pip
|
||||
docker exec --user root mlos-core-devcontainer chown -R vscode /opt/conda/pkgs/cache /var/cache/pip
|
||||
displayName: 'Fixup vscode uid/gid in the running container'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v licenseheaders
|
||||
# licenseheaders changes the contents of the files, so make this check fail if there are any changes detected
|
||||
git diff --no-pager --exit-code
|
||||
displayName: 'Check that all of the python files have licenseheaders'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v conda-env
|
||||
displayName: 'Update the conda env in the devcontainer'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer conda run -n mlos_core pip install pytest-azurepipelines
|
||||
displayName: 'Install pytest-azurepipelines'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v check
|
||||
displayName: 'Run lint checks'
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v test
|
||||
displayName: 'Run tests'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v dist dist-test
|
||||
displayName: 'Generate and test binary distribution files'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v devcontainer
|
||||
displayName: 'Test rebuilding the devcontainer in the devcontainer'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v doc
|
||||
displayName: 'Generate docs and test check them'
|
||||
- task: AzureStaticWebApp@0
|
||||
# Only publish docs on the main branch.
|
||||
condition: eq(variables['Build.SourceBranch'], 'refs/heads/main')
|
||||
displayName: 'Publish documentation'
|
||||
inputs:
|
||||
app_location: '/doc/build/html'
|
||||
api_location: 'api'
|
||||
output_location: '/'
|
||||
azure_static_web_apps_api_token: $(deployment_token)
|
||||
|
||||
- publish: $(System.DefaultWorkingDirectory)/doc/source/badges/coverage.svg
|
||||
artifact: coverage-status-badge
|
||||
condition: eq(variables['Build.SourceBranch'], 'refs/heads/main')
|
||||
displayName: 'Publish the coverage status badge'
|
||||
|
||||
- bash: |
|
||||
set -x
|
||||
docker stop -t 1 mlos-core-devcontainer || true
|
||||
docker rm --force mlos-core-devcontainer || true
|
||||
displayName: 'Cleanup the devcontainer'
|
||||
|
||||
- bash: |
|
||||
echo $(ACR_PASSWORD) | docker login --username $(ACR_USERNAME) --password-stdin $(ACR_LOGINURL)
|
||||
condition: and(eq(variables['Build.SourceBranch'], 'refs/heads/main'), ne(variables['ACR_LOGINURL'], ''))
|
||||
displayName: Login to ACR
|
||||
- bash: |
|
||||
set -x
|
||||
docker tag devcontainer-cli $(ACR_LOGINURL)/devcontainer-cli
|
||||
docker push $(ACR_LOGINURL)/devcontainer-cli
|
||||
docker tag mlos-core-devcontainer $(ACR_LOGINURL)/mlos-core-devcontainer
|
||||
docker push $(ACR_LOGINURL)/mlos-core-devcontainer
|
||||
condition: and(eq(variables['Build.SourceBranch'], 'refs/heads/main'), ne(variables['ACR_LOGINURL'], ''))
|
||||
displayName: Push Docker images
|
||||
|
|
@ -1,196 +0,0 @@
|
|||
name: 'MlosCore Windows'
|
||||
|
||||
trigger:
|
||||
- main
|
||||
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
- '*'
|
||||
|
||||
schedules:
|
||||
- cron: "0 0 * * *"
|
||||
displayName: Nightly build
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
always: "true"
|
||||
|
||||
jobs:
|
||||
- job: 'Windows'
|
||||
|
||||
pool:
|
||||
vmImage: windows-latest
|
||||
|
||||
# Test multiple versions of python.
|
||||
strategy:
|
||||
matrix:
|
||||
DefaultPython:
|
||||
# Floating most recent version of python (useful to catch new compatibility issues in nightly builds).
|
||||
conda_env: 'mlos_core'
|
||||
conda_env_yml: 'mlos_core-windows.yml'
|
||||
PythonVersion: ''
|
||||
# Skipping testing of additional versions on Windows for now.
|
||||
|
||||
variables:
|
||||
- name: CONDA_CACHE_DIR
|
||||
value: $(CONDA)/envs/$(conda_env)
|
||||
#value: $(CONDA)/envs
|
||||
- name: cache_cur_date
|
||||
value: "0000-00-00"
|
||||
- name: cache_cur_hour
|
||||
value: "00"
|
||||
- name: cache_prev_hour
|
||||
value: "23"
|
||||
|
||||
steps:
|
||||
- pwsh: |
|
||||
Write-Host "##vso[task.prependpath]$env:CONDA\condabin"
|
||||
Write-Host "##vso[task.prependpath]$env:CONDA\bin"
|
||||
Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
|
||||
Write-Host "##vso[task.prependpath]$env:CONDA\Library\bin"
|
||||
Write-Host "##vso[task.prependpath]$env:CONDA\Library\usr\bin"
|
||||
Write-Host "##vso[task.prependpath]$env:CONDA\Library\mingw-w64\bin"
|
||||
Write-Host "##vso[task.prependpath]$env:CONDA"
|
||||
displayName: Add conda to PATH
|
||||
|
||||
- pwsh: |
|
||||
Write-Host "##vso[task.setvariable variable=cache_cur_date;]" (Get-Date -Format yyyy-MM-dd)
|
||||
Write-Host "##vso[task.setvariable variable=cache_cur_hour;]" (Get-Date -Format HH)
|
||||
Write-Host "##vso[task.setvariable variable=cache_prev_hour;]" (Get-Date).AddHours(-1).ToString('HH')
|
||||
displayName: 'Set cache timestamp variables'
|
||||
- pwsh: |
|
||||
echo "cache_cur_date: $(cache_cur_date)"
|
||||
echo "cache_cur_hour: $(cache_cur_hour)"
|
||||
echo "cache_prev_hour: $(cache_prev_hour)"
|
||||
echo "CONDA_CACHE_DIR: $(CONDA_CACHE_DIR)"
|
||||
echo "CONDA: $env:CONDA"
|
||||
echo "CONDA_PREFIX: $env:CONDA_PREFIX"
|
||||
echo "CONDA_ROOT: $env:CONDA_ROOT"
|
||||
echo "PATH: $env:PATH"
|
||||
displayName: 'Print some pipeline and environment variables'
|
||||
- task: Cache@2
|
||||
displayName: Restore cached conda environment
|
||||
inputs:
|
||||
key: 'conda | "$(Agent.OS)" | "$(conda_env)" | conda-envs/$(conda_env_yml) | mlos_core/setup.py | mlos_bench/setup.py | "$(cache_cur_date)" | "$(cache_cur_hour)" | .azure-pipelines/windows.yml'
|
||||
path: $(CONDA_CACHE_DIR)
|
||||
# Prefer cached environments from the last hour or day.
|
||||
restoreKeys: |
|
||||
conda | "$(Agent.OS)" | "$(conda_env)" | conda-envs/$(conda_env_yml) | mlos_core/setup.py | mlos_bench/setup.py | "$(cache_cur_date)" | "$(cache_cur_hour)"
|
||||
conda | "$(Agent.OS)" | "$(conda_env)" | conda-envs/$(conda_env_yml) | mlos_core/setup.py | mlos_bench/setup.py | "$(cache_cur_date)" | "$(cache_prev_hour)"
|
||||
# conda | "$(Agent.OS)" | "$(conda_env)" | conda-envs/$(conda_env_yml) | mlos_core/setup.py | mlos_bench/setup.py | "$(cache_cur_date)"
|
||||
cacheHitVar: CONDA_CACHE_RESTORED
|
||||
|
||||
# NOTE: these steps are handled by the Makefile in Linux environments for reusability.
|
||||
|
||||
- pwsh: |
|
||||
Get-ChildItem -Recurse Env:
|
||||
displayName: 'Log some environment variables for debugging'
|
||||
|
||||
- pwsh: |
|
||||
conda config --set channel_priority strict
|
||||
#conda update -v -y -n base conda
|
||||
displayName: Update and configure conda
|
||||
|
||||
# Try and speed up the pipeline by using a faster solver:
|
||||
- pwsh: |
|
||||
conda install -v -y -n base conda-libmamba-solver
|
||||
conda config --set experimental_solver libmamba
|
||||
conda config --set solver libmamba
|
||||
Write-Host "##vso[task.setvariable variable=CONDA_EXPERIMENTAL_SOLVER;]libmamba"
|
||||
Write-Host "##vso[task.setvariable variable=EXPERIMENTAL_SOLVER;]libmamba"
|
||||
ignoreLASTEXITCODE: true
|
||||
errorActionPreference: continue
|
||||
displayName: Install and default to experimental mamba solver
|
||||
- pwsh: Remove-Item -Recurse -Force $env:CONDA\pkgs\cache\*.json
|
||||
displayName: Workaround a mamba cache bug in ADO
|
||||
# https://github.com/mamba-org/mamba/issues/488
|
||||
|
||||
- pwsh: |
|
||||
if (! (conda env list | Select-String -Pattern "^$(conda_env) ") ) { conda env create -v -n $(conda_env) -f conda-envs/$(conda_env_yml) }
|
||||
conda env update -v -n $(conda_env) -f conda-envs/$(conda_env_yml) --prune
|
||||
displayName: 'Create/update mlos_core conda environment'
|
||||
|
||||
- pwsh: conda run -n $(conda_env) pip install pytest-azurepipelines
|
||||
displayName: 'Install pytest-azurepipelines'
|
||||
|
||||
- pwsh: conda run -n $(conda_env) pylint -j0 mlos_core/mlos_core mlos_bench/mlos_bench
|
||||
displayName: 'Run lint checks'
|
||||
- pwsh: |
|
||||
conda run -n $(conda_env) pytest --cov=. --cov-append --cov-report=xml --cov-report=html --junitxml=junit/test-results.xml --napoleon-docstrings --test-run-title="$(System.JobDisplayName)" mlos_core/ mlos_bench/
|
||||
displayName: 'Run tests'
|
||||
|
||||
# Note: unlike the Makefile version, the pwsh version of these rules are all run within a single shell context, so we can
|
||||
# split commands across lines with CWD maintained (hence we also require the "cd .." here).
|
||||
- pwsh: |
|
||||
# Build the mlos_core wheel.
|
||||
cd mlos_core
|
||||
conda run -n $(conda_env) python setup.py bdist_wheel
|
||||
cd ..
|
||||
$mlos_core_whl = (Resolve-Path mlos_core/dist/mlos_core-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
|
||||
echo $mlos_core_whl
|
||||
# Build the mlos_bench wheel.
|
||||
cd mlos_bench
|
||||
conda run -n $(conda_env) python setup.py bdist_wheel
|
||||
cd ..
|
||||
$mlos_bench_whl = (Resolve-Path mlos_bench/dist/mlos_bench-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
|
||||
echo $mlos_bench_whl
|
||||
# Setup a clean environment to test installing/using them.
|
||||
$PythonVersReq = (conda list -n $(conda_env) | Select-String -AllMatches -Pattern '^python\s+([0-9.]+)\s+').Matches.Groups[1].Value
|
||||
conda create -y -v -n mlos-dist-test-$(PythonVersion) python=${PythonVersReq}
|
||||
conda install -y -v -n mlos-dist-test-$(PythonVersion) vswhere vs2019_win-64
|
||||
conda install -y -v -n mlos-dist-test-$(PythonVersion) conda-forge::GPy
|
||||
conda run -n mlos-dist-test-$(PythonVersion) pip install pytest pytest-timeout pytest-forked pytest-xdist
|
||||
# Install mlos_core wheel.
|
||||
conda run -n mlos-dist-test-$(PythonVersion) pip install "${mlos_core_whl}[full]"
|
||||
# Install mlos_bench wheel.
|
||||
conda run -n mlos-dist-test-$(PythonVersion) pip install "${mlos_bench_whl}[full]"
|
||||
# Just pick one simple test to run for now.
|
||||
# The rest should have been handled in a separate step.
|
||||
conda run -n mlos-dist-test-$(PythonVersion) python -m pytest mlos_core/mlos_core/tests/spaces/spaces_test.py
|
||||
# Run a simple mlos_bench test.
|
||||
conda run -n mlos-dist-test-$(PythonVersion) python -m pytest mlos_bench/mlos_bench/tests/environment/mock_env_test.py
|
||||
displayName: 'Generate and test binary distribution files'
|
||||
|
||||
# NOTE: Windows ADO agents don't support Linux Docker containers.
|
||||
# This will have to be tested manually for now using the ".devcontainer/build/build-devcontainer.ps1" script.
|
||||
#
|
||||
#- job: WindowsDevContainer
|
||||
# pool:
|
||||
# vmImage: windows-latest
|
||||
# steps:
|
||||
# - pwsh: |
|
||||
# if ("$(Build.Reason)" -eq "Schedule") {
|
||||
# Write-Host "##vso[task.setvariable variable=NO_CACHE;]true"
|
||||
# }
|
||||
# displayName: 'Set NO_CACHE variable for nightly builds'
|
||||
# - pwsh: |
|
||||
# Write-Host "NO_CACHE: $env:NO_CACHE"
|
||||
# .devcontainer/build/build-devcontainer.ps1
|
||||
# displayName: 'Build devcontainer in Windows'
|
||||
# - pwsh: |
|
||||
# docker run -d --rm --user root `
|
||||
# --volume /var/run/docker.sock:/var/run/docker.sock `
|
||||
# --volume ${PWD}:/workspaces/MlosCore \
|
||||
# --env CONTAINER_WORKSPACE_FOLDER=/workspaces/MlosCore \
|
||||
# --env LOCAL_WORKSPACE_FOLDER=${PWD} \
|
||||
# --workdir /workspaces/MlosCore \
|
||||
# --name mlos-core-devcontainer mlos-core-devcontainer sleep infinity
|
||||
# displayName: 'Start the devcontainer in the background'
|
||||
#
|
||||
# - pwsh: |
|
||||
# docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v conda-env
|
||||
# displayName: 'Update the conda env in the devcontainer'
|
||||
#
|
||||
# - pwsh: |
|
||||
# docker exec --user vscode mlos-core-devcontainer conda run -n $(conda_env) pip show mlos_core
|
||||
# docker exec --user vscode mlos-core-devcontainer conda run -n $(conda_env) pip show mlos_bench
|
||||
# displayName: 'Basic devcontainer test'
|
||||
#
|
||||
# # Skip other tests - they should get run identically in the Linux job.
|
||||
#
|
||||
# - pwsh: |
|
||||
# docker stop -t 1 mlos-core-devcontainer || true
|
||||
# docker rm --force mlos-core-devcontainer || true
|
||||
# displayName: 'Cleanup the devcontainer'
|
||||
# ignoreLASTEXITCODE: true
|
||||
|
|
@ -23,6 +23,9 @@ RUN npm install -g @devcontainers/cli
|
|||
# Also add the cspell tool to image.
|
||||
RUN npm install -g cspell
|
||||
|
||||
# Also add the markdown-link-check tool to image.
|
||||
RUN npm install -g markdown-link-check
|
||||
|
||||
# Adjust the uid/gid of the node user to match the host user
|
||||
ARG NODE_UID=1000
|
||||
ARG NODE_GID=1000
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
.PHONY: all devcontainer-cli devcontainer cspell
|
||||
.PHONY: all devcontainer-cli devcontainer cspell markdown-link-check
|
||||
|
||||
all: devcontainer-cli devcontainer
|
||||
|
||||
devcontainer-cli cspell:
|
||||
devcontainer-cli cspell markdown-link-check:
|
||||
./build-devcontainer-cli.sh
|
||||
|
||||
devcontainer:
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ else
|
|||
rmdir "$tmpdir"
|
||||
fi
|
||||
|
||||
docker build -t devcontainer-cli:latest -t cspell:latest \
|
||||
docker build -t devcontainer-cli:latest -t cspell:latest -t markdown-link-check:latest \
|
||||
$devcontainer_cli_build_args \
|
||||
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
||||
--build-arg NODE_UID=$(id -u) \
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@
|
|||
"trond-snekvik.simple-rst",
|
||||
"DavidAnson.vscode-markdownlint", // Linter for markdown files
|
||||
"huntertran.auto-markdown-toc", // Auto-generated Markdown Table of Contents
|
||||
"ms-azure-devops.azure-pipelines", // Azure Pipelines integration
|
||||
"cschleiden.vscode-github-actions", // GitHub Actions integration
|
||||
"ms-azuretools.vscode-docker",
|
||||
"waderyan.gitblame", // Enhances git blame experience
|
||||
"donjayamanne.githistory", // Enhanced git history experience
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
#!/bin/bash
|
||||
##
|
||||
## Copyright (c) Microsoft Corporation.
|
||||
## Licensed under the MIT License.
|
||||
##
|
||||
|
||||
set -x
|
||||
|
||||
set -eu
|
||||
scriptdir=$(dirname "$(readlink -f "$0")")
|
||||
cd "$scriptdir/"
|
||||
|
||||
# Build the helper container that has the cspell CLI.
|
||||
#../build/build-devcontainer-cli.sh
|
||||
|
||||
# Make this work inside a devcontainer as well.
|
||||
reporoot=$(readlink -f "$scriptdir/../../")
|
||||
if [ -n "${LOCAL_WORKSPACE_FOLDER:-}" ]; then
|
||||
reporoot="$LOCAL_WORKSPACE_FOLDER"
|
||||
fi
|
||||
|
||||
# Basically does what the markdown-link-check github action would do, but locally too.
|
||||
# See Also: ~/.github/workflows/markdown-link-check.yml
|
||||
|
||||
docker run -i --rm \
|
||||
--user $(id -u):$(id -g) \
|
||||
-v "$reporoot":/src:ro \
|
||||
--workdir /src \
|
||||
markdown-link-check:latest \
|
||||
find ./doc ./mlos_core ./mlos_bench ./.devcontainer -name '*.md' -not -path './node_modules/*' \
|
||||
-exec markdown-link-check '{}' --config ./.github/workflows/markdown-link-check-config.json -q -v ';'
|
||||
|
||||
docker run -i --rm \
|
||||
--user $(id -u):$(id -g) \
|
||||
-v "$reporoot":/src:ro \
|
||||
--workdir /src \
|
||||
markdown-link-check:latest \
|
||||
find . -type f '(' -wholename ./CODE_OF_CONDUCT.md -o -wholename ./CONTRIBUTING.md -o -wholename ./README.md -o -wholename ./SECURITY.md ')' -not -path './node_modules/*' \
|
||||
-exec markdown-link-check '{}' --config ./.github/workflows/markdown-link-check-config.json -q -v ';'
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
name: MLOS DevContainer
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tags:
|
||||
description: Manual MLOS DevContainer run
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
schedule:
|
||||
- cron: "1 0 * * *"
|
||||
|
||||
jobs:
|
||||
DevContainer:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Here we only test a single (latest) version of python.
|
||||
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
BUILDKIT_INLINE_CACHE: 1
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set NO_CACHE variable for nightly builds
|
||||
if: github.event_name == 'schedule'
|
||||
run: |
|
||||
echo "NO_CACHE=true" >> $GITHUB_ENV
|
||||
|
||||
- name: Log some environment variables for debugging
|
||||
run: |
|
||||
set -x
|
||||
printenv
|
||||
echo "NO_CACHE: ${NO_CACHE:-}"
|
||||
|
||||
- name: Build the devcontainer image
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
set -x
|
||||
.devcontainer/build/build-devcontainer.sh
|
||||
|
||||
- name: Start the devcontainer in the background
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
set -x
|
||||
docker run -d --rm --user root \
|
||||
--volume /var/run/docker.sock:/var/run/docker.sock \
|
||||
--env DOCKER_BUILDKIT=$DOCKER_BUILDKIT \
|
||||
--volume $(pwd):/workspaces/MlosCore \
|
||||
--env CONTAINER_WORKSPACE_FOLDER=/workspaces/MlosCore \
|
||||
--env LOCAL_WORKSPACE_FOLDER=$(pwd) \
|
||||
--workdir /workspaces/MlosCore \
|
||||
--name mlos-core-devcontainer mlos-core-devcontainer sleep infinity
|
||||
- name: Fixup vscode uid/gid in the running container
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
set -x
|
||||
docker exec --user root mlos-core-devcontainer groupmod -g `id -g` vscode
|
||||
docker exec --user root mlos-core-devcontainer usermod -u `id -u` -g `id -g` vscode
|
||||
docker exec --user root mlos-core-devcontainer chown -R vscode:vscode /home/vscode
|
||||
docker exec --user root mlos-core-devcontainer mkdir -p /opt/conda/pkgs/cache /var/cache/pip
|
||||
docker exec --user root mlos-core-devcontainer chown -R vscode /opt/conda/pkgs/cache /var/cache/pip
|
||||
|
||||
- name: Update the conda env in the devcontainer
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v conda-env
|
||||
|
||||
- name: Check for missing licenseheaders
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v licenseheaders
|
||||
# licenseheaders changes the contents of the files, so make this check fail if there are any changes detected
|
||||
git --no-pager diff --exit-code
|
||||
|
||||
- name: Run lint checks
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v check
|
||||
|
||||
- name: Run tests
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v test
|
||||
|
||||
- name: Generate and test binary distribution files
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v dist dist-test
|
||||
|
||||
- name: Test rebuilding the devcontainer in the devcontainer
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v devcontainer
|
||||
|
||||
- name: Generate docs and test check them
|
||||
run: |
|
||||
set -x
|
||||
docker exec --user vscode mlos-core-devcontainer make CONDA_INFO_LEVEL=-v doc
|
||||
|
||||
- name: Deploy to GitHub pages
|
||||
if: github.ref == 'refs/heads/main'
|
||||
uses: JamesIves/github-pages-deploy-action@v4
|
||||
with:
|
||||
branch: gh-pages
|
||||
folder: doc/build/html
|
||||
clean: true
|
||||
|
||||
# TODO: Add code coverage upload support
|
||||
# - name: Upload coverage to codecov
|
||||
# if: github.repository == 'microsoft/mlos' && github.ref == 'refs/heads/main'
|
||||
# uses: codecov/codecov-action@v1
|
||||
# with:
|
||||
# token: ${{ secrets.CODECOV_TOKEN }}
|
||||
# file: ./coverage.xml
|
||||
# flags: unittests
|
||||
# fail_ci_if_error: true
|
||||
|
||||
- name: Cleanup the devcontainer
|
||||
run: |
|
||||
set -x
|
||||
docker stop -t 1 mlos-core-devcontainer || true
|
||||
docker rm --force mlos-core-devcontainer || true
|
||||
|
||||
- name: Container Registry Login
|
||||
if: github.repository == 'microsoft/mlos' && github.ref == 'refs/heads/main'
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ${{ secrets.ACR_LOGINURL }}
|
||||
username: ${{ secrets.ACR_USERNAME }}
|
||||
password: ${{ secrets.ACR_PASSWORD }}
|
||||
- name: Publish the container images
|
||||
if: github.repository == 'microsoft/mlos' && github.ref == 'refs/heads/main'
|
||||
timeout-minutes: 15
|
||||
# We only push to the :latest tag, to avoid needing to cleanup the
|
||||
# registry manually (there's currently no API for that).
|
||||
run: |
|
||||
set -x
|
||||
docker tag devcontainer-cli:latest ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:latest
|
||||
docker push ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:latest
|
||||
docker tag mlos-core-devcontainer:latest ${{ secrets.ACR_LOGINURL }}/mlos-core-devcontainer:latest
|
||||
docker push ${{ secrets.ACR_LOGINURL }}/mlos-core-devcontainer:latest
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
name: MLOS Linux
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tags:
|
||||
description: Manual MLOS Linux run
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
schedule:
|
||||
- cron: "1 0 * * *"
|
||||
|
||||
jobs:
|
||||
Linux:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
# Test multiple versions of python.
|
||||
strategy:
|
||||
matrix:
|
||||
python_version:
|
||||
# Empty string is the floating most recent version of python
|
||||
# (useful to catch new compatibility issues in nightly builds)
|
||||
- ""
|
||||
- "3.8"
|
||||
- "3.9"
|
||||
- "3.10"
|
||||
# FIXME: There are currently several missing/broken dependencies for python 3.11.
|
||||
#- "3.11"
|
||||
|
||||
env:
|
||||
cache_cur_date: 0000-00-00
|
||||
cache_cur_hour: 00
|
||||
cache_prev_hour: 23
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: conda-incubator/setup-miniconda@v2
|
||||
|
||||
- name: Set cache timestamp variables
|
||||
id: set_cache_vars
|
||||
run: |
|
||||
set -x
|
||||
if [ -z "${{ matrix.python_version }}" ]; then
|
||||
CONDA_ENV_NAME=mlos_core
|
||||
else
|
||||
CONDA_ENV_NAME="mlos_core-${{ matrix.python_version }}"
|
||||
fi
|
||||
echo "CONDA_ENV_NAME=$CONDA_ENV_NAME" >> $GITHUB_ENV
|
||||
echo "cache_cur_date=$(date -u +%Y-%m-%d)" >> $GITHUB_ENV
|
||||
echo "cache_cur_hour=$(date -u +%H)" >> $GITHUB_ENV
|
||||
echo "cache_prev_hour=$(date -u -d'1 hour ago' +%H)" >> $GITHUB_ENV
|
||||
echo "CONDA=$CONDA" >> $GITHUB_ENV
|
||||
|
||||
- name: Restore cached conda environment
|
||||
id: restore-conda-cache
|
||||
if: github.event_name != 'schedule'
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ${{ env.CONDA }}/envs/${{ env.CONDA_ENV_NAME }}
|
||||
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
|
||||
restore-keys: |
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_NAME }}.yml') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}
|
||||
|
||||
- name: Log some environment variables for debugging
|
||||
run: |
|
||||
set -x
|
||||
printenv
|
||||
echo "cache_cur_date: $cache_cur_date"
|
||||
echo "cache_cur_hour: $cache_cur_hour"
|
||||
echo "cache_prev_hour: $cache_prev_hour"
|
||||
echo "cache-hit: ${{ steps.restore-conda-cache.outputs.cache-hit }}"
|
||||
|
||||
- name: Update and configure conda
|
||||
run: |
|
||||
set -x
|
||||
conda config --set channel_priority strict
|
||||
#conda update -v -y -n base conda
|
||||
|
||||
# Try and speed up the pipeline by using a faster solver:
|
||||
- name: Install and default to mamba solver
|
||||
run: |
|
||||
set -x
|
||||
conda install -v -y -n base conda-libmamba-solver
|
||||
# Try to set either of the configs for the solver.
|
||||
conda config --set experimental_solver libmamba || true
|
||||
conda config --set solver libmamba || true
|
||||
echo "CONDA_EXPERIMENTAL_SOLVER=libmamba" >> $GITHUB_ENV
|
||||
echo "EXPERIMENTAL_SOLVER=libmamba" >> $GITHUB_ENV
|
||||
|
||||
- name: Create/update mlos_core conda environment
|
||||
run: make CONDA_ENV_NAME=$CONDA_ENV_NAME CONDA_INFO_LEVEL=-v conda-env
|
||||
|
||||
# This is moreso about code cleanliness, which is a dev thing, not a
|
||||
# functionality thing, and the rules for that change between python versions,
|
||||
# so only do this for the default in the devcontainer.
|
||||
#- name: Run lint checks
|
||||
# run: make CONDA_ENV_NAME=$CONDA_ENV_NAME check
|
||||
|
||||
# Only run the coverage checks on the devcontainer job.
|
||||
- name: Run tests
|
||||
run: make CONDA_ENV_NAME=$CONDA_ENV_NAME SKIP_COVERAGE=true test
|
||||
|
||||
- name: Generate and test binary distribution files
|
||||
run: make CONDA_ENV_NAME=$CONDA_ENV_NAME CONDA_INFO_LEVEL=-v dist dist-test
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"timeout": "20s",
|
||||
"retryOn429": true,
|
||||
"retryCount": 5,
|
||||
"fallbackRetryDelay": "30s",
|
||||
"aliveStatusCodes": [200, 203, 206, 403, 429]
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
name: MLOS Markdown Link Check
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tags:
|
||||
description: Manual MLOS Markdown Link Check run
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
|
||||
jobs:
|
||||
# Check in-repo markdown links
|
||||
markdown-link-check:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: gaurav-nelson/github-action-markdown-link-check@v1
|
||||
with:
|
||||
use-quiet-mode: yes
|
||||
use-verbose-mode: yes
|
||||
folder-path: './doc, ./mlos_core, ./mlos_bench, ./.devcontainer'
|
||||
file-path: './CODE_OF_CONDUCT.md, ./CONTRIBUTING.md, ./README.md, ./SECURITY.md'
|
||||
config-file: './.github/workflows/markdown-link-check-config.json'
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
name: MLOS Windows
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tags:
|
||||
description: Manual MLOS Linux run
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
schedule:
|
||||
- cron: "1 0 * * *"
|
||||
|
||||
jobs:
|
||||
Windows:
|
||||
runs-on: windows-latest
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: pwsh
|
||||
|
||||
env:
|
||||
CONDA_ENV_NAME: mlos_core
|
||||
CONDA_ENV_YML: mlos_core-windows.yml
|
||||
cache_cur_date: 0000-00-00
|
||||
cache_cur_hour: 00
|
||||
cache_prev_hour: 23
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: conda-incubator/setup-miniconda@v2
|
||||
|
||||
- name: Set cache timestamp variables
|
||||
id: set_cache_vars
|
||||
run: |
|
||||
"cache_cur_date=$(date -u +%Y-%m-%d)" >> $GITHUB_ENV
|
||||
"cache_cur_hour=$(date -u +%H)" >> $GITHUB_ENV
|
||||
"cache_prev_hour=$(date -u -d'1 hour ago' +%H)" >> $GITHUB_ENV
|
||||
"CONDA=$env:CONDA" >> $GITHUB_ENV
|
||||
|
||||
- name: Add conda libs to the PATH
|
||||
run: |
|
||||
$env:PATH = "$env:CONDA\condabin;$env:PATH"
|
||||
$env:PATH = "$env:CONDA\bin;$env:PATH"
|
||||
$env:PATH = "$env:CONDA\Scripts;$env:PATH"
|
||||
$env:PATH = "$env:CONDA\Library\bin;$env:PATH"
|
||||
$env:PATH = "$env:CONDA\Library\usr\bin;$env:PATH"
|
||||
$env:PATH = "$env:CONDA\Library\bin;$env:PATH"
|
||||
$env:PATH = "$env:CONDA\Library\mingw-w64\bin;$env:PATH"
|
||||
$env:PATH = "$env:CONDA;$env:PATH"
|
||||
"$env:PATH" >> $GITHUB_ENV
|
||||
|
||||
- name: Restore cached conda environment
|
||||
id: restore-conda-cache
|
||||
if: github.event_name != 'schedule'
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ${{ env.CONDA }}/envs/mlos_core
|
||||
key: conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_cur_hour }}
|
||||
restore-keys: |
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}-${{ env.cache_prev_hour }}
|
||||
conda-${{ runner.os }}-${{ env.CONDA_ENV_NAME }}-${{ hashFiles('conda-envs/${{ env.CONDA_ENV_YML }}') }}-${{ hashFiles('mlos_core/setup.py') }}-${{ hashFiles('mlos_bench/setup.py') }}-${{ env.cache_cur_date }}
|
||||
|
||||
- name: Log some environment variables for debugging
|
||||
run: |
|
||||
Get-ChildItem -Recurse Env:
|
||||
Write-Host "cache_cur_date: $env:cache_cur_date"
|
||||
Write-Host "cache_cur_hour: $env:cache_cur_hour"
|
||||
Write-Host "cache_prev_hour: $env:cache_prev_hour"
|
||||
Write-Host "cache-hit: ${{ steps.restore-conda-cache.outputs.cache-hit }}"
|
||||
|
||||
- name: Update and configure conda
|
||||
run: |
|
||||
conda config --set channel_priority strict
|
||||
#conda update -v -y -n base conda
|
||||
|
||||
# Try and speed up the pipeline by using a faster solver:
|
||||
- name: Install and default to mamba solver
|
||||
run: |
|
||||
conda install -v -y -n base conda-libmamba-solver
|
||||
# Try to set either of the configs for the solver.
|
||||
if (!(conda config --set experimental_solver libmamba)) { conda config --set solver libmamba }
|
||||
"CONDA_EXPERIMENTAL_SOLVER=libmamba" >> $GITHUB_ENV
|
||||
"EXPERIMENTAL_SOLVER=libmamba" >> $GITHUB_ENV
|
||||
|
||||
- name: Create/update mlos_core conda environment
|
||||
run: |
|
||||
if (! (conda env list | Select-String -Pattern "^$env:CONDA_ENV_NAME ") ) { conda env create -v -n $env:CONDA_ENV_NAME -f conda-envs/$env:CONDA_ENV_YML }
|
||||
conda env update -v -n $env:CONDA_ENV_NAME -f conda-envs/$env:CONDA_ENV_YML --prune
|
||||
|
||||
# This is moreso about code cleanliness, which is a dev thing, not a
|
||||
# functionality thing, and the rules for that change between python versions,
|
||||
# so only do this for the default in the devcontainer.
|
||||
#- name: Run lint checks
|
||||
# run: conda run -n $env:CONDA_ENV_NAME pylint -j0 mlos_core/mlos_core mlos_bench/mlos_bench
|
||||
|
||||
# Only run the coverage checks on the devcontainer job.
|
||||
- name: Run tests
|
||||
run: |
|
||||
conda run -n $env:CONDA_ENV_NAME pytest --junitxml=junit/test-results.xml mlos_core/ mlos_bench/
|
||||
|
||||
# Note: unlike the Makefile version, the pwsh version of these rules are all run within a single shell context, so we can
|
||||
# split commands across lines with CWD maintained (hence we also require the "cd .." here).
|
||||
- name: Generate and test binary distribution files
|
||||
run: |
|
||||
# Build the mlos_core wheel.
|
||||
cd mlos_core
|
||||
conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
|
||||
cd ..
|
||||
$mlos_core_whl = (Resolve-Path mlos_core/dist/mlos_core-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
|
||||
echo $mlos_core_whl
|
||||
# Build the mlos_bench wheel.
|
||||
cd mlos_bench
|
||||
conda run -n $env:CONDA_ENV_NAME python setup.py bdist_wheel
|
||||
cd ..
|
||||
$mlos_bench_whl = (Resolve-Path mlos_bench/dist/mlos_bench-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
|
||||
echo $mlos_bench_whl
|
||||
# Setup a clean environment to test installing/using them.
|
||||
$PythonVersReq = (conda list -n $env:CONDA_ENV_NAME | Select-String -AllMatches -Pattern '^python\s+([0-9.]+)\s+').Matches.Groups[1].Value
|
||||
conda create -y -v -n mlos-dist-test-$env:PythonVersion python=${PythonVersReq}
|
||||
conda install -y -v -n mlos-dist-test-$env:PythonVersion vswhere vs2019_win-64
|
||||
conda install -y -v -n mlos-dist-test-$env:PythonVersion conda-forge::GPy
|
||||
conda run -n mlos-dist-test-$env:PythonVersion pip install pytest pytest-timeout pytest-forked pytest-xdist
|
||||
# Install mlos_core wheel.
|
||||
conda run -n mlos-dist-test-$env:PythonVersion pip install "${mlos_core_whl}[full]"
|
||||
# Install mlos_bench wheel.
|
||||
conda run -n mlos-dist-test-$env:PythonVersion pip install "${mlos_bench_whl}[full]"
|
||||
# Just pick one simple test to run for now.
|
||||
# The rest should have been handled in a separate step.
|
||||
conda run -n mlos-dist-test-$env:PythonVersion python -m pytest mlos_core/mlos_core/tests/spaces/spaces_test.py
|
||||
# Run a simple mlos_bench test.
|
||||
conda run -n mlos-dist-test-$env:PythonVersion python -m pytest mlos_bench/mlos_bench/tests/environment/mock_env_test.py
|
||||
|
|
@ -11,7 +11,7 @@
|
|||
"trond-snekvik.simple-rst",
|
||||
"DavidAnson.vscode-markdownlint", // Linter for markdown files
|
||||
"huntertran.auto-markdown-toc", // Auto-generated Markdown Table of Contents
|
||||
"ms-azure-devops.azure-pipelines", // Azure Pipelines integration
|
||||
"cschleiden.vscode-github-actions", // GitHub Actions integration
|
||||
"ms-azuretools.vscode-docker",
|
||||
"waderyan.gitblame", // Enhances git blame experience
|
||||
"donjayamanne.githistory", // Enhanced git history experience
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
# Microsoft Open Source Code of Conduct
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
|
||||
Resources:
|
||||
|
||||
- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
|
||||
- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
|
||||
- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
|
||||
# Microsoft Open Source Code of Conduct
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
|
||||
Resources:
|
||||
|
||||
- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
|
||||
- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
|
||||
- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
|
||||
|
|
|
|||
|
|
@ -1,37 +1,37 @@
|
|||
# Contributing to MLOS
|
||||
|
||||
This project welcomes contributions and suggestions.
|
||||
Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution.
|
||||
For details, visit https://cla.opensource.microsoft.com.
|
||||
|
||||
When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment).
|
||||
Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
|
||||
## Details
|
||||
|
||||
[`main`](https://github.com/microsoft/MLOS/tree/main) is considered the primary development branch.
|
||||
|
||||
We expect development to follow a typical "gitflow" style workflow:
|
||||
|
||||
1. Fork a copy of the [MLOS repo in Github](https://github.com/microsoft/MLOS).
|
||||
2. Create a development (a.k.a. topic) branch off of `main` to work on changes.
|
||||
|
||||
```shell
|
||||
git checkout -b YourDevName/some-topic-description main
|
||||
```
|
||||
|
||||
3. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls).
|
||||
4. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass).
|
||||
5. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history.
|
||||
|
||||
### Caveats
|
||||
|
||||
There are consumers of MLOS internal to Microsoft that use an internal copy of the Github repo targetting code that is not open-sourced.
|
||||
This arrangement sometimes means porting changes from the internal repo to Github (and vise-versa).
|
||||
When that happens, the changes are submitted as a PR as described above, with the slight modification of (once approved and passing tests) using a rebase based merge instead of a squash merge in order to allow detecting duplicate patches between the public and private repos.
|
||||
|
||||
Additionally, to try and catch breaking changes we run some extra internal integration tests as well.
|
||||
If they do find issues, we encourage a conversation to occur on how to resolve them in the PRs.
|
||||
# Contributing to MLOS
|
||||
|
||||
This project welcomes contributions and suggestions.
|
||||
Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution.
|
||||
For details, visit https://cla.opensource.microsoft.com.
|
||||
|
||||
When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment).
|
||||
Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
|
||||
## Details
|
||||
|
||||
[`main`](https://github.com/microsoft/MLOS/tree/main) is considered the primary development branch.
|
||||
|
||||
We expect development to follow a typical "gitflow" style workflow:
|
||||
|
||||
1. Fork a copy of the [MLOS repo in Github](https://github.com/microsoft/MLOS).
|
||||
2. Create a development (a.k.a. topic) branch off of `main` to work on changes.
|
||||
|
||||
```shell
|
||||
git checkout -b YourDevName/some-topic-description main
|
||||
```
|
||||
|
||||
3. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls).
|
||||
4. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass).
|
||||
5. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history.
|
||||
|
||||
### Caveats
|
||||
|
||||
There are consumers of MLOS internal to Microsoft that use an internal copy of the Github repo targetting code that is not open-sourced.
|
||||
This arrangement sometimes means porting changes from the internal repo to Github (and vise-versa).
|
||||
When that happens, the changes are submitted as a PR as described above, with the slight modification of (once approved and passing tests) using a rebase based merge instead of a squash merge in order to allow detecting duplicate patches between the public and private repos.
|
||||
|
||||
Additionally, to try and catch breaking changes we run some extra internal integration tests as well.
|
||||
If they do find issues, we encourage a conversation to occur on how to resolve them in the PRs.
|
||||
|
|
|
|||
40
Makefile
40
Makefile
|
|
@ -21,7 +21,7 @@ MKDIR_BUILD := $(shell mkdir -p build)
|
|||
MAKEFLAGS += -j$(shell nproc)
|
||||
|
||||
.PHONY: all
|
||||
all: check test dist dist-test # doc
|
||||
all: check test dist dist-test doc licenseheaders
|
||||
|
||||
.PHONY: conda-env
|
||||
conda-env: build/conda-env.${CONDA_ENV_NAME}.build-stamp
|
||||
|
|
@ -41,7 +41,7 @@ clean-conda-env:
|
|||
rm -f build/conda-env.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
.PHONY: check
|
||||
check: pycodestyle pydocstyle pylint # cspell licenseheaders
|
||||
check: pycodestyle pydocstyle pylint # cspell licenseheaders markdown-link-check
|
||||
|
||||
.PHONY: pycodestyle
|
||||
pycodestyle: conda-env build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp build/pycodestyle.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
|
|
@ -91,6 +91,20 @@ build/cspell-container.build-stamp:
|
|||
$(MAKE) -C .devcontainer/build cspell
|
||||
touch $@
|
||||
|
||||
.PHONY: markdown-link-check
|
||||
ifeq ($(DOCKER),)
|
||||
markdown-link-check:
|
||||
@echo "NOTE: docker is not available. Skipping markdown-link-check check."
|
||||
else
|
||||
markdown-link-check: build/markdown-link-check-container.build-stamp
|
||||
./.devcontainer/scripts/run-markdown-link-check.sh
|
||||
endif
|
||||
|
||||
build/markdown-link-check-container.build-stamp:
|
||||
# Build the docker image with markdown-link-check in it.
|
||||
$(MAKE) -C .devcontainer/build markdown-link-check
|
||||
touch $@
|
||||
|
||||
.PHONY: pylint
|
||||
pylint: conda-env build/pylint.mlos_core.${CONDA_ENV_NAME}.build-stamp build/pylint.mlos_bench.${CONDA_ENV_NAME}.build-stamp
|
||||
|
||||
|
|
@ -132,24 +146,10 @@ PYTEST_OPTIONS :=
|
|||
# Allow optionally skipping coverage calculations during local testing to skip up inner dev loop.
|
||||
SKIP_COVERAGE := $(shell echo $${SKIP_COVERAGE:-} | grep -i -x -e 1 -e true)
|
||||
|
||||
# When running in Azure pipeline, add some additional arguments for pytest-azurepipelines plugin.
|
||||
ifneq ($(SYSTEM_JOBDISPLAYNAME),)
|
||||
PYTEST_OPTIONS += --napoleon-docstrings --test-run-title="$(SYSTEM_JOBDISPLAYNAME)"
|
||||
ifeq ($(SYSTEM_JOBDISPLAYNAME),DevContainer)
|
||||
SKIP_COVERAGE :=
|
||||
else
|
||||
SKIP_COVERAGE := true
|
||||
PYTEST_OPTIONS += --no-coverage-upload
|
||||
endif
|
||||
endif
|
||||
|
||||
ifeq ($(SKIP_COVERAGE),)
|
||||
PYTEST_OPTIONS += --cov=. --cov-append --cov-report=xml --cov-report=html --junitxml=junit/test-results.xml --local-badge-output-dir=doc/source/badges/
|
||||
endif
|
||||
|
||||
testing:
|
||||
@echo "$(PYTEST_OPTIONS)"
|
||||
|
||||
# Run the pytest target on only the modules that have changed recently, but
|
||||
# make sure the coverage report is for both of them when used in the pipeline.
|
||||
# NOTE: When run locally, the junit/test-results.xml will only include the
|
||||
|
|
@ -297,7 +297,7 @@ doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile
|
|||
# See check-doc
|
||||
|
||||
.PHONY: doc
|
||||
doc: doc/build/html/staticwebapp.config.json build/check-doc.build-stamp build/linklint-doc.build-stamp
|
||||
doc: doc/build/html/.nojekyll build/check-doc.build-stamp build/linklint-doc.build-stamp
|
||||
|
||||
doc/build/html/htmlcov/index.html: doc/build/html/index.html
|
||||
# Make the codecov html report available for the site.
|
||||
|
|
@ -305,9 +305,9 @@ doc/build/html/htmlcov/index.html: doc/build/html/index.html
|
|||
mkdir -p doc/build/html/htmlcov
|
||||
touch doc/build/html/htmlcov/index.html
|
||||
|
||||
doc/build/html/staticwebapp.config.json: doc/build/html/index.html doc/build/html/htmlcov/index.html doc/staticwebapp.config.json
|
||||
# Copy the azure static web app config file to the doc build directory.
|
||||
cp doc/staticwebapp.config.json doc/build/html/
|
||||
doc/build/html/.nojekyll: doc/build/html/index.html doc/build/html/htmlcov/index.html
|
||||
# Make sure that github pages doesn't try to run jekyll on the docs.
|
||||
touch doc/build/html/.nojekyll
|
||||
|
||||
.PHONY: check-doc
|
||||
check-doc: build/check-doc.build-stamp
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# MlosCore
|
||||
|
||||
[](https://msgsl.visualstudio.com/MLOS/_build/latest?definitionId=34&branchName=main)
|
||||
[](https://msgsl.visualstudio.com/MLOS/_build/latest?definitionId=61&branchName=main)
|
||||
[](https://microsoft.github.io/MLOS/_images/tests.svg)
|
||||
[](https://microsoft.github.io/MLOS/_images/coverage.svg)
|
||||
|
||||
This repository contains a stripped down implementation of essentially just the core optimizer and config space description APIs from the original [MLOS](https://github.com/microsoft/MLOS) as well as the `mlos-bench` module intended to help automate and manage running experiments for autotuning systems with `mlos-core`.
|
||||
|
||||
|
|
@ -56,7 +56,7 @@ Simply open the project in VSCode and follow the prompts to build and open the d
|
|||
conda activate mlos_core
|
||||
```
|
||||
|
||||
2. Run the [`BayesianOptimization.ipynb`](./Notebooks/BayesianOptimization.ipynb) notebook.
|
||||
2. Run the [`BayesianOptimization.ipynb`](./mlos_core/notebooks/BayesianOptimization.ipynb) notebook.
|
||||
|
||||
## Distributing
|
||||
|
||||
|
|
@ -83,5 +83,8 @@ Simply open the project in VSCode and follow the prompts to build and open the d
|
|||
|
||||
## See Also
|
||||
|
||||
<!-- TODO: Reenable checking these once they no longer require authentication. -->
|
||||
<!-- markdown-link-check-disable -->
|
||||
- API and Examples Documentation: <https://aka.ms/mlos-core/docs>
|
||||
- Source Code Repository: <https://aka.ms/mlos-core/src>
|
||||
<!-- markdown-link-check-enable -->
|
||||
|
|
|
|||
82
SECURITY.md
82
SECURITY.md
|
|
@ -1,41 +1,41 @@
|
|||
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.5 BLOCK -->
|
||||
|
||||
## Security
|
||||
|
||||
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
|
||||
|
||||
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below.
|
||||
|
||||
## Reporting Security Issues
|
||||
|
||||
**Please do not report security vulnerabilities through public GitHub issues.**
|
||||
|
||||
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).
|
||||
|
||||
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
|
||||
|
||||
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
|
||||
|
||||
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
|
||||
|
||||
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
|
||||
* Full paths of source file(s) related to the manifestation of the issue
|
||||
* The location of the affected source code (tag/branch/commit or direct URL)
|
||||
* Any special configuration required to reproduce the issue
|
||||
* Step-by-step instructions to reproduce the issue
|
||||
* Proof-of-concept or exploit code (if possible)
|
||||
* Impact of the issue, including how an attacker might exploit the issue
|
||||
|
||||
This information will help us triage your report more quickly.
|
||||
|
||||
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.
|
||||
|
||||
## Preferred Languages
|
||||
|
||||
We prefer all communications to be in English.
|
||||
|
||||
## Policy
|
||||
|
||||
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd).
|
||||
|
||||
<!-- END MICROSOFT SECURITY.MD BLOCK -->
|
||||
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.5 BLOCK -->
|
||||
|
||||
## Security
|
||||
|
||||
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
|
||||
|
||||
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below.
|
||||
|
||||
## Reporting Security Issues
|
||||
|
||||
**Please do not report security vulnerabilities through public GitHub issues.**
|
||||
|
||||
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).
|
||||
|
||||
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
|
||||
|
||||
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
|
||||
|
||||
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
|
||||
|
||||
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
|
||||
* Full paths of source file(s) related to the manifestation of the issue
|
||||
* The location of the affected source code (tag/branch/commit or direct URL)
|
||||
* Any special configuration required to reproduce the issue
|
||||
* Step-by-step instructions to reproduce the issue
|
||||
* Proof-of-concept or exploit code (if possible)
|
||||
* Impact of the issue, including how an attacker might exploit the issue
|
||||
|
||||
This information will help us triage your report more quickly.
|
||||
|
||||
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.
|
||||
|
||||
## Preferred Languages
|
||||
|
||||
We prefer all communications to be in English.
|
||||
|
||||
## Policy
|
||||
|
||||
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd).
|
||||
|
||||
<!-- END MICROSOFT SECURITY.MD BLOCK -->
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
comment: false
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
{
|
||||
"auth": {
|
||||
"identityProviders": {
|
||||
"azureActiveDirectory": {
|
||||
"registration": {
|
||||
"openIdIssuer": "https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/v2.0",
|
||||
"clientIdSettingName": "AZURE_CLIENT_ID",
|
||||
"clientSecretSettingName": "AZURE_CLIENT_SECRET"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"routes": [
|
||||
{
|
||||
"route": "/.auth/login/github",
|
||||
"statusCode": 404
|
||||
},
|
||||
{
|
||||
"route": "/.auth/login/twitter",
|
||||
"statusCode": 404
|
||||
},
|
||||
{
|
||||
"route": "/*",
|
||||
"allowedRoles": ["authenticated"]
|
||||
}
|
||||
],
|
||||
"responseOverrides": {
|
||||
"401": {
|
||||
"statusCode": 302,
|
||||
"redirect": "/.auth/login/aad"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -12,7 +12,7 @@ Which are implemented using [`Services`](../service/).
|
|||
|
||||
Environments also have [`Tunables`](../tunables/) and [`TunableGroups`](../tunables/) for controlling their configuration.
|
||||
|
||||
Environments can also be stackable via the [`CompositeEnvironment`](./composite.py) class.
|
||||
Environments can also be stackable via the [`CompositeEnvironment`](./composite_env.py) class.
|
||||
|
||||
For instance, a VM, OS, and Application Environment can be stacked together to form a full benchmarking environment, each with their own tunables.
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче