Merged PR 860: Various Build Fixups

- add nightly builds so we can catch issues with upstream changes sooner - allow reading conda envs out of the environment - handle only setting one of CONDA_EXPERIMENTAL_SOLVER and CONDA_SOLVER envs - adjust conda envs to avoid python 3.11 for now - adjust Makefile to use the same version of python from the current CONDA_ENV for our test environment (this is quick way to respect that version constraint and only specify it in one place) - adjust the expected version of configspace - remove configspace from conda env file (upstream doesn't have the version we need outside of conda-forge) - move all other version restrictions to setup.py files - add a 'full' mode for installing all extra_requires for each package - use that in the conda envs instead Related to additional build/test issues discovered in !850
2023-01-30 23:49:26 +00:00 · 2023-01-30 23:49:26 +00:00 · e26f2919eb
--- a/.azure-pipelines/linux.yml
+++ b/.azure-pipelines/linux.yml
@ -11,7 +11,7 @@ jobs:
  strategy:
    matrix:
      DefaultPython:
-        # Currently 3.9
+        # Floating most recent version of python (useful to catch new compatibility issues in nightly builds).
        conda_env: 'mlos_core'
  # Python 3.6 is currently incompatible.
  #   Python36:
@ -20,6 +20,11 @@ jobs:
        conda_env: 'mlos_core-3.8'
      Python39:
        conda_env: 'mlos_core-3.9'
+      Python310:
+        conda_env: 'mlos_core-3.10'
+      # FIXME: There are currently several missing/broken dependencies for python 3.11.
+      #Python311:
+      #  conda_env: 'mlos_core-3.11'

  variables:
    - name: CONDA_CACHE_DIR
@ -87,7 +92,7 @@ jobs:
  - bash: make CONDA_ENV_NAME=$(conda_env) test
    displayName: 'Run tests'

-  - bash: make CONDA_ENV_NAME=$(conda_env) dist dist-test
+  - bash: make CONDA_ENV_NAME=$(conda_env) CONDA_INFO_LEVEL=-v dist dist-test
    displayName: 'Generate and test binary distribution files'

  - bash: make CONDA_ENV_NAME=$(conda_env) doc
--- a/.azure-pipelines/windows.yml
+++ b/.azure-pipelines/windows.yml
@ -11,20 +11,10 @@ jobs:
  strategy:
    matrix:
      DefaultPython:
-        # Currently 3.9
+        # Floating most recent version of python (useful to catch new compatibility issues in nightly builds).
        conda_env: 'mlos_core'
        PythonVersion: ''
-  # Python 3.6 is currently incompatible.
-  #   Python36:
-  #     conda_env: 'mlos_core-3.6'
-  #     PythonVersion: '3.6'
  # Skipping testing of additional versions on Windows for now.
-  #    Python38:
-  #      conda_env: 'mlos_core-3.8'
-  #     PythonVersion: '3.8'
-  #    Python39:
-  #      conda_env: 'mlos_core-3.9'
-  #     PythonVersion: '3.9'

  variables:
    - name: CONDA_CACHE_DIR
@ -127,14 +117,14 @@ jobs:
      $mlos_bench_whl = (Resolve-Path mlos_bench/dist/mlos_bench-*-py3-none-any.whl | Select-Object -ExpandProperty Path)
      echo $mlos_bench_whl
      # Setup a clean environment to test installing/using them.
-      conda create -y -v -n mlos-dist-test-$(PythonVersion) python=$(PythonVersion)
-      conda install -y -v -n mlos-dist-test-$(PythonVersion) pytest
+      $PythonVersReq = (conda list -n $(conda_env) | Select-String -AllMatches -Pattern '^python\s+([0-9.]+)\s+').Matches.Groups[1].Value
+      conda create -y -v -n mlos-dist-test-$(PythonVersion) python=${PythonVersReq}
      conda install -y -v -n mlos-dist-test-$(PythonVersion) conda-forge::gpy
-      conda run -n mlos-dist-test-$(PythonVersion) pip install -r test-requirements.txt
+      conda run -n mlos-dist-test-$(PythonVersion) pip install pytest
      # Install mlos_core wheel.
-      conda run -n mlos-dist-test-$(PythonVersion) pip install $mlos_core_whl
+      conda run -n mlos-dist-test-$(PythonVersion) pip install "${mlos_core_whl}[full]"
      # Install mlos_bench wheel.
-      conda run -n mlos-dist-test-$(PythonVersion) pip install $mlos_bench_whl
+      conda run -n mlos-dist-test-$(PythonVersion) pip install "${mlos_bench_whl}[full]"
      # Just pick one simple test to run for now.
      # The rest should have been handled in a separate step.
      conda run -n mlos-dist-test-$(PythonVersion) python -m pytest mlos_core/mlos_core/spaces/tests/spaces_test.py
--- a/.gitignore
+++ b/.gitignore
@ -150,6 +150,7 @@ junit/test-results.xml
 .dist-test-env.*.build-stamp
 .dist-test.*.build-stamp
 .doc-prereqs.build-stamp
+.doc-prereqs.*.build-stamp
 .pycodestyle.build-stamp
 .pylint.build-stamp
 .pytest.build-stamp
--- a/51
+++ b/51
@ -1,4 +1,4 @@
-CONDA_ENV_NAME := mlos_core
+CONDA_ENV_NAME ?= mlos_core
 PYTHON_VERSION := $(shell echo "${CONDA_ENV_NAME}" | sed -r -e 's/^mlos_core[-]?//')
 ENV_YML := conda-envs/${CONDA_ENV_NAME}.yml

@ -7,12 +7,24 @@ PYTHON_FILES := $(shell find ./ -type f -name '*.py' 2>/dev/null | egrep -v -e '
 MLOS_CORE_PYTHON_FILES := $(shell find ./mlos_core/ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./mlos_core/build/')
 MLOS_BENCH_PYTHON_FILES := $(shell find ./mlos_bench/ -type f -name '*.py' 2>/dev/null | egrep -v -e '^./mlos_bench/build/')

-# If available, use the mamba solver to speed things up.
-CONDA_SOLVER := $(shell conda list -n base | grep -q '^conda-libmamba-solver\s' && echo libmamba || echo classic)
-export CONDA_EXPERIMENTAL_SOLVER := ${CONDA_SOLVER}
-export EXPERIMENTAL_SOLVER := ${CONDA_SOLVER}
+# If available, and not already set to something else, use the mamba solver to speed things up.
+CONDA_SOLVER ?= $(shell conda list -n base | grep -q '^conda-libmamba-solver\s' && echo libmamba || echo classic)
+# To handle multiple versions of conda, make sure we only have one of the environment variables set.
+ifneq ($(shell conda config --show | grep '^experimental_solver:'),)
+    export CONDA_EXPERIMENTAL_SOLVER := ${CONDA_SOLVER}
+    export EXPERIMENTAL_SOLVER := ${CONDA_SOLVER}
+    undefine CONDA_SOLVER
+    unexport CONDA_SOLVER
+else
+    export CONDA_SOLVER := ${CONDA_SOLVER}
+    undefine CONDA_EXPERIMENTAL_SOLVER
+    unexport CONDA_EXPERIMENTAL_SOLVER
+    undefine EXPERIMENTAL_SOLVER
+    unexport EXPERIMENTAL_SOLVER
+endif
+
 # Allow overriding the default verbosity of conda for CI jobs.
-CONDA_INFO_LEVEL := -q
+CONDA_INFO_LEVEL ?= -q

 .PHONY: all
 all: check test dist # doc
@ -92,21 +104,27 @@ dist-test-env-clean:
 .PHONY: dist-test-env
 dist-test-env: dist .dist-test-env.$(PYTHON_VERSION).build-stamp

+.dist-test-env.$(PYTHON_VERSION).build-stamp: .conda-env.${CONDA_ENV_NAME}.build-stamp
+# Use the same version of python as the one we used to build the wheels.
+.dist-test-env.$(PYTHON_VERSION).build-stamp: PYTHON_VERS_REQ=$(shell conda list -n ${CONDA_ENV_NAME} | egrep '^python\s+' | sed -r -e 's/^python\s+//' | cut -d' ' -f1)
 .dist-test-env.$(PYTHON_VERSION).build-stamp: mlos_core/dist/mlos_core-*-py3-none-any.whl mlos_bench/dist/mlos_bench-*-py3-none-any.whl
 	# Check to make sure we only have a single wheel version availble.
 	# Else, run `make dist-clean` to remove prior versions.
 	ls mlos_core/dist/mlos_core-*-py3-none-any.whl | wc -l | grep -q -x 1
 	ls mlos_bench/dist/mlos_bench-*-py3-none-any.whl | wc -l | grep -q -x 1
+	# Symlink them to make the install step easier.
+	rm -rf mlos_core/dist/tmp && mkdir -p mlos_core/dist/tmp
+	cd mlos_core/dist/tmp && ln -s ../mlos_core-*-py3-none-any.whl mlos_core-latest-py3-none-any.whl
+	rm -rf mlos_bench/dist/tmp && mkdir -p mlos_bench/dist/tmp
+	cd mlos_bench/dist/tmp && ln -s ../mlos_bench-*-py3-none-any.whl mlos_bench-latest-py3-none-any.whl
 	# Create a clean test environment for checking the wheel files.
 	$(MAKE) dist-test-env-clean
-	conda create -y ${CONDA_INFO_LEVEL} -n mlos-dist-test-$(PYTHON_VERSION) python=$(PYTHON_VERSION)
-	conda install -y ${CONDA_INFO_LEVEL} -n mlos-dist-test-$(PYTHON_VERSION) pytest
+	conda create -y ${CONDA_INFO_LEVEL} -n mlos-dist-test-$(PYTHON_VERSION) python=$(PYTHON_VERS_REQ)
+	conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install pytest
 	# Test a clean install of the mlos_core wheel.
-	conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install mlos_core/dist/mlos_core-*-py3-none-any.whl
-	# Install the necessary extra requirements for the tests.
-	conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install -r test-requirements.txt
+	conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install "mlos_core/dist/tmp/mlos_core-latest-py3-none-any.whl[full]"
 	# Test a clean install of the mlos_bench wheel.
-	conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install mlos_bench/dist/mlos_bench-*-py3-none-any.whl
+	conda run -n mlos-dist-test-$(PYTHON_VERSION) pip install "mlos_bench/dist/tmp/mlos_bench-latest-py3-none-any.whl[full]"
 	touch .dist-test-env.$(PYTHON_VERSION).build-stamp

 .PHONY: dist-test
@ -130,16 +148,17 @@ dist-test: dist-test-env .dist-test.$(PYTHON_VERSION).build-stamp
 dist-test-clean: dist-test-env-clean
 	rm -f .dist-test.$(PYTHON_VERSION).build-stamp

-.doc-prereqs.build-stamp: doc/requirements.txt
-	conda run -n ${CONDA_ENV_NAME} pip install -r doc/requirements.txt
-	touch .doc-prereqs.build-stamp
+.doc-prereqs.${CONDA_ENV_NAME}.build-stamp: doc/requirements.txt
+	conda run -n ${CONDA_ENV_NAME} pip install -U -r doc/requirements.txt
+	touch .doc-prereqs.${CONDA_ENV_NAME}.build-stamp

 .PHONY: doc-prereqs
-doc-prereqs: .doc-prereqs.build-stamp
+doc-prereqs: .doc-prereqs.${CONDA_ENV_NAME}.build-stamp

 .PHONY: clean-doc-env
 clean-doc-env:
 	rm -f .doc-prereqs.build-stamp
+	rm -f .doc-prereqs.${CONDA_ENV_NAME}.build-stamp

 .PHONY: doc
 doc: conda-env doc-prereqs clean-doc
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -6,6 +6,13 @@ pr:
    include:
    - '*'

+schedules:
+- cron: "0 0 * * *"
+  displayName: Nightly build
+  branches:
+    include:
+    - main
+
 jobs:
  - template: .azure-pipelines/linux.yml
    parameters:
--- a/conda-envs/mlos_core-3.10.yml
+++ b/conda-envs/mlos_core-3.10.yml
@ -0,0 +1,31 @@
+name: mlos_core-3.10
+channels:
+  - defaults
+#  - conda-forge
+dependencies:
+  - scipy
+  - numpy
+  - pandas
+  - pip
+  - pylint
+  - pycodestyle
+  - autopep8
+  - pytest
+  - setuptools
+  - json5
+  - jupyterlab
+  - jupyter
+  - ipykernel
+  - nb_conda_kernels
+  - matplotlib
+  - seaborn
+  - python=3.10
+  # See comments in mlos_core.yml.
+  - conda-forge::gpy
+  - pip:
+    - "--editable ../mlos_core[full]"
+    - "--editable ../mlos_bench[full]"
+    - pytest-cov
+    - pytest-forked
+    - pytest-xdist
+    - pytest-timeout
--- a/conda-envs/mlos_core-3.11.yml
+++ b/conda-envs/mlos_core-3.11.yml
@ -0,0 +1,32 @@
+name: mlos_core-3.11
+channels:
+  - defaults
+#  - conda-forge
+dependencies:
+  - scipy
+  - numpy
+  - pandas
+  - pip
+  - pylint
+  - pycodestyle
+  - autopep8
+  - pytest
+  - setuptools
+  - json5
+  - jupyterlab
+  - jupyter
+  - ipykernel
+  - nb_conda_kernels
+  - matplotlib
+  - seaborn
+  - python=3.11
+  # See comments in mlos_core.yml.
+  # FIXME: Doesn't exist for 3.11 yet
+  #- conda-forge::gpy
+  - pip:
+    - "--editable ../mlos_core[full]"
+    - "--editable ../mlos_bench[full]"
+    - pytest-cov
+    - pytest-forked
+    - pytest-xdist
+    - pytest-timeout
--- a/conda-envs/mlos_core-3.8.yml
+++ b/conda-envs/mlos_core-3.8.yml
@ -3,11 +3,9 @@ channels:
  - defaults
 #  - conda-forge
 dependencies:
-  - scikit-learn
  - scipy
  - numpy
  - pandas
-  - configspace
  - pip
  - pylint
  - pycodestyle
@ -22,16 +20,11 @@ dependencies:
  - matplotlib
  - seaborn
  - python=3.8
-  #- conda-forge::c-compiler
-  #- conda-forge::cxx-compiler
-  # Install gpy via conda (not pip) for emukit so we can side step some compiler issues on Windows platforms.
+  # See comments in mlos_core.yml.
  - conda-forge::gpy
  - pip:
-    - scikit-optimize
-    - emukit
-    - azure-storage-file-share
-    - "--editable ../mlos_core"
-    - "--editable ../mlos_bench"
+    - "--editable ../mlos_core[full]"
+    - "--editable ../mlos_bench[full]"
    - pytest-cov
    - pytest-forked
    - pytest-xdist
--- a/conda-envs/mlos_core-3.9.yml
+++ b/conda-envs/mlos_core-3.9.yml
@ -3,11 +3,9 @@ channels:
  - defaults
 #  - conda-forge
 dependencies:
-  - scikit-learn
  - scipy
  - numpy
  - pandas
-  - configspace
  - pip
  - pylint
  - pycodestyle
@ -22,16 +20,11 @@ dependencies:
  - matplotlib
  - seaborn
  - python=3.9
-  #- conda-forge::c-compiler
-  #- conda-forge::cxx-compiler
-  # Install gpy via conda (not pip) for emukit so we can side step some compiler issues on Windows platforms.
+  # See comments in mlos_core.yml.
  - conda-forge::gpy
  - pip:
-    - scikit-optimize
-    - emukit
-    - azure-storage-file-share
-    - "--editable ../mlos_core"
-    - "--editable ../mlos_bench"
+    - "--editable ../mlos_core[full]"
+    - "--editable ../mlos_bench[full]"
    - pytest-cov
    - pytest-forked
    - pytest-xdist
--- a/conda-envs/mlos_core.yml
+++ b/conda-envs/mlos_core.yml
@ -3,11 +3,9 @@ channels:
  - defaults
 #  - conda-forge
 dependencies:
-  - scikit-learn
  - scipy
  - numpy
  - pandas
-  - configspace
  - pip
  - pylint
  - pycodestyle
@ -21,17 +19,19 @@ dependencies:
  - nb_conda_kernels
  - matplotlib
  - seaborn
-  - python
+  # FIXME: for now, we are disabling support for python 3.11 due to various dependency issues.
+  - python<3.11
+  # Virtual packages for compiler dependencies.
+  # Note: this doesn't actually install the compilers on Windows.
+  # It just adds the vs2019_win-64 and vswhere packages to help find the
+  # compilers which still (seem) to need to be installed manually.
  #- conda-forge::c-compiler
  #- conda-forge::cxx-compiler
-  # Install gpy via conda (not pip) for emukit so we can side step some compiler issues on Windows platforms.
+  # Instead, install gpy via conda (not pip) for emukit so we can side step some compiler issues on Windows platforms.
  - conda-forge::gpy
  - pip:
-    - scikit-optimize
-    - emukit
-    - azure-storage-file-share
-    - "--editable ../mlos_core"
-    - "--editable ../mlos_bench"
+    - "--editable ../mlos_core[full]"
+    - "--editable ../mlos_bench[full]"
    - pytest-cov
    - pytest-forked
    - pytest-xdist
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@ -1,6 +1,7 @@
 sphinx
 nbsphinx
 jupyter_core>=4.11.2    # nbsphix dependency - addresses CVE-2022-39286
+nbconvert==7.2.1        # Work around an issue with mistune attributes with newer versions
 mistune>=2.0.3          # Address CVE-2022-34749
 numpydoc
 sphinx-rtd-theme
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@ -77,7 +77,8 @@ Environments
   :template: class.rst

   Environment
-   AppEnv
+   LocalEnv
+   RemoteEnv
   CompositeEnv
   Service
   Status
--- a/mlos_bench/setup.py
+++ b/mlos_bench/setup.py
@ -2,10 +2,23 @@
 Setup instructions for the mlos_bench package.
 """

+from itertools import chain
 from setuptools import setup, find_packages

 _VERSION = '0.0.4'

+extra_requires = {
+    # Additional tools for extra functionality.
+    'azure': 'azure-storage-file-share',
+    # Transitive extra_requires from mlos-core.
+    'emukit': 'emukit',
+    'skopt': 'scikit-optimize',
+}
+
+# construct special 'full' extra that adds requirements for all built-in
+# backend integrations and additional extra features.
+extra_requires['full'] = list(set(chain(extra_requires.values())))
+
 # pylint: disable=duplicate-code
 setup(
    name='mlos-bench',
@ -16,15 +29,10 @@ setup(
        'requests',
        'json5',
    ],
-    # Transitive extra_requires from mlos-core.
-    extras_require={
-        'emukit': 'emukit',
-        'skopt': 'scikit-optimize',
-        'azure': 'azure-storage-file-share',
-    },
+    extras_require=extra_requires,
    author='Microsoft',
    author_email='mlos-maintainers@service.microsoft.com',
-    description=('MLOS Core Python interface for parameter optimization.'),
+    description=('MLOS Bench Python interface for benchmark automation and optimization.'),
    license='MIT',
    keywords='',
    python_requires='>=3.8',
--- a/mlos_core/setup.py
+++ b/mlos_core/setup.py
@ -2,31 +2,38 @@
 Setup instructions for the mlos_core package.
 """

+from itertools import chain
 from setuptools import setup, find_packages

 _VERSION = '0.0.4'

+extra_requires = {
+    'emukit': 'emukit',
+    'skopt': 'scikit-optimize<=0.9.0',  # temporarily work around some version mismatch issues (PR 850)
+}
+
+# construct special 'full' extra that adds requirements for all built-in
+# backend integrations and additional extra features.
+extra_requires['full'] = list(set(chain(extra_requires.values())))
+
 # pylint: disable=duplicate-code
 setup(
    name='mlos-core',
    version=_VERSION,
    packages=find_packages(),
    install_requires=[
-        'scikit-learn>=1.1.3',
+        'scikit-learn<1.2', # temporarily work around some version mismatch issues (PR 850)
        'joblib>=1.1.1',    # CVE-2022-21797: scikit-learn dependency, addressed in 1.2.0dev0, which isn't currently released
        'scipy>=1.3.2',
        'numpy>=1.18.1',
        'pandas>=1.0.3',
-        'ConfigSpace>=0.4.21',
+        'ConfigSpace>=0.6.1',
    ],
-    extras_require={
-        'emukit': 'emukit',
-        'skopt': 'scikit-optimize',
-    },
+    extras_require=extra_requires,
    author='Microsoft',
    author_email='mlos-maintainers@service.microsoft.com',
    description=('MLOS Core Python interface for parameter optimization.'),
    license='MIT',
    keywords='',
-    # python_requires='>=3.7',
+    python_requires='>=3.8',
 )
--- a/test-requirements.txt
+++ b/test-requirements.txt
@ -1,4 +0,0 @@
-# Extra requirements that are needed for running tests
-emukit
-scikit-optimize
-azure-storage-file-share