Added ai-ml-automl environment (#3292)

* Added ai-ml-automl registry environment to replace the AzureML-AutoML environment

assets/training/automl/environments/ai-ml-automl/README.md

@@ -0,0 +1,5 @@
+# ai-ml-automl Docker Environment
+
+## Overview
+This environment is used by Azure ML AutoML for training models.
+It is not intended for use in other scenarios and is subject to change without notice.

assets/training/automl/environments/ai-ml-automl/asset.yaml

@@ -0,0 +1,11 @@
+name: ai-ml-automl
+version: auto
+type: environment
+spec: spec.yaml
+extra_config: environment.yaml
+test:
+  pytest:
+    enabled: true 
+    pip_requirements: tests/requirements.txt
+    tests_dir: tests
+categories: ["AutoML", "Training"]

assets/training/automl/environments/ai-ml-automl/context/Dockerfile

@@ -0,0 +1,62 @@
+FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:{{latest-image-tag}}
+
+
+ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/azureml-automl
+ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH
+
+COPY --from=mcr.microsoft.com/azureml/mlflow-ubuntu20.04-py38-cpu-inference:20230306.v3 /var/mlflow_resources/mlflow_score_script.py /var/mlflow_resources/mlflow_score_script.py
+
+ENV MLFLOW_MODEL_FOLDER="mlflow-model"
+# ENV AML_APP_ROOT="/var/mlflow_resources"
+# ENV AZUREML_ENTRY_SCRIPT="mlflow_score_script.py"
+
+ENV ENABLE_METADATA=true
+
+# begin conda create
+# Create conda environment
+RUN conda create -p $AZUREML_CONDA_ENVIRONMENT_PATH \
+    python=3.9 \
+    # begin conda dependencies
+    pip \
+    py-cpuinfo=5.0.0 \
+    joblib=1.2.0 \
+    setuptools-git \
+    'psutil>5.0.0,<6.0.0' \
+    pytorch=2.2.2 \
+    # end conda dependencies
+    -c conda-forge -c pytorch -c anaconda -c nvidia && \
+    conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH && \
+    conda clean -a -y
+# end conda create
+
+# begin pip install
+# Install pip dependencies
+RUN pip install \
+                # begin pypi dependencies
+                azureml-core=={{latest-pypi-version}} \
+                azureml-mlflow=={{latest-pypi-version}} \
+                azureml-pipeline-core=={{latest-pypi-version}} \
+                azureml-telemetry=={{latest-pypi-version}} \
+                azureml-interpret=={{latest-pypi-version}} \
+                azureml-responsibleai=={{latest-pypi-version}} \
+                azureml-automl-core=={{latest-pypi-version}} \
+                azureml-automl-runtime=={{latest-pypi-version}} \
+                azureml-train-automl-client=={{latest-pypi-version}} \
+                azureml-train-automl-runtime=={{latest-pypi-version}} \
+                azureml-train-automl=={{latest-pypi-version}} \
+                azureml-dataset-runtime=={{latest-pypi-version}} \
+                azureml-defaults=={{latest-pypi-version}} \
+                # TODO: replace the hard coded above by a referenceto azureml-train-automl[tabular]
+                'mlflow-skinny==2.9.2' \
+                'xgboost==1.5.2' \
+                'prophet==1.1.4' \
+                'inference-schema' \
+                'mltable>=1.0.0'
+                # end pypi dependencies
+# end pip install
+
+# begin pip ad-hoc
+# Install pip ad-hoc dependencies for security updates
+RUN pip install --upgrade 'pyarrow==14.0.2'
+# end pip ad-hoc
+

assets/training/automl/environments/ai-ml-automl/environment.yaml

@@ -0,0 +1,11 @@
+image:
+  name: azureml/curated/ai-ml-automl
+  os: linux
+  context:
+    dir: context
+    dockerfile: Dockerfile
+    template_files:
+    - Dockerfile
+  publish:
+    location: mcr
+    visibility: public

assets/training/automl/environments/ai-ml-automl/spec.yaml

@@ -0,0 +1,20 @@
+$schema: https://azuremlschemas.azureedge.net/latest/environment.schema.json
+
+description: >-
+  An environment used by Azure ML AutoML for training models.
+
+name: "{{asset.name}}"
+version: "{{asset.version}}"
+
+build:
+  path: "{{image.context.path}}"
+  dockerfile_path: "{{image.dockerfile.path}}"
+
+os_type: linux
+
+tags:
+  OS: Ubuntu20.04
+  Training: ""
+  Preview: ""
+  OpenMpi: "4.1.0"
+  Python: "3.9"

assets/training/automl/environments/ai-ml-automl/tests/automl_sample_test.py

@@ -0,0 +1,87 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""Smoke tests running a job in the ai-ml-automl environment."""
+# This only tests that scikit-learn training can be dome in the environment.
+# After the environment is deployed, notebooks tests should be run before updating
+# the "Prod" label in the Jasmine service
+import os
+import time
+from pathlib import Path
+from azure.ai.ml import command, Input, MLClient
+from azure.ai.ml._restclient.models import JobStatus
+from azure.ai.ml.entities import Environment, BuildContext
+from azure.identity import AzureCliCredential
+
+BUILD_CONTEXT = Path("../context")
+JOB_SOURCE_CODE = "src"
+TIMEOUT_MINUTES = os.environ.get("timeout_minutes", 45)
+STD_LOG = Path("artifacts/user_logs/std_log.txt")
+
+
+def test_azure_ai_ml_automl():
+    """Tests a sample job using ai-ml-automl as the environment."""
+    this_dir = Path(__file__).parent
+
+    subscription_id = os.environ.get("subscription_id")
+    resource_group = os.environ.get("resource_group")
+    workspace_name = os.environ.get("workspace")
+
+    ml_client = MLClient(
+        AzureCliCredential(), subscription_id, resource_group, workspace_name
+    )
+
+    env_name = "ai-ml-automl"
+
+    env_docker_context = Environment(
+        build=BuildContext(path=this_dir / BUILD_CONTEXT),
+        name="ai-ml-automl",
+        description="ai-ml-automl environment created from a Docker context.",
+    )
+    ml_client.environments.create_or_update(env_docker_context)
+
+    # create the command
+    job = command(
+        code=this_dir / JOB_SOURCE_CODE,  # local path where the code is stored
+        command="python main.py --diabetes-csv ${{inputs.diabetes}}",
+        inputs={
+            "diabetes": Input(
+                type="uri_file",
+                path="https://azuremlexamples.blob.core.windows.net/datasets/diabetes.csv",
+            )
+        },
+        environment=f"{env_name}@latest",
+        compute=os.environ.get("cpu_cluster"),
+        display_name="sklearn-diabetes-example",
+        description="A test run of the ai-ml-automl curated environment",
+        experiment_name="sklearnExperiment"
+    )
+
+    returned_job = ml_client.create_or_update(job)
+    assert returned_job is not None
+
+    # Poll until final status is reached or timed out
+    timeout = time.time() + (TIMEOUT_MINUTES * 60)
+    while time.time() <= timeout:
+        job = ml_client.jobs.get(returned_job.name)
+        status = job.status
+        if status in [JobStatus.COMPLETED, JobStatus.FAILED]:
+            break
+        time.sleep(30)  # sleep 30 seconds
+    else:
+        # Timeout
+        ml_client.jobs.cancel(returned_job.name)
+        raise Exception(f"Test aborted because the job took longer than {TIMEOUT_MINUTES} minutes. "
+                        f"Last status was {status}.")
+
+    if status == JobStatus.FAILED:
+        ml_client.jobs.download(returned_job.name)
+        if STD_LOG.exists():
+            print(f"*** BEGIN {STD_LOG} ***")
+            with open(STD_LOG, "r") as f:
+                print(f.read(), end="")
+            print(f"*** END {STD_LOG} ***")
+        else:
+            ml_client.jobs.stream(returned_job.name)
+
+    assert status == JobStatus.COMPLETED

assets/training/automl/environments/ai-ml-automl/tests/requirements.txt

@@ -0,0 +1,2 @@
+azure-ai-ml==1.2.0
+azure.identity==1.10.0

assets/training/automl/environments/ai-ml-automl/tests/src/main.py

@@ -0,0 +1,87 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+"""Simple Sklearn Test."""
+# imports
+import mlflow
+import argparse
+import pandas as pd
+
+from sklearn.linear_model import LinearRegression
+from sklearn.model_selection import train_test_split
+
+
+# define functions
+def main(args):
+    """Run and evaluate model."""
+    # enable auto logging
+    mlflow.autolog()
+    # setup parameters
+    params = {
+        "fit_intercept": args.fit_intercept,
+        "positive": args.positive,
+    }
+    # read in data
+    df = pd.read_csv(args.diabetes_csv)
+
+    # process data
+    X_train, X_test, y_train, y_test = process_data(df, args.random_state)
+
+    # train model
+    model = train_model(params, X_train, X_test, y_train, y_test)
+
+    print(model)
+
+
+def process_data(df, random_state):
+    """Process data."""
+    # split dataframe into X and y
+    X = df.drop(["target"], axis=1)
+    y = df["target"]
+
+    # train/test split
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=0.2, random_state=random_state
+    )
+
+    # return splits and encoder
+    return X_train, X_test, y_train, y_test
+
+
+def train_model(params, X_train, X_test, y_train, y_test):
+    """Train the model."""
+    # train model
+    model = LinearRegression(**params)
+    model = model.fit(X_train, y_train)
+
+    # return model
+    return model
+
+
+def parse_args():
+    """Parse arguments."""
+    # setup arg parser
+    parser = argparse.ArgumentParser()
+
+    # add arguments
+    parser.add_argument("--diabetes-csv", type=str)
+    parser.add_argument("--random_state", type=int, default=42)
+    parser.add_argument("--fit_intercept", type=bool, default=True)
+    parser.add_argument("--positive", type=bool, default=False)
+    parser.add_argument("--intel-extension", type=bool, default=False)
+
+    # parse args
+    args = parser.parse_args()
+
+    # return args
+    return args
+
+
+# run script
+if __name__ == "__main__":
+    # parse args
+    args = parse_args()
+    if (args.intel_extension):
+        from sklearnex import patch_sklearn
+        patch_sklearn()
+    # run main function
+    main(args)