Make release dir optional for update_assets (#821)

* Make release dir optional for update_assets
* Add a test that doesn't require a release directory

.github/workflows/assets-docs.yaml

@@ -57,7 +57,6 @@ jobs:
           token: ${{ secrets.AZUREML_BOT_PAT }}
           path: ${{ env.wiki_dir }}
 
-      # TODO: Not needed unless calling update_assets.py
       - name: Use Python 3.8 or newer
         uses: actions/setup-python@v4
         with:

.github/workflows/assets-release.yaml

@@ -222,7 +222,6 @@ jobs:
           token: ${{ secrets.AZUREML_BOT_PAT }}
           path: ${{ env.release_dir }}
 
-      # TODO: Not needed unless calling update_assets.py
       - name: Use Python 3.8 or newer
         uses: actions/setup-python@v4
         with:
@@ -231,10 +230,9 @@ jobs:
       - name: Install dependencies
         run: pip install -e $main_dir/$scripts_azureml_assets_dir
       
-      # TODO: This may be overkill, consider copying directories without involving this script
       - name: Update release branch
         if: steps.download-artifact.outputs.download-path
-        run: python -u $main_dir/$scripts_assets_dir/update_assets.py -i ${{ runner.temp }}/$releasable_assets_artifact -r $release_dir -c
+        run: python -u $main_dir/$scripts_assets_dir/copy_assets.py -i ${{ runner.temp }}/$releasable_assets_artifact -o $release_dir/latest
   
       - name: Delete deprecated assets
         run: python -u $main_dir/$scripts_assets_dir/asset_utils.py delete -i $release_dir -r ${{ runner.temp }}/$asset_list_artifact/$asset_list_file

scripts/azureml-assets/CHANGELOG.md

@@ -1,8 +1,12 @@
-## 1.6.0 (Unreleased)
+## 1.7.0 (Unreleased)
 ### 🚀 New Features
 
 ### 🐛 Bugs Fixed
 
+## 1.6.0 (2023-07-17)
+### 🚀 New Features
+- [#821](https://github.com/Azure/azureml-assets/pull/821) Make release directory optional for update_assets.py
+
 ## 1.5.2 (2023-07-13)
 ### 🐛 Bugs Fixed
 - [#882](https://github.com/Azure/azureml-assets/pull/882) Create package list using pip if conda is unavailable

scripts/azureml-assets/azureml/assets/update_assets.py

@@ -89,92 +89,91 @@ def get_latest_release_tag_version(asset_config: assets.AssetConfig, release_dir
 
 
 def update_asset(asset_config: assets.AssetConfig,
-                 release_directory_root: Path,
-                 copy_only: bool,
-                 skip_unreleased: bool,
-                 output_directory_root: Path = None) -> str:
+                 output_directory_root: Path,
+                 release_directory_root: Path = None,
+                 skip_unreleased: bool = False,
+                 use_version_dir: bool = False) -> str:
     """Update asset to prepare for release.
 
     Args:
         asset_config (assets.AssetConfig): Asset config
-        release_directory_root (Path): Release branch location
-        copy_only (bool): Only copy assets, don't actually update
-        skip_unreleased (bool): Skip unreleased explicitly-versioned assets
-        output_directory_root (Path, optional): Output directory for updated assets, otherwise update them in-place
+        output_directory_root (Path): Output directory for updated assets
+        release_directory_root (Path, optional): Release branch location
+        skip_unreleased (bool, optional): Skip unreleased explicitly-versioned assets. Defaults to False.
+        use_version_dir (bool, optional): Use version directory for output. Defaults to False.
 
     Returns:
         str: Version of updated asset, or None if not updated
     """
-    # Determine asset's release directory
-    release_dir = util.get_asset_release_dir(asset_config, release_directory_root)
+    # The release directory is required if auto-versioning
+    if asset_config.auto_version and release_directory_root is None:
+        logger.log_error(f"Asset {asset_config.name} is auto-versioned but can't be updated because no release "
+                         "directory was specified to compare against")
+        exit(1)
 
-    # Define output directory, which may be different from the release directory
-    if output_directory_root:
-        output_directory = util.get_asset_output_dir(asset_config, output_directory_root)
-    else:
-        output_directory = release_dir
-
-    # Simpler operation that just copies the directory
-    if copy_only:
-        util.copy_asset_to_output_dir(asset_config=asset_config, output_directory=output_directory)
-        return asset_config.spec_as_object().version
-
-    # Get version from main branch, set a few defaults
-    main_version = asset_config.version
-    auto_version = asset_config.auto_version
-    release_version = None
-    check_contents = False
-    pending_release = False
-
-    # Check existing release dir
-    if release_dir.exists():
-        release_asset_configs = util.find_assets(input_dirs=release_dir,
-                                                 asset_config_filename=asset_config.file_name)
-        if not release_asset_configs:
-            logger.log_error(f"Release directory {release_dir} exists, but it's missing an asset config file")
-            exit(1)
-        release_asset_config = release_asset_configs[0]
-        release_version = release_asset_config.version
-        check_contents = True
-
-        # See if the asset version is unreleased
-        pending_release = not release_tag_exists(release_asset_config, release_directory_root)
-        if pending_release and not auto_version and main_version != release_version and skip_unreleased:
-            # Skip the unreleased asset version
-            logger.log_warning(f"Skipping {release_asset_config.type.value} {release_asset_config.name} because "
-                               f"version {release_version} hasn't been released yet")
-            return None
-
-    # Determine new version
-    if not auto_version:
-        # Use explicit version
-        new_version = main_version
-    elif pending_release:
-        # Reuse existing auto version
-        new_version = release_version
-    else:
-        # Increment auto version
-        new_version = int(release_version) + 1 if release_version else 1
+    # Identify output directory
+    output_is_release = release_directory_root is not None and output_directory_root.samefile(release_directory_root)
+    if output_is_release:
+        # Prevent release directory corruption
+        use_version_dir = False
+    output_directory = util.get_asset_output_dir(asset_config, output_directory_root, use_version_dir)
 
+    # To keep things simple, we'll create a temporary directory for each update
     with tempfile.TemporaryDirectory() as temp_dir:
         temp_dir_path = Path(temp_dir)
+
         # Copy asset to temp directory and pin image/package versions
-        util.copy_asset_to_output_dir(asset_config=asset_config, output_directory=temp_dir_path, add_subdir=True)
+        temp_asset_dir = util.copy_asset_to_output_dir(asset_config=asset_config, output_directory=temp_dir_path,
+                                                       add_subdir=True)
         temp_asset_config = util.find_assets(input_dirs=temp_dir_path, asset_config_filename=asset_config.file_name)[0]
         if temp_asset_config.type == assets.AssetType.ENVIRONMENT:
             temp_env_config = temp_asset_config.extra_config_as_object()
-            if temp_env_config:
-                pin_env_files(temp_env_config)
+            pin_env_files(temp_env_config)
 
-        temp_asset_dir = util.get_asset_output_dir(asset_config=asset_config, output_directory_root=temp_dir_path)
+        # Get version info, set a few defaults
+        main_version = asset_config.version
+        auto_version = asset_config.auto_version
+        release_version = None
+        pending_release = False
 
-        # Compare temporary version with one in release
-        if check_contents:
-            assets.update_spec(temp_asset_config, version=release_version)
-            dirs_equal = util.are_dir_trees_equal(temp_asset_dir, release_dir)
-            if dirs_equal:
+        # Look in release directory for existing asset
+        if release_directory_root is not None:
+            release_dir = util.get_asset_release_dir(asset_config, release_directory_root)
+            if release_dir.exists():
+                # Check existing release dir
+                release_asset_configs = util.find_assets(input_dirs=release_dir,
+                                                         asset_config_filename=asset_config.file_name)
+                if not release_asset_configs:
+                    logger.log_error(f"Release directory {release_dir} exists, but it's missing an asset config file")
+                    exit(1)
+                release_asset_config = release_asset_configs[0]
+                release_version = release_asset_config.version
+
+                # Compare temporary version with one in release
+                assets.update_spec(temp_asset_config, version=release_version)
+                dirs_equal = util.are_dir_trees_equal(temp_asset_dir, release_dir)
+                if dirs_equal and output_is_release:
+                    return None
+
+            # See if the asset version is unreleased
+            pending_release = not release_tag_exists(release_asset_config, release_directory_root)
+            if pending_release and not auto_version and main_version != release_version and skip_unreleased:
+                # Skip the unreleased asset version
+                logger.log_warning(f"Skipping {release_asset_config.type.value} {release_asset_config.name} because "
+                                   f"version {release_version} hasn't been released yet")
                 return None
 
+        # Determine new version
+        if not auto_version:
+            # Use explicit version
+            new_version = main_version
+        elif pending_release:
+            # Reuse existing auto version
+            new_version = release_version
+        else:
+            # Increment auto version
+            new_version = int(release_version) + 1 if release_version else 1
+
         # Copy and replace any existing directory
         util.copy_replace_dir(source=temp_asset_dir, dest=output_directory)
 
@@ -189,19 +188,19 @@ def update_asset(asset_config: assets.AssetConfig,
 
 def update_assets(input_dirs: List[Path],
                   asset_config_filename: str,
-                  release_directory_root: Path,
-                  copy_only: bool,
-                  skip_unreleased: bool,
-                  output_directory_root: Path = None):
+                  output_directory_root: Path,
+                  release_directory_root: Path = None,
+                  skip_unreleased: bool = False,
+                  use_version_dirs: bool = False):
     """Update assets to prepare for release.
 
     Args:
         input_dirs (List[Path]): List of directories to search for assets.
         asset_config_filename (str): Asset config filename to search for
-        release_directory_root (Path): Release directory location
-        copy_only (bool): Only copy assets, don't actually update
-        skip_unreleased (bool): Skip unreleased explicitly-versioned assets
-        output_directory_root (Path, optional): Output directory for updated assets, otherwise update them in-place
+        output_directory_root (Path): Output directory for updated assets
+        release_directory_root (Path, optional): Release directory location
+        skip_unreleased (bool, optional): Skip unreleased explicitly-versioned assets. Defaults to False.
+        use_version_dirs (bool, optional): Use version directories for output. Defaults to False.
     """
     # Find assets under input dirs
     counters = Counter()
@@ -210,15 +209,15 @@ def update_assets(input_dirs: List[Path],
 
         # Update asset if it's changed
         new_version = update_asset(asset_config=asset_config,
+                                   output_directory_root=output_directory_root,
                                    release_directory_root=release_directory_root,
-                                   copy_only=copy_only,
                                    skip_unreleased=skip_unreleased,
-                                   output_directory_root=output_directory_root)
+                                   use_version_dir=use_version_dirs)
         if new_version:
             logger.print(f"Updated {asset_config.type.value} {asset_config.name} version {new_version}")
             counters[UPDATED_COUNT] += 1
 
-            # Track updated environments by OS
+            # Track updated environments
             if asset_config.type == assets.AssetType.ENVIRONMENT:
                 counters[UPDATED_ENV_COUNT] += 1
         else:
@@ -237,23 +236,27 @@ if __name__ == '__main__':
                         help="Comma-separated list of directories containing assets")
     parser.add_argument("-a", "--asset-config-filename", default=assets.DEFAULT_ASSET_FILENAME,
                         help="Asset config file name to search for")
-    parser.add_argument("-r", "--release-directory", required=True, type=Path,
+    parser.add_argument("-o", "--output-directory", required=True, type=Path,
+                        help="Copy new/updated assets into this directory, can be the same as --release-directory")
+    parser.add_argument("-r", "--release-directory", type=Path,
                         help="Directory to which the release branch has been cloned")
-    parser.add_argument("-o", "--output-directory", type=Path,
-                        help="Directory to which new/updated assets will be written, defaults to release directory")
-    parser.add_argument("-c", "--copy-only", action="store_true",
-                        help="Just copy assets into the release directory")
     parser.add_argument("-s", "--skip-unreleased", action="store_true",
                         help="Skip unreleased explicitly-versioned assets in the release branch")
+    parser.add_argument("-v", "--use-version-dirs", action="store_true",
+                        help="Use version directories when storing assets in output directory")
     args = parser.parse_args()
 
+    # Check interdependencies
+    if args.skip_unreleased and args.release_directory is None:
+        parser.error("--skip-unreleased requires --release-directory")
+
     # Convert comma-separated values to lists
     input_dirs = [Path(d) for d in args.input_dirs.split(",")]
 
     # Update assets
     update_assets(input_dirs=input_dirs,
                   asset_config_filename=args.asset_config_filename,
+                  output_directory_root=args.output_directory,
                   release_directory_root=args.release_directory,
-                  copy_only=args.copy_only,
                   skip_unreleased=args.skip_unreleased,
-                  output_directory_root=args.output_directory)
+                  use_version_dirs=args.use_version_dirs)

scripts/azureml-assets/azureml/assets/util/util.py

@@ -184,7 +184,7 @@ def get_asset_release_dir_from_parts(type: assets.AssetType, name: str, release_
 
 
 def copy_asset_to_output_dir(asset_config: assets.AssetConfig, output_directory: Path, add_subdir: bool = False,
-                             use_version_dir: bool = False):
+                             use_version_dir: bool = False) -> Path:
     """Copy asset directory to output directory.
 
     Args:
@@ -192,6 +192,9 @@ def copy_asset_to_output_dir(asset_config: assets.AssetConfig, output_directory:
         output_directory_root (Path): Output directory root
         add_subdir (bool, optional): Add asset-specific subdirectories to output_directory
         use_version_dir (bool, optional): Store asset in version-specific directory
+
+    Returns:
+        Path: The asset's output directory
     """
     if add_subdir:
         output_directory = get_asset_output_dir(asset_config, output_directory, use_version_dir)
@@ -200,6 +203,7 @@ def copy_asset_to_output_dir(asset_config: assets.AssetConfig, output_directory:
 
     common_dir, relative_release_paths = find_common_directory(asset_config.release_paths)
     copy_replace_dir(source=common_dir, dest=output_directory, paths=relative_release_paths)
+    return output_directory
 
 
 def apply_tag_template(full_image_name: str, template: str = None) -> str:

scripts/azureml-assets/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup, find_packages
 
 setup(
    name="azureml-assets",
-   version="1.5.2",
+   version="1.6.0",
    description="Utilities for publishing assets to Azure Machine Learning system registries.",
    author="Microsoft Corp",
    packages=find_packages(),

test/resources/update/manual-version-no-release-dir/expected/environment/environment-in-subdir/asset.yaml

@@ -0,0 +1,5 @@
+name: environment-in-subdir
+version: 1
+type: environment
+spec: spec.yaml
+extra_config: environment.yaml

test/resources/update/manual-version-no-release-dir/expected/environment/environment-in-subdir/context/Dockerfile

@@ -0,0 +1,16 @@
+FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:{{latest-image-tag}}
+
+WORKDIR /
+
+ENV CONDA_PREFIX=/azureml-envs/build-test-good
+ENV CONDA_DEFAULT_ENV=$CONDA_PREFIX
+ENV PATH=$CONDA_PREFIX/bin:$PATH
+
+# This is needed for mpi to locate libpython
+ENV LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
+
+# Create conda environment
+COPY conda_dependencies.yaml .
+RUN conda env create -p $CONDA_PREFIX -f conda_dependencies.yaml -q && \
+    rm conda_dependencies.yaml && \
+    conda clean -a -y

test/resources/update/manual-version-no-release-dir/expected/environment/environment-in-subdir/context/conda_dependencies.yaml

@@ -0,0 +1,9 @@
+name: build-test-good
+channels:
+- anaconda
+- conda-forge
+dependencies:
+- python=3.7
+- pip=20.2.4
+- pip:
+  - azureml-core=={{latest-pypi-version}}

test/resources/update/manual-version-no-release-dir/expected/environment/environment-in-subdir/environment.yaml

@@ -0,0 +1,9 @@
+image:
+  name: environment-in-parent-dir
+  os: linux
+  context:
+    dir: context
+    dockerfile: Dockerfile
+    template_files:
+    - Dockerfile
+    - conda_dependencies.yaml

test/resources/update/manual-version-no-release-dir/expected/environment/environment-in-subdir/spec.yaml

@@ -0,0 +1,11 @@
+$schema: https://azuremlschemas.azureedge.net/latest/environment.schema.json
+
+description: >-
+  Test environment.
+
+name: "environment-in-subdir"
+version: "1"
+
+image: "environment-in-parent-dir:1"
+
+os_type: linux

test/resources/update/manual-version-no-release-dir/main/asset.yaml

@@ -0,0 +1,5 @@
+name: environment-in-subdir
+version: 1
+type: environment
+spec: spec.yaml
+extra_config: environment.yaml

test/resources/update/manual-version-no-release-dir/main/context/Dockerfile

@@ -0,0 +1,16 @@
+FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:{{latest-image-tag}}
+
+WORKDIR /
+
+ENV CONDA_PREFIX=/azureml-envs/build-test-good
+ENV CONDA_DEFAULT_ENV=$CONDA_PREFIX
+ENV PATH=$CONDA_PREFIX/bin:$PATH
+
+# This is needed for mpi to locate libpython
+ENV LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
+
+# Create conda environment
+COPY conda_dependencies.yaml .
+RUN conda env create -p $CONDA_PREFIX -f conda_dependencies.yaml -q && \
+    rm conda_dependencies.yaml && \
+    conda clean -a -y

test/resources/update/manual-version-no-release-dir/main/context/conda_dependencies.yaml

@@ -0,0 +1,9 @@
+name: build-test-good
+channels:
+- anaconda
+- conda-forge
+dependencies:
+- python=3.7
+- pip=20.2.4
+- pip:
+  - azureml-core=={{latest-pypi-version}}

test/resources/update/manual-version-no-release-dir/main/environment.yaml

@@ -0,0 +1,9 @@
+image:
+  name: environment-in-parent-dir
+  os: linux
+  context:
+    dir: context
+    dockerfile: Dockerfile
+    template_files:
+    - Dockerfile
+    - conda_dependencies.yaml

test/resources/update/manual-version-no-release-dir/main/spec.yaml

@@ -0,0 +1,11 @@
+$schema: https://azuremlschemas.azureedge.net/latest/environment.schema.json
+
+description: >-
+  Test environment.
+
+name: "{{asset.name}}"
+version: "{{asset.version}}"
+
+image: "{{image.name}}:{{asset.version}}"
+
+os_type: linux

test/test_update_assets.py

@@ -16,23 +16,25 @@ RESOURCES_DIR = Path("resources/update")
 
 
 @pytest.mark.parametrize(
-    "test_subdir,skip_unreleased,create_tag",
+    "test_subdir,skip_unreleased,create_tag,use_release_dir",
     [
-        ("in-subdir", False, True),
-        ("in-parent-dir", False, False),
-        ("manual-version", False, True),
-        ("manual-version-unreleased", False, False),
-        ("manual-version-unreleased-skip", True, False),
-        ("with-description", False, True),
+        ("in-subdir", False, True, True),
+        ("in-parent-dir", False, False, True),
+        ("manual-version", False, True, True),
+        ("manual-version-unreleased", False, False, True),
+        ("manual-version-unreleased-skip", True, False, True),
+        ("manual-version-no-release-dir", False, False, False),
+        ("with-description", False, True, True),
     ]
 )
-def test_update_assets(test_subdir: str, skip_unreleased: bool, create_tag: bool):
+def test_update_assets(test_subdir: str, skip_unreleased: bool, create_tag: bool, use_release_dir: bool):
     """Test update_assets function.
 
     Args:
         test_subdir (str): Test subdirectory
         skip_unreleased (bool): Value to pass to update_assets
         create_tag (bool): Create release tag in temp repo
+        use_release_dir (bool): Use release directory
     """
     this_dir = Path(__file__).parent
     test_dir = this_dir / RESOURCES_DIR / test_subdir
@@ -48,22 +50,23 @@ def test_update_assets(test_subdir: str, skip_unreleased: bool, create_tag: bool
         temp_output_path = Path(temp_dir2)
         temp_expected_path = Path(temp_dir3)
 
-        # Create fake release branch
-        shutil.copytree(release_dir, temp_release_path, dirs_exist_ok=True)
-        repo = Repo.init(temp_release_path)
-        for path in (temp_release_path / "latest").rglob("*"):
-            if path.is_dir():
-                continue
-            rel_path = path.relative_to(temp_release_path)
-            repo.index.add(str(rel_path))
-        repo.git.config("user.email", "<>")
-        repo.git.config("user.name", "Unit Test")
-        repo.git.commit("-m", "Initial commit")
+        if use_release_dir:
+            # Create fake release branch
+            shutil.copytree(release_dir, temp_release_path, dirs_exist_ok=True)
+            repo = Repo.init(temp_release_path)
+            for path in (temp_release_path / "latest").rglob("*"):
+                if path.is_dir():
+                    continue
+                rel_path = path.relative_to(temp_release_path)
+                repo.index.add(str(rel_path))
+            repo.git.config("user.email", "<>")
+            repo.git.config("user.name", "Unit Test")
+            repo.git.commit("-m", "Initial commit")
 
-        # Create tag
-        if create_tag:
-            asset_config = util.find_assets(input_dirs=temp_release_path)[0]
-            repo.create_tag(asset_config.full_name)
+            # Create tag
+            if create_tag:
+                asset_config = util.find_assets(input_dirs=temp_release_path)[0]
+                repo.create_tag(asset_config.full_name)
 
         # Create updatable expected dir
         if expected_dir.exists():
@@ -72,8 +75,9 @@ def test_update_assets(test_subdir: str, skip_unreleased: bool, create_tag: bool
             if expected_asset_config.type == assets.AssetType.ENVIRONMENT:
                 assets.pin_env_files(expected_asset_config.extra_config_as_object())
 
+        release_directory_root = temp_release_path if use_release_dir else None
         assets.update_assets(input_dirs=main_dir, asset_config_filename=assets.DEFAULT_ASSET_FILENAME,
-                             release_directory_root=temp_release_path, copy_only=False,
-                             skip_unreleased=skip_unreleased, output_directory_root=temp_output_path)
+                             output_directory_root=temp_output_path, release_directory_root=release_directory_root,
+                             skip_unreleased=skip_unreleased, use_version_dirs=False)
 
         assert util.are_dir_trees_equal(temp_output_path, temp_expected_path, True)