Bug 1790453 - Selectively expire public/build artifacts early. r=jmaher

Differential Revision: https://phabricator.services.mozilla.com/D163647

taskcluster/ci/artifact-build/kind.yml

@@ -11,6 +11,7 @@ transforms:
     - gecko_taskgraph.transforms.build_attrs:transforms
     - gecko_taskgraph.transforms.build_lints:transforms
     - gecko_taskgraph.transforms.job:transforms
+    - gecko_taskgraph.transforms.artifact:transforms
     - gecko_taskgraph.transforms.task:transforms
 
 jobs:

taskcluster/ci/build-fat-aar/kind.yml

@@ -14,6 +14,7 @@ transforms:
     - gecko_taskgraph.transforms.build_lints:transforms
     - gecko_taskgraph.transforms.build_fat_aar:transforms
     - gecko_taskgraph.transforms.job:transforms
+    - gecko_taskgraph.transforms.artifact:transforms
     - gecko_taskgraph.transforms.task:transforms
 
 job-defaults:

taskcluster/ci/build/kind.yml

@@ -15,6 +15,7 @@ transforms:
     - gecko_taskgraph.transforms.build_lints:transforms
     - gecko_taskgraph.transforms.release_notifications:transforms
     - gecko_taskgraph.transforms.job:transforms
+    - gecko_taskgraph.transforms.artifact:transforms
     - gecko_taskgraph.transforms.task:transforms
 
 jobs-from:

taskcluster/ci/config.yml

@@ -691,14 +691,17 @@ expiration-policy:
     by-project:
         try:
             default: 1 month
+            short: 14 days
             medium: 1 month
             long: 1 month
         autoland:
             default: 1 year
+            short: 3 months
             medium: 1 year
             # To avoid keeping shippable builds for over a year
             long: 1 year
         default:
             default: 3 months
+            short: 1 month
             medium: 1 year
             long: 1 year

taskcluster/ci/instrumented-build/kind.yml

@@ -13,6 +13,7 @@ transforms:
     - gecko_taskgraph.transforms.build_attrs:transforms
     - gecko_taskgraph.transforms.build_lints:transforms
     - gecko_taskgraph.transforms.job:transforms
+    - gecko_taskgraph.transforms.artifact:transforms
     - gecko_taskgraph.transforms.task:transforms
 
 job-defaults:

taskcluster/ci/upload-symbols/kind.yml

@@ -43,7 +43,7 @@ job-template:
             MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system
     run:
         using: mach
-        mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py <build/public/build/target.crashreporter-symbols-full.tar.zst>"}
+        mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py <build/public/cidata/target.crashreporter-symbols-full.tar.zst>"}
         sparse-profile: upload-symbols
     scopes:
         - secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-upload

taskcluster/gecko_taskgraph/transforms/artifact.py

@@ -0,0 +1,115 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply different expiration dates to different artifacts based on a manifest file (artifacts.yml)
+"""
+import logging
+import os
+import sys
+
+import yaml
+from gecko_taskgraph.transforms.job.common import get_expiration
+from gecko_taskgraph.util.workertypes import worker_type_implementation
+from taskgraph.transforms.base import TransformSequence
+from yaml import YAMLError
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+def read_artifact_manifest(manifest_path):
+    """Read the artifacts.yml manifest file and return it."""
+    # logger.info(f"The current directory is {os.getcwd()}")
+    try:
+        with open(manifest_path, "r") as ymlf:
+            yml = yaml.safe_load(ymlf.read())
+            return yml
+    except YAMLError as ye:
+        err = 'Failed to parse manifest "{manifest_path}". Invalid Yaml:'
+        err += ye
+        raise SystemExit(err)
+    except FileNotFoundError:
+        err = f'Failed to load manifest "{manifest_path}". File not found'
+        raise SystemExit(err)
+    except PermissionError:
+        err = f'Failed to load manifest "{manifest_path}". Permission Error'
+        raise SystemExit(err)
+
+
+@transforms.add
+def set_artifact_expiration(config, jobs):
+    """Set the expiration for certain artifacts based on a manifest file."""
+    """---
+       win:
+         - build_resources.json: short
+
+       linux:
+         - target.crashreporter-symbols-full.tar.zst: medium
+    """
+    transform_dir = os.path.dirname(__file__)
+    manifest = read_artifact_manifest(os.path.join(transform_dir, "artifacts.yml"))
+
+    for job in jobs:
+        try:
+            platform = job["attributes"]["build_platform"]
+        except KeyError:
+            err = "Tried to get build_platfrom for job, but it does not exist. Exiting."
+            raise SystemExit(err)
+        if "worker" in job:
+            if "env" in job["worker"]:
+                if isinstance(job["worker"]["env"], dict):
+                    job["worker"]["env"]["MOZ_ARTIFACT_PLATFORM"] = platform
+                else:
+                    raise SystemExit(
+                        f"Expected env to be a dict, but it was {type(job['worker']['env'])}"
+                    )
+            if "artifacts" in job["worker"]:
+                plat = platform.lower()
+                if "plain" in plat or "ccov" in plat or "rusttest" in plat:
+                    art_dict = None
+                elif (
+                    plat == "toolchain-wasm32-wasi-compiler-rt-trunk"
+                    or plat == "toolchain-linux64-x64-compiler-rt-trunk"
+                    or plat == "toolchain-linux64-x86-compiler-rt-trunk"
+                    or plat == "android-geckoview-docs"
+                ):
+                    art_dict = None
+                elif plat.startswith("win"):
+                    art_dict = manifest["win"]
+                elif plat.startswith("linux"):
+                    art_dict = manifest["linux"]
+                elif plat.startswith("mac"):
+                    art_dict = manifest["macos"]
+                elif plat.startswith("android"):
+                    art_dict = manifest["android"]
+                else:
+                    print(
+                        'The platform name "{plat}" didn\'t start with',
+                        '"win", "mac", "android", or "linux".',
+                        file=sys.stderr,
+                    )
+                    art_dict = None
+                worker_implementation, _ = worker_type_implementation(
+                    config.graph_config, config.params, job["worker-type"]
+                )
+                if worker_implementation == "docker-worker":
+                    artifact_dest = "/builds/worker/cidata/{}"
+                else:
+                    artifact_dest = "cidata/{}"
+
+                if art_dict is not None:
+                    for art_name in art_dict.keys():
+                        # The 'artifacts' key of a job is a list at this stage.
+                        # So, must append a new dict to the list
+                        expiry_policy = art_dict[art_name]
+                        expires = get_expiration(config, policy=expiry_policy)
+                        new_art = {
+                            "name": f"public/cidata/{art_name}",
+                            "path": artifact_dest.format(art_name),
+                            "type": "file",
+                            "expires-after": expires,
+                        }
+                        job["worker"]["artifacts"].append(new_art)
+        yield job

taskcluster/gecko_taskgraph/transforms/artifacts.yml

@@ -0,0 +1,12 @@
+---
+win:
+    target.crashreporter-symbols-full.tar.zst: short
+
+linux:
+    target.crashreporter-symbols-full.tar.zst: short
+
+macos:
+    target.crashreporter-symbols-full.tar.zst: short
+
+android:
+    target.crashreporter-symbols-full.tar.zst: short

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -22,6 +22,7 @@ import uuid
 from datetime import datetime
 
 import six
+import yaml
 from mozharness.base.config import DEFAULT_CONFIG_PATH, BaseConfig, parse_config_file
 from mozharness.base.errors import MakefileErrorList
 from mozharness.base.log import ERROR, FATAL, OutputParser
@@ -38,6 +39,7 @@ from mozharness.mozilla.automation import (
     AutomationMixin,
 )
 from mozharness.mozilla.secrets import SecretsMixin
+from yaml import YAMLError
 
 AUTOMATION_EXIT_CODES = sorted(EXIT_STATUS_DICT.values())
 
@@ -1434,3 +1436,92 @@ items from that key's value."
             os.path.join("testing", "parse_build_tests_ccov.py"),
         ]
         self.run_command(command=cmd, cwd=topsrcdir, env=env, halt_on_failure=True)
+
+    @PostScriptRun
+    def _relocate_artifacts(self):
+        """Move certain artifacts out of the default upload directory.
+
+        These artifacts will be moved to a secondary directory called `cidata`.
+        Then they will be uploaded with different expiration values."""
+        dirs = self.query_abs_dirs()
+        topsrcdir = dirs["abs_src_dir"]
+        base_work_dir = dirs["base_work_dir"]
+
+        build_platform = os.environ.get("MOZ_ARTIFACT_PLATFORM")
+        if build_platform is not None:
+            build_platform = build_platform.lower()
+        else:
+            err = "The MOZ_ARTIFACT_PLATFORM env var is not set.\n"
+            err += "That means this build was not modified by the artifact transform. Exiting."
+            self.error(err)
+            return
+        try:
+            upload_dir = os.environ["UPLOAD_DIR"]
+        except KeyError:
+            self.fatal("The env. var. UPLOAD_DIR is not set.")
+
+        artifact_yml_path = os.path.join(
+            topsrcdir, "taskcluster/gecko_taskgraph/transforms/artifacts.yml"
+        )
+
+        upload_short_dir = os.path.join(base_work_dir, "cidata")
+
+        # Choose artifacts based on build platform
+        if build_platform.startswith("win"):
+            main_platform = "win"
+        elif build_platform.startswith("linux"):
+            main_platform = "linux"
+        elif build_platform.startswith("mac"):
+            main_platform = "macos"
+        elif build_platform.startswith("android"):
+            if build_platform == "android-geckoview-docs":
+                return
+            main_platform = "android"
+        else:
+            err = "Build platform {} didn't start with 'mac', 'linux', 'win', or 'android'".format(
+                build_platform
+            )
+            self.fatal(err)
+        try:
+            with open(artifact_yml_path) as artfile:
+                arts = []
+                platforms = yaml.safe_load(artfile.read())
+                for artifact in platforms[main_platform]:
+                    arts.append(artifact)
+        except FileNotFoundError:
+            self.fatal("Could not read artifacts.yml; file not found. Exiting.")
+        except PermissionError:
+            self.fatal("Could not read artifacts.yml; permission error.")
+        except YAMLError as ye:
+            self.fatal(f"Failed to parse artifacts.yml with error:\n{ye}")
+
+        try:
+            os.makedirs(upload_short_dir)
+        except FileExistsError:
+            pass
+        except PermissionError:
+            self.fatal(f'Failed to create dir. "{upload_short_dir}"; permission error.')
+
+        for art in arts:
+            source_file = os.path.join(upload_dir, art)
+            if not os.path.exists(source_file):
+                self.info(
+                    f"The artifact {source_file} is not present in this build. Skipping"
+                )
+                continue
+            dest_file = os.path.join(upload_short_dir, art)
+            try:
+                os.rename(source_file, dest_file)
+                if os.path.exists(dest_file):
+                    self.info(
+                        f"Successfully moved artifact {source_file} to {dest_file}"
+                    )
+                else:
+                    self.fatal(
+                        f"Move of {source_file} to {dest_file} was not successful."
+                    )
+            except (PermissionError, FileNotFoundError) as err:
+                self.fatal(
+                    f'Failed to move file "{art}" from {source_file} to {dest_file}:\n{err}'
+                )
+                continue