From cabafb474605017bc9ee287989fcbcfb5786b253 Mon Sep 17 00:00:00 2001 From: Brenden Hyde Date: Fri, 2 Dec 2022 21:31:03 +0000 Subject: [PATCH] Bug 1790453 - Selectively expire public/build artifacts early. r=jmaher Differential Revision: https://phabricator.services.mozilla.com/D163647 --- taskcluster/ci/artifact-build/kind.yml | 1 + taskcluster/ci/build-fat-aar/kind.yml | 1 + taskcluster/ci/build/kind.yml | 1 + taskcluster/ci/config.yml | 3 + taskcluster/ci/instrumented-build/kind.yml | 1 + taskcluster/ci/upload-symbols/kind.yml | 2 +- .../gecko_taskgraph/transforms/artifact.py | 115 ++++++++++++++++++ .../gecko_taskgraph/transforms/artifacts.yml | 12 ++ .../mozharness/mozilla/building/buildbase.py | 91 ++++++++++++++ 9 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 taskcluster/gecko_taskgraph/transforms/artifact.py create mode 100644 taskcluster/gecko_taskgraph/transforms/artifacts.yml diff --git a/taskcluster/ci/artifact-build/kind.yml b/taskcluster/ci/artifact-build/kind.yml index bf6d79a65d01..007370467e2d 100644 --- a/taskcluster/ci/artifact-build/kind.yml +++ b/taskcluster/ci/artifact-build/kind.yml @@ -11,6 +11,7 @@ transforms: - gecko_taskgraph.transforms.build_attrs:transforms - gecko_taskgraph.transforms.build_lints:transforms - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.artifact:transforms - gecko_taskgraph.transforms.task:transforms jobs: diff --git a/taskcluster/ci/build-fat-aar/kind.yml b/taskcluster/ci/build-fat-aar/kind.yml index 3d4d518b1e1a..3c5c563b63df 100644 --- a/taskcluster/ci/build-fat-aar/kind.yml +++ b/taskcluster/ci/build-fat-aar/kind.yml @@ -14,6 +14,7 @@ transforms: - gecko_taskgraph.transforms.build_lints:transforms - gecko_taskgraph.transforms.build_fat_aar:transforms - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.artifact:transforms - gecko_taskgraph.transforms.task:transforms job-defaults: diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml index e7fdd0a5353c..53cc565d056e 100644 --- a/taskcluster/ci/build/kind.yml +++ b/taskcluster/ci/build/kind.yml @@ -15,6 +15,7 @@ transforms: - gecko_taskgraph.transforms.build_lints:transforms - gecko_taskgraph.transforms.release_notifications:transforms - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.artifact:transforms - gecko_taskgraph.transforms.task:transforms jobs-from: diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index ed9667fa1bdc..883dd87cd375 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -691,14 +691,17 @@ expiration-policy: by-project: try: default: 1 month + short: 14 days medium: 1 month long: 1 month autoland: default: 1 year + short: 3 months medium: 1 year # To avoid keeping shippable builds for over a year long: 1 year default: default: 3 months + short: 1 month medium: 1 year long: 1 year diff --git a/taskcluster/ci/instrumented-build/kind.yml b/taskcluster/ci/instrumented-build/kind.yml index ba4aaea29dc6..7c6862a5d17c 100644 --- a/taskcluster/ci/instrumented-build/kind.yml +++ b/taskcluster/ci/instrumented-build/kind.yml @@ -13,6 +13,7 @@ transforms: - gecko_taskgraph.transforms.build_attrs:transforms - gecko_taskgraph.transforms.build_lints:transforms - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.artifact:transforms - gecko_taskgraph.transforms.task:transforms job-defaults: diff --git a/taskcluster/ci/upload-symbols/kind.yml b/taskcluster/ci/upload-symbols/kind.yml index 1dd619cfea99..1b912247f4b9 100644 --- a/taskcluster/ci/upload-symbols/kind.yml +++ b/taskcluster/ci/upload-symbols/kind.yml @@ -43,7 +43,7 @@ job-template: MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system run: using: mach - mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py "} + mach: {artifact-reference: "python toolkit/crashreporter/tools/upload_symbols.py "} sparse-profile: upload-symbols scopes: - secrets:get:project/releng/gecko/build/level-{level}/gecko-symbol-upload diff --git a/taskcluster/gecko_taskgraph/transforms/artifact.py b/taskcluster/gecko_taskgraph/transforms/artifact.py new file mode 100644 index 000000000000..8501bf9d8b2b --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/artifact.py @@ -0,0 +1,115 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply different expiration dates to different artifacts based on a manifest file (artifacts.yml) +""" +import logging +import os +import sys + +import yaml +from gecko_taskgraph.transforms.job.common import get_expiration +from gecko_taskgraph.util.workertypes import worker_type_implementation +from taskgraph.transforms.base import TransformSequence +from yaml import YAMLError + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +def read_artifact_manifest(manifest_path): + """Read the artifacts.yml manifest file and return it.""" + # logger.info(f"The current directory is {os.getcwd()}") + try: + with open(manifest_path, "r") as ymlf: + yml = yaml.safe_load(ymlf.read()) + return yml + except YAMLError as ye: + err = 'Failed to parse manifest "{manifest_path}". Invalid Yaml:' + err += ye + raise SystemExit(err) + except FileNotFoundError: + err = f'Failed to load manifest "{manifest_path}". File not found' + raise SystemExit(err) + except PermissionError: + err = f'Failed to load manifest "{manifest_path}". Permission Error' + raise SystemExit(err) + + +@transforms.add +def set_artifact_expiration(config, jobs): + """Set the expiration for certain artifacts based on a manifest file.""" + """--- + win: + - build_resources.json: short + + linux: + - target.crashreporter-symbols-full.tar.zst: medium + """ + transform_dir = os.path.dirname(__file__) + manifest = read_artifact_manifest(os.path.join(transform_dir, "artifacts.yml")) + + for job in jobs: + try: + platform = job["attributes"]["build_platform"] + except KeyError: + err = "Tried to get build_platfrom for job, but it does not exist. Exiting." + raise SystemExit(err) + if "worker" in job: + if "env" in job["worker"]: + if isinstance(job["worker"]["env"], dict): + job["worker"]["env"]["MOZ_ARTIFACT_PLATFORM"] = platform + else: + raise SystemExit( + f"Expected env to be a dict, but it was {type(job['worker']['env'])}" + ) + if "artifacts" in job["worker"]: + plat = platform.lower() + if "plain" in plat or "ccov" in plat or "rusttest" in plat: + art_dict = None + elif ( + plat == "toolchain-wasm32-wasi-compiler-rt-trunk" + or plat == "toolchain-linux64-x64-compiler-rt-trunk" + or plat == "toolchain-linux64-x86-compiler-rt-trunk" + or plat == "android-geckoview-docs" + ): + art_dict = None + elif plat.startswith("win"): + art_dict = manifest["win"] + elif plat.startswith("linux"): + art_dict = manifest["linux"] + elif plat.startswith("mac"): + art_dict = manifest["macos"] + elif plat.startswith("android"): + art_dict = manifest["android"] + else: + print( + 'The platform name "{plat}" didn\'t start with', + '"win", "mac", "android", or "linux".', + file=sys.stderr, + ) + art_dict = None + worker_implementation, _ = worker_type_implementation( + config.graph_config, config.params, job["worker-type"] + ) + if worker_implementation == "docker-worker": + artifact_dest = "/builds/worker/cidata/{}" + else: + artifact_dest = "cidata/{}" + + if art_dict is not None: + for art_name in art_dict.keys(): + # The 'artifacts' key of a job is a list at this stage. + # So, must append a new dict to the list + expiry_policy = art_dict[art_name] + expires = get_expiration(config, policy=expiry_policy) + new_art = { + "name": f"public/cidata/{art_name}", + "path": artifact_dest.format(art_name), + "type": "file", + "expires-after": expires, + } + job["worker"]["artifacts"].append(new_art) + yield job diff --git a/taskcluster/gecko_taskgraph/transforms/artifacts.yml b/taskcluster/gecko_taskgraph/transforms/artifacts.yml new file mode 100644 index 000000000000..217f617edf5a --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/artifacts.yml @@ -0,0 +1,12 @@ +--- +win: + target.crashreporter-symbols-full.tar.zst: short + +linux: + target.crashreporter-symbols-full.tar.zst: short + +macos: + target.crashreporter-symbols-full.tar.zst: short + +android: + target.crashreporter-symbols-full.tar.zst: short diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index ee27dcf5fbc4..082a73f94675 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -22,6 +22,7 @@ import uuid from datetime import datetime import six +import yaml from mozharness.base.config import DEFAULT_CONFIG_PATH, BaseConfig, parse_config_file from mozharness.base.errors import MakefileErrorList from mozharness.base.log import ERROR, FATAL, OutputParser @@ -38,6 +39,7 @@ from mozharness.mozilla.automation import ( AutomationMixin, ) from mozharness.mozilla.secrets import SecretsMixin +from yaml import YAMLError AUTOMATION_EXIT_CODES = sorted(EXIT_STATUS_DICT.values()) @@ -1434,3 +1436,92 @@ items from that key's value." os.path.join("testing", "parse_build_tests_ccov.py"), ] self.run_command(command=cmd, cwd=topsrcdir, env=env, halt_on_failure=True) + + @PostScriptRun + def _relocate_artifacts(self): + """Move certain artifacts out of the default upload directory. + + These artifacts will be moved to a secondary directory called `cidata`. + Then they will be uploaded with different expiration values.""" + dirs = self.query_abs_dirs() + topsrcdir = dirs["abs_src_dir"] + base_work_dir = dirs["base_work_dir"] + + build_platform = os.environ.get("MOZ_ARTIFACT_PLATFORM") + if build_platform is not None: + build_platform = build_platform.lower() + else: + err = "The MOZ_ARTIFACT_PLATFORM env var is not set.\n" + err += "That means this build was not modified by the artifact transform. Exiting." + self.error(err) + return + try: + upload_dir = os.environ["UPLOAD_DIR"] + except KeyError: + self.fatal("The env. var. UPLOAD_DIR is not set.") + + artifact_yml_path = os.path.join( + topsrcdir, "taskcluster/gecko_taskgraph/transforms/artifacts.yml" + ) + + upload_short_dir = os.path.join(base_work_dir, "cidata") + + # Choose artifacts based on build platform + if build_platform.startswith("win"): + main_platform = "win" + elif build_platform.startswith("linux"): + main_platform = "linux" + elif build_platform.startswith("mac"): + main_platform = "macos" + elif build_platform.startswith("android"): + if build_platform == "android-geckoview-docs": + return + main_platform = "android" + else: + err = "Build platform {} didn't start with 'mac', 'linux', 'win', or 'android'".format( + build_platform + ) + self.fatal(err) + try: + with open(artifact_yml_path) as artfile: + arts = [] + platforms = yaml.safe_load(artfile.read()) + for artifact in platforms[main_platform]: + arts.append(artifact) + except FileNotFoundError: + self.fatal("Could not read artifacts.yml; file not found. Exiting.") + except PermissionError: + self.fatal("Could not read artifacts.yml; permission error.") + except YAMLError as ye: + self.fatal(f"Failed to parse artifacts.yml with error:\n{ye}") + + try: + os.makedirs(upload_short_dir) + except FileExistsError: + pass + except PermissionError: + self.fatal(f'Failed to create dir. "{upload_short_dir}"; permission error.') + + for art in arts: + source_file = os.path.join(upload_dir, art) + if not os.path.exists(source_file): + self.info( + f"The artifact {source_file} is not present in this build. Skipping" + ) + continue + dest_file = os.path.join(upload_short_dir, art) + try: + os.rename(source_file, dest_file) + if os.path.exists(dest_file): + self.info( + f"Successfully moved artifact {source_file} to {dest_file}" + ) + else: + self.fatal( + f"Move of {source_file} to {dest_file} was not successful." + ) + except (PermissionError, FileNotFoundError) as err: + self.fatal( + f'Failed to move file "{art}" from {source_file} to {dest_file}:\n{err}' + ) + continue