mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Backed out 2 changesets (bug 1563090) for Linting failure on run-visual-metrics.py . CLOSED TREE 

Backed out changeset 1b2389663474 (bug 1563090)
Backed out changeset 9071c90b2e84 (bug 1563090)
This commit is contained in:
Narcis Beleuzu 2019-09-04 19:03:58 +03:00
Родитель b1a47abb5e
Коммит 3fca079bae
16 изменённых файлов: 1 добавлений и 591 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
taskcluster/ci/config.yml
Просмотреть файл

@ -137,7 +137,6 @@ treeherder:
'java': 'Java checks'
'SS': 'Shadow scheduler'
'test-info': 'Test manifest skip/fail information'
'vismet': 'Visual Metrics Analsyis'
index:
products:

3
taskcluster/ci/docker-image/kind.yml
Просмотреть файл

@ -233,6 +233,3 @@ jobs:
webrender-updater:
symbol: I(wrupdater)
parent: debian9-base
visual-metrics:
symbol: I(visual-metrics)
parent: debian9-base

1
taskcluster/ci/fetch/kind.yml
Просмотреть файл

@ -14,4 +14,3 @@ jobs-from:
- benchmarks.yml
- toolchains.yml
- chromium-fetch.yml
- visual-metrics.yml

11
taskcluster/ci/fetch/visual-metrics.yml
Просмотреть файл

@ -1,11 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
visual-metrics:
description: "Browsertime visual metrics analsyis script"
fetch:
type: static-url
url: https://raw.githubusercontent.com/mozilla/browsertime/4745d29bb5f8cd60bebe5287d5ba4c996a9d0ae4/vendor/visualmetrics.py
sha256: 9e587fb43c46dd0c37a15bc7688cf061c82592196eafe3ddd54e2ead27997d66
size: 82756

43
taskcluster/ci/visual-metrics/kind.yml
Просмотреть файл

@ -1,43 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
kind-dependencies:
- fetch
transforms:
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
- taskgraph.transforms.visual_metrics:transforms
jobs:
visual-metrics:
label: visual-metrics
description: "Run visual metrics calculations"
run-on-projects: ['try']
worker-type: t-linux-xlarge
treeherder:
symbol: vismet(visual-metrics)
platform: visualmetrics/opt
tier: 2
kind: other
worker:
docker-image: {in-tree: visual-metrics}
max-run-time: 9000
artifacts:
- type: file
name: public/visual-metrics.tar.xz
path: /builds/worker/visual-metrics.tar.xz
fetches:
fetch:
- visual-metrics
run:
using: run-task
command: /builds/worker/bin/run-visual-metrics.py -- --orange --perceptual --contentful --force --renderignore 5 --json --viewport
checkout: false

23
taskcluster/docker/visual-metrics/Dockerfile
Просмотреть файл

@ -1,23 +0,0 @@
# %ARG DOCKER_IMAGE_PARENT
FROM $DOCKER_IMAGE_PARENT
MAINTAINER Barret Rennie <barret@mozilla.com>
RUN apt-get update && \
apt-get install \
ffmpeg \
imagemagick \
pyssim \
python \
python-pil \
python3 \
python3-pip
COPY requirements.txt /builds/worker/requirements.txt
RUN pip3 install --require-hashes -r /builds/worker/requirements.txt && \
rm /builds/worker/requirements.txt
COPY run-visual-metrics.py /builds/worker/bin/run-visual-metrics.py
RUN chmod +x /builds/worker/bin/run-visual-metrics.py
VOLUME /builds/worker/workspace/
VOLUME /builds/worker/artifacts/

12
taskcluster/docker/visual-metrics/requirements.txt
Просмотреть файл

@ -1,12 +0,0 @@
# Direct dependencies
attrs==19.1.0 --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79
requests==2.22.0 --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31
structlog==19.1.0 --hash=sha256:db441b81c65b0f104a7ce5d86c5432be099956b98b8a2c8be0b3fb3a7a0b1536
voluptuous==0.11.5 --hash=sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1
# Transitive dependencies
certifi==2019.6.16 --hash=sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939
chardet==3.0.4 --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
idna==2.8 --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
six==1.12.0 --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c
urllib3==1.25.3 --hash=sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1

385
taskcluster/docker/visual-metrics/run-visual-metrics.py
Просмотреть файл

@ -1,385 +0,0 @@
#!/usr/bin/env python3
"""Instrument visualmetrics.py to run in parallel.
Environment variables:
VISUAL_METRICS_JOBS_JSON:
A JSON blob containing the job descriptions.
Can be overridden with the --jobs-json-path option set to a local file
path.
"""
import argparse
import os
import json
import sys
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from functools import partial
from multiprocessing import cpu_count
from pathlib import Path
import attr
import requests
import structlog
import subprocess
from voluptuous import Required, Schema, Url
#: The workspace directory where files will be downloaded, etc.
WORKSPACE_DIR = Path("/", "builds", "worker", "workspace")
#: The directory where job artifacts will be stored.
WORKSPACE_JOBS_DIR = WORKSPACE_DIR / "jobs"
#: The directory where artifacts from this job will be placed.
OUTPUT_DIR = Path("/", "builds", "worker", "artifacts")
#: A job to process through visualmetrics.py
@attr.s
class Job:
#: The directory for all the files pertaining to the job.
job_dir = attr.ib(type=Path)
#: json_path: The path to the ``browsertime.json`` file on disk.
json_path = attr.ib(type=Path)
#: json_url: The URL of the ``browsertime.json`` file.
json_url = attr.ib(type=str)
#: video_path: The path of the video file on disk.
video_path = attr.ib(type=Path)
#: video_url: The URl of the video file.
video_url = attr.ib(type=str)
# NB: Keep in sync with try_task_config_schema in
# taskcluster/taskgraph.decision.py
#: The schema for validating jobs.
JOB_SCHEMA = Schema(
{
Required("jobs"): [
{
Required("browsertime_json_url"): Url(),
Required("video_url"): Url(),
}
]
}
)
def main(log, args):
"""Run visualmetrics.py in parallel.
Args:
log: The structlog logger instance.
args: The parsed arguments from the argument parser.
Returns:
The return code that the program will exit with.
"""
fetch_dir = os.getenv("MOZ_FETCHES_DIR")
if not fetch_dir:
log.error("Expected MOZ_FETCHES_DIR environment variable.")
return 1
visualmetrics_path = Path(fetch_dir) / "visualmetrics.py"
if not visualmetrics_path.exists():
log.error(
"Could not locate visualmetrics.py: expected it at %s"
% visualmetrics_path
)
return 1
if args.jobs_json_path:
try:
with open(args.jobs_json_path, "r") as f:
jobs_json = json.load(f)
except Exception as e:
log.error(
"Could not read jobs.json file: %s" % e,
path=args.jobs_json_path,
exc_info=True,
)
return 1
log.info(
"Loaded jobs.json from file",
path=args.jobs_json_path,
jobs_json=jobs_json,
)
else:
raw_jobs_json = os.getenv("VISUAL_METRICS_JOBS_JSON")
if raw_jobs_json is not None and isinstance(raw_jobs_json, bytes):
raw_jobs_json = raw_jobs_json.decode("utf-8")
elif raw_jobs_json is None:
log.error(
"Expected one of --jobs-json-path or "
"VISUAL_METRICS_JOBS_JSON environment variable."
)
return 1
try:
jobs_json = json.loads(raw_jobs_json)
except (TypeError, ValueError) as e:
log.error(
"Failed to decode VISUAL_METRICS_JOBS_JSON environment "
"variable: %s" % e,
value=raw_jobs_json,
)
return 1
log.info("Parsed jobs.json from environment", jobs_json=jobs_json)
try:
JOB_SCHEMA(jobs_json)
except Exception as e:
log.error("Failed to parse jobs.json: %s" % e)
return 1
try:
downloaded_jobs, failed_jobs = download_inputs(log, jobs_json["jobs"])
except Exception as e:
log.error("Failed to download jobs: %s" % e, exc_info=True)
return 1
with ProcessPoolExecutor(max_workers=cpu_count()) as executor:
for job, result in zip(
downloaded_jobs,
executor.map(
partial(
run_visual_metrics,
visualmetrics_path=visualmetrics_path,
options=args.visual_metrics_options,
),
downloaded_jobs,
),
):
if isinstance(result, Exception):
log.error(
"Failed to run visualmetrics.py",
video_url=job.video_url,
error=result,
)
else:
with (job.job_dir / "visual-metrics.json").open("wb") as f:
f.write(result)
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
with Path(WORKSPACE_DIR, "jobs.json").open("w") as f:
json.dump(
{
"successful_jobs": [
{
"video_url": job.video_url,
"browsertime_json_url": job.json_url,
"path": (
str(job.job_dir.relative_to(WORKSPACE_DIR)) + "/"
),
}
for job in downloaded_jobs
],
"failed_jobs": [
{
"video_url": job.video_url,
"browsertime_json_url": job.json_url,
}
for job in failed_jobs
],
},
f,
)
subprocess.check_output(
[
"tar",
"cJf",
str(OUTPUT_DIR / "visual-metrics.tar.xz"),
"-C",
str(WORKSPACE_DIR),
".",
]
)
def download_inputs(log, raw_jobs):
"""Download the inputs for all jobs in parallel.
Args:
log: The structlog logger instance.
raw_jobs: The list of unprocessed jobs from the ``jobs.json`` input file.
Returns:
A tuple of the successfully downloaded jobs and the failed to download jobs.
"""
WORKSPACE_DIR.mkdir(parents=True, exist_ok=True)
pending_jobs = []
for i, job in enumerate(raw_jobs):
job_dir = WORKSPACE_JOBS_DIR / str(i)
job_dir.mkdir(parents=True, exist_ok=True)
pending_jobs.append(
Job(
job_dir,
job_dir / "browsertime.json",
job["browsertime_json_url"],
job_dir / "video",
job["video_url"],
)
)
downloaded_jobs = []
failed_jobs = []
with ThreadPoolExecutor(max_workers=8) as executor:
for job, success in executor.map(
partial(download_job, log), pending_jobs
):
if success:
downloaded_jobs.append(job)
else:
job.job_dir.rmdir()
failed_jobs.append(job)
return downloaded_jobs, failed_jobs
def download_job(log, job):
"""Download the files for a given job.
Args:
log: The structlog logger instance.
job: The job to download.
Returns:
A tuple of the job and whether or not the download was successful.
The returned job will be updated so that it's :attr:`Job.video_path`
attribute is updated to match the file path given by the video file
in the ``browsertime.json`` file.
"""
log = log.bind(json_url=job.json_url)
try:
download(job.video_url, job.video_path)
download(job.json_url, job.json_path)
except Exception as e:
log.error(
"Failed to download files for job: %s" % e,
video_url=job.video_url,
exc_info=True,
)
return job, False
try:
with job.json_path.open("r") as f:
browsertime_json = json.load(f)
except OSError as e:
log.error("Could not read browsertime.json: %s" % e)
return job, False
except ValueError as e:
log.error("Could not parse browsertime.json as JSON: %s" % e)
return job, False
try:
video_path = job.job_dir / browsertime_json[0]["files"]["video"][0]
except KeyError:
log.error("Could not read video path from browsertime.json file")
return job, False
video_path.parent.mkdir(parents=True, exist_ok=True)
job.video_path.rename(video_path)
job.video_path = video_path
return job, True
def download(url, path):
"""Download the resource at the given URL to the local path.
Args:
url: The URL of the resource to download.
path: The local path to download the resource to.
Raises:
OSError:
Raised if an IO error occurs while writing the file.
requests.exceptions.HTTPError:
Raised when an HTTP error (including e.g., HTTP 404) occurs.
"""
request = requests.get(url, stream=True)
request.raise_for_status()
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("wb") as f:
for chunk in request:
f.write(chunk)
def run_visual_metrics(job, visualmetrics_path, options):
"""Run visualmetrics.py on the input job.
Returns:
Either a string containing the JSON output of visualmetrics.py or an
exception raised by :func:`subprocess.check_output`.
"""
cmd = [
"/usr/bin/python",
str(visualmetrics_path),
"--video",
str(job.video_path),
]
cmd.extend(options)
try:
return subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
return e
if __name__ == "__main__":
structlog.configure(
processors=[
structlog.processors.TimeStamper(fmt="iso"),
structlog.processors.format_exc_info,
structlog.dev.ConsoleRenderer(colors=False),
],
cache_logger_on_first_use=True,
)
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--jobs-json-path",
type=Path,
metavar="PATH",
help=(
"The path to the jobs.josn file. If not present, the "
"VISUAL_METRICS_JOBS_JSON environment variable will be used "
"instead."
),
)
parser.add_argument(
"visual_metrics_options",
type=str,
metavar="VISUAL-METRICS-OPTIONS",
help="Options to pass to visualmetrics.py",
nargs="*",
)
args = parser.parse_args()
log = structlog.get_logger()
try:
sys.exit(main(log, args))
except Exception as e:
log.error("Unhandled exception: %s" % e, exc_info=True)
sys.exit(1)

5
taskcluster/docs/kinds.rst
Просмотреть файл

@ -598,8 +598,3 @@ test archive into it's own archive.
geckodriver-signing
-------------------
Signing for geckodriver binary.
visual-metrics
--------------
Tasks that compute visual performance metrics from videos and images captured
by other tasks.

13
taskcluster/taskgraph/decision.py
Просмотреть файл

@ -27,7 +27,7 @@ from .util.schema import validate_schema, Schema
from .util.taskcluster import get_artifact
from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds
from .util.yaml import load_yaml
from voluptuous import Required, Optional, Url
from voluptuous import Required, Optional
logger = logging.getLogger(__name__)
@ -117,22 +117,11 @@ PER_PROJECT_PARAMETERS = {
}
}
visual_metrics_jobs_schema = Schema({
Required('jobs'): [
{
Required('browsertime_json_url'): Url(),
Required('video_url'): Url(),
}
]
})
try_task_config_schema = Schema({
Required('tasks'): [basestring],
Optional('templates'): {basestring: object},
Optional('disable-pgo'): bool,
Optional('browsertime'): bool,
# Keep in sync with JOB_SCHEMA in taskcluster/docker/visual-metrics/run-visual-metrics.py.
Optional('visual-metrics-jobs'): visual_metrics_jobs_schema,
})

31
taskcluster/taskgraph/transforms/visual_metrics.py
Просмотреть файл

@ -1,31 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
These transformations take a task description for a visual metrics task and
add the necessary environment variables to run on the given inputs.
"""
from __future__ import absolute_import, print_function, unicode_literals
import json
from taskgraph.transforms.base import TransformSequence
transforms = TransformSequence()
@transforms.add
def set_visual_metrics_jobs(config, jobs):
"""Set the visual metrics configuration for the given jobs."""
vismet_jobs = config.params['try_task_config'].get('visual-metrics-jobs')
if vismet_jobs:
vismet_jobs = json.dumps(vismet_jobs)
for job in jobs:
if vismet_jobs:
job['task']['payload'].setdefault('env', {}).update(
VISUAL_METRICS_JOBS_JSON=vismet_jobs)
yield job

1
taskcluster/taskgraph/util/schema.py
Просмотреть файл

@ -129,7 +129,6 @@ def resolve_keyed_by(item, field, item_name, **extra_values):
WHITELISTED_SCHEMA_IDENTIFIERS = [
# upstream-artifacts are handed directly to scriptWorker, which expects interCaps
lambda path: "[u'upstream-artifacts']" in path,
lambda path: "[u'browsertime_json_url']" in path or "[u'video_url']" in path,
]

1
tools/lint/py2.yml
Просмотреть файл

@ -19,7 +19,6 @@ py2:
- security
- servo
- taskcluster/docker/funsize-update-generator
- taskcluster/docker/visual-metrics
- testing/gtest
- testing/mochitest
- testing/mozharness

1
tools/tryselect/selectors/chooser/__init__.py
Просмотреть файл

@ -28,7 +28,6 @@ class ChooserParser(BaseTryParser):
'gecko-profile',
'path',
'rebuild',
'visual-metrics-jobs',
]

1
tools/tryselect/selectors/fuzzy.py
Просмотреть файл

@ -130,7 +130,6 @@ class FuzzyParser(BaseTryParser):
'gecko-profile',
'path',
'rebuild',
'visual-metrics-jobs',
]

60
tools/tryselect/templates.py
Просмотреть файл

@ -14,11 +14,8 @@ import os
import sys
from abc import ABCMeta, abstractmethod, abstractproperty
from argparse import Action, SUPPRESS
from textwrap import dedent
import mozpack.path as mozpath
import voluptuous
from taskgraph.decision import visual_metrics_jobs_schema
from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
from .tasks import resolve_tests_by_suite
@ -254,62 +251,6 @@ class DisablePgo(TryConfig):
}
visual_metrics_jobs_description = dedent("""\
The file should be a JSON file of the format:
{
"jobs": [
{
"browsertime_json_url": "http://example.com/browsertime.json",
"video_url": "http://example.com/video.mp4"
}
]
}
""")
class VisualMetricsJobs(TryConfig):
arguments = [
[['--visual-metrics-jobs'],
{'dest': 'visual_metrics_jobs',
'metavar': 'PATH',
'help': (
'The path to a visual metrics jobs file. Only required when '
'running a "visual-metrics" job.\n'
'%s' % visual_metrics_jobs_description
)}],
]
def try_config(self, **kwargs):
file_path = kwargs.get('visual_metrics_jobs')
if not file_path:
return None
try:
with open(file_path) as f:
visual_metrics_jobs = json.load(f)
visual_metrics_jobs_schema(visual_metrics_jobs)
except (IOError, OSError) as e:
print('Failed to read file %s: %s' % (file_path, f))
sys.exit(1)
except TypeError:
print('Failed to parse file %s as JSON: %s' % (file_path, f))
sys.exit(1)
except voluptuous.Error as e:
print(
'The file %s does not match the expected format: %s\n'
'%s'
% (file_path, e, visual_metrics_jobs_description)
)
sys.exit(1)
return {
'visual-metrics-jobs': visual_metrics_jobs,
}
all_templates = {
'artifact': Artifact,
'browsertime': Browsertime,
@ -319,5 +260,4 @@ all_templates = {
'gecko-profile': GeckoProfile,
'path': Path,
'rebuild': Rebuild,
'visual-metrics-jobs': VisualMetricsJobs,
}