зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1655031 - Enable HTTP/3 live site tests in CI. r=tarek
Differential Revision: https://phabricator.services.mozilla.com/D84932
This commit is contained in:
Gregory Mierzwinski
Родитель
ee2aa57e5c
Коммит
21952648e0
|
|
@ -159,6 +159,7 @@ treeherder:
|
|||
'doc': 'Documentation'
|
||||
'GhS': 'GitHub Synchronization'
|
||||
'perftest': 'Performance tests'
|
||||
'perftest-http3': 'Performance tests with HTTP/3'
|
||||
|
||||
index:
|
||||
products:
|
||||
|
|
|
|||
|
|
@ -34,3 +34,88 @@ domcount:
|
|||
--perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count
|
||||
--browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
||||
livesites:
|
||||
description: Live site performance testing
|
||||
variants: [http3]
|
||||
treeherder:
|
||||
symbol: perftest({symbol})
|
||||
attributes:
|
||||
batch: false
|
||||
cron: true
|
||||
perftest:
|
||||
- [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js]
|
||||
- [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js]
|
||||
- [g-image, netwerk/test/perf/perftest_http3_google_image.js]
|
||||
- [g-search, netwerk/test/perf/perftest_http3_google_search.js]
|
||||
- [lq-fetch, netwerk/test/perf/perftest_http3_lucasquicfetch.js]
|
||||
- [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js]
|
||||
- [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js]
|
||||
perftest-metrics:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
# Example of how the specify settings for each metric
|
||||
resources:
|
||||
extraOptions: []
|
||||
firstPaint: {}
|
||||
navigationTiming: {}
|
||||
pageTimings: {}
|
||||
timeToContentfulPaint: {}
|
||||
fb-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, requestsPerSecond]
|
||||
g-image: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, imagesPerSecond, imageLoadTime]
|
||||
g-search: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint]
|
||||
lq-fetch: [navigationTiming, pageTimings, resources, timeToContentfulPaint, resourceLoadTime, imagesLoaded, imagesMissed]
|
||||
ytw: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames]
|
||||
ytw-scroll: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint, droppedFrames, decodedFrames]
|
||||
default: [navigationTiming, pageTimings, resources, firstPaint, timeToContentfulPaint]
|
||||
# Leave this here so people know this exists when looking
|
||||
# for examples
|
||||
perftest-perfherder-global:
|
||||
extraOptions: []
|
||||
perftest-extra-options:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
- --perfherder-split-by browserScripts.pageinfo.url
|
||||
g-search:
|
||||
- --perfherder-split-by browserScripts.pageinfo.url
|
||||
lq-fetch:
|
||||
- --perfherder-split-by browserScripts.pageinfo.url
|
||||
default: []
|
||||
perftest-btime-variants:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
fb-scroll:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
g-search:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
lq-fetch:
|
||||
- [null, "browsertime.waitTime=1000,browsertime.cycles=5"]
|
||||
ytw:
|
||||
- [null, browsertime.waitTime=20000]
|
||||
ytw-scroll:
|
||||
- [null, browsertime.waitTime=20000]
|
||||
default:
|
||||
- [null, browsertime.waitTime=1000]
|
||||
run:
|
||||
command: >-
|
||||
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
|
||||
cd $MOZ_FETCHES_DIR &&
|
||||
python3.8 python/mozperftest/mozperftest/runner.py
|
||||
{perftest_testname}
|
||||
--browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin
|
||||
--browsertime-iterations 10
|
||||
--flavor desktop-browser
|
||||
--perfherder
|
||||
--perfherder-metrics {perftest_metrics}
|
||||
--browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
|
|
|||
|
|
@ -37,3 +37,81 @@ domcount:
|
|||
--perfherder-metrics name:totalDOMCount,unit:count name:panelMenuCount,unit:count name:lightDOMCount,unit:count name:lightDOMDetails,unit:count
|
||||
--browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
||||
|
||||
livesites:
|
||||
description: Live site performance testing
|
||||
variants: [http3]
|
||||
treeherder:
|
||||
symbol: perftest({symbol})
|
||||
attributes:
|
||||
batch: false
|
||||
cron: true
|
||||
perftest:
|
||||
- [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js]
|
||||
- [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js]
|
||||
- [g-image, netwerk/test/perf/perftest_http3_google_image.js]
|
||||
- [g-search, netwerk/test/perf/perftest_http3_google_search.js]
|
||||
- [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js]
|
||||
- [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js]
|
||||
perftest-metrics:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
# Example of how the specify settings for each metric
|
||||
resourceTiming:
|
||||
extraOptions: []
|
||||
firstPaint: {}
|
||||
fb-scroll: [resourceTiming, firstPaint, firstContentfulPaint, requestsPerSecond]
|
||||
g-image: [resourceTiming, firstPaint, firstContentfulPaint, imagesPerSecond, imageLoadTime]
|
||||
g-search: [resourceTiming, firstPaint, firstContentfulPaint]
|
||||
ytw: [resourceTiming, firstPaint, firstContentfulPaint, droppedFrames, decodedFrames]
|
||||
ytw-scroll: [resourceTiming, firstPaint, firstContentfulPaint, droppedFrames, decodedFrames]
|
||||
default: [firstPaint]
|
||||
# Leave this here so people know this exists when looking
|
||||
# for examples
|
||||
perftest-perfherder-global:
|
||||
extraOptions: []
|
||||
perftest-extra-options:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
- --perfherder-split-by browserScripts.pageinfo.url
|
||||
g-search:
|
||||
- --perfherder-split-by browserScripts.pageinfo.url
|
||||
default: []
|
||||
perftest-btime-variants:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
g-search:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
fb-scroll:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
ytw:
|
||||
- [null, browsertime.waitTime=20000]
|
||||
ytw-scroll:
|
||||
- [null, browsertime.waitTime=20000]
|
||||
default:
|
||||
- [null, browsertime.waitTime=1000]
|
||||
run:
|
||||
command: >-
|
||||
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
|
||||
cd $MOZ_FETCHES_DIR &&
|
||||
python3 -m venv . &&
|
||||
bin/python3 python/mozperftest/mozperftest/runner.py
|
||||
{perftest_testname}
|
||||
--browsertime-binary ${MOZ_FETCHES_DIR}/target.dmg
|
||||
--browsertime-node ${MOZ_FETCHES_DIR}/node/bin/node
|
||||
--flavor desktop-browser
|
||||
--perfherder
|
||||
--perfherder-metrics {perftest_metrics}
|
||||
--browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
|
|
|||
|
|
@ -35,3 +35,80 @@ domcount:
|
|||
--browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe
|
||||
--browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
||||
|
||||
livesites:
|
||||
description: Live site performance testing
|
||||
variants: [http3]
|
||||
treeherder:
|
||||
symbol: perftest({symbol})
|
||||
attributes:
|
||||
batch: false
|
||||
cron: true
|
||||
perftest:
|
||||
- [cloudflare, netwerk/test/perf/perftest_http3_cloudflareblog.js]
|
||||
- [fb-scroll, netwerk/test/perf/perftest_http3_facebook_scroll.js]
|
||||
- [g-image, netwerk/test/perf/perftest_http3_google_image.js]
|
||||
- [g-search, netwerk/test/perf/perftest_http3_google_search.js]
|
||||
- [ytw, netwerk/test/perf/perftest_http3_youtube_watch.js]
|
||||
- [ytw-scroll, netwerk/test/perf/perftest_http3_youtube_watch_scroll.js]
|
||||
perftest-metrics:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
# Example of how the specify settings for each metric
|
||||
resourceTiming:
|
||||
extraOptions: []
|
||||
firstPaint: {}
|
||||
fb-scroll: [resourceTiming, firstPaint, firstContentfulPaint, requestsPerSecond]
|
||||
g-image: [resourceTiming, firstPaint, firstContentfulPaint, imagesPerSecond, imageLoadTime]
|
||||
g-search: [resourceTiming, firstPaint, firstContentfulPaint]
|
||||
ytw: [resourceTiming, firstPaint, firstContentfulPaint, droppedFrames, decodedFrames]
|
||||
ytw-scroll: [resourceTiming, firstPaint, firstContentfulPaint, droppedFrames, decodedFrames]
|
||||
default: [firstPaint]
|
||||
# Leave this here so people know this exists when looking
|
||||
# for examples
|
||||
perftest-perfherder-global:
|
||||
extraOptions: []
|
||||
perftest-extra-options:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
- --perfherder-split-by browserScripts.pageinfo.url
|
||||
g-search:
|
||||
- --perfherder-split-by browserScripts.pageinfo.url
|
||||
default: []
|
||||
perftest-btime-variants:
|
||||
by-perftest:
|
||||
cloudflare:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
g-search:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
fb-scroll:
|
||||
- ["10s", browsertime.waitTime=10000]
|
||||
- ["25s", browsertime.waitTime=25000]
|
||||
- ["35s", browsertime.waitTime=35000]
|
||||
- ["60s", browsertime.waitTime=60000]
|
||||
ytw:
|
||||
- [null, browsertime.waitTime=20000]
|
||||
ytw-scroll:
|
||||
- [null, browsertime.waitTime=20000]
|
||||
default:
|
||||
- [null, browsertime.waitTime=1000]
|
||||
run:
|
||||
command: >-
|
||||
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
|
||||
cd $MOZ_FETCHES_DIR &&
|
||||
python3.exe python/mozperftest/mozperftest/runner.py
|
||||
{perftest_testname}
|
||||
--browsertime-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe
|
||||
--flavor desktop-browser
|
||||
--perfherder
|
||||
--perfherder-metrics {perftest_metrics}
|
||||
--browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe
|
||||
--browsertime-node ${MOZ_FETCHES_DIR}/node/node.exe
|
||||
--output $MOZ_FETCHES_DIR/../artifacts
|
||||
|
|
|
|||
|
|
@ -6,22 +6,281 @@ This transform passes options from `mach perftest` to the corresponding task.
|
|||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import date, timedelta
|
||||
import json
|
||||
|
||||
import six
|
||||
from six import ensure_text, text_type
|
||||
|
||||
from voluptuous import (
|
||||
Any,
|
||||
Optional,
|
||||
Extra,
|
||||
)
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
|
||||
from taskgraph.util.treeherder import split_symbol, join_symbol
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
perftest_description_schema = Schema({
|
||||
|
||||
# The test names and the symbols to use for them: [test-symbol, test-path]
|
||||
Optional('perftest'): [[text_type]],
|
||||
|
||||
# Metrics to gather for the test. These will be merged
|
||||
# with options specified through perftest-perfherder-global
|
||||
Optional('perftest-metrics'): optionally_keyed_by(
|
||||
'perftest',
|
||||
Any(
|
||||
[text_type],
|
||||
{text_type: Any(
|
||||
None,
|
||||
{text_type: Any(None, text_type, [text_type])}
|
||||
)}
|
||||
)
|
||||
),
|
||||
|
||||
# Perfherder data options that will be applied to
|
||||
# all metrics gathered.
|
||||
Optional('perftest-perfherder-global'): optionally_keyed_by(
|
||||
'perftest',
|
||||
{text_type: Any(None, text_type, [text_type])}
|
||||
),
|
||||
|
||||
# Extra options to add to the test's command
|
||||
Optional('perftest-extra-options'): optionally_keyed_by(
|
||||
'perftest',
|
||||
[text_type]
|
||||
),
|
||||
|
||||
|
||||
# Variants of the test to make based on extra browsertime
|
||||
# arguments. Expecting:
|
||||
# [variant-suffix, options-to-use]
|
||||
# If variant-suffix is `null` then the options will be added
|
||||
# to the existing task. Otherwise, a new variant is created
|
||||
# with the given suffix and with its options replaced.
|
||||
Optional('perftest-btime-variants'): optionally_keyed_by(
|
||||
'perftest',
|
||||
[[Any(None, text_type)]]
|
||||
),
|
||||
|
||||
# These options will be parsed in the next schemas
|
||||
Extra: object,
|
||||
})
|
||||
|
||||
|
||||
transforms.add_validate(perftest_description_schema)
|
||||
|
||||
|
||||
@transforms.add
|
||||
def split_tests(config, jobs):
|
||||
for job in jobs:
|
||||
if job.get("perftest") is None:
|
||||
yield job
|
||||
continue
|
||||
|
||||
for test_symbol, test_name in job.pop("perftest"):
|
||||
job_new = deepcopy(job)
|
||||
|
||||
job_new["perftest"] = test_symbol
|
||||
job_new["name"] += "-" + test_symbol
|
||||
job_new["treeherder"]["symbol"] = job["treeherder"]["symbol"].format(
|
||||
symbol=test_symbol
|
||||
)
|
||||
job_new["run"]["command"] = job["run"]["command"].replace(
|
||||
"{perftest_testname}",
|
||||
test_name
|
||||
)
|
||||
|
||||
yield job_new
|
||||
|
||||
|
||||
@transforms.add
|
||||
def handle_keyed_by_perftest(config, jobs):
|
||||
fields = [
|
||||
'perftest-metrics',
|
||||
'perftest-extra-options',
|
||||
'perftest-btime-variants'
|
||||
]
|
||||
for job in jobs:
|
||||
if job.get("perftest") is None:
|
||||
yield job
|
||||
continue
|
||||
|
||||
for field in fields:
|
||||
resolve_keyed_by(job, field, item_name=job['name'])
|
||||
|
||||
job.pop("perftest")
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def parse_perftest_metrics(config, jobs):
|
||||
"""Parse the metrics into a dictionary immediately.
|
||||
|
||||
This way we can modify the extraOptions field (and others) entry through the
|
||||
transforms that come later. The metrics aren't formatted until the end of the
|
||||
transforms.
|
||||
"""
|
||||
for job in jobs:
|
||||
if job.get("perftest-metrics") is None:
|
||||
yield job
|
||||
continue
|
||||
perftest_metrics = job.pop("perftest-metrics")
|
||||
|
||||
# If perftest metrics is a string, split it up first
|
||||
if isinstance(perftest_metrics, list):
|
||||
new_metrics_info = [
|
||||
{"name": metric}
|
||||
for metric in perftest_metrics
|
||||
]
|
||||
else:
|
||||
new_metrics_info = []
|
||||
for metric, options in perftest_metrics.items():
|
||||
entry = {"name": metric}
|
||||
entry.update(options)
|
||||
new_metrics_info.append(entry)
|
||||
|
||||
job["perftest-metrics"] = new_metrics_info
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def split_perftest_variants(config, jobs):
|
||||
for job in jobs:
|
||||
if job.get("variants") is None:
|
||||
yield job
|
||||
continue
|
||||
|
||||
for variant in job.pop("variants"):
|
||||
job_new = deepcopy(job)
|
||||
|
||||
group, symbol = split_symbol(job_new["treeherder"]["symbol"])
|
||||
group += "-" + variant
|
||||
job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
|
||||
job_new["name"] += "-" + variant
|
||||
job_new.setdefault("perftest-perfherder-global", {}).setdefault(
|
||||
"extraOptions", []
|
||||
).append(variant)
|
||||
job_new[variant] = True
|
||||
|
||||
yield job_new
|
||||
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def split_btime_variants(config, jobs):
|
||||
for job in jobs:
|
||||
if job.get("perftest-btime-variants") is None:
|
||||
yield job
|
||||
continue
|
||||
|
||||
variants = job.pop("perftest-btime-variants")
|
||||
if not variants:
|
||||
yield job
|
||||
continue
|
||||
|
||||
yield_existing = False
|
||||
for suffix, options in variants:
|
||||
if suffix is None:
|
||||
# Append options to the existing job
|
||||
job.setdefault("perftest-btime-variants", []).append(options)
|
||||
yield_existing = True
|
||||
else:
|
||||
job_new = deepcopy(job)
|
||||
group, symbol = split_symbol(job_new["treeherder"]["symbol"])
|
||||
symbol += "-" + suffix
|
||||
job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
|
||||
job_new["name"] += "-" + suffix
|
||||
job_new.setdefault("perftest-perfherder-global", {}).setdefault(
|
||||
"extraOptions", []
|
||||
).append(suffix)
|
||||
# Replace the existing options with the new ones
|
||||
job_new["perftest-btime-variants"] = [options]
|
||||
yield job_new
|
||||
|
||||
# The existing job has been modified so we should also return it
|
||||
if yield_existing:
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def setup_http3_tests(config, jobs):
|
||||
for job in jobs:
|
||||
if job.get("http3") is None or not job.pop("http3"):
|
||||
yield job
|
||||
continue
|
||||
job.setdefault("perftest-btime-variants", []).append(
|
||||
"firefox.preference=network.http.http3.enabled:true"
|
||||
)
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def setup_perftest_metrics(config, jobs):
|
||||
for job in jobs:
|
||||
if job.get("perftest-metrics") is None:
|
||||
yield job
|
||||
continue
|
||||
perftest_metrics = job.pop("perftest-metrics")
|
||||
|
||||
# Options to apply to each metric
|
||||
global_options = job.pop("perftest-perfherder-global", {})
|
||||
for metric_info in perftest_metrics:
|
||||
for opt, val in global_options.items():
|
||||
if isinstance(val, list) and opt in metric_info:
|
||||
metric_info[opt].extend(val)
|
||||
elif not (isinstance(val, list) and len(val) == 0):
|
||||
metric_info[opt] = val
|
||||
|
||||
job["run"]["command"] = job["run"]["command"].replace(
|
||||
"{perftest_metrics}",
|
||||
" ".join([
|
||||
",".join([
|
||||
":".join([option, str(value).replace(" ", "")])
|
||||
for option, value in metric_info.items()
|
||||
])
|
||||
for metric_info in perftest_metrics
|
||||
])
|
||||
)
|
||||
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def setup_perftest_browsertime_variants(config, jobs):
|
||||
for job in jobs:
|
||||
if job.get("perftest-btime-variants") is None:
|
||||
yield job
|
||||
continue
|
||||
|
||||
job["run"]["command"] += " --browsertime-extra-options %s" % \
|
||||
",".join([opt.strip() for opt in job.pop("perftest-btime-variants")])
|
||||
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def setup_perftest_extra_options(config, jobs):
|
||||
for job in jobs:
|
||||
if job.get("perftest-extra-options") is None:
|
||||
yield job
|
||||
continue
|
||||
job["run"]["command"] += " " + " ".join(job.pop("perftest-extra-options"))
|
||||
yield job
|
||||
|
||||
|
||||
@transforms.add
|
||||
def pass_perftest_options(config, jobs):
|
||||
for job in jobs:
|
||||
env = job.setdefault('worker', {}).setdefault('env', {})
|
||||
env['PERFTEST_OPTIONS'] = six.ensure_text(
|
||||
env['PERFTEST_OPTIONS'] = ensure_text(
|
||||
json.dumps(config.params["try_task_config"].get('perftest-options'))
|
||||
)
|
||||
yield job
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче