N806: variables in functions should be lowercase

This commit is contained in:
Yoann Schneider 2024-02-26 18:36:50 +01:00 коммит произвёл Sebastian Hengst
Родитель 3d812c391c
Коммит 642cff92bf
24 изменённых файлов: 293 добавлений и 291 удалений

Просмотреть файл

@ -53,8 +53,8 @@ def print_url_to_taskcluster(job_guid):
job_guid = job["job_guid"]
(decoded_task_id, _) = job_guid.split("/")
# As of slugid v2, slugid.encode() returns a string not bytestring under Python 3.
taskId = slugid.encode(uuid.UUID(decoded_task_id))
logger.info("https://firefox-ci-tc.services.mozilla.com/tasks/%s", taskId)
task_id = slugid.encode(uuid.UUID(decoded_task_id))
logger.info("https://firefox-ci-tc.services.mozilla.com/tasks/%s", task_id)
if __name__ == "__main__":
@ -95,13 +95,13 @@ if __name__ == "__main__":
th_instance_not_found.append(job)
else:
# You can use this value in a url with &selectedJob=
jobId = job["id"]
job_id = job["id"]
remove_some_attributes(job, production_job)
differences = DeepDiff(job, production_dict[job["job_guid"]])
if differences:
pprint.pprint(differences)
logger.info(jobId)
logger.info(job_id)
else:
# Delete jobs that don"t have any differences
del production_dict[job["job_guid"]]

Просмотреть файл

@ -39,6 +39,8 @@ select = [
"F",
# pyupgrade
"UP",
# pep-naming
"N806",
]
ignore = [

Просмотреть файл

@ -48,7 +48,7 @@ def test_bz_reopen_bugs(request, mock_bugzilla_reopen_request, client, test_job,
import json
EXPECTED_REOPEN_ATTEMPTS = {
expected_reopen_attempts = {
"https://thisisnotbugzilla.org/rest/bug/202": json.dumps(
{
"status": "REOPENED",
@ -68,4 +68,4 @@ def test_bz_reopen_bugs(request, mock_bugzilla_reopen_request, client, test_job,
}
),
}
assert reopened_bugs == EXPECTED_REOPEN_ATTEMPTS
assert reopened_bugs == expected_reopen_attempts

Просмотреть файл

@ -38,12 +38,12 @@ def transformed_pulse_jobs(sample_data, test_repository):
def mock_artifact(taskId, runId, artifactName):
# Mock artifact with empty body
baseUrl = (
base_url = (
"https://taskcluster.net/api/queue/v1/task/{taskId}/runs/{runId}/artifacts/{artifactName}"
)
responses.add(
responses.GET,
baseUrl.format(taskId=taskId, runId=runId, artifactName=artifactName),
base_url.format(taskId=taskId, runId=runId, artifactName=artifactName),
body="",
content_type="text/plain",
status=200,
@ -53,20 +53,20 @@ def mock_artifact(taskId, runId, artifactName):
@pytest.fixture
async def new_pulse_jobs(sample_data, test_repository, push_stored):
revision = push_stored[0]["revisions"][0]["revision"]
pulseMessages = copy.deepcopy(sample_data.taskcluster_pulse_messages)
pulse_messages = copy.deepcopy(sample_data.taskcluster_pulse_messages)
tasks = copy.deepcopy(sample_data.taskcluster_tasks)
jobs = []
# Over here we transform the Pulse messages into the intermediary taskcluster-treeherder
# generated messages
for message in list(pulseMessages.values()):
taskId = message["payload"]["status"]["taskId"]
task = tasks[taskId]
for message in list(pulse_messages.values()):
task_id = message["payload"]["status"]["taskId"]
task = tasks[task_id]
# If we pass task to handleMessage we won't hit the network
taskRuns = await handleMessage(message, task)
task_runs = await handleMessage(message, task)
# handleMessage returns [] when it is a task that is not meant for Treeherder
for run in reversed(taskRuns):
mock_artifact(taskId, run["retryId"], "public/logs/live_backing.log")
for run in reversed(task_runs):
mock_artifact(task_id, run["retryId"], "public/logs/live_backing.log")
run["origin"]["project"] = test_repository.name
run["origin"]["revision"] = revision
jobs.append(run)
@ -99,11 +99,11 @@ def test_new_job_transformation(new_pulse_jobs, new_transformed_jobs, failure_cl
job_guid = message["taskId"]
(decoded_task_id, _) = job_guid.split("/")
# As of slugid v2, slugid.encode() returns a string not bytestring under Python 3.
taskId = slugid.encode(uuid.UUID(decoded_task_id))
task_id = slugid.encode(uuid.UUID(decoded_task_id))
transformed_job = jl.process_job(message, "https://firefox-ci-tc.services.mozilla.com")
# Not all messages from Taskcluster will be processed
if transformed_job:
assert new_transformed_jobs[taskId] == transformed_job
assert new_transformed_jobs[task_id] == transformed_job
def test_ingest_pulse_jobs(

Просмотреть файл

@ -78,8 +78,8 @@ def test_smaller_than_bigger():
def test_extra_option_max_length():
with open(os.path.join("schemas", "performance-artifact.json")) as f:
PERFHERDER_SCHEMA = json.load(f)
assert 100 == _lookup_extra_options_max(PERFHERDER_SCHEMA)
perfherder_schema = json.load(f)
assert 100 == _lookup_extra_options_max(perfherder_schema)
def test_validate_perf_schema_no_exception():

Просмотреть файл

@ -190,20 +190,20 @@ def test_import(mock_bugscache_bugzilla_request):
assert bug.dupe_of == 1662628
# key: open bug, values: duplicates
EXPECTED_BUG_DUPE_OF_DATA = {
expected_bug_dupe_of_data = {
1392106: [1442991, 1443801],
1411358: [1204281],
1662628: [1652208, 1660324, 1660719, 1660765, 1663081, 1663118, 1702255],
1736534: [],
}
for open_bug, duplicates in EXPECTED_BUG_DUPE_OF_DATA.items():
for open_bug, duplicates in expected_bug_dupe_of_data.items():
assert Bugscache.objects.get(id=open_bug).dupe_of is None
assert set(Bugscache.objects.filter(dupe_of=open_bug).values_list("id", flat=True)) == set(
duplicates
)
EXPECTED_BUG_COUNT = sum(
[1 + len(duplicates) for duplicates in EXPECTED_BUG_DUPE_OF_DATA.values()]
expected_bug_count = sum(
[1 + len(duplicates) for duplicates in expected_bug_dupe_of_data.values()]
)
assert len(Bugscache.objects.all()) == EXPECTED_BUG_COUNT
assert len(Bugscache.objects.all()) == expected_bug_count

Просмотреть файл

@ -40,7 +40,7 @@ def test_data_ingestion(setup_repository_data, mock_file_bugzilla_map_request):
import_process.run()
assert FilesBugzillaMap.objects.count() == 7
EXPECTED_FILES_BUGZILLA_DATA_IMPORT_1 = [
expected_files_bugzilla_data_import_1 = [
("AUTHORS", "AUTHORS", "mozilla.org", "Licensing"),
("browser/components/BrowserGlue.jsm", "BrowserGlue.jsm", "Firefox", "General"),
(
@ -74,7 +74,7 @@ def test_data_ingestion(setup_repository_data, mock_file_bugzilla_map_request):
"File first seen on mozilla-beta",
),
]
assert EXPECTED_FILES_BUGZILLA_DATA_IMPORT_1 == list(
assert expected_files_bugzilla_data_import_1 == list(
FilesBugzillaMap.objects.all()
.values_list(
"path", "file_name", "bugzilla_component__product", "bugzilla_component__component"
@ -82,7 +82,7 @@ def test_data_ingestion(setup_repository_data, mock_file_bugzilla_map_request):
.order_by("path")
)
EXPECTED_BUGZILLA_COMPONENTS_IMPORT_1 = [
expected_bugzilla_components_import_1 = [
("Core", "Storage: IndexedDB"),
("Firefox", "General"),
("Mock", "File first seen on mozilla-beta"),
@ -91,7 +91,7 @@ def test_data_ingestion(setup_repository_data, mock_file_bugzilla_map_request):
("mozilla.org", "Different path, same product, different component"),
("mozilla.org", "Licensing"),
]
assert EXPECTED_BUGZILLA_COMPONENTS_IMPORT_1 == sorted(
assert expected_bugzilla_components_import_1 == sorted(
list(
BugzillaComponent.objects.all()
.values_list("product", "component")
@ -103,7 +103,7 @@ def test_data_ingestion(setup_repository_data, mock_file_bugzilla_map_request):
import_process.run()
assert FilesBugzillaMap.objects.count() == 6
EXPECTED_FILES_BUGZILLA_DATA_IMPORT_2 = [
expected_files_bugzilla_data_import_2 = [
("AUTHORS", "AUTHORS", "mozilla.org", "Import 2: same product, different component"),
("browser/components/BrowserGlue.jsm", "BrowserGlue.jsm", "Firefox", "General"),
(
@ -131,7 +131,7 @@ def test_data_ingestion(setup_repository_data, mock_file_bugzilla_map_request):
"File first seen on mozilla-beta",
),
]
assert EXPECTED_FILES_BUGZILLA_DATA_IMPORT_2 == sorted(
assert expected_files_bugzilla_data_import_2 == sorted(
list(
FilesBugzillaMap.objects.all()
.values_list(
@ -141,7 +141,7 @@ def test_data_ingestion(setup_repository_data, mock_file_bugzilla_map_request):
)
)
EXPECTED_BUGZILLA_COMPONENTS_IMPORT_2 = [
expected_bugzilla_components_import_2 = [
("Core", "Storage: IndexedDB"),
("Core", "Storage: IndexedDB2"),
("Firefox", "General"),
@ -149,7 +149,7 @@ def test_data_ingestion(setup_repository_data, mock_file_bugzilla_map_request):
("Testing", "web-platform-tests"),
("mozilla.org", "Import 2: same product, different component"),
]
assert EXPECTED_BUGZILLA_COMPONENTS_IMPORT_2 == sorted(
assert expected_bugzilla_components_import_2 == sorted(
list(
BugzillaComponent.objects.all()
.values_list("product", "component")

Просмотреть файл

@ -132,42 +132,42 @@ def test_formula_demands_at_least_framework_and_suite(FormulaClass, betamax_reco
def test_breakdown_updates_between_calculations(FormulaClass, betamax_recorder):
formula = FormulaClass(betamax_recorder.session)
test_moniker_A = ("build_metrics", "build times")
test_moniker_B = ("talos", "tp5n", "nonmain_startup_fileio")
test_moniker_a = ("build_metrics", "build times")
test_moniker_b = ("talos", "tp5n", "nonmain_startup_fileio")
cassette_preffix_A = "-".join(filter(None, test_moniker_A))
cassette_preffix_B = "-".join(filter(None, test_moniker_B))
cassette_preffix_a = "-".join(filter(None, test_moniker_a))
cassette_preffix_b = "-".join(filter(None, test_moniker_b))
with betamax_recorder.use_cassette(f"{cassette_preffix_A}", serialize_with="prettyjson"):
formula(*test_moniker_A) # let it perform calculus & cache breakdown
breakdown_A = formula.breakdown()
with betamax_recorder.use_cassette(f"{cassette_preffix_a}", serialize_with="prettyjson"):
formula(*test_moniker_a) # let it perform calculus & cache breakdown
breakdown_a = formula.breakdown()
with betamax_recorder.use_cassette(f"{cassette_preffix_B}", serialize_with="prettyjson"):
formula(*test_moniker_B) # let it perform calculus & cache breakdown
breakdown_B = formula.breakdown()
with betamax_recorder.use_cassette(f"{cassette_preffix_b}", serialize_with="prettyjson"):
formula(*test_moniker_b) # let it perform calculus & cache breakdown
breakdown_b = formula.breakdown()
assert breakdown_A != breakdown_B
assert breakdown_a != breakdown_b
@pytest.mark.parametrize("FormulaClass", concrete_formula_classes())
def test_breakdown_resets_to_null_when_calculus_errors_out(FormulaClass, betamax_recorder):
formula = FormulaClass(betamax_recorder.session)
test_moniker_A = ("build_metrics", "build times")
test_moniker_B = ("nonexistent_framework", "nonexistent_suite")
test_moniker_a = ("build_metrics", "build times")
test_moniker_b = ("nonexistent_framework", "nonexistent_suite")
cassette_preffix_A = "-".join(filter(None, test_moniker_A))
cassette_preffix_B = "-".join(filter(None, test_moniker_B))
cassette_preffix_a = "-".join(filter(None, test_moniker_a))
cassette_preffix_b = "-".join(filter(None, test_moniker_b))
# run happy path calculus
with betamax_recorder.use_cassette(f"{cassette_preffix_A}", serialize_with="prettyjson"):
formula(*test_moniker_A) # let it perform calculus & cache breakdown
with betamax_recorder.use_cassette(f"{cassette_preffix_a}", serialize_with="prettyjson"):
formula(*test_moniker_a) # let it perform calculus & cache breakdown
_ = formula.breakdown()
# now run alternated path calculus
with betamax_recorder.use_cassette(f"{cassette_preffix_B}", serialize_with="prettyjson"):
with betamax_recorder.use_cassette(f"{cassette_preffix_b}", serialize_with="prettyjson"):
with pytest.raises(NoFiledBugs):
formula(*test_moniker_B) # intentionally blows up while doing calculus
formula(*test_moniker_b) # intentionally blows up while doing calculus
# cached breakdown got invalidated & can no longer be obtained
with pytest.raises(RuntimeError):

Просмотреть файл

@ -83,22 +83,22 @@ def test_detect_alerts_in_series(
mock_deviance,
):
base_time = time.time() # generate it based off current time
INTERVAL = 30
interval = 30
_generate_performance_data(
test_repository,
test_perf_signature,
base_time,
1,
0.5,
int(INTERVAL / 2),
int(interval / 2),
)
_generate_performance_data(
test_repository,
test_perf_signature,
base_time,
int(INTERVAL / 2) + 1,
int(interval / 2) + 1,
1.0,
int(INTERVAL / 2),
int(interval / 2),
)
generate_new_alerts_in_series(test_perf_signature)
@ -107,8 +107,8 @@ def test_detect_alerts_in_series(
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(
1,
(INTERVAL / 2) + 1,
(INTERVAL / 2),
(interval / 2) + 1,
(interval / 2),
test_perf_signature,
0.5,
1.0,
@ -125,8 +125,8 @@ def test_detect_alerts_in_series(
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(
1,
(INTERVAL / 2) + 1,
(INTERVAL / 2),
(interval / 2) + 1,
(interval / 2),
test_perf_signature,
0.5,
1.0,
@ -142,9 +142,9 @@ def test_detect_alerts_in_series(
test_repository,
test_perf_signature,
base_time,
(INTERVAL + 1),
(interval + 1),
2.0,
INTERVAL,
interval,
)
generate_new_alerts_in_series(test_perf_signature)
@ -152,8 +152,8 @@ def test_detect_alerts_in_series(
assert PerformanceAlertSummary.objects.count() == 2
_verify_alert(
2,
INTERVAL + 1,
INTERVAL,
interval + 1,
interval,
test_perf_signature,
1.0,
2.0,
@ -232,22 +232,22 @@ def test_no_alerts_with_old_data(
test_perf_signature,
):
base_time = 0 # 1970, too old!
INTERVAL = 30
interval = 30
_generate_performance_data(
test_repository,
test_perf_signature,
base_time,
1,
0.5,
int(INTERVAL / 2),
int(interval / 2),
)
_generate_performance_data(
test_repository,
test_perf_signature,
base_time,
int(INTERVAL / 2) + 1,
int(interval / 2) + 1,
1.0,
int(INTERVAL / 2),
int(interval / 2),
)
generate_new_alerts_in_series(test_perf_signature)
@ -269,7 +269,7 @@ def test_custom_alert_threshold(
# under default settings, this set of data would generate
# 2 alerts, but we'll set an artificially high threshold
# of 200% that should only generate 1
INTERVAL = 60
interval = 60
base_time = time.time()
_generate_performance_data(
test_repository,
@ -277,23 +277,23 @@ def test_custom_alert_threshold(
base_time,
1,
0.5,
int(INTERVAL / 3),
int(interval / 3),
)
_generate_performance_data(
test_repository,
test_perf_signature,
base_time,
int(INTERVAL / 3) + 1,
int(interval / 3) + 1,
0.6,
int(INTERVAL / 3),
int(interval / 3),
)
_generate_performance_data(
test_repository,
test_perf_signature,
base_time,
2 * int(INTERVAL / 3) + 1,
2 * int(interval / 3) + 1,
2.0,
int(INTERVAL / 3),
int(interval / 3),
)
generate_new_alerts_in_series(test_perf_signature)
@ -319,22 +319,22 @@ def test_alert_change_type_absolute(
test_perf_signature.save()
base_time = time.time() # generate it based off current time
INTERVAL = 30
interval = 30
_generate_performance_data(
test_repository,
test_perf_signature,
base_time,
1,
0.5,
int(INTERVAL / 2),
int(interval / 2),
)
_generate_performance_data(
test_repository,
test_perf_signature,
base_time,
int(INTERVAL / 2) + 1,
int(interval / 2) + 1,
new_value,
int(INTERVAL / 2),
int(interval / 2),
)
generate_new_alerts_in_series(test_perf_signature)

Просмотреть файл

@ -123,10 +123,10 @@ def test_detect_changes_few_revisions_many_values():
def test_detect_changes_historical_data(filename, expected_timestamps):
"""Parse JSON produced by http://graphs.mozilla.org/api/test/runs"""
# Configuration for Analyzer
FORE_WINDOW = 12
MIN_BACK_WINDOW = 12
MAX_BACK_WINDOW = 24
THRESHOLD = 7
fore_window = 12
min_back_window = 12
max_back_window = 24
threshold = 7
payload = SampleData.get_perf_data(os.path.join("graphs", filename))
runs = payload["test_runs"]
@ -134,10 +134,10 @@ def test_detect_changes_historical_data(filename, expected_timestamps):
results = detect_changes(
data,
min_back_window=MIN_BACK_WINDOW,
max_back_window=MAX_BACK_WINDOW,
fore_window=FORE_WINDOW,
t_threshold=THRESHOLD,
min_back_window=min_back_window,
max_back_window=max_back_window,
fore_window=fore_window,
t_threshold=threshold,
)
regression_timestamps = [d.push_timestamp for d in results if d.change_detected]
assert regression_timestamps == expected_timestamps

Просмотреть файл

@ -18,37 +18,37 @@ def test_bugzilla_components_for_path(client, test_job):
bugzilla_component=BugzillaComponent.objects.last(),
)
URL_BASE = reverse("bugzilla-component-list")
url_base = reverse("bugzilla-component-list")
EXPECTED_MOCK1 = [{"product": "Mock Product 1", "component": "Mock Component 1"}]
expected_mock1 = [{"product": "Mock Product 1", "component": "Mock Component 1"}]
resp = client.get(URL_BASE + "?path=file_1.extension")
resp = client.get(url_base + "?path=file_1.extension")
assert resp.status_code == 200
assert resp.json() == EXPECTED_MOCK1
assert resp.json() == expected_mock1
resp = client.get(URL_BASE + "?path=file_2.extension")
resp = client.get(url_base + "?path=file_2.extension")
assert resp.json() == []
resp = client.get(URL_BASE + "?path=ile_2.extension")
resp = client.get(url_base + "?path=ile_2.extension")
assert resp.json() == []
resp = client.get(URL_BASE + "?path=file_1")
assert resp.json() == EXPECTED_MOCK1
resp = client.get(url_base + "?path=file_1")
assert resp.json() == expected_mock1
resp = client.get(URL_BASE + "?path=mock/folder/file_1.extension")
assert resp.json() == EXPECTED_MOCK1
resp = client.get(url_base + "?path=mock/folder/file_1.extension")
assert resp.json() == expected_mock1
resp = client.get(URL_BASE + "?path=other_mock/other_folder/file_1.extension")
resp = client.get(url_base + "?path=other_mock/other_folder/file_1.extension")
# Should also pass because search falls back to file name if no match for path.
assert resp.json() == EXPECTED_MOCK1
assert resp.json() == expected_mock1
resp = client.get(URL_BASE + "?path=folder/file_1.extension")
assert resp.json() == EXPECTED_MOCK1
resp = client.get(url_base + "?path=folder/file_1.extension")
assert resp.json() == expected_mock1
resp = client.get(URL_BASE + "?path=folder/file_1.other_extension")
assert resp.json() == EXPECTED_MOCK1
resp = client.get(url_base + "?path=folder/file_1.other_extension")
assert resp.json() == expected_mock1
resp = client.get(URL_BASE + "?path=completely.unrelated")
resp = client.get(url_base + "?path=completely.unrelated")
assert resp.json() == []
BugzillaComponent.objects.create(product="Mock Product 1", component="Mock Component 2")
@ -59,25 +59,25 @@ def test_bugzilla_components_for_path(client, test_job):
bugzilla_component=BugzillaComponent.objects.last(),
)
EXPECTED_MOCK2 = [{"product": "Mock Product 1", "component": "Mock Component 2"}]
expected_mock2 = [{"product": "Mock Product 1", "component": "Mock Component 2"}]
EXPECTED_MOCK1_MOCK2 = [
expected_mock1_mock2 = [
{"product": "Mock Product 1", "component": "Mock Component 1"},
{"product": "Mock Product 1", "component": "Mock Component 2"},
]
resp = client.get(URL_BASE + "?path=file_1.extension")
assert resp.json() == EXPECTED_MOCK1_MOCK2
resp = client.get(url_base + "?path=file_1.extension")
assert resp.json() == expected_mock1_mock2
resp = client.get(URL_BASE + "?path=mock/folder/file_1.extension")
assert resp.json() == EXPECTED_MOCK1
resp = client.get(url_base + "?path=mock/folder/file_1.extension")
assert resp.json() == expected_mock1
resp = client.get(URL_BASE + "?path=mock/folder_2/file_1.extension")
assert resp.json() == EXPECTED_MOCK2
resp = client.get(url_base + "?path=mock/folder_2/file_1.extension")
assert resp.json() == expected_mock2
resp = client.get(URL_BASE + "?path=other_mock/other_folder/file_1.extension")
resp = client.get(url_base + "?path=other_mock/other_folder/file_1.extension")
# Should also pass because search falls back to file name if no match for path.
assert resp.json() == EXPECTED_MOCK1_MOCK2
assert resp.json() == expected_mock1_mock2
BugzillaComponent.objects.create(product="Mock Product 3", component="Mock Component 3")
@ -87,16 +87,16 @@ def test_bugzilla_components_for_path(client, test_job):
bugzilla_component=BugzillaComponent.objects.last(),
)
EXPECTED_MOCK3 = [{"product": "Mock Product 3", "component": "Mock Component 3"}]
expected_mock3 = [{"product": "Mock Product 3", "component": "Mock Component 3"}]
resp = client.get(URL_BASE + "?path=other.file.js")
assert resp.json() == EXPECTED_MOCK3
resp = client.get(url_base + "?path=other.file.js")
assert resp.json() == expected_mock3
resp = client.get(URL_BASE + "?path=other.file")
assert resp.json() == EXPECTED_MOCK3
resp = client.get(url_base + "?path=other.file")
assert resp.json() == expected_mock3
resp = client.get(URL_BASE + "?path=other")
assert resp.json() == EXPECTED_MOCK3
resp = client.get(url_base + "?path=other")
assert resp.json() == expected_mock3
BugzillaComponent.objects.create(product="Mock Product 4", component="Mock Component 4")
@ -106,23 +106,23 @@ def test_bugzilla_components_for_path(client, test_job):
bugzilla_component=BugzillaComponent.objects.last(),
)
EXPECTED_MOCK4 = [{"product": "Mock Product 4", "component": "Mock Component 4"}]
expected_mock4 = [{"product": "Mock Product 4", "component": "Mock Component 4"}]
EXPECTED_MOCK3_MOCK4 = [
expected_mock3_mock4 = [
{"product": "Mock Product 3", "component": "Mock Component 3"},
{"product": "Mock Product 4", "component": "Mock Component 4"},
]
resp = client.get(URL_BASE + "?path=other.file.js")
assert resp.json() == EXPECTED_MOCK3
resp = client.get(url_base + "?path=other.file.js")
assert resp.json() == expected_mock3
resp = client.get(URL_BASE + "?path=other.extension")
assert resp.json() == EXPECTED_MOCK4
resp = client.get(url_base + "?path=other.extension")
assert resp.json() == expected_mock4
resp = client.get(URL_BASE + "?path=other")
assert resp.json() == EXPECTED_MOCK3_MOCK4
resp = client.get(url_base + "?path=other")
assert resp.json() == expected_mock3_mock4
resp = client.get(URL_BASE + "?path=another")
resp = client.get(url_base + "?path=another")
assert resp.json() == []
BugzillaComponent.objects.create(
@ -166,19 +166,19 @@ def test_bugzilla_components_for_path(client, test_job):
bugzilla_component=BugzillaComponent.objects.last(),
)
EXPECTED_MOCK_ORG_MOZILLA = [
expected_mock_org_mozilla = [
{
"product": "Mock Product org.mozilla.*.<TestName>",
"component": "Mock Component File Match",
}
]
resp = client.get(URL_BASE + "?path=org.mozilla.geckoview.test.MockTestName#Subtest")
assert resp.json() == EXPECTED_MOCK_ORG_MOZILLA
resp = client.get(url_base + "?path=org.mozilla.geckoview.test.MockTestName#Subtest")
assert resp.json() == expected_mock_org_mozilla
# Only take test name into account.
resp = client.get(URL_BASE + "?path=org.mozilla.otherproduct.otherfolder.MockTestName")
assert resp.json() == EXPECTED_MOCK_ORG_MOZILLA
resp = client.get(url_base + "?path=org.mozilla.otherproduct.otherfolder.MockTestName")
assert resp.json() == expected_mock_org_mozilla
BugzillaComponent.objects.create(product="Testing", component="Mochitest")
@ -189,5 +189,5 @@ def test_bugzilla_components_for_path(client, test_job):
)
# Respect the ignore list of product and component combinations.
resp = client.get(URL_BASE + "?path=mock/mochitest/mochitest.test")
resp = client.get(url_base + "?path=mock/mochitest/mochitest.test")
assert resp.json() == []

Просмотреть файл

@ -27,9 +27,9 @@ def reopen_intermittent_bugs():
)
# Intermittent bugs get closed after 3 weeks of inactivity if other conditions don't apply:
# https://github.com/mozilla/relman-auto-nag/blob/c7439e247677333c1cd8c435234b3ef3adc49680/auto_nag/scripts/close_intermittents.py#L17
RECENT_DAYS = 7
recent_days = 7
recently_used_bugs = set(
BugJobMap.objects.filter(created__gt=datetime.now() - timedelta(RECENT_DAYS)).values_list(
BugJobMap.objects.filter(created__gt=datetime.now() - timedelta(recent_days)).values_list(
"bug_id", flat=True
)
)

Просмотреть файл

@ -118,8 +118,8 @@ async def ingest_task(taskId, root_url):
# Remove default timeout limit of 5 minutes
timeout = aiohttp.ClientTimeout(total=0)
async with taskcluster.aio.createSession(connector=conn, timeout=timeout) as session:
asyncQueue = taskcluster.aio.Queue({"rootUrl": root_url}, session=session)
results = await asyncio.gather(asyncQueue.status(taskId), asyncQueue.task(taskId))
async_queue = taskcluster.aio.Queue({"rootUrl": root_url}, session=session)
results = await asyncio.gather(async_queue.status(taskId), async_queue.task(taskId))
await handleTask(
{
"status": results[0]["status"],
@ -130,7 +130,7 @@ async def ingest_task(taskId, root_url):
async def handleTask(task, root_url):
taskId = task["status"]["taskId"]
task_id = task["status"]["taskId"]
runs = task["status"]["runs"]
# If we iterate in order of the runs, we will not be able to mark older runs as
# "retry" instead of exception
@ -139,7 +139,7 @@ async def handleTask(task, root_url):
"exchange": stateToExchange[run["state"]],
"payload": {
"status": {
"taskId": taskId,
"taskId": task_id,
"runs": runs,
},
"runId": run["runId"],
@ -148,35 +148,35 @@ async def handleTask(task, root_url):
}
try:
taskRuns = await handleMessage(message, task["task"])
task_runs = await handleMessage(message, task["task"])
except Exception as e:
logger.exception(e)
if taskRuns:
if task_runs:
# Schedule and run jobs inside the thread pool executor
jobFutures = [
routine_to_future(process_job_with_threads, run, root_url) for run in taskRuns
job_futures = [
routine_to_future(process_job_with_threads, run, root_url) for run in task_runs
]
await await_futures(jobFutures)
await await_futures(job_futures)
async def fetchGroupTasks(taskGroupId, root_url):
tasks = []
query = {}
continuationToken = ""
continuation_token = ""
# Limiting the connection pool just in case we have too many
conn = aiohttp.TCPConnector(limit=10)
# Remove default timeout limit of 5 minutes
timeout = aiohttp.ClientTimeout(total=0)
async with taskcluster.aio.createSession(connector=conn, timeout=timeout) as session:
asyncQueue = taskcluster.aio.Queue({"rootUrl": root_url}, session=session)
async_queue = taskcluster.aio.Queue({"rootUrl": root_url}, session=session)
while True:
if continuationToken:
query = {"continuationToken": continuationToken}
response = await asyncQueue.listTaskGroup(taskGroupId, query=query)
if continuation_token:
query = {"continuationToken": continuation_token}
response = await async_queue.listTaskGroup(taskGroupId, query=query)
tasks.extend(response["tasks"])
continuationToken = response.get("continuationToken")
if continuationToken is None:
continuation_token = response.get("continuationToken")
if continuation_token is None:
break
logger.info("Requesting more tasks. %s tasks so far...", len(tasks))
return tasks
@ -193,8 +193,8 @@ async def processTasks(taskGroupId, root_url):
return
# Schedule and run tasks inside the thread pool executor
taskFutures = [routine_to_future(handleTask, task, root_url) for task in tasks]
await await_futures(taskFutures)
task_futures = [routine_to_future(handleTask, task, root_url) for task in tasks]
await await_futures(task_futures)
async def routine_to_future(func, *args):
@ -249,12 +249,12 @@ def get_decision_task_id(project, revision, root_url):
def repo_meta(project):
_repo = Repository.objects.filter(name=project)[0]
assert _repo, f"The project {project} you specified is incorrect"
splitUrl = _repo.url.split("/")
split_url = _repo.url.split("/")
return {
"url": _repo.url,
"branch": _repo.branch,
"owner": splitUrl[3],
"repo": splitUrl[4],
"owner": split_url[3],
"repo": split_url[4],
"tc_root_url": _repo.tc_root_url,
}
@ -270,16 +270,16 @@ def query_data(repo_meta, commit):
event_base_sha = repo_meta["branch"]
# First we try with `master` being the base sha
# e.g. https://api.github.com/repos/servo/servo/compare/master...1418c0555ff77e5a3d6cf0c6020ba92ece36be2e
compareResponse = github.compare_shas(
compare_response = github.compare_shas(
repo_meta["owner"], repo_meta["repo"], repo_meta["branch"], commit
)
merge_base_commit = compareResponse.get("merge_base_commit")
merge_base_commit = compare_response.get("merge_base_commit")
if merge_base_commit:
commiter_date = merge_base_commit["commit"]["committer"]["date"]
# Since we don't use PushEvents that contain the "before" or "event.base.sha" fields [1]
# we need to discover the right parent which existed in the base branch.
# [1] https://github.com/taskcluster/taskcluster/blob/3dda0adf85619d18c5dcf255259f3e274d2be346/services/github/src/api.js#L55
parents = compareResponse["merge_base_commit"]["parents"]
parents = compare_response["merge_base_commit"]["parents"]
if len(parents) == 1:
parent = parents[0]
commit_info = fetch_json(parent["url"])
@ -301,12 +301,12 @@ def query_data(repo_meta, commit):
assert event_base_sha != repo_meta["branch"]
logger.info("We have a new base: %s", event_base_sha)
# When using the correct event_base_sha the "commits" field will be correct
compareResponse = github.compare_shas(
compare_response = github.compare_shas(
repo_meta["owner"], repo_meta["repo"], event_base_sha, commit
)
commits = []
for _commit in compareResponse["commits"]:
for _commit in compare_response["commits"]:
commits.append(
{
"message": _commit["commit"]["message"],
@ -453,7 +453,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
loop = asyncio.get_event_loop()
typeOfIngestion = options["ingestion_type"][0]
type_of_ingestion = options["ingestion_type"][0]
root_url = options["root_url"]
if not options["enable_eager_celery"]:
@ -462,22 +462,22 @@ class Command(BaseCommand):
# Make sure all tasks are run synchronously / immediately
settings.CELERY_TASK_ALWAYS_EAGER = True
if typeOfIngestion == "task":
if type_of_ingestion == "task":
assert options["taskId"]
loop.run_until_complete(ingest_task(options["taskId"], root_url))
elif typeOfIngestion == "prUrl":
elif type_of_ingestion == "prUrl":
assert options["prUrl"]
ingest_pr(options["prUrl"], root_url)
elif typeOfIngestion.find("git") > -1:
elif type_of_ingestion.find("git") > -1:
if not os.environ.get("GITHUB_TOKEN"):
logger.warning(
"If you don't set up GITHUB_TOKEN you might hit Github's rate limiting. See docs for info."
)
if typeOfIngestion == "git-push":
if type_of_ingestion == "git-push":
ingest_push(options["project"], options["commit"])
elif typeOfIngestion == "git-pushes":
elif type_of_ingestion == "git-pushes":
ingest_git_pushes(options["project"], options["dryRun"])
elif typeOfIngestion == "push":
elif type_of_ingestion == "push":
ingest_hg_push(options)
else:
raise Exception("Please check the code for valid ingestion types.")

Просмотреть файл

@ -38,17 +38,17 @@ def stateFromRun(jobRun):
def resultFromRun(jobRun):
RUN_TO_RESULT = {
run_to_result = {
"completed": "success",
"failed": "fail",
}
state = jobRun["state"]
if state in list(RUN_TO_RESULT.keys()):
return RUN_TO_RESULT[state]
if state in list(run_to_result.keys()):
return run_to_result[state]
elif state == "exception":
reasonResolved = jobRun.get("reasonResolved")
if reasonResolved in ["canceled", "superseded"]:
return reasonResolved
reason_resolved = jobRun.get("reasonResolved")
if reason_resolved in ["canceled", "superseded"]:
return reason_resolved
return "exception"
else:
return "unknown"
@ -57,12 +57,12 @@ def resultFromRun(jobRun):
# Creates a log entry for Treeherder to retrieve and parse. This log is
# displayed on the Treeherder Log Viewer once parsed.
def createLogReference(root_url, taskId, runId):
logUrl = taskcluster_urls.api(
log_url = taskcluster_urls.api(
root_url, "queue", "v1", "task/{taskId}/runs/{runId}/artifacts/public/logs/live_backing.log"
).format(taskId=taskId, runId=runId)
return {
"name": "live_backing_log",
"url": logUrl,
"url": log_url,
}
@ -71,27 +71,27 @@ def createLogReference(root_url, taskId, runId):
# Treeherder job message.
# TODO: Refactor https://bugzilla.mozilla.org/show_bug.cgi?id=1560596
def parseRouteInfo(prefix, taskId, routes, task):
matchingRoutes = list(filter(lambda route: route.split(".")[0] == "tc-treeherder", routes))
matching_routes = list(filter(lambda route: route.split(".")[0] == "tc-treeherder", routes))
if len(matchingRoutes) != 1:
if len(matching_routes) != 1:
raise PulseHandlerError(
"Could not determine Treeherder route. Either there is no route, "
+ "or more than one matching route exists."
+ f"Task ID: {taskId} Routes: {routes}"
)
parsedRoute = parseRoute(matchingRoutes[0])
parsed_route = parseRoute(matching_routes[0])
return parsedRoute
return parsed_route
def validateTask(task):
treeherderMetadata = task.get("extra", {}).get("treeherder")
if not treeherderMetadata:
treeherder_metadata = task.get("extra", {}).get("treeherder")
if not treeherder_metadata:
logger.debug("Task metadata is missing Treeherder job configuration.")
return False
try:
jsonschema.validate(treeherderMetadata, get_json_schema("task-treeherder-config.yml"))
jsonschema.validate(treeherder_metadata, get_json_schema("task-treeherder-config.yml"))
except (jsonschema.ValidationError, jsonschema.SchemaError) as e:
logger.error("JSON Schema validation error during Taskcluser message ingestion: %s", e)
return False
@ -169,26 +169,26 @@ def ignore_task(task, taskId, rootUrl, project):
async def handleMessage(message, taskDefinition=None):
async with taskcluster.aio.createSession() as session:
jobs = []
taskId = message["payload"]["status"]["taskId"]
asyncQueue = taskcluster.aio.Queue({"rootUrl": message["root_url"]}, session=session)
task = (await asyncQueue.task(taskId)) if not taskDefinition else taskDefinition
task_id = message["payload"]["status"]["taskId"]
async_queue = taskcluster.aio.Queue({"rootUrl": message["root_url"]}, session=session)
task = (await async_queue.task(task_id)) if not taskDefinition else taskDefinition
try:
parsedRoute = parseRouteInfo("tc-treeherder", taskId, task["routes"], task)
parsed_route = parseRouteInfo("tc-treeherder", task_id, task["routes"], task)
except PulseHandlerError as e:
logger.debug("%s", str(e))
return jobs
if ignore_task(task, taskId, message["root_url"], parsedRoute["project"]):
if ignore_task(task, task_id, message["root_url"], parsed_route["project"]):
return jobs
logger.debug("Message received for task %s", taskId)
logger.debug("Message received for task %s", task_id)
# Validation failures are common and logged, so do nothing more.
if not validateTask(task):
return jobs
taskType = EXCHANGE_EVENT_MAP.get(message["exchange"])
task_type = EXCHANGE_EVENT_MAP.get(message["exchange"])
# Originally this code was only within the "pending" case, however, in order to support
# ingesting all tasks at once which might not have "pending" case
@ -196,18 +196,18 @@ async def handleMessage(message, taskDefinition=None):
# This will only work if the previous run has not yet been processed by Treeherder
# since _remove_existing_jobs() will prevent it
if message["payload"]["runId"] > 0:
jobs.append(await handleTaskRerun(parsedRoute, task, message, session))
jobs.append(await handleTaskRerun(parsed_route, task, message, session))
if not taskType:
if not task_type:
raise Exception("Unknown exchange: {exchange}".format(exchange=message["exchange"]))
elif taskType == "pending":
jobs.append(handleTaskPending(parsedRoute, task, message))
elif taskType == "running":
jobs.append(handleTaskRunning(parsedRoute, task, message))
elif taskType in ("completed", "failed"):
jobs.append(await handleTaskCompleted(parsedRoute, task, message, session))
elif taskType == "exception":
jobs.append(await handleTaskException(parsedRoute, task, message, session))
elif task_type == "pending":
jobs.append(handleTaskPending(parsed_route, task, message))
elif task_type == "running":
jobs.append(handleTaskRunning(parsed_route, task, message))
elif task_type in ("completed", "failed"):
jobs.append(await handleTaskCompleted(parsed_route, task, message, session))
elif task_type == "exception":
jobs.append(await handleTaskException(parsed_route, task, message, session))
return jobs
@ -218,30 +218,30 @@ async def handleMessage(message, taskDefinition=None):
# Specific handlers for each message type will add/remove information necessary
# for the type of task event..
def buildMessage(pushInfo, task, runId, payload):
taskId = payload["status"]["taskId"]
jobRun = payload["status"]["runs"][runId]
treeherderConfig = task["extra"]["treeherder"]
task_id = payload["status"]["taskId"]
job_run = payload["status"]["runs"][runId]
treeherder_config = task["extra"]["treeherder"]
job = {
"buildSystem": "taskcluster",
"owner": task["metadata"]["owner"],
"taskId": f"{slugid.decode(taskId)}/{runId}",
"taskId": f"{slugid.decode(task_id)}/{runId}",
"retryId": runId,
"isRetried": False,
"display": {
# jobSymbols could be an integer (i.e. Chunk ID) but need to be strings
# for treeherder
"jobSymbol": str(treeherderConfig["symbol"]),
"groupSymbol": treeherderConfig.get("groupSymbol", "?"),
"jobSymbol": str(treeherder_config["symbol"]),
"groupSymbol": treeherder_config.get("groupSymbol", "?"),
# Maximum job name length is 140 chars...
"jobName": task["metadata"]["name"][0:139],
},
"state": stateFromRun(jobRun),
"result": resultFromRun(jobRun),
"tier": treeherderConfig.get("tier", 1),
"state": stateFromRun(job_run),
"result": resultFromRun(job_run),
"tier": treeherder_config.get("tier", 1),
"timeScheduled": task["created"],
"jobKind": treeherderConfig.get("jobKind", "other"),
"reason": treeherderConfig.get("reason", "scheduled"),
"jobKind": treeherder_config.get("jobKind", "other"),
"reason": treeherder_config.get("reason", "scheduled"),
"jobInfo": {
"links": [],
"summary": task["metadata"]["description"],
@ -263,28 +263,28 @@ def buildMessage(pushInfo, task, runId, payload):
# Transform "collection" into an array of labels if task doesn't
# define "labels".
labels = treeherderConfig.get("labels", [])
labels = treeherder_config.get("labels", [])
if not labels:
if not treeherderConfig.get("collection"):
if not treeherder_config.get("collection"):
labels = ["opt"]
else:
labels = list(treeherderConfig["collection"].keys())
labels = list(treeherder_config["collection"].keys())
job["labels"] = labels
machine = treeherderConfig.get("machine", {})
machine = treeherder_config.get("machine", {})
job["buildMachine"] = {
"name": jobRun.get("workerId", "unknown"),
"name": job_run.get("workerId", "unknown"),
"platform": machine.get("platform", task["workerType"]),
"os": machine.get("os", "-"),
"architecture": machine.get("architecture", "-"),
}
if treeherderConfig.get("productName"):
job["productName"] = treeherderConfig["productName"]
if treeherder_config.get("productName"):
job["productName"] = treeherder_config["productName"]
if treeherderConfig.get("groupName"):
job["display"]["groupName"] = treeherderConfig["groupName"]
if treeherder_config.get("groupName"):
job["display"]["groupName"] = treeherder_config["groupName"]
return job
@ -318,13 +318,13 @@ def handleTaskRunning(pushInfo, task, message):
async def handleTaskCompleted(pushInfo, task, message, session):
payload = message["payload"]
jobRun = payload["status"]["runs"][payload["runId"]]
job_run = payload["status"]["runs"][payload["runId"]]
job = buildMessage(pushInfo, task, payload["runId"], payload)
job["timeStarted"] = jobRun["started"]
job["timeCompleted"] = jobRun["resolved"]
job["timeStarted"] = job_run["started"]
job["timeCompleted"] = job_run["resolved"]
job["logs"] = [
createLogReference(message["root_url"], payload["status"]["taskId"], jobRun["runId"]),
createLogReference(message["root_url"], payload["status"]["taskId"], job_run["runId"]),
]
job = await addArtifactUploadedLinks(
message["root_url"], payload["status"]["taskId"], payload["runId"], job, session
@ -334,17 +334,17 @@ async def handleTaskCompleted(pushInfo, task, message, session):
async def handleTaskException(pushInfo, task, message, session):
payload = message["payload"]
jobRun = payload["status"]["runs"][payload["runId"]]
job_run = payload["status"]["runs"][payload["runId"]]
# Do not report runs that were created as an exception. Such cases
# are deadline-exceeded
if jobRun["reasonCreated"] == "exception":
if job_run["reasonCreated"] == "exception":
return
job = buildMessage(pushInfo, task, payload["runId"], payload)
# Jobs that get cancelled before running don't have a started time
if jobRun.get("started"):
job["timeStarted"] = jobRun["started"]
job["timeCompleted"] = jobRun["resolved"]
if job_run.get("started"):
job["timeStarted"] = job_run["started"]
job["timeCompleted"] = job_run["resolved"]
# exceptions generally have no logs, so in the interest of not linking to a 404'ing artifact,
# don't include a link
job["logs"] = []
@ -355,21 +355,21 @@ async def handleTaskException(pushInfo, task, message, session):
async def fetchArtifacts(root_url, taskId, runId, session):
asyncQueue = taskcluster.aio.Queue({"rootUrl": root_url}, session=session)
res = await asyncQueue.listArtifacts(taskId, runId)
async_queue = taskcluster.aio.Queue({"rootUrl": root_url}, session=session)
res = await async_queue.listArtifacts(taskId, runId)
artifacts = res["artifacts"]
continuationToken = res.get("continuationToken")
while continuationToken is not None:
continuation_token = res.get("continuationToken")
while continuation_token is not None:
continuation = {"continuationToken": res["continuationToken"]}
try:
res = await asyncQueue.listArtifacts(taskId, runId, continuation)
res = await async_queue.listArtifacts(taskId, runId, continuation)
except Exception:
break
artifacts = artifacts.concat(res["artifacts"])
continuationToken = res.get("continuationToken")
continuation_token = res.get("continuationToken")
return artifacts

Просмотреть файл

@ -14,28 +14,28 @@
def parseRoute(route):
id = None
owner = None
parsedProject = None
parsedRoute = route.split(".")
project = parsedRoute[2]
parsed_project = None
parsed_route = route.split(".")
project = parsed_route[2]
if len(project.split("/")) == 2:
[owner, parsedProject] = project.split("/")
[owner, parsed_project] = project.split("/")
else:
parsedProject = project
parsed_project = project
if len(parsedRoute) == 5:
id = parsedRoute[4]
if len(parsed_route) == 5:
id = parsed_route[4]
pushInfo = {
"destination": parsedRoute[0],
push_info = {
"destination": parsed_route[0],
"id": int(id) if id else 0,
"project": parsedProject,
"revision": parsedRoute[3],
"project": parsed_project,
"revision": parsed_route[3],
}
if owner and parsedProject:
pushInfo["owner"] = owner
pushInfo["origin"] = "github.com"
if owner and parsed_project:
push_info["owner"] = owner
push_info["origin"] = "github.com"
else:
pushInfo["origin"] = "hg.mozilla.org"
push_info["origin"] = "hg.mozilla.org"
return pushInfo
return push_info

Просмотреть файл

@ -37,8 +37,8 @@ def add_headers_function(headers, path, url):
if report_uri not in CSP_DIRECTIVES:
CSP_DIRECTIVES.append(report_uri)
CSP_HEADER = "; ".join(CSP_DIRECTIVES)
headers["Content-Security-Policy"] = CSP_HEADER
csp_header = "; ".join(CSP_DIRECTIVES)
headers["Content-Security-Policy"] = csp_header
class CustomWhiteNoise(WhiteNoiseMiddleware):

Просмотреть файл

@ -48,8 +48,8 @@ def get_error_summary(job, queryset=None):
dates = list(line_cache.keys())
dates.sort()
for d in dates:
dTime = datetime.datetime.strptime(d, "%Y-%m-%d")
if dTime <= (job.submit_time - datetime.timedelta(days=LINE_CACHE_TIMEOUT_DAYS)):
date_time = datetime.datetime.strptime(d, "%Y-%m-%d")
if date_time <= (job.submit_time - datetime.timedelta(days=LINE_CACHE_TIMEOUT_DAYS)):
del line_cache[d]
else:
break

Просмотреть файл

@ -28,7 +28,7 @@ Type 'yes' to continue, or 'no' to cancel: """
# verbose, so let's do that programmatically
s = PerformanceSignature.objects.get(id=1)
PerformanceDatum.objects.filter(signature=s).delete()
INTERVAL = 30
interval = 30
now = time.time()
# create a push first as need a push_id
@ -40,8 +40,8 @@ Type 'yes' to continue, or 'no' to cancel: """
)
for t, v in zip(
[i for i in range(INTERVAL)],
([0.5 for i in range(int(INTERVAL / 2))] + [1.0 for i in range(int(INTERVAL / 2))]),
[i for i in range(interval)],
([0.5 for i in range(int(interval / 2))] + [1.0 for i in range(int(interval / 2))]),
):
PerformanceDatum.objects.create(
repository=s.repository,

Просмотреть файл

@ -10,14 +10,14 @@ def get_commit_history(repository, revision, push):
from mozci.push import Push as MozciPush
from mozci.errors import ParentPushNotFound
mozciPush = MozciPush([revision], repository.name)
mozci_push = MozciPush([revision], repository.name)
parent = None
parent_sha = None
parent_repo = None
parent_push = None
try:
parent = mozciPush.parent
parent = mozci_push.parent
except ParentPushNotFound:
pass

Просмотреть файл

@ -73,7 +73,7 @@ def get_history(
# For each failure item in ``tests``, we group all jobs of the exact same type into
# a field called `jobs`. So it has passed and failed jobs in there.
#
def get_current_test_failures(push, option_map, jobs, investigatedTests=None):
def get_current_test_failures(push, option_map, jobs, investigated_tests=None):
# Using .distinct(<fields>) here would help by removing duplicate FailureLines
# for the same job (with different sub-tests), but it's only supported by
# postgres. Just using .distinct() has no effect.
@ -107,19 +107,19 @@ def get_current_test_failures(push, option_map, jobs, investigatedTests=None):
all_failed_jobs[job.id] = job
# The 't' ensures the key starts with a character, as required for a query selector
test_key = re.sub(r"\W+", "", f"t{test_name}{config}{platform}{job_name}{job_group}")
isClassifiedIntermittent = any(
is_classified_intermittent = any(
job["failure_classification_id"] == 4 for job in jobs[job_name]
)
isInvestigated = False
investigatedTestId = None
for investigatedTest in investigatedTests:
is_investigated = False
investigated_test_id = None
for investigated_test in investigated_tests:
if (
investigatedTest.test == test_name
and job.job_type.id == investigatedTest.job_type.id
investigated_test.test == test_name
and job.job_type.id == investigated_test.job_type.id
):
isInvestigated = True
investigatedTestId = investigatedTest.id
is_investigated = True
investigated_test_id = investigated_test.id
break
if test_key not in tests:
@ -140,16 +140,16 @@ def get_current_test_failures(push, option_map, jobs, investigatedTests=None):
"totalFailures": 0,
"totalJobs": 0,
"failedInParent": False,
"isClassifiedIntermittent": isClassifiedIntermittent,
"isInvestigated": isInvestigated,
"investigatedTestId": investigatedTestId,
"isClassifiedIntermittent": is_classified_intermittent,
"isInvestigated": is_investigated,
"investigatedTestId": investigated_test_id,
}
tests[test_key] = line
countJobs = len(
count_jobs = len(
list(filter(lambda x: x["result"] in ["success", "testfailed"], jobs[job_name]))
)
tests[test_key]["totalFailures"] += 1
tests[test_key]["totalJobs"] = countJobs
tests[test_key]["totalJobs"] = count_jobs
# Each line of the sorted list that is returned here represents one test file per platform/
# config. Each line will have at least one failing job, but may have several
@ -232,13 +232,13 @@ def get_test_failures(
fixed_by_commit_history = get_history(
2, push_date, fixed_by_commit_history_days, option_map, repository_ids
)
investigatedTests = InvestigatedTests.objects.filter(push=push)
investigated_tests = InvestigatedTests.objects.filter(push=push)
# ``push_failures`` are tests that have FailureLine records created by our Log Parser.
# These are tests we are able to show to examine to see if we can determine they are
# intermittent. If they are not, we tell the user they need investigation.
# These are failures ONLY for the current push, not relative to history.
push_failures = get_current_test_failures(push, option_map, jobs, investigatedTests)
push_failures = get_current_test_failures(push, option_map, jobs, investigated_tests)
filtered_push_failures = [failure for failure in push_failures if filter_failure(failure)]
# Based on the intermittent and FixedByCommit history, set the appropriate classification

Просмотреть файл

@ -19,11 +19,11 @@ class FilesBugzillaMapViewSet(viewsets.ReadOnlyModelViewSet):
# combinations can be in the failure line, it might not be a test and
# the real issue gets logged earlier but not detected as failure line.
# Require user input for the product and component to use.
IGNORE_LIST_PRODUCT_COMPONENT = [
ignore_list_product_component = [
{product: "Testing", component: "Mochitest"},
]
for product_component in queryset:
if product_component not in IGNORE_LIST_PRODUCT_COMPONENT:
if product_component not in ignore_list_product_component:
filtered_queryset.append(product_component)
return filtered_queryset[:5]
@ -40,8 +40,8 @@ class FilesBugzillaMapViewSet(viewsets.ReadOnlyModelViewSet):
# Drop parameters
path = (path.split("?"))[0]
file = (path.split("/"))[-1]
fileNameParts = file.split(".")
file_without_extension = fileNameParts[0] + ("." if len(fileNameParts) > 1 else "")
file_name_parts = file.split(".")
file_without_extension = file_name_parts[0] + ("." if len(file_name_parts) > 1 else "")
queryset = (
FilesBugzillaMap.objects.select_related("bugzilla_component")
.filter(path__endswith=path)

Просмотреть файл

@ -35,13 +35,13 @@ class InvestigatedViewSet(viewsets.ModelViewSet):
project = kwargs["project"]
revision = request.query_params.get("revision")
test = request.data["test"]
jobName = request.data["jobName"]
jobSymbol = request.data["jobSymbol"]
job_name = request.data["jobName"]
job_symbol = request.data["jobSymbol"]
try:
repository = Repository.objects.get(name=project)
push = Push.objects.get(revision=revision, repository=repository)
job_type = JobType.objects.get(name=jobName, symbol=jobSymbol)
job_type = JobType.objects.get(name=job_name, symbol=job_symbol)
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
serializer.save(push=push, job_type=job_type, test=test)
@ -54,7 +54,7 @@ class InvestigatedViewSet(viewsets.ModelViewSet):
return Response(f"No push with revision: {revision}", status=HTTP_404_NOT_FOUND)
except JobType.DoesNotExist:
return Response(f"No JobType with job name: {jobName}", status=HTTP_404_NOT_FOUND)
return Response(f"No JobType with job name: {job_name}", status=HTTP_404_NOT_FOUND)
def destroy(self, request, project, pk=None):
try:

Просмотреть файл

@ -313,7 +313,7 @@ class JobsProjectViewSet(viewsets.ViewSet):
- count (10)
- return_type (dict)
"""
MAX_JOBS_COUNT = 2000
max_jobs_count = 2000
filter_params = {}
@ -348,8 +348,8 @@ class JobsProjectViewSet(viewsets.ViewSet):
return Response("Invalid value for offset or count", status=HTTP_400_BAD_REQUEST)
return_type = filter_params.get("return_type", "dict").lower()
if count > MAX_JOBS_COUNT:
msg = f"Specified count exceeds API MAX_JOBS_COUNT value: {MAX_JOBS_COUNT}"
if count > max_jobs_count:
msg = f"Specified count exceeds API MAX_JOBS_COUNT value: {max_jobs_count}"
return Response({"detail": msg}, status=HTTP_400_BAD_REQUEST)
try:

Просмотреть файл

@ -35,7 +35,7 @@ class PushViewSet(viewsets.ViewSet):
GET method for list of ``push`` records with revisions
"""
# What is the upper limit on the number of pushes returned by the api
MAX_PUSH_COUNT = 1000
max_push_count = 1000
# make a mutable copy of these params
filter_params = request.query_params.copy()
@ -167,8 +167,8 @@ class PushViewSet(viewsets.ViewSet):
except ValueError:
return Response({"detail": "Valid count value required"}, status=HTTP_400_BAD_REQUEST)
if count > MAX_PUSH_COUNT:
msg = f"Specified count exceeds api limit: {MAX_PUSH_COUNT}"
if count > max_push_count:
msg = f"Specified count exceeds api limit: {max_push_count}"
return Response({"detail": msg}, status=HTTP_400_BAD_REQUEST)
# we used to have a "full" parameter for this endpoint so you could