Bug 1755569 - Update vendored taskgraph to 1.1.7, r=releng-reviewers,gbrown

Differential Revision: https://phabricator.services.mozilla.com/D139157

third_party/python/requirements.in

@@ -39,7 +39,7 @@ setuptools==51.2.0
 six==1.13.0
 slugid==2.0.0
 taskcluster==44.2.2
-taskcluster-taskgraph==1.1.5
+taskcluster-taskgraph==1.1.7
 taskcluster-urls==13.0.1
 tqdm==4.62.3
 urllib3==1.26

third_party/python/requirements.txt

@@ -385,9 +385,9 @@ slugid==2.0.0 \
     #   -r requirements-mach-vendor-python.in
     #   taskcluster
     #   taskcluster-taskgraph
-taskcluster-taskgraph==1.1.5 \
-    --hash=sha256:d55564691d211c550319fae5964d67400b4c1ebd5e8423b2548d48240b8498ad \
-    --hash=sha256:e162cbefb765fd500166ede24172f4520c0662589b6218586b9eab4cbfeed0da
+taskcluster-taskgraph==1.1.7 \
+    --hash=sha256:648e2cd5a92ce3412d830d1a28fb41ba3415956f44fe6013a53d920fc313ebdf \
+    --hash=sha256:7025856ed543c8e4a42fae580d579feadb53a54ce91b6e6b994e678449be3aee
     # via -r requirements-mach-vendor-python.in
 taskcluster-urls==13.0.1 \
     --hash=sha256:5e25e7e6818e8877178b175ff43d2e6548afad72694aa125f404a7329ece0973 \

third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.1.7.dist-info/METADATA

@@ -1,14 +1,19 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 1.1.5
+Version: 1.1.7
 Summary: Build taskcluster taskgraphs
 Home-page: https://hg.mozilla.org/ci/taskgraph
 License: UNKNOWN
 Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
 Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Software Development
 Requires-Dist: appdirs (>=1.4)
 Requires-Dist: attrs (>=19.1.0)
 Requires-Dist: json-e (>=2.7)

third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.1.7.dist-info/RECORD

@@ -1,16 +1,16 @@
 taskgraph/__init__.py,sha256=jwOtU7TkmU317LP_IsgIswpj2T1OPUXXgMRv4sIU7nE,707
-taskgraph/config.py,sha256=7gYepXBtCJFvvfugD8KotMSgIs1pkJIk7hEfmlpovP8,4381
+taskgraph/config.py,sha256=Bm0RU1LyEj4RuX_lBnpQrr5fGmx7zCbKleN1Zy6fQ5w,4410
 taskgraph/create.py,sha256=VeAYxtLK8f5HufgUSjzDRP7WBSwQza--_O7RzRRhzA4,5190
-taskgraph/decision.py,sha256=hfs6UQtgyfJLy0blDLUkLFh2yAz_37ZNcnYW5OHUYtg,9346
+taskgraph/decision.py,sha256=GlK1rs1VoWqoTfHM9oJzf0LS608HJVtEYWIPmQ3iEhc,9418
 taskgraph/docker.py,sha256=HlCEtfW2coUAyGce8ToVfn8OdLfQJT7Vq_mYQntO4gQ,7485
 taskgraph/files_changed.py,sha256=30ONF-m1hbGQnQ31E-TCsHG1jhdNjLTOolSXu2jzOCs,2159
 taskgraph/filter_tasks.py,sha256=ty4SK0ujcZ8824F4ikwdCaGX1TRSq_90TFArDogJHTo,875
 taskgraph/generator.py,sha256=Axwun0FeuEXKZpCKD3F4HHCk5K3STeYrR2gr15DY9DM,14549
 taskgraph/graph.py,sha256=W7I-lfDiBMqSj5ZKgqVmebwTaNxdmbKLx2MqFT_cD_I,4666
-taskgraph/main.py,sha256=8R6zwOKzK7W4khjxeShbbHstubMOgznbi4QKgGjCKMc,22397
+taskgraph/main.py,sha256=YIK1epKTS91riC0yLp9AR7HBuIg_eD-8GBf6P1sBTeQ,22465
 taskgraph/morph.py,sha256=EH_kP5FAMb0F8Oap4VZeVpMNXGr0QhDVz_lxiY-ksgA,9562
 taskgraph/optimize.py,sha256=dxeS3BVaRpPrhC0oK4vmZpcSfdMQTeKiWqvb_Lnx1dM,12518
-taskgraph/parameters.py,sha256=zEPrnbnxnCX1riu2_aSgfxBcx2BGOCvFzz-3JRHxS6A,10661
+taskgraph/parameters.py,sha256=QSVVMhoOQ0Uck4rE9_McFkDByUvidAG0yJFLLxLGuEE,10744
 taskgraph/target_tasks.py,sha256=mspItlKD-HCuQR1x_UD6HT_Qd1v5kEHvgRWIUbweRDg,3166
 taskgraph/task.py,sha256=OiYCsYhVukqLdQR2olQSG-l7oP2Z_2XaIZxovBPqxCc,2733
 taskgraph/taskgraph.py,sha256=DfldF0Fp_hQeDK2lQs4SoWgrp_9tG6y2VfuBEp8C3OM,2397
@@ -29,7 +29,7 @@ taskgraph/run-task/robustcheckout.py,sha256=P41ZGrec5aa8hVTEqOkKJ9wTygsgaXtjoQnl
 taskgraph/run-task/run-task,sha256=ORv9ImlVaSNZfvwsqRLrCkW3Tjsd1Gg7B9X7M8eMOts,39029
 taskgraph/test/__init__.py,sha256=7LTScvkVcLPqivjL-wCb_Znk2GGajaJi1fJ4SjcLMoA,289
 taskgraph/test/automationrelevance.json,sha256=ttGejNZeVhcTWXFdoU0xME0YEYRYvaYcAr8zBz0ewrw,17980
-taskgraph/test/conftest.py,sha256=MiCe5gyqGDIARdzauK4wgB4jlyJzhfwiAElBxfTYtlg,3462
+taskgraph/test/conftest.py,sha256=urHJNFOw7mVSr8WzIDZtOZPbXykdR--iw3J_hiF2rjg,3584
 taskgraph/test/mockedopen.py,sha256=Ccr2qGJSLeWIUYd03Poy8eKKRSW2aTpRGI-0AA7xYrw,4055
 taskgraph/test/test_create.py,sha256=oY7DeVW8usjVNe-QPPyTrE3hsvwMo9HvMxPSCllOsMQ,3657
 taskgraph/test/test_decision.py,sha256=dxlMnRtpKZrOqSZo-znPLw6TwlFJOB9Yp0sadEnJdaQ,2658
@@ -45,18 +45,18 @@ taskgraph/test/test_taskgraph.py,sha256=KuBulN0QGHON3_ksVE3hhNyenk_QkjWjoGYghrB-
 taskgraph/test/test_transforms_base.py,sha256=Vo9slzCB2GePvMoLmkrSdhYVWh2nQYn5bRxMjsx40Mw,873
 taskgraph/test/test_transforms_job.py,sha256=pHO_Ea32b3OjCSIYhRHK3CqHsugJizD43_a3jMFtG_A,4490
 taskgraph/test/test_util_attributes.py,sha256=K_Wro-p5oA-S4yrsLqT8HMBlOAN4L0krQQQ82WYgGAQ,3596
-taskgraph/test/test_util_docker.py,sha256=j1eIYDcL8cch9OjCH-bQrK9t9ljPVZeTpaQNdKJ2nq8,8803
+taskgraph/test/test_util_docker.py,sha256=TKe9D5d3q8rK0iA5rMhzbmhQ4c4OSnVKrjioeZ1uNuo,8800
 taskgraph/test/test_util_memoize.py,sha256=yq-PTegHBdQZkRm6Iv1NdCRcKCVzsb4pmZPvIz0P6j8,2340
-taskgraph/test/test_util_parameterization.py,sha256=rXZhIDDAGyZH5p55AhNzjuYdkekAN6HftRRN41YcIOg,7948
+taskgraph/test/test_util_parameterization.py,sha256=IW8zWFMi7INVuAkpT5rk674nZVZ4BhHrKRTz8HKN-O0,7945
 taskgraph/test/test_util_path.py,sha256=icJyH1DyMNVuZ5xfPXvrEQwQ0pQYTg4ORlZq3RK6_V8,5906
 taskgraph/test/test_util_python_path.py,sha256=VQo4hwsJ0It-jLIqe4nErPmZn9AQ7rliN25h2oO_zMg,1216
 taskgraph/test/test_util_readonlydict.py,sha256=KRgjLvSBsZZj4EUhwcqeUsM1T--iGklVE0QJuC6Xv4o,1234
 taskgraph/test/test_util_schema.py,sha256=6lLB-ToEyEt6OC6qQFPZ_yKcNAdv5hyApK-skA4B-KA,5683
-taskgraph/test/test_util_taskcluster.py,sha256=yRF_wUpPjYDe6lrrDhTs6HONead97AMNEEBC2slgwug,481
+taskgraph/test/test_util_taskcluster.py,sha256=hZqtF7OC58OO4nW4URxF5ZTyxnNiwh2Jkqbtx1ZZjww,9457
 taskgraph/test/test_util_templates.py,sha256=u3ckrzmx1eyk7vXiqRiQlETtVZvSx4FHEXN5xB4GZDQ,1676
 taskgraph/test/test_util_time.py,sha256=SG4WmSupTQiL1UhE2UMsMNZEugfPdoaxO4aWSxLGXBM,1803
 taskgraph/test/test_util_treeherder.py,sha256=20zzGcMd0BL0ayTFQj6Etj39afdxZPgtZxSGUZ0iL5M,912
-taskgraph/test/test_util_vcs.py,sha256=EueicUhHOqsQuRW-VXbiP3-zO20MUBN_GidOJbvs2jc,6522
+taskgraph/test/test_util_vcs.py,sha256=zL9j_PaSBPTuncgCpjvR5SDL0wIAkblPSTabn9qU-qs,6625
 taskgraph/test/test_util_yaml.py,sha256=zymZxaAZBIBn5u-p91QsA---IqCH_CVVk3YqMoshLlQ,1019
 taskgraph/transforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 taskgraph/transforms/base.py,sha256=oIQdBKrHG_FZXAoRyiazxgLa8zbAI4TEVlduqz8H12I,5277
@@ -68,7 +68,7 @@ taskgraph/transforms/task.py,sha256=zAfcqIVegJh2OScqQR0mgLPhxjD4WCzXmRgW4-jC0Ng,
 taskgraph/transforms/job/__init__.py,sha256=imtb3MHVQbKtcCngSnvgumtBfOwxOPiRsJDwHKUtYn0,16891
 taskgraph/transforms/job/common.py,sha256=onHnerPcmmvbSk0oHt8mvJmOo7AnjHQya0ombgMNLG8,7106
 taskgraph/transforms/job/index_search.py,sha256=zPldmHSalHJjvULAMF9_QAeOZzIeWpr89kOVeP2IJAE,1220
-taskgraph/transforms/job/run_task.py,sha256=GqR1ZPMnpoFi_d8HlzpomxB7nWIHctgthYZ_ve3jM6M,8871
+taskgraph/transforms/job/run_task.py,sha256=UjFkPaULuMW3-nyJJuzRNubLFnmnN4o9W7G5kdUFxvw,9017
 taskgraph/transforms/job/toolchain.py,sha256=z2Z7sxI4yn_dI8zzcMWcrcmfTHeK6mgfSNSM6MAgrCU,4649
 taskgraph/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 taskgraph/util/archive.py,sha256=Hcm8YHsCtazX7efDSd8vXm8Pw87Z1UP_Bo-gIUEzb_4,2856
@@ -84,17 +84,17 @@ taskgraph/util/path.py,sha256=lZKXxtxjbVTSvto0X-M6zKaNVRwpdoR5Y8IaJg-_Xag,4688
 taskgraph/util/python_path.py,sha256=93R0mADSe1MeTTOsrDWEjLTW6MVpf2COuf1jXbxuQOk,821
 taskgraph/util/readonlydict.py,sha256=XzTG-gqGqWVlSkDxSyOL6Ur7Z0ONhIJ9DVLWV3q4q1w,787
 taskgraph/util/schema.py,sha256=rtb3ebLsC0aOdoc2g6DaMZF-Md2G9jfIXAoN65oPDjg,6829
-taskgraph/util/taskcluster.py,sha256=Xn4Oly9XStt4j0zVO5QmWb2SN9UF_tFRNyWEPeKOlrc,11042
+taskgraph/util/taskcluster.py,sha256=zOoMKRxHrASJfdv-ni4evRskjanZUnoSNiAMkXPBDlQ,11395
 taskgraph/util/taskgraph.py,sha256=OfaclpaZzVfxViMXaBIAYmSJlI-ZFsBudGVndgH7znw,1982
 taskgraph/util/templates.py,sha256=Dqxfl244u-PX7dnsk3_vYyzDwpDgJtANK6NmZwN3Qow,1417
 taskgraph/util/time.py,sha256=cMRYsBiz7rgPwgZk77p0P7h9JzeEJENBZCoetBaEHqY,3490
 taskgraph/util/treeherder.py,sha256=XrdE-Je0ZvXe6_8f0DvvqNbrHherUk-hUuxirImPEIo,2138
-taskgraph/util/vcs.py,sha256=0IH2BD3oADHNRAJYZpEG64uXl7oeGt4bh2uyrE__MvM,6497
+taskgraph/util/vcs.py,sha256=p_cQ9iIqALzVNBVPUIG9nDz4IGMdqom_5o3eidL1vCY,6510
 taskgraph/util/verify.py,sha256=e0j_Ec2gDDIVlnEO9tHQ0gjxzGm00NJbPVI5q4BuiHc,5653
 taskgraph/util/workertypes.py,sha256=5g2mgIbEKMzDpZNnmPMoMNyy7Wahi-jmWcV1amDAcPo,2341
 taskgraph/util/yaml.py,sha256=hfKI_D8Q7dimq4_VvO3WEh8CJsTrsIMwN6set7HIQbY,990
-taskcluster_taskgraph-1.1.5.dist-info/METADATA,sha256=Wpb67ZvvQdG8FPjq_P1Zrc5OI8dDNnujTjsh6HzIvjk,730
-taskcluster_taskgraph-1.1.5.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-taskcluster_taskgraph-1.1.5.dist-info/entry_points.txt,sha256=VoXNtZpN4LvyXYB1wq47AU9CO-DMYMJ0VktKxjugzbY,51
-taskcluster_taskgraph-1.1.5.dist-info/top_level.txt,sha256=3JNeYn_hNiNXC7DrdH_vcv-WYSE7QdgGjdvUYvSjVp0,10
-taskcluster_taskgraph-1.1.5.dist-info/RECORD,,
+taskcluster_taskgraph-1.1.7.dist-info/METADATA,sha256=6mI04JHq80YlL1sYj3Xooy7c7jNKUyWl4SpjDmcbTF8,990
+taskcluster_taskgraph-1.1.7.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+taskcluster_taskgraph-1.1.7.dist-info/entry_points.txt,sha256=VoXNtZpN4LvyXYB1wq47AU9CO-DMYMJ0VktKxjugzbY,51
+taskcluster_taskgraph-1.1.7.dist-info/top_level.txt,sha256=3JNeYn_hNiNXC7DrdH_vcv-WYSE7QdgGjdvUYvSjVp0,10
+taskcluster_taskgraph-1.1.7.dist-info/RECORD,,

third_party/python/taskcluster_taskgraph/taskgraph/config.py

@@ -70,6 +70,7 @@ graph_config_schema = Schema(
         Extra: object,
     }
 )
+"""Schema for GraphConfig"""
 
 
 @attr.s(frozen=True, cmp=False)

third_party/python/taskcluster_taskgraph/taskgraph/decision.py

@@ -153,7 +153,10 @@ def get_decision_parameters(graph_config, options):
     }
 
     repo = get_repository(os.getcwd())
-    commit_message = repo.get_commit_message()
+    try:
+        commit_message = repo.get_commit_message()
+    except UnicodeDecodeError:
+        commit_message = ""
 
     # Define default filter list, as most configurations shouldn't need
     # custom filters.

third_party/python/taskcluster_taskgraph/taskgraph/main.py

@@ -314,7 +314,7 @@ def generate_taskgraph(options, parameters, logdir):
     "the hash or `.~1` (hg) or `HEAD~1` (git) can be used as well.",
 )
 def show_taskgraph(options):
-    from taskgraph.parameters import Parameters
+    from taskgraph.parameters import Parameters, parameters_loader
     from taskgraph.util.vcs import get_repository
 
     if options.pop("verbose", False):
@@ -352,10 +352,12 @@ def show_taskgraph(options):
 
     parameters: List[Any[str, Parameters]] = options.pop("parameters")
     if not parameters:
-        kwargs = {
+        overrides = {
             "target-kind": options.get("target_kind"),
         }
-        parameters = [Parameters(strict=False, **kwargs)]  # will use default values
+        parameters = [
+            parameters_loader(None, strict=False, overrides=overrides)
+        ]  # will use default values
 
     for param in parameters[:]:
         if isinstance(param, str) and os.path.isdir(param):

third_party/python/taskcluster_taskgraph/taskgraph/parameters.py

@@ -65,7 +65,7 @@ def _get_defaults(repo_root=None):
         repo_url = repo.get_url()
         project = repo_url.rsplit("/", 1)[1]
     except (CalledProcessError, IndexError):
-        # IndexError because not all repo_urls have slashes.
+        # IndexError is raised if repo url doesn't have any slashes.
         repo_url = ""
         project = ""
 

third_party/python/taskcluster_taskgraph/taskgraph/transforms/job/run_task.py

@@ -12,7 +12,7 @@ import attr
 
 from taskgraph.transforms.task import taskref_or_string
 from taskgraph.transforms.job import run_job_using
-from taskgraph.util import path
+from taskgraph.util import path, taskcluster
 from taskgraph.util.schema import Schema
 from taskgraph.transforms.job.common import support_vcs_checkout
 from voluptuous import Required, Any, Optional
@@ -115,10 +115,12 @@ def script_url(config, script):
     # want to use the run-task/fetch-content corresponding to the taskgraph
     # version we are running, and otherwise, we aren't going to run the task we
     # generate, so the exact version doesn't matter.
+    # TASK_ID is also set for test-action-callback, so we check for that with
+    # taskcluster.testing.
     # If we checked out the taskgraph code with run-task in the decision task,
     # we can use TASKGRAPH_* to find the right version, which covers the
     # existing use case.
-    if "TASK_ID" in os.environ:
+    if "TASK_ID" in os.environ and not taskcluster.testing:
         if (
             "TASKGRAPH_HEAD_REPOSITORY" not in os.environ
             or "TASKGRAPH_HEAD_REV" not in os.environ

third_party/python/taskcluster_taskgraph/taskgraph/util/taskcluster.py

@@ -26,9 +26,6 @@ PRODUCTION_TASKCLUSTER_ROOT_URL = "https://taskcluster.net"
 # the maximum number of parallel Taskcluster API calls to make
 CONCURRENCY = 50
 
-# the maximum number of parallel Taskcluster API calls to make
-CONCURRENCY = 50
-
 
 @memoize
 def get_root_url(use_proxy):
@@ -39,7 +36,7 @@ def get_root_url(use_proxy):
     is not set."""
     if use_proxy:
         try:
-            return os.environ["TASKCLUSTER_PROXY_URL"]
+            return liburls.normalize_root_url(os.environ["TASKCLUSTER_PROXY_URL"])
         except KeyError:
             if "TASK_ID" not in os.environ:
                 raise RuntimeError(
@@ -55,40 +52,61 @@ def get_root_url(use_proxy):
             )
         else:
             logger.debug("Using default TASKCLUSTER_ROOT_URL (Firefox CI production)")
-            return PRODUCTION_TASKCLUSTER_ROOT_URL
+            return liburls.normalize_root_url(PRODUCTION_TASKCLUSTER_ROOT_URL)
     logger.debug(
         "Running in Taskcluster instance {}{}".format(
             os.environ["TASKCLUSTER_ROOT_URL"],
             " with taskcluster-proxy" if "TASKCLUSTER_PROXY_URL" in os.environ else "",
         )
     )
-    return os.environ["TASKCLUSTER_ROOT_URL"]
+    return liburls.normalize_root_url(os.environ["TASKCLUSTER_ROOT_URL"])
 
 
-@memoize
-def get_session():
-    session = requests.Session()
-
-    retry = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
+def requests_retry_session(
+    retries,
+    backoff_factor=0.1,
+    status_forcelist=(500, 502, 503, 504),
+    concurrency=CONCURRENCY,
+    session=None,
+):
+    session = session or requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+    )
 
     # Default HTTPAdapter uses 10 connections. Mount custom adapter to increase
     # that limit. Connections are established as needed, so using a large value
     # should not negatively impact performance.
     http_adapter = requests.adapters.HTTPAdapter(
-        pool_connections=CONCURRENCY, pool_maxsize=CONCURRENCY, max_retries=retry
+        pool_connections=concurrency,
+        pool_maxsize=concurrency,
+        max_retries=retry,
     )
-    session.mount("https://", http_adapter)
     session.mount("http://", http_adapter)
+    session.mount("https://", http_adapter)
 
     return session
 
 
-def _do_request(url, force_get=False, **kwargs):
+@memoize
+def get_session():
+    return requests_retry_session(retries=5)
+
+
+def _do_request(url, method=None, **kwargs):
+    if method is None:
+        method = "post" if kwargs else "get"
+
     session = get_session()
-    if kwargs and not force_get:
-        response = session.post(url, **kwargs)
-    else:
-        response = session.get(url, stream=True, **kwargs)
+    if method == "get":
+        kwargs["stream"] = True
+
+    response = getattr(session, method)(url, **kwargs)
+
     if response.status_code >= 400:
         # Consume content before raise_for_status, so that the connection can be
         # reused.
@@ -304,7 +322,7 @@ def list_task_group_incomplete_tasks(task_group_id):
             "v1",
             f"task-group/{task_group_id}/list",
         )
-        resp = _do_request(url, force_get=True, params=params).json()
+        resp = _do_request(url, method="get", params=params).json()
         for task in [t["status"] for t in resp["tasks"]]:
             if task["state"] in ["running", "pending", "unscheduled"]:
                 yield task["taskId"]

third_party/python/taskcluster_taskgraph/taskgraph/util/vcs.py

@@ -25,10 +25,10 @@ class Repository(ABC):
 
         self._env = os.environ.copy()
 
-    def run(self, *args: str):
+    def run(self, *args: str, **kwargs):
         cmd = (self.binary,) + args
         return subprocess.check_output(
-            cmd, cwd=self.path, env=self._env, universal_newlines=True
+            cmd, cwd=self.path, env=self._env, encoding="utf-8", **kwargs
         )
 
     @abstractproperty