Bug 1636871 - allow URLs and aliases for --android-install-apk r=sparky

This patch let us use urls for that option

Differential Revision: https://phabricator.services.mozilla.com/D74629

python/mozperftest/mozperftest/system/android.py

@@ -1,11 +1,21 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import os
+import tempfile
+from pathlib import Path
+
 from mozdevice import ADBDevice, ADBError
 from mozperftest.layers import Layer
+from mozperftest.utils import download_file
 
-HERE = os.path.dirname(__file__)
+
+_ROOT_URL = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/"
+_FENNEC_BUILDS = "project.mobile.fenix.v2.fennec-nightly.latest/artifacts/public/build/"
+_PERMALINKS = {
+    "fennec_nightly_armeabi_v7a": _ROOT_URL
+    + _FENNEC_BUILDS
+    + "armeabi-v7a/geckoNightly/target.apk"
+}
 
 
 class DeviceError(Exception):
@@ -30,7 +40,11 @@ class AndroidDevice(Layer):
         "install-apk": {
             "nargs": "*",
             "default": [],
-            "help": "APK to install to the device",
+            "help": (
+                "APK to install to the device "
+                "Can be a file, an url or an alias url from "
+                " %s" % ", ".join(_PERMALINKS.keys())
+            ),
         },
     }
 
@@ -57,7 +71,17 @@ class AndroidDevice(Layer):
         # install APKs
         for apk in self.get_arg("android-install-apk"):
             self.info("Installing %s" % apk)
-            self.device.install_app(apk, replace=True)
+            if apk in _PERMALINKS:
+                apk = _PERMALINKS[apk]
+            if apk.startswith("http"):
+                with tempfile.TemporaryDirectory() as tmpdirname:
+                    target = Path(tmpdirname, "target.apk")
+                    self.info("Downloading %s" % apk)
+                    download_file(apk, target)
+                    self.info("Installing downloaded APK")
+                    self.device.install_app(str(target), replace=True)
+            else:
+                self.device.install_app(apk, replace=True)
             self.info("Done.")
 
         # checking that the app is installed

python/mozperftest/mozperftest/tests/support.py

@@ -57,3 +57,18 @@ def get_running_env(**kwargs):
     env = MachEnvironment(mach_cmd, **mach_args)
     metadata = Metadata(mach_cmd, env, "script")
     return mach_cmd, metadata, env
+
+
+def requests_content(chunks=None):
+    if chunks is None:
+        chunks = [b"some ", b"content"]
+
+    def _content(*args, **kw):
+        class Resp:
+            def iter_content(self, **kw):
+                for chunk in chunks:
+                    yield chunk
+
+        return Resp()
+
+    return _content

python/mozperftest/mozperftest/tests/test_android.py

@@ -3,7 +3,7 @@ import mozunit
 import pytest
 import mock
 
-from mozperftest.tests.support import get_running_env
+from mozperftest.tests.support import get_running_env, requests_content
 from mozperftest.environment import SYSTEM
 from mozperftest.system.android import DeviceError
 
@@ -48,5 +48,22 @@ def test_android_failure():
         android(metadata)
 
 
+@mock.patch("mozperftest.utils.requests.get", new=requests_content())
+@mock.patch("mozperftest.system.android.ADBDevice")
+def test_android_apk_alias(device):
+    args = {
+        "android-install-apk": ["fennec_nightly_armeabi_v7a"],
+        "android": True,
+        "android-app-name": "org.mozilla.fenned_aurora",
+    }
+
+    mach_cmd, metadata, env = get_running_env(**args)
+    system = env.layers[SYSTEM]
+    with system as android:
+        android(metadata)
+    # XXX really ?
+    assert device.mock_calls[1][1][0].endswith("target.apk")
+
+
 if __name__ == "__main__":
     mozunit.main()

python/mozperftest/mozperftest/tests/test_utils.py

@@ -4,7 +4,12 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import sys
 import mozunit
-from mozperftest.utils import host_platform, silence
+import mock
+import pytest
+from pathlib import Path
+
+from mozperftest.utils import host_platform, silence, download_file
+from mozperftest.tests.support import temp_file, requests_content
 
 
 def test_silence():
@@ -25,5 +30,23 @@ def test_host_platform():
             assert "64" not in plat
 
 
+def get_raise(*args, **kw):
+    raise Exception()
+
+
+@mock.patch("mozperftest.utils.requests.get", new=get_raise)
+def test_download_file_fails():
+    with temp_file() as target, pytest.raises(Exception):
+        download_file("http://I don't exist", Path(target), retry_sleep=0.1)
+
+
+@mock.patch("mozperftest.utils.requests.get", new=requests_content())
+def test_download_file_success():
+    with temp_file() as target:
+        download_file("http://content", Path(target), retry_sleep=0.1)
+        with open(target) as f:
+            assert f.read() == "some content"
+
+
 if __name__ == "__main__":
     mozunit.main()

python/mozperftest/mozperftest/utils.py

@@ -6,8 +6,12 @@ import contextlib
 import sys
 import os
 import random
+from io import StringIO
+from redo import retry
+import requests
 
-from six import StringIO
+
+RETRY_SLEEP = 10
 
 
 @contextlib.contextmanager
@@ -96,3 +100,32 @@ def build_test_list(tests, randomized=False):
         # we don't always run tests in the same order
         random.shuffle(res)
     return res
+
+
+def download_file(url, target, retry_sleep=RETRY_SLEEP, attempts=3):
+    """Downloads a file, given an URL in the target path.
+
+    The function will attempt several times on failures.
+    """
+
+    def _download_file(url, target):
+        req = requests.get(url, stream=True, timeout=30)
+        target_dir = target.parent.resolve()
+        if str(target_dir) != "":
+            target_dir.mkdir(exist_ok=True)
+
+        with target.open("wb") as f:
+            for chunk in req.iter_content(chunk_size=1024):
+                if not chunk:
+                    continue
+                f.write(chunk)
+                f.flush()
+        return target
+
+    return retry(
+        _download_file,
+        args=(url, target),
+        attempts=attempts,
+        sleeptime=retry_sleep,
+        jitter=0,
+    )