Add recording information to json file
Generate zip and manifest Auto generate recordings
This commit is contained in:
Florin Strugariu
Родитель
a01e17ed84
Коммит
6b2e405b73
2
Pipfile
2
Pipfile
|
|
@ -11,6 +11,8 @@ mozdevice = "*"
|
|||
click = "*"
|
||||
click-config-file = "*"
|
||||
selenium = "*"
|
||||
mozversion = "*"
|
||||
tldextract = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "2.7"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "2b461ebbd7ddec1721e23ad287de6942ac5cb9a2ee046b8959f0e722669a1d66"
|
||||
"sha256": "903e785e8704b2eb496a7c47629326f9519173bb770b294dacd01449a57ff25e"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
|
@ -24,6 +24,19 @@
|
|||
],
|
||||
"version": "==1.7"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939"
|
||||
],
|
||||
"version": "==2019.6.16"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||
],
|
||||
"version": "==3.0.4"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
|
||||
|
|
@ -46,6 +59,13 @@
|
|||
],
|
||||
"version": "==5.0.6"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
|
||||
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
|
||||
],
|
||||
"version": "==2.8"
|
||||
},
|
||||
"mozdevice": {
|
||||
"hashes": [
|
||||
"sha256:a7b582331448f28c9f44d5a113a152537e5666b0f5ce1052dc569ab516ec99a8",
|
||||
|
|
@ -63,10 +83,10 @@
|
|||
},
|
||||
"mozlog": {
|
||||
"hashes": [
|
||||
"sha256:8c32f07b939960f769df891fbb30cabb86822e4d3a8d0a16ebe693d0f46eae0e",
|
||||
"sha256:a9e84e44113ba3cfde217d4e941979d37445ee48166a79583f9fc1e74770d5e1"
|
||||
"sha256:ad433902b5865a76706750ffc7119b32286e97b971e6d325d2909d0bf0670801",
|
||||
"sha256:dc85cfb9d47af6811f2367f471de7028c36204340c5e68a928115409ea75d9a9"
|
||||
],
|
||||
"version": "==4.0"
|
||||
"version": "==4.2.0"
|
||||
},
|
||||
"mozprofile": {
|
||||
"hashes": [
|
||||
|
|
@ -83,6 +103,28 @@
|
|||
],
|
||||
"version": "==1.0.0"
|
||||
},
|
||||
"mozversion": {
|
||||
"hashes": [
|
||||
"sha256:35911badaaf02715e56c6062379688724e7afeffc2d25be8567312d24054cdd4",
|
||||
"sha256:65f41d7dc14002f83d8f147c82ca34f7213ad07065d250939daaeeb3787dc0fa"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.1.0"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
|
||||
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
|
||||
],
|
||||
"version": "==2.22.0"
|
||||
},
|
||||
"requests-file": {
|
||||
"hashes": [
|
||||
"sha256:75c175eed739270aec3c5279ffd74e6527dada275c5c0d76b5817e9c86bb7dea",
|
||||
"sha256:8f04aa6201bacda0567e7ac7f677f1499b0fc76b22140c54bc06edf1ba92e2fa"
|
||||
],
|
||||
"version": "==1.4.3"
|
||||
},
|
||||
"selenium": {
|
||||
"hashes": [
|
||||
"sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c",
|
||||
|
|
@ -98,12 +140,20 @@
|
|||
],
|
||||
"version": "==1.12.0"
|
||||
},
|
||||
"tldextract": {
|
||||
"hashes": [
|
||||
"sha256:2c1c5d9d454f79734b4f3da0d603856dd9f820753410a3e9abf0a0c9fde33e97",
|
||||
"sha256:b72bef6013de67c7fa181250bc2c2e089a994d259c09ca95a9771f2f97e29ed1"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.2.1"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:a53063d8b9210a7bdec15e7b272776b9d42b2fd6816401a0d43006ad2f9902db",
|
||||
"sha256:d363e3607d8de0c220d31950a8f38b18d5ba7c0830facd71a1c6b1036b7ce06c"
|
||||
"sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
|
||||
"sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
|
||||
],
|
||||
"version": "==1.25.2"
|
||||
"version": "==1.25.3"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
|
|
|||
20
README.md
20
README.md
|
|
@ -16,6 +16,26 @@ $ pipenv install
|
|||
|
||||
```
|
||||
$ pipenv run python studio.py --help
|
||||
|
||||
Usage: studio.py [OPTIONS] [PATH]
|
||||
|
||||
Options:
|
||||
--app [GeckoViewExample|Firefox|Fenix|Chrome|Refbrow|Fennec]
|
||||
App type to launch. [required]
|
||||
--binary FILE Path to the app to launch. If Android app
|
||||
path to APK file to install
|
||||
--proxy [mitm2|mitm4|wpr] Proxy Service to use. [required]
|
||||
--record / --replay
|
||||
--certutil FILE Path to certutil. Note: Only when recording
|
||||
and Only on Android
|
||||
--sites FILE JSON file containing the websites
|
||||
information we want ro record. Note: Only
|
||||
when recording
|
||||
--url URL Site to load. Note: Only when replaying.
|
||||
--config FILE Read configuration from FILE.
|
||||
--help Show this message and exit.
|
||||
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@ import subprocess
|
|||
|
||||
from mozdevice import ADBAndroid
|
||||
from mozprofile import create_profile
|
||||
from mozversion import mozversion
|
||||
|
||||
|
||||
class AbstractAndroidFirefox(object):
|
||||
def __init__(self, proxy, certutil):
|
||||
def __init__(self, proxy, certutil, binary):
|
||||
self.proxy = proxy
|
||||
self.certutil = certutil
|
||||
self.app_args = [
|
||||
|
|
@ -18,12 +19,45 @@ class AbstractAndroidFirefox(object):
|
|||
"env1",
|
||||
"R_LOG_LEVEL=6",
|
||||
]
|
||||
self.binary = binary
|
||||
self.profile = None
|
||||
|
||||
def set_profile(self):
|
||||
self.profile = create_profile("firefox")
|
||||
print("Created profile: {}".format(self.profile.profile))
|
||||
|
||||
device_storage = "/sdcard/raptor"
|
||||
device_profile = os.path.join(device_storage, "profile")
|
||||
if self.device.is_dir(device_storage):
|
||||
self.device.rm(device_storage, recursive=True)
|
||||
self.device.mkdir(device_storage)
|
||||
self.device.mkdir(device_profile)
|
||||
|
||||
self.app_args.extend(["-profile", device_profile])
|
||||
|
||||
userjs = os.path.join(self.profile.profile, "user.js")
|
||||
with open(userjs) as f:
|
||||
prefs = f.readlines()
|
||||
|
||||
prefs = [p for p in prefs if "network.proxy" not in p]
|
||||
|
||||
with open(userjs, "w") as f:
|
||||
f.writelines(prefs)
|
||||
|
||||
self.profile.set_preferences(
|
||||
{
|
||||
"network.proxy.type": 1,
|
||||
"network.proxy.http": "127.0.0.1",
|
||||
"network.proxy.http_port": 8080,
|
||||
"network.proxy.ssl": "127.0.0.1",
|
||||
"network.proxy.ssl_port": 8080,
|
||||
"network.proxy.no_proxies_on": "localhost, 127.0.0.1",
|
||||
}
|
||||
)
|
||||
|
||||
self.device.push(self.profile.profile, device_profile)
|
||||
self.device.chmod(device_storage, recursive=True)
|
||||
|
||||
def create_certificate(self):
|
||||
certdb = "sql:{}/".format(self.profile.profile)
|
||||
print("Creating certificate database")
|
||||
|
|
@ -51,55 +85,42 @@ class AbstractAndroidFirefox(object):
|
|||
assert "mitmproxy-cert" in subprocess.check_output(command)
|
||||
|
||||
def setup_device(self):
|
||||
# setup device
|
||||
self.device = ADBAndroid()
|
||||
|
||||
self.device.stop_application(self.APP_NAME)
|
||||
if self.binary:
|
||||
if self.device.is_app_installed(self.APP_NAME):
|
||||
print("Uninstalling app %s" % self.APP_NAME)
|
||||
self.device.uninstall_app(self.APP_NAME)
|
||||
self.device.install_app(apk_path=self.binary)
|
||||
|
||||
self.device.shell("pm clear {}".format(self.APP_NAME))
|
||||
self.device.create_socket_connection("reverse", "tcp:8080", "tcp:8080")
|
||||
|
||||
device_storage = "/sdcard/raptor"
|
||||
device_profile = os.path.join(device_storage, "profile")
|
||||
if self.device.is_dir(device_storage):
|
||||
self.device.rm(device_storage, recursive=True)
|
||||
self.device.mkdir(device_storage)
|
||||
self.device.mkdir(device_profile)
|
||||
self.app_args.extend(["-profile", device_profile])
|
||||
|
||||
userjs = os.path.join(self.profile.profile, "user.js")
|
||||
with open(userjs) as f:
|
||||
prefs = f.readlines()
|
||||
|
||||
prefs = [p for p in prefs if "network.proxy" not in p]
|
||||
|
||||
with open(userjs, "w") as f:
|
||||
f.writelines(prefs)
|
||||
|
||||
self.profile.set_preferences(
|
||||
{
|
||||
"network.proxy.type": 1,
|
||||
"network.proxy.http": "127.0.0.1",
|
||||
"network.proxy.http_port": 8080,
|
||||
"network.proxy.ssl": "127.0.0.1",
|
||||
"network.proxy.ssl_port": 8080,
|
||||
"network.proxy.no_proxies_on": "localhost, 127.0.0.1",
|
||||
}
|
||||
)
|
||||
|
||||
self.device.push(self.profile.profile, device_profile)
|
||||
self.device.chmod(device_storage, recursive=True)
|
||||
|
||||
def run_android_app(self, url):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def start(self, url="about:blank"):
|
||||
# create profile
|
||||
self.set_profile()
|
||||
# create certificate database
|
||||
self.create_certificate()
|
||||
# setup device
|
||||
self.setup_device()
|
||||
# start app
|
||||
self.set_profile()
|
||||
self.create_certificate()
|
||||
self.run_android_app(url)
|
||||
|
||||
def stop(self):
|
||||
self.device.stop_application(self.APP_NAME)
|
||||
|
||||
def screen_shot(self, path):
|
||||
self.device.rm("/sdcard/screen.png")
|
||||
self.device.shell("screencap -p /sdcard/screen.png")
|
||||
self.device.pull("/sdcard/screen.png", path)
|
||||
self.device.rm("/sdcard/screen.png")
|
||||
|
||||
def app_information(self):
|
||||
if self.binary:
|
||||
return mozversion.get_version(binary=self.binary)
|
||||
return None
|
||||
|
||||
|
||||
class GeckoViewExample(AbstractAndroidFirefox):
|
||||
APP_NAME = "org.mozilla.geckoview_example"
|
||||
|
|
@ -113,7 +134,7 @@ class GeckoViewExample(AbstractAndroidFirefox):
|
|||
extra_args=self.app_args,
|
||||
url=url,
|
||||
e10s=True,
|
||||
fail_if_running=False
|
||||
fail_if_running=False,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -123,10 +144,7 @@ class Fenix(AbstractAndroidFirefox):
|
|||
INTENT = "android.intent.action.VIEW"
|
||||
|
||||
def run_android_app(self, url):
|
||||
extras = {
|
||||
"args": " ".join(self.app_args),
|
||||
"use_multiprocess": True
|
||||
}
|
||||
extras = {"args": " ".join(self.app_args), "use_multiprocess": True}
|
||||
|
||||
# start app
|
||||
self.device.stop_application(self.APP_NAME)
|
||||
|
|
@ -136,7 +154,7 @@ class Fenix(AbstractAndroidFirefox):
|
|||
self.INTENT,
|
||||
extras=extras,
|
||||
url=url,
|
||||
fail_if_running=False
|
||||
fail_if_running=False,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -148,10 +166,7 @@ class Fennec(AbstractAndroidFirefox):
|
|||
def run_android_app(self, url):
|
||||
self.device.stop_application(self.APP_NAME)
|
||||
self.device.launch_fennec(
|
||||
self.APP_NAME,
|
||||
extra_args=self.app_args,
|
||||
url=url,
|
||||
fail_if_running=False
|
||||
self.APP_NAME, extra_args=self.app_args, url=url, fail_if_running=False
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -161,10 +176,7 @@ class RefBrow(AbstractAndroidFirefox):
|
|||
INTENT = "android.intent.action.MAIN"
|
||||
|
||||
def run_android_app(self, url):
|
||||
extras = {
|
||||
"args": " ".join(self.app_args),
|
||||
"use_multiprocess": True
|
||||
}
|
||||
extras = {"args": " ".join(self.app_args), "use_multiprocess": True}
|
||||
|
||||
# start app
|
||||
self.device.stop_application(self.APP_NAME)
|
||||
|
|
@ -174,5 +186,5 @@ class RefBrow(AbstractAndroidFirefox):
|
|||
self.INTENT,
|
||||
extras=extras,
|
||||
url=url,
|
||||
fail_if_running=False
|
||||
fail_if_running=False,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
class AbstractDesktop(object):
|
||||
def __init__(self, proxy, certutil, binary=None):
|
||||
self.proxy = proxy
|
||||
self.binary = binary
|
||||
self.certutil = certutil
|
||||
self.driver = None
|
||||
|
||||
def screen_shot(self, path):
|
||||
self.driver.save_screenshot(path)
|
||||
|
||||
def stop(self):
|
||||
self.driver.quit()
|
||||
|
|
@ -1,10 +1,9 @@
|
|||
from selenium.webdriver import Chrome, ChromeOptions
|
||||
|
||||
from apps.desktop import AbstractDesktop
|
||||
|
||||
class DesktopChrome(object):
|
||||
def __init__(self, proxy, *args):
|
||||
self.proxy = proxy
|
||||
|
||||
class DesktopChrome(AbstractDesktop):
|
||||
def start(self, url="about:blank"):
|
||||
options = ChromeOptions()
|
||||
options.add_argument("--proxy-server=127.0.0.1:8080")
|
||||
|
|
@ -12,5 +11,11 @@ class DesktopChrome(object):
|
|||
options.add_argument("--ignore-certificate-errors")
|
||||
options.add_argument("--no-default-browser-check")
|
||||
|
||||
driver = Chrome(options=options)
|
||||
driver.get(url)
|
||||
self.driver = Chrome(options=options)
|
||||
self.driver.get(url)
|
||||
|
||||
def app_information(self):
|
||||
app_information = {}
|
||||
app_information["browserName"] = self.driver.capabilities["browserName"]
|
||||
app_information["browserVersion"] = self.driver.capabilities["browserVersion"]
|
||||
return app_information
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from mozversion import mozversion
|
||||
from selenium.webdriver import Firefox, FirefoxOptions
|
||||
|
||||
from apps.desktop import AbstractDesktop
|
||||
|
||||
class DesktopFirefox(object):
|
||||
def __init__(self, proxy, *args):
|
||||
self.proxy = proxy
|
||||
|
||||
class DesktopFirefox(AbstractDesktop):
|
||||
def start(self, url="about:blank"):
|
||||
options = FirefoxOptions()
|
||||
options.set_preference("network.proxy.type", 1)
|
||||
|
|
@ -14,5 +14,18 @@ class DesktopFirefox(object):
|
|||
options.set_preference("network.proxy.ssl_port", 8080)
|
||||
options.set_preference("security.csp.enable", True)
|
||||
|
||||
driver = Firefox(options=options)
|
||||
driver.get(url)
|
||||
self.driver = Firefox(options=options, firefox_binary=self.binary)
|
||||
self.driver.get(url)
|
||||
|
||||
def app_information(self):
|
||||
app_information = {}
|
||||
if self.binary:
|
||||
app_information = mozversion.get_version(binary=self.binary)
|
||||
else:
|
||||
app_information["browserName"] = self.driver.capabilities["browserName"]
|
||||
app_information["browserVersion"] = self.driver.capabilities[
|
||||
"browserVersion"
|
||||
]
|
||||
app_information["buildID"] = self.driver.capabilities["moz:buildID"]
|
||||
|
||||
return app_information
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
certutil="/usr/bin/certutil"
|
||||
|
||||
app="Chrome"
|
||||
proxy="mitm4"
|
||||
|
||||
sites="mobile-sites.json."
|
||||
path="Recordings"
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
certutil="/usr/bin/certutil"
|
||||
|
||||
app="Chrome"
|
||||
proxy="mitm4"
|
||||
|
||||
path="Recordings\google.mp"
|
||||
url="https://www.google.com"
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
[
|
||||
{
|
||||
"label": "search",
|
||||
"url": "https://www.google.com/search?hl=en&q=barack+obama&cad=h",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://www.youtube.com"
|
||||
},
|
||||
{
|
||||
"url": "https://www.facebook.com",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1"
|
||||
},
|
||||
{
|
||||
"url": "https://www.paypal.com/myaccount/summary/",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://www.tumblr.com/dashboard",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"label": "docs",
|
||||
"url": "https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing"
|
||||
},
|
||||
{
|
||||
"label": "slides",
|
||||
"url": "https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing"
|
||||
},
|
||||
{
|
||||
"label": "sheets",
|
||||
"url": "https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing"
|
||||
},
|
||||
{
|
||||
"url": "https://www.fandom.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase"
|
||||
},
|
||||
{
|
||||
"url": "https://imgur.com/gallery/m5tYJL6"
|
||||
},
|
||||
{
|
||||
"url": "https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2"
|
||||
},
|
||||
{
|
||||
"url": "https://yandex.ru/search/?text=barack%20obama&lr=10115"
|
||||
},
|
||||
{
|
||||
"url": "https://www.bing.com/search?q=barack+obama"
|
||||
},
|
||||
{
|
||||
"url": "https://www.apple.com/macbook-pro/"
|
||||
},
|
||||
{
|
||||
"url": "https://www.microsoft.com/en-us/"
|
||||
},
|
||||
{
|
||||
"url": "https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/"
|
||||
},
|
||||
{
|
||||
"url": "https://www.yahoo.com/lifestyle/police-respond-noise-complaint-end-playing-video-games-respectful-tenants-002329963.html"
|
||||
},
|
||||
{
|
||||
"url": "https://www.netflix.com/title/80117263",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"label": "mail",
|
||||
"url": "https://mail.yahoo.com/",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://twitter.com/BarackObama"
|
||||
},
|
||||
{
|
||||
"url": "https://www.instagram.com/",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://www.linkedin.com/in/thommy-harris-hk-385723106/",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://www.ebay.com/"
|
||||
},
|
||||
{
|
||||
"url": "https://en.wikipedia.org/wiki/Barack_Obama"
|
||||
},
|
||||
{
|
||||
"label": "mail",
|
||||
"url": "https://mail.google.com/",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://outlook.live.com/mail/inbox",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://office.live.com/start/Word.aspx?omkt=en-US&auth=1&nf=1",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://pinterest.com/",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"label": "binast",
|
||||
"url": "https://www.instagram.com/",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://www.twitch.tv/videos/326804629"
|
||||
},
|
||||
{
|
||||
"url": "https://www.blogger.com/u/1/blogger.g?blogID={testblogID}",
|
||||
"login": true
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
[
|
||||
{
|
||||
"url": "https://www.amazon.com"
|
||||
},
|
||||
{
|
||||
"url": "https://www.google.com",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://m.facebook.com",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://www.youtube.com"
|
||||
},
|
||||
{
|
||||
"url": "https://www.instagram.com",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://m.ebay-kleinanzeigen.de"
|
||||
},
|
||||
{
|
||||
"url": "https://www.bing.com/search?q=restaurants"
|
||||
},
|
||||
{
|
||||
"label": "search",
|
||||
"url": "https://www.google.com/search?q=restaurants+near+me",
|
||||
"login": true
|
||||
},
|
||||
{
|
||||
"url": "https://booking.com"
|
||||
},
|
||||
{
|
||||
"url": "https://cnn.com"
|
||||
},
|
||||
{
|
||||
"label": "ampstories",
|
||||
"url": " https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other"
|
||||
},
|
||||
{
|
||||
"label": "search",
|
||||
"url": "https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone"
|
||||
},
|
||||
{
|
||||
"url": "https://en.m.wikipedia.org/wiki/Main_Page"
|
||||
},
|
||||
{
|
||||
"label": "video",
|
||||
"url": "https://www.youtube.com/watch?v=COU5T-Wafa4"
|
||||
},
|
||||
{
|
||||
"url": "https://www.reddit.com"
|
||||
},
|
||||
{
|
||||
"url": "https://stackoverflow.com/"
|
||||
},
|
||||
{
|
||||
"url": "https://www.bbc.com/news/business-47245877"
|
||||
},
|
||||
{
|
||||
"url": "https://support.microsoft.com/en-us"
|
||||
},
|
||||
{
|
||||
"url": "https://www.jianshu.com/"
|
||||
},
|
||||
{
|
||||
"url": "https://m.imdb.com/"
|
||||
},
|
||||
{
|
||||
"url": "https://www.allrecipes.com/"
|
||||
},
|
||||
{
|
||||
"url": "http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words"
|
||||
},
|
||||
{
|
||||
"url": "https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566"
|
||||
},
|
||||
{
|
||||
"url": "https://aframe.io/examples/showcase/animation/"
|
||||
},
|
||||
{
|
||||
"label": "cristiano",
|
||||
"url": "https://m.facebook.com/Cristiano"
|
||||
}
|
||||
]
|
||||
|
|
@ -2,6 +2,7 @@ import os
|
|||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
dirname = os.path.dirname(__file__)
|
||||
|
||||
|
|
@ -37,17 +38,27 @@ class MITMProxyBase(object):
|
|||
return self.process
|
||||
|
||||
def stop(self):
|
||||
self.process.send_signal(signal.SIGINT)
|
||||
if self.mode is "record":
|
||||
# Record mode. Send proxy stop command and wait for it to close.
|
||||
# If is's not closed kill the process
|
||||
self.process.send_signal(signal.SIGINT)
|
||||
time.sleep(10)
|
||||
|
||||
if self.process.poll() is None:
|
||||
self.process.kill()
|
||||
else:
|
||||
# Replay mode. Wait for the process to be killed and then make sure the proxy is closed
|
||||
try:
|
||||
self.process.wait()
|
||||
finally:
|
||||
self.process.send_signal(signal.SIGINT)
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
try:
|
||||
self.process.wait()
|
||||
finally:
|
||||
self.stop()
|
||||
self.stop()
|
||||
|
||||
|
||||
class MITMProxy202(MITMProxyBase):
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ class AlternateServerPlayback:
|
|||
"""
|
||||
self.flowmap = {}
|
||||
for i in flows:
|
||||
if i.type == 'websocket':
|
||||
if i.type == "websocket":
|
||||
ctx.log.info(
|
||||
"Request is a WebSocketFlow. Removing from request list as WebSockets"
|
||||
" are disabled. Bug 1559117"
|
||||
|
|
|
|||
|
|
@ -1,28 +1,65 @@
|
|||
import datetime
|
||||
import os
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import os
|
||||
import urllib
|
||||
from urllib import parse
|
||||
|
||||
from mitmproxy import ctx
|
||||
|
||||
|
||||
class HttpProtocolExtractor:
|
||||
def __init__(self):
|
||||
self.request_protocol = {}
|
||||
self.hashes = []
|
||||
self.request_count = 0
|
||||
|
||||
ctx.log.info("Init Http Protocol extractor JS")
|
||||
|
||||
def _hash(self, flow):
|
||||
"""
|
||||
Calculates a loose hash of the flow request.
|
||||
"""
|
||||
r = flow.request
|
||||
|
||||
# unquote url
|
||||
# See Bug 1509835
|
||||
_, _, path, _, query, _ = urllib.parse.urlparse(parse.unquote(r.url))
|
||||
queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
|
||||
|
||||
key = [str(r.port), str(r.scheme), str(r.method), str(path)]
|
||||
key.append(str(r.raw_content))
|
||||
key.append(r.host)
|
||||
|
||||
for p in queriesArray:
|
||||
key.append(p[0])
|
||||
key.append(p[1])
|
||||
|
||||
return hashlib.sha256(repr(key).encode("utf8", "surrogateescape")).digest()
|
||||
|
||||
def response(self, flow):
|
||||
ctx.log.info("Response using protocol: %s" % flow.response.data.http_version)
|
||||
self.request_protocol[
|
||||
urllib.parse.urlparse(flow.request.url).netloc
|
||||
] = flow.response.data.http_version.decode("utf-8")
|
||||
self.request_count += 1
|
||||
hash = self._hash(flow)
|
||||
if not hash in self.hashes:
|
||||
self.hashes.append(hash)
|
||||
|
||||
if flow.type == "websocket":
|
||||
ctx.log.info("Response is a WebSocketFlow. Bug 1559117")
|
||||
else:
|
||||
ctx.log.info(
|
||||
"Response using protocol: %s" % flow.response.data.http_version
|
||||
)
|
||||
self.request_protocol[
|
||||
urllib.parse.urlparse(flow.request.url).netloc
|
||||
] = flow.response.data.http_version.decode("utf-8")
|
||||
|
||||
def done(self):
|
||||
output_json = {}
|
||||
|
||||
output_json["recording_date"] = str(datetime.datetime.now())
|
||||
output_json["http_protocol"] = self.request_protocol
|
||||
output_json["recorded_requests"] = self.request_count
|
||||
output_json["recorded_requests_unique"] = len(self.hashes)
|
||||
|
||||
try:
|
||||
# Mitmproxy 4.0.4
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ class WebPageReplay(object):
|
|||
self.certificate_path = os.path.join(self.port_fw.mitm_home, "mitmproxy-ca.pem")
|
||||
|
||||
def __enter__(self):
|
||||
|
||||
self.port_fw.start()
|
||||
print("Starting WebPageReplay")
|
||||
print(" ".join(self.command()))
|
||||
|
|
@ -51,7 +50,6 @@ class WebPageReplay(object):
|
|||
return os.path.join(self.binary_path, name)
|
||||
|
||||
def command(self):
|
||||
|
||||
command = [
|
||||
self.binary,
|
||||
"--https_cert_file",
|
||||
|
|
|
|||
217
studio.py
217
studio.py
|
|
@ -1,10 +1,24 @@
|
|||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import time
|
||||
from zipfile import ZipFile
|
||||
|
||||
import click
|
||||
import click_config_file
|
||||
from mozdevice import ADBAndroid
|
||||
from tldextract import tldextract
|
||||
|
||||
|
||||
from apps.android.firefox import GeckoViewExample, Fenix, Fennec, RefBrow
|
||||
from apps.desktop.firefox import DesktopFirefox as Firefox
|
||||
from apps.android.firefox import (
|
||||
GeckoViewExample,
|
||||
Fenix,
|
||||
Fennec,
|
||||
RefBrow,
|
||||
AbstractAndroidFirefox,
|
||||
)
|
||||
from apps.desktop.chrome import DesktopChrome as Chrome
|
||||
from apps.desktop.firefox import DesktopFirefox as Firefox
|
||||
from proxy.mitmproxy import MITMProxy202, MITMProxy404
|
||||
from proxy.webpagereplay import WebPageReplay
|
||||
|
||||
|
|
@ -14,14 +28,181 @@ APPS = {
|
|||
"Fenix": Fenix,
|
||||
"Fennec": Fennec,
|
||||
"Refbrow": RefBrow,
|
||||
"Chrome": Chrome
|
||||
"Chrome": Chrome,
|
||||
}
|
||||
PROXYS = {"mitm2": MITMProxy202, "mitm": MITMProxy404, "wpr": WebPageReplay}
|
||||
|
||||
PROXYS = {"mitm2": MITMProxy202, "mitm4": MITMProxy404, "wpr": WebPageReplay}
|
||||
|
||||
RECORD_TIMEOUT = 30
|
||||
|
||||
|
||||
class Mode:
|
||||
def __init__(self, app, binary, proxy, certutil, path, sites, url):
|
||||
self.app = app
|
||||
self.binary = binary
|
||||
self.proxy = proxy
|
||||
self.certutil = certutil
|
||||
self.path = path
|
||||
self.sites_path = sites
|
||||
self.url = url
|
||||
self.information = {}
|
||||
|
||||
def _digest_file(self, file, algorithm):
|
||||
"""I take a file like object 'f' and return a hex-string containing
|
||||
of the result of the algorithm 'a' applied to 'f'."""
|
||||
with open(file, "rb") as f:
|
||||
h = hashlib.new(algorithm)
|
||||
chunk_size = 1024 * 10
|
||||
data = f.read(chunk_size)
|
||||
while data:
|
||||
h.update(data)
|
||||
data = f.read(chunk_size)
|
||||
name = repr(f.name) if hasattr(f, "name") else "a file"
|
||||
print("hashed %s with %s to be %s" % (name, algorithm, h.hexdigest()))
|
||||
return h.hexdigest()
|
||||
|
||||
def replaying(self):
|
||||
with PROXYS[self.proxy](path=self.path, mode="replay") as proxy_service:
|
||||
app_service = APPS[self.app](proxy_service, self.certutil)
|
||||
app_service.start(self.url)
|
||||
|
||||
def recording(self):
|
||||
print("Starting record mode!!!")
|
||||
if not os.path.exists(self.path):
|
||||
print("Creating recording path: %s" % self.path)
|
||||
os.mkdir(self.path)
|
||||
|
||||
for site in self.parse_sites_json():
|
||||
if not os.path.exists(os.path.dirname(site["recording_path"])):
|
||||
print(
|
||||
"Creating recording path: %s"
|
||||
% os.path.dirname(site["recording_path"])
|
||||
)
|
||||
os.mkdir(os.path.dirname(site["recording_path"]))
|
||||
|
||||
with PROXYS[self.proxy](
|
||||
path=site["recording_path"], mode="record"
|
||||
) as proxy_service:
|
||||
app_service = APPS[self.app](proxy_service, self.certutil, self.binary)
|
||||
print("Recording %s..." %site["url"])
|
||||
app_service.start(site["url"])
|
||||
|
||||
if not site.get("login", None):
|
||||
print("Waiting %s for the site to load..." % RECORD_TIMEOUT)
|
||||
time.sleep(RECORD_TIMEOUT)
|
||||
else:
|
||||
time.sleep(5)
|
||||
raw_input("Do user input and press <Return>")
|
||||
|
||||
app_service.screen_shot(site["screen_shot_path"])
|
||||
self.information["app_info"] = app_service.app_information()
|
||||
app_service.stop()
|
||||
|
||||
self.update_json_information(site)
|
||||
self.generate_zip_file(site)
|
||||
self.generate_manifest_file(site)
|
||||
|
||||
def parse_sites_json(self):
|
||||
print("Parsing sites json")
|
||||
sites = []
|
||||
if self.sites_path is not None:
|
||||
with open(self.sites_path, "r") as sites_file:
|
||||
sites_json = json.loads(sites_file.read())
|
||||
|
||||
self.information["app"] = self.app.lower()
|
||||
|
||||
self.information["platform"] = {
|
||||
"system": platform.system(),
|
||||
"release": platform.release(),
|
||||
"version": platform.version(),
|
||||
"machine": platform.machine(),
|
||||
"processor": platform.processor(),
|
||||
}
|
||||
|
||||
platform_name = platform.system().lower()
|
||||
|
||||
if isinstance(self.app, AbstractAndroidFirefox):
|
||||
device = ADBAndroid()
|
||||
|
||||
for property in ["ro.product.model", "ro.build.user", "ro.build.version.release"]:
|
||||
self.information[property] = device.shell_output("getprop {}".format(property))
|
||||
|
||||
platform_name = "".join(
|
||||
e for e in self.information["ro.product.model"] if e.isalnum()
|
||||
)
|
||||
|
||||
for site in sites_json:
|
||||
name = [self.proxy, platform_name, self.app.lower(), site["domain"]]
|
||||
label = site.get("label")
|
||||
if label:
|
||||
name.append(label)
|
||||
name = "-".join(name)
|
||||
|
||||
site["path"] = os.path.join(self.path, name, name)
|
||||
site["name"] = name
|
||||
|
||||
site["recording_path"] = "%s.mp" % site["path"]
|
||||
site["json_path"] = "%s.json" % site["path"]
|
||||
site["screen_shot_path"] = "%s.png" % site["path"]
|
||||
site["zip_path"] = os.path.join(self.path, "%s.zip" % site["name"])
|
||||
site["manifest_path"] = os.path.join(
|
||||
self.path, "%s.manifest" % site["name"]
|
||||
)
|
||||
|
||||
sites.append(site)
|
||||
else:
|
||||
raise Exception("No site JSON file found!!!")
|
||||
|
||||
return sites
|
||||
|
||||
def update_json_information(self, site):
|
||||
time.sleep(3)
|
||||
print("Updating json with recording information")
|
||||
|
||||
with open(site["json_path"], "r") as f:
|
||||
json_data = json.loads(f.read())
|
||||
|
||||
self.information["proxy"] = self.proxy
|
||||
|
||||
self.information["url"] = site["url"]
|
||||
self.information["domain"] = tldextract.extract(site["url"]).domain
|
||||
|
||||
self.information["label"] = site.get("label")
|
||||
|
||||
json_data["info"] = self.information
|
||||
with open(site["json_path"], "w") as f:
|
||||
f.write(json.dumps(json_data))
|
||||
|
||||
def generate_zip_file(self, site):
|
||||
print("Generating zip file")
|
||||
|
||||
with ZipFile(site["zip_path"], "w") as zf:
|
||||
zf.write(site["recording_path"], os.path.basename(site["recording_path"]))
|
||||
zf.write(site["json_path"], os.path.basename(site["json_path"]))
|
||||
zf.write(
|
||||
site["screen_shot_path"], os.path.basename(site["screen_shot_path"])
|
||||
)
|
||||
|
||||
def generate_manifest_file(self, site):
|
||||
print("Generating manifest file")
|
||||
with open(site["manifest_path"], "w") as f:
|
||||
manifest = {}
|
||||
manifest["size"] = os.path.getsize(site["zip_path"])
|
||||
manifest["visibility"] = "public"
|
||||
manifest["digest"] = self._digest_file(site["zip_path"], "sha512")
|
||||
manifest["algorithm"] = "sha512"
|
||||
manifest["filename"] = os.path.basename(site["zip_path"])
|
||||
f.write(json.dumps(manifest))
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--app", required=True, type=click.Choice(APPS.keys()), help="App to launch."
|
||||
"--app", required=True, type=click.Choice(APPS.keys()), help="App type to launch."
|
||||
)
|
||||
@click.option(
|
||||
"--binary",
|
||||
default=None,
|
||||
help="Path to the app to launch. If Android app path to APK file to install ",
|
||||
)
|
||||
@click.option(
|
||||
"--proxy",
|
||||
|
|
@ -30,14 +211,24 @@ PROXYS = {"mitm2": MITMProxy202, "mitm": MITMProxy404, "wpr": WebPageReplay}
|
|||
help="Proxy Service to use.",
|
||||
)
|
||||
@click.option("--record/--replay", default=False)
|
||||
@click.option("--certutil", help="Path to certutil.")
|
||||
@click.option("--url", default="about:blank", help="Site to load.")
|
||||
@click.argument("path")
|
||||
@click.option(
|
||||
"--certutil", help="Path to certutil. Note: Only when recording and Only on Android"
|
||||
)
|
||||
@click.option(
|
||||
"--sites",
|
||||
help="JSON file containing the websites information we want ro record. Note: Only when recording",
|
||||
)
|
||||
@click.argument("path", default="Recordings")
|
||||
@click.option(
|
||||
"--url", default="about:blank", help="Site to load. Note: Only when replaying."
|
||||
)
|
||||
@click_config_file.configuration_option()
|
||||
def cli(app, proxy, record, certutil, url, path):
|
||||
with PROXYS[proxy](path=path, mode="record" if record else "replay") as proxy:
|
||||
app = APPS[app](proxy, certutil)
|
||||
app.start(url)
|
||||
def cli(app, binary, proxy, record, certutil, sites, path, url):
|
||||
mode = Mode(app, binary, proxy, certutil, path, sites, url)
|
||||
if record:
|
||||
mode.recording()
|
||||
else:
|
||||
mode.replaying()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче