From 4c874857eda2e6150123a05865eb88735b150def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tarek=20Ziad=C3=A9?= Date: Thu, 17 Dec 2020 13:22:16 +0000 Subject: [PATCH] Bug 1682829 - Automate a fuzzing smoke test in the CI r=decoder Adds a smoke test script we can use in the CI to smoke test Firefox prior to bigger tests. Differential Revision: https://phabricator.services.mozilla.com/D100008 --- build/gen_test_packages_manifest.py | 2 + .../mozbuild/mozbuild/action/test_archive.py | 7 ++ python/mozbuild/mozbuild/schedules.py | 1 + taskcluster/ci/config.yml | 1 + taskcluster/ci/fuzzing/kind.yml | 47 ++++++++++++ taskcluster/docs/kinds.rst | 7 +- testing/testsuite-targets.mk | 1 + tools/fuzzing/smoke/js.py | 27 +++++++ tools/fuzzing/smoke/smoke.py | 72 +++++++++++++++++++ tools/fuzzing/smoke/tests.py | 35 +++++++++ 10 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 taskcluster/ci/fuzzing/kind.yml create mode 100755 tools/fuzzing/smoke/js.py create mode 100644 tools/fuzzing/smoke/smoke.py create mode 100644 tools/fuzzing/smoke/tests.py diff --git a/build/gen_test_packages_manifest.py b/build/gen_test_packages_manifest.py index b6f2353684b9..afaae8c56ac3 100644 --- a/build/gen_test_packages_manifest.py +++ b/build/gen_test_packages_manifest.py @@ -25,6 +25,7 @@ ALL_HARNESSES = [ "updater-dep", "jsreftest", "perftests", + "fuzztest", ] PACKAGE_SPECIFIED_HARNESSES = [ @@ -41,6 +42,7 @@ PACKAGE_SPECIFIED_HARNESSES = [ "jittest", "jsreftest", "perftests", + "fuzztest", ] # These packages are not present for every build configuration. diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py index 6e467b0b693c..49b698f3fb41 100644 --- a/python/mozbuild/mozbuild/action/test_archive.py +++ b/python/mozbuild/mozbuild/action/test_archive.py @@ -98,6 +98,7 @@ ARCHIVE_FILES = { "jit-test/**", "jittest/**", # To make the ignore checker happy "perftests/**", + "fuzztest/**", ], }, { @@ -670,6 +671,12 @@ ARCHIVE_FILES = { "pattern": "jsreftest/**", }, ], + "fuzztest": [ + { + "source": buildconfig.topsrcdir, + "pattern": "tools/fuzzing/smoke/**", + }, + ], "jittest": [ { "source": buildconfig.topsrcdir, diff --git a/python/mozbuild/mozbuild/schedules.py b/python/mozbuild/mozbuild/schedules.py index abeb3ce10295..66fe623f7daf 100644 --- a/python/mozbuild/mozbuild/schedules.py +++ b/python/mozbuild/mozbuild/schedules.py @@ -49,6 +49,7 @@ EXCLUSIVE_COMPONENTS = [ "condprofile", "cppunittest", "firefox-ui", + "fuzztest", "geckoview-junit", "gtest", "marionette", diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 6683fb5fca45..08a9e4230b14 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -174,6 +174,7 @@ treeherder: 'GhS': 'GitHub Synchronization' 'perftest': 'Performance tests' 'perftest-http3': 'Performance tests with HTTP/3' + 'fuzzing': 'Fuzzing checks' index: products: diff --git a/taskcluster/ci/fuzzing/kind.yml b/taskcluster/ci/fuzzing/kind.yml new file mode 100644 index 000000000000..37ecc1f3837e --- /dev/null +++ b/taskcluster/ci/fuzzing/kind.yml @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.source_test:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - fetch + - toolchain + - build + +job-defaults: + platform: linux64-shippable/opt + require-build: + linux64-shippable/opt: build-linux64-shippable/opt + dependencies: + linux64-opt: build-linux64/opt + worker-type: b-linux + worker: + max-run-time: 3600 + docker-image: {in-tree: debian10-amd64-build} + env: + LD_LIBRARY_PATH: /builds/worker/fetches + JSSHELL: ./js + treeherder: + kind: test + tier: 3 + fetches: + build: + - target.jsshell.zip + - target.fuzztest.tests.tar.gz + +jobs: + simple: + description: Simple Fuzzing Test + treeherder: + symbol: simple-fuzzing + run: + using: run-task + command: >- + cd $MOZ_FETCHES_DIR && + python tools/fuzzing/smoke/smoke.py diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index a16dacb35b99..d84810c05d42 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -715,4 +715,9 @@ Push tasks to try to test new scriptworker deployments. updatebot ------------------ -Check for updates to (supported) third party libraries, and manage their lifecycle. \ No newline at end of file +Check for updates to (supported) third party libraries, and manage their lifecycle. + +fuzzing +------- + +Performs fuzzing smoke tests diff --git a/testing/testsuite-targets.mk b/testing/testsuite-targets.mk index 732ee1157abb..f5195367bb65 100644 --- a/testing/testsuite-targets.mk +++ b/testing/testsuite-targets.mk @@ -122,6 +122,7 @@ TEST_PKGS_TARGZ := \ jsreftest \ jittest \ perftests \ + fuzztest \ $(NULL) ifdef LINK_GTEST_DURING_COMPILE diff --git a/tools/fuzzing/smoke/js.py b/tools/fuzzing/smoke/js.py new file mode 100755 index 000000000000..c787e072e771 --- /dev/null +++ b/tools/fuzzing/smoke/js.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Hello I am a fake jsshell for testing purpose. +Add more features! +""" +from __future__ import absolute_import +import argparse +import sys + + +def run(): + parser = argparse.ArgumentParser(description="Process some integers.") + parser.add_argument("-e", type=str, default=None) + + parser.add_argument("--fuzzing-safe", action="store_true", default=False) + + args = parser.parse_args() + + if args.e is not None: + if "crash()" in args.e: + sys.exit(1) + + +if __name__ == "__main__": + run() diff --git a/tools/fuzzing/smoke/smoke.py b/tools/fuzzing/smoke/smoke.py new file mode 100644 index 000000000000..b81fa5394175 --- /dev/null +++ b/tools/fuzzing/smoke/smoke.py @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Smoke test script for Fuzzing + +This script can be used to perform simple calls using `jsshell` +or whatever other tools you may add. + +The call is done via `taskcluster/ci/fuzzing/kind.yml` and +files contained in the `target.jsshell.zip` and `target.fuzztest.tests.tar.gz` +build artifacts are downloaded to run things. + +Everything included in this directory will be added in +`target.fuzztest.tests.tar.gz` at build time, so you can add more scripts and +tools if you need. They will be located in `$MOZ_FETCHES_DIR` and follow the +same directory structure than the source tree. +""" +from __future__ import absolute_import +from distutils.spawn import find_executable +import os +import os.path +import subprocess +import shlex +import sys + + +def run_jsshell(command, label=None): + """Invokes `jsshell` with command. + + This function will use the `JSSHELL` environment variable, + and fallback to a `js` executable if it finds one + """ + shell = os.environ.get("JSSHELL") + if shell is None: + shell = find_executable("js") + if shell is None: + raise FileNotFoundError(shell) + else: + if not os.path.exists(shell) or not os.path.isfile(shell): + raise FileNotFoundError(shell) + + if label is None: + label = command + sys.stdout.write(label) + cmd = [shell] + shlex.split(command) + sys.stdout.flush() + try: + subprocess.check_call(cmd) + finally: + sys.stdout.write("\n") + sys.stdout.flush() + + +def smoke_test(): + # first, let's make sure it catches crashes so we don't have false + # positives. + try: + run_jsshell("-e 'crash();'", "Testing for crash\n") + except subprocess.CalledProcessError: + pass + else: + raise Exception("Could not get the process to crash") + + # now let's proceed with some tests + run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'", "Simple Fuzzing...") + + # add more smoke tests here + + +if __name__ == "__main__": + # if this calls raises an error, the job will turn red in the CI. + smoke_test() diff --git a/tools/fuzzing/smoke/tests.py b/tools/fuzzing/smoke/tests.py new file mode 100644 index 000000000000..9b49380cadc5 --- /dev/null +++ b/tools/fuzzing/smoke/tests.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import +import os +import pytest +from contextlib import contextmanager + +import smoke + +JS = os.path.join(os.path.dirname(__file__), "js.py") + + +@contextmanager +def fake_js(): + os.environ["JSSHELL"] = JS + try: + yield + finally: + del os.environ["JSSHELL"] + + +def test_run_no_jsshell(): + with pytest.raises(FileNotFoundError): + smoke.run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'") + + +def test_run_jsshell_set(): + with fake_js(): + smoke.run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'") + + +def test_smoke_test(): + with fake_js(): + smoke.smoke_test()