From 1a56460275855d4d6ba5eddcec5e54be460f8927 Mon Sep 17 00:00:00 2001 From: Robert Bartlensky Date: Fri, 6 Jul 2018 17:37:16 +0100 Subject: [PATCH 01/29] Bug 1473951: Add infer to taskcluster and build. r=gps MozReview-Commit-ID: BHi3E6J3nuH --HG-- extra : rebase_source : a59180efe4fed56222d2847d60133739f38c8ca8 --- .flake8 | 1 + build/build-infer/README | 36 ++++++ build/build-infer/build-infer.py | 116 ++++++++++++++++++ build/build-infer/infer-linux64.json | 5 + taskcluster/ci/docker-image/kind.yml | 3 + .../ci/static-analysis-autotest/kind.yml | 1 + taskcluster/ci/toolchain/linux.yml | 22 ++++ taskcluster/docker/infer-build/Dockerfile | 30 +++++ taskcluster/scripts/misc/build-infer-linux.sh | 22 ++++ 9 files changed, 236 insertions(+) create mode 100644 build/build-infer/README create mode 100755 build/build-infer/build-infer.py create mode 100644 build/build-infer/infer-linux64.json create mode 100644 taskcluster/docker/infer-build/Dockerfile create mode 100755 taskcluster/scripts/misc/build-infer-linux.sh diff --git a/.flake8 b/.flake8 index 9676dac2658f..2ef78e1d5a37 100644 --- a/.flake8 +++ b/.flake8 @@ -21,3 +21,4 @@ exclude = security/nss/, testing/mochitest/pywebsocket, tools/lint/test/files, + build/build-infer/build-infer.py, diff --git a/build/build-infer/README b/build/build-infer/README new file mode 100644 index 000000000000..af11d9af3c54 --- /dev/null +++ b/build/build-infer/README @@ -0,0 +1,36 @@ +build-infer.py +============== + +A script to build infer from source. + +``` +usage: build-infer.py [-h] -c CONFIG [--clean] + +optional arguments: + -h, --help show this help message and exit + -c CONFIG, --config CONFIG + infer configuration file + --clean Clean the build directory +``` + +Pre-requisites +-------------- +* Working build toolchain. +* ocam +* git +* autoconf +* libsqlite-dev +* CMake +* Ninja +* Python 2.7 + +Please use the latest available CMake for your platform to avoid surprises. + +Config file format +------------------ + +build-clang.py accepts a JSON config format with the following fields: + +* infer_revision: The infer revision to build. +* infer_repo: git repository for infer. +* patches: Optional list of patches to apply. \ No newline at end of file diff --git a/build/build-infer/build-infer.py b/build/build-infer/build-infer.py new file mode 100755 index 000000000000..cf3e1d501e71 --- /dev/null +++ b/build/build-infer/build-infer.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import subprocess +import json +import argparse +import sys +import shutil +from functools import reduce + + +def check_run(args, path): + print(' '.join(args) + ' in ' + path, file=sys.stderr) + subprocess.run(args, cwd=path, check=True) + + +def run_in(path, args, extra_env=None): + ''' + Runs the given commands in the directory specified by . + ''' + env = dict(os.environ) + env.update(extra_env or {}) + check_run(args, path) + subprocess.run(args, cwd=path) + + +def build_tar_package(tar, name, base, directory): + name = os.path.realpath(name) + run_in(base, [tar, + '-c', + '-%s' % ('J' if '.xz' in name else 'j'), + '-f', + name, directory]) + + +def is_git_repo(dir): + '''Check whether the given directory is a git repository.''' + from subprocess import CalledProcessError + try: + check_run(['git', 'rev-parse'], dir) + return True + except CalledProcessError: + return False + + +def git_clone(main_dir, url, clone_dir, commit): + ''' + Clones the repository from into , and brings the + repository to the state of . + ''' + run_in(main_dir, ['git', 'clone', url, clone_dir]) + run_in(clone_dir, ['git', 'checkout', commit]) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', required=True, + type=argparse.FileType('r'), + help='Infer configuration file') + parser.add_argument('-b', '--base-dir', + help="Base directory for code and build artifacts") + parser.add_argument('--clean', action='store_true', + help='Clean the build directory') + parser.add_argument('--skip-tar', action='store_true', + help='Skip tar packaging stage') + + args = parser.parse_args() + + # The directories end up in the debug info, so the easy way of getting + # a reproducible build is to run it in a know absolute directory. + # We use a directory that is registered as a volume in the Docker image. + if args.base_dir: + base_dir = args.base_dir + else: + base_dir = reduce(os.path.join, + [os.sep + 'builds', 'worker', + 'workspace', 'moz-toolchain']) + infer_dir = os.path.join(base_dir, 'infer') + source_dir = os.path.join(infer_dir, 'src') + build_dir = os.path.join(infer_dir, 'build') + + if args.clean: + shutil.rmtree(build_dir) + os.sys.exit(0) + + config = json.load(args.config) + infer_revision = config['infer_revision'] + infer_repo = config['infer_repo'] + + for folder in [infer_dir, source_dir, build_dir]: + os.makedirs(folder, exist_ok=True) + + # clone infer + if not is_git_repo(source_dir): + # git doesn't like cloning into a non-empty folder. If src is not a git + # repo then just remove it in order to reclone + shutil.rmtree(source_dir) + git_clone(infer_dir, infer_repo, source_dir, infer_revision) + # apply a few patches + dir_path = os.path.dirname(os.path.realpath(__file__)) + # clean the git directory by reseting all changes + git_commands = [['clean', '-f'], ['reset', '--hard']] + for command in git_commands: + run_in(source_dir, ['git']+command) + for p in config.get('patches', []): + run_in(source_dir, ['git', 'apply', os.path.join(dir_path, p)]) + # build infer + run_in(source_dir, ['./build-infer.sh', 'java'], + extra_env={'NO_CMAKE_STRIP': '1'}) + package_name = 'infer' + if not args.skip_tar: + build_tar_package('tar', '%s.tar.xz' % (package_name), + source_dir, os.path.join('infer', 'bin')) diff --git a/build/build-infer/infer-linux64.json b/build/build-infer/infer-linux64.json new file mode 100644 index 000000000000..5b5e51ef3b16 --- /dev/null +++ b/build/build-infer/infer-linux64.json @@ -0,0 +1,5 @@ +{ + "infer_repo": "https://github.com/facebook/infer", + "infer_revision": "14aa1edbf532675f48ba0a35a2e55d07b79b1f3c", + "patches": [] +} diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index ba0919ed6c1c..253c5fdc7790 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -86,6 +86,9 @@ jobs: fetch: symbol: I(fetch) parent: debian9-base + infer-build: + symbol: I(infer) + parent: debian9-base mingw32-build: symbol: I(mingw) parent: debian9-base diff --git a/taskcluster/ci/static-analysis-autotest/kind.yml b/taskcluster/ci/static-analysis-autotest/kind.yml index acc42d0af601..c5540dcd9693 100644 --- a/taskcluster/ci/static-analysis-autotest/kind.yml +++ b/taskcluster/ci/static-analysis-autotest/kind.yml @@ -48,6 +48,7 @@ jobs: toolchains: - linux64-clang - linux64-clang-tidy + - linux64-infer - linux64-rust - linux64-sccache diff --git a/taskcluster/ci/toolchain/linux.yml b/taskcluster/ci/toolchain/linux.yml index 0887d4ff5d52..41136474e389 100755 --- a/taskcluster/ci/toolchain/linux.yml +++ b/taskcluster/ci/toolchain/linux.yml @@ -125,6 +125,28 @@ linux64-clang-tidy: toolchains: - linux64-gcc-4.9 +linux64-infer: + description: "infer build" + index: + product: static-analysis + job-name: linux64-infer + treeherder: + kind: build + platform: toolchains/opt + symbol: TL(infer) + tier: 1 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + docker-image: {in-tree: infer-build} + max-run-time: 3600 + run: + using: toolchain-script + script: build-infer-linux.sh + resources: + - 'build/build-infer/build-infer.py' + - 'build/build-infer/infer-linux64.json' + toolchain-artifact: public/build/infer.tar.xz + linux64-gcc-4.9: description: "GCC 4.9 toolchain build" treeherder: diff --git a/taskcluster/docker/infer-build/Dockerfile b/taskcluster/docker/infer-build/Dockerfile new file mode 100644 index 000000000000..84d46433f4d0 --- /dev/null +++ b/taskcluster/docker/infer-build/Dockerfile @@ -0,0 +1,30 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT +MAINTAINER Mike Hommey + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/workspace +VOLUME /builds/worker/tooltool-cache + +ENV XZ_OPT=-T0 + +RUN apt-get update && \ + apt-get install \ + autoconf \ + bison \ + bzip2 \ + flex \ + curl \ + git \ + opam \ + libsqlite3-dev \ + autoconf \ + automake \ + cmake \ + libc6-dev \ + openjdk-8-jdk-headless \ + pkg-config \ + patch \ + tar \ + unzip \ + zlib1g-dev diff --git a/taskcluster/scripts/misc/build-infer-linux.sh b/taskcluster/scripts/misc/build-infer-linux.sh new file mode 100755 index 000000000000..c42a925cbadf --- /dev/null +++ b/taskcluster/scripts/misc/build-infer-linux.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -x -e -v + +# This script is for building infer for Linux. + +WORKSPACE=$HOME/workspace +HOME_DIR=$WORKSPACE/build +UPLOAD_DIR=$HOME/artifacts + +cd $HOME_DIR/src + +# gets a bit too verbose here +set +x + +cd build/build-infer +./build-infer.py -c infer-linux64.json + +set -x + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +cp infer.tar.* $UPLOAD_DIR From 86dd2d64bf34915f543cfc07577dad50c80105d7 Mon Sep 17 00:00:00 2001 From: Nicolas Chevobbe Date: Thu, 26 Jul 2018 15:18:08 +0200 Subject: [PATCH 02/29] Bug 1476605 - Fix Error rep failure when stacktrace is a longString; r=sole. The actual fix for this issue will come in a future reps bundle when https://github.com/devtools-html/debugger.html/pull/6705 gets merged. But, I do want this change to be uplifted to release, and it should be easier to only apply this simple change than the whole next release which would contain unrelated code changes. Also, this patch introduces a test to make sure we don't ever regress this. MozReview-Commit-ID: 3QPrx3TLln0 --HG-- extra : rebase_source : 9a62da36af01506e9290d059a2aa3a4a7c8bc75f --- .../client/shared/components/reps/reps.js | 2 +- .../webconsole/test/mochitest/browser.ini | 1 + ..._webconsole_error_with_longstring_stack.js | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 devtools/client/webconsole/test/mochitest/browser_webconsole_error_with_longstring_stack.js diff --git a/devtools/client/shared/components/reps/reps.js b/devtools/client/shared/components/reps/reps.js index a6bca5769c31..3509a39dc170 100644 --- a/devtools/client/shared/components/reps/reps.js +++ b/devtools/client/shared/components/reps/reps.js @@ -1978,7 +1978,7 @@ function getStacktraceElements(props, preview) { // Result: // ["scriptLocation:2:100", "scriptLocation", "2", "100"] const locationParts = location.match(/^(.*):(\d+):(\d+)$/); - if (props.onViewSourceInDebugger && location && !IGNORED_SOURCE_URLS.includes(locationParts[1]) && locationParts) { + if (props.onViewSourceInDebugger && location && locationParts && !IGNORED_SOURCE_URLS.includes(locationParts[1])) { const [, url, line, column] = locationParts; onLocationClick = e => { // Don't trigger ObjectInspector expand/collapse. diff --git a/devtools/client/webconsole/test/mochitest/browser.ini b/devtools/client/webconsole/test/mochitest/browser.ini index c9228ef84403..455f07374787 100644 --- a/devtools/client/webconsole/test/mochitest/browser.ini +++ b/devtools/client/webconsole/test/mochitest/browser.ini @@ -276,6 +276,7 @@ subsuite = clipboard [browser_webconsole_cspro.js] [browser_webconsole_document_focus.js] [browser_webconsole_duplicate_errors.js] +[browser_webconsole_error_with_longstring_stack.js] [browser_webconsole_error_with_unicode.js] [browser_webconsole_errors_after_page_reload.js] [browser_webconsole_eval_in_debugger_stackframe.js] diff --git a/devtools/client/webconsole/test/mochitest/browser_webconsole_error_with_longstring_stack.js b/devtools/client/webconsole/test/mochitest/browser_webconsole_error_with_longstring_stack.js new file mode 100644 index 000000000000..1222e261d87e --- /dev/null +++ b/devtools/client/webconsole/test/mochitest/browser_webconsole_error_with_longstring_stack.js @@ -0,0 +1,29 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Check if an error with a longString stack is displayed as expected. + +"use strict"; + +const MESSAGE = "Error with longString stack"; +const TEST_URI = `data:text/html;charset=utf8,`; + +add_task(async function() { + const hud = await openNewTabAndConsole(TEST_URI); + + info("Wait for the error to be logged"); + const msgNode = await waitFor(() => findMessage(hud, MESSAGE)); + ok(msgNode, `Error logged`); + + const errorNode = msgNode.querySelector(".objectBox-stackTrace"); + ok(errorNode, "The error object is logged as expected"); + ok(errorNode.textContent.includes("longString stack")); + ok(errorNode.querySelectorAll(".objectBox-stackTrace-fn").length > 0, + "Frames functions are displayed"); + ok(errorNode.querySelectorAll(".objectBox-stackTrace-location").length > 0, + "Frames location are displayed"); +}); From 417af36aff345490638c513c490d14d69f05473c Mon Sep 17 00:00:00 2001 From: Nicolas Chevobbe Date: Tue, 24 Jul 2018 14:41:50 +0200 Subject: [PATCH 03/29] Bug 1470922 - Make sure jsterm tests run in both old and new jsterm; r=bgrins. Since codeMirror jsterm is becoming the default, we explicitely set the pref to false before each jsterm test so they run with the 2 version of the component. MozReview-Commit-ID: 2qxIcRlIS9M --HG-- extra : rebase_source : 025dcf3ee32969db87621f6c92e284a21e2a51e2 --- devtools/client/webconsole/test/mocha-test-setup.js | 2 +- .../mochitest/browser_jsterm_add_edited_input_to_history.js | 1 + ...sterm_autocomplete-properties-with-non-alphanumeric-names.js | 1 + .../mochitest/browser_jsterm_autocomplete_array_no_index.js | 1 + .../test/mochitest/browser_jsterm_autocomplete_arrow_keys.js | 1 + .../mochitest/browser_jsterm_autocomplete_cached_results.js | 1 + .../test/mochitest/browser_jsterm_autocomplete_commands.js | 1 + .../mochitest/browser_jsterm_autocomplete_crossdomain_iframe.js | 1 + .../test/mochitest/browser_jsterm_autocomplete_escape_key.js | 1 + .../browser_jsterm_autocomplete_extraneous_closing_brackets.js | 1 + .../test/mochitest/browser_jsterm_autocomplete_helpers.js | 1 + .../test/mochitest/browser_jsterm_autocomplete_in_chrome_tab.js | 1 + .../browser_jsterm_autocomplete_in_debugger_stackframe.js | 1 + .../test/mochitest/browser_jsterm_autocomplete_inside_text.js | 1 + .../mochitest/browser_jsterm_autocomplete_native_getters.js | 1 + .../mochitest/browser_jsterm_autocomplete_nav_and_tab_key.js | 1 + .../test/mochitest/browser_jsterm_autocomplete_paste_undo.js | 1 + .../test/mochitest/browser_jsterm_autocomplete_return_key.js | 1 + .../browser_jsterm_autocomplete_return_key_no_selection.js | 1 + .../webconsole/test/mochitest/browser_jsterm_completion.js | 1 + .../test/mochitest/browser_jsterm_content_defined_helpers.js | 1 + .../webconsole/test/mochitest/browser_jsterm_copy_command.js | 1 + .../test/mochitest/browser_jsterm_ctrl_a_select_all.js | 1 + .../webconsole/test/mochitest/browser_jsterm_ctrl_key_nav.js | 1 + .../test/mochitest/browser_jsterm_document_no_xray.js | 1 + .../webconsole/test/mochitest/browser_jsterm_error_docs.js | 1 + .../test/mochitest/browser_jsterm_error_outside_valid_range.js | 1 + .../webconsole/test/mochitest/browser_jsterm_helper_clear.js | 1 + .../webconsole/test/mochitest/browser_jsterm_helper_dollar.js | 1 + .../test/mochitest/browser_jsterm_helper_dollar_dollar.js | 1 + .../webconsole/test/mochitest/browser_jsterm_helper_dollar_x.js | 1 + .../webconsole/test/mochitest/browser_jsterm_helper_help.js | 1 + .../test/mochitest/browser_jsterm_helper_keys_values.js | 1 + .../webconsole/test/mochitest/browser_jsterm_helper_pprint.js | 1 + .../browser_jsterm_hide_when_devtools_chrome_enabled_false.js | 1 + .../client/webconsole/test/mochitest/browser_jsterm_history.js | 1 + .../test/mochitest/browser_jsterm_history_arrow_keys.js | 1 + .../webconsole/test/mochitest/browser_jsterm_history_nav.js | 1 + .../webconsole/test/mochitest/browser_jsterm_history_persist.js | 1 + .../webconsole/test/mochitest/browser_jsterm_input_expansion.js | 1 + .../client/webconsole/test/mochitest/browser_jsterm_inspect.js | 1 + .../webconsole/test/mochitest/browser_jsterm_instance_of.js | 1 + .../webconsole/test/mochitest/browser_jsterm_multiline.js | 1 + .../browser_jsterm_no_autocompletion_on_defined_variables.js | 1 + .../mochitest/browser_jsterm_no_input_and_tab_key_pressed.js | 1 + .../webconsole/test/mochitest/browser_jsterm_null_undefined.js | 1 + .../test/mochitest/browser_jsterm_popup_close_on_tab_switch.js | 1 + .../mochitest/browser_jsterm_screenshot_command_clipboard.js | 1 + .../test/mochitest/browser_jsterm_screenshot_command_file.js | 1 + .../client/webconsole/test/mochitest/browser_jsterm_selfxss.js | 1 + 50 files changed, 50 insertions(+), 1 deletion(-) diff --git a/devtools/client/webconsole/test/mocha-test-setup.js b/devtools/client/webconsole/test/mocha-test-setup.js index 890f51d2c8a0..a90fcaf15928 100644 --- a/devtools/client/webconsole/test/mocha-test-setup.js +++ b/devtools/client/webconsole/test/mocha-test-setup.js @@ -24,7 +24,7 @@ pref("devtools.webconsole.inputHistoryCount", 50); pref("devtools.webconsole.persistlog", false); pref("devtools.webconsole.timestampMessages", false); pref("devtools.webconsole.sidebarToggle", true); -pref("devtools.webconsole.jsterm.codeMirror", false); +pref("devtools.webconsole.jsterm.codeMirror", true); global.loader = { lazyServiceGetter: () => {}, diff --git a/devtools/client/webconsole/test/mochitest/browser_jsterm_add_edited_input_to_history.js b/devtools/client/webconsole/test/mochitest/browser_jsterm_add_edited_input_to_history.js index f49a3cdcfef0..97185424411a 100644 --- a/devtools/client/webconsole/test/mochitest/browser_jsterm_add_edited_input_to_history.js +++ b/devtools/client/webconsole/test/mochitest/browser_jsterm_add_edited_input_to_history.js @@ -13,6 +13,7 @@ const TEST_URI = "data:text/html;charset=utf-8,Web Console test for bug 817834"; add_task(async function() { // Run test with legacy JsTerm + await pushPref("devtools.webconsole.jsterm.codeMirror", false); await performTests(); // And then run it with the CodeMirror-powered one. await pushPref("devtools.webconsole.jsterm.codeMirror", true); diff --git a/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete-properties-with-non-alphanumeric-names.js b/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete-properties-with-non-alphanumeric-names.js index e501514e0257..7051a1504a35 100644 --- a/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete-properties-with-non-alphanumeric-names.js +++ b/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete-properties-with-non-alphanumeric-names.js @@ -11,6 +11,7 @@ const TEST_URI = "data:text/html;charset=utf8,test autocompletion with $ or _"; add_task(async function() { // Run test with legacy JsTerm + await pushPref("devtools.webconsole.jsterm.codeMirror", false); await performTests(); // And then run it with the CodeMirror-powered one. await pushPref("devtools.webconsole.jsterm.codeMirror", true); diff --git a/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete_array_no_index.js b/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete_array_no_index.js index a2068612dcf5..d99d367a26f4 100644 --- a/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete_array_no_index.js +++ b/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete_array_no_index.js @@ -17,6 +17,7 @@ const TEST_URI = `data:text/html;charset=utf-8, add_task(async function() { // Run test with legacy JsTerm + await pushPref("devtools.webconsole.jsterm.codeMirror", false); await performTests(); // And then run it with the CodeMirror-powered one. await pushPref("devtools.webconsole.jsterm.codeMirror", true); diff --git a/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete_arrow_keys.js b/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete_arrow_keys.js index 8245a2e525c4..76a3ea93e5c4 100644 --- a/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete_arrow_keys.js +++ b/devtools/client/webconsole/test/mochitest/browser_jsterm_autocomplete_arrow_keys.js @@ -17,6 +17,7 @@ const TEST_URI = `data:text/html;charset=utf-8,