From 110d1587aece68f8facc977cacde3899c5831a5c Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Wed, 4 May 2016 10:57:30 -0700 Subject: [PATCH] Added stability to OSS instrumentation tests Summary: This change makes all instrumentation tests to be executed in sequence in independent retriable processes. With a new test being open sourced recently our CI stability degraded. This PR should bring back stability because tests won't affect each other and will have shorter lifetime Closes https://github.com/facebook/react-native/pull/7353 Differential Revision: D3259081 fb-gh-sync-id: 48ccdb5dbd561d416526497ff474378db9ca3c60 fbshipit-source-id: 48ccdb5dbd561d416526497ff474378db9ca3c60 --- .../react/tests/ReactPickerTestCase.java | 2 +- circle.yml | 2 +- scripts/run-android-instrumentation-tests.sh | 21 +++++-- .../run-ci-android-instrumentation-tests.js | 56 +++++++++++++++++++ scripts/run-ci-e2e-tests.js | 28 +--------- scripts/try-n-times.js | 27 +++++++++ 6 files changed, 103 insertions(+), 33 deletions(-) create mode 100644 scripts/run-ci-android-instrumentation-tests.js create mode 100644 scripts/try-n-times.js diff --git a/ReactAndroid/src/androidTest/java/com/facebook/react/tests/ReactPickerTestCase.java b/ReactAndroid/src/androidTest/java/com/facebook/react/tests/ReactPickerTestCase.java index fcba6412f6..b27d9ecc07 100644 --- a/ReactAndroid/src/androidTest/java/com/facebook/react/tests/ReactPickerTestCase.java +++ b/ReactAndroid/src/androidTest/java/com/facebook/react/tests/ReactPickerTestCase.java @@ -6,7 +6,7 @@ * of patent rights can be found in the PATENTS file in the same directory. */ -package com.facebook.react; +package com.facebook.react.tests; import java.util.ArrayList; import java.util.List; diff --git a/circle.yml b/circle.yml index 569cf98b5c..8979d3fb6f 100644 --- a/circle.yml +++ b/circle.yml @@ -69,7 +69,7 @@ test: # build test APK - buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests --config build.threads=1 # run installed apk with tests - - source scripts/circle-ci-android-setup.sh && retry3 ./scripts/run-android-instrumentation-tests.sh com.facebook.react.tests + - node ./scripts/run-ci-android-instrumentation-tests.js --retries 3 --path ./ReactAndroid/src/androidTest/java/com/facebook/react/tests --package com.facebook.react.tests - ./gradlew :ReactAndroid:testDebugUnitTest # Deprecated: these tests are executed using Buck above, while we support Gradle we just make sure the test code compiles diff --git a/scripts/run-android-instrumentation-tests.sh b/scripts/run-android-instrumentation-tests.sh index fb2960847a..2f7647dc81 100755 --- a/scripts/run-android-instrumentation-tests.sh +++ b/scripts/run-android-instrumentation-tests.sh @@ -1,12 +1,16 @@ #!/bin/bash +# Python script to run instrumentation tests, copied from https://github.com/circleci/circle-dummy-android +# Example: ./scripts/run-android-instrumentation-tests.sh com.facebook.react.tests com.facebook.react.tests.ReactPickerTestCase +# export PATH="$ANDROID_HOME/platform-tools:$ANDROID_HOME/tools:$PATH" # clear the logs adb logcat -c # run tests and check output -python - $1 << END +python - $1 $2 << END + import re import subprocess as sp import sys @@ -14,8 +18,13 @@ import threading import time done = False -test_app = sys.argv[1] +test_app = sys.argv[1] +test_class = None + +if len(sys.argv) > 2: + test_class = sys.argv[2] + def update(): # prevent CircleCI from killing the process for inactivity while not done: @@ -28,8 +37,12 @@ t.start() def run(): sp.Popen(['adb', 'wait-for-device']).communicate() - p = sp.Popen('adb shell am instrument -w %s/android.support.test.runner.AndroidJUnitRunner' % test_app, - shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE) + if (test_class != None): + p = sp.Popen('adb shell am instrument -w -e class %s %s/android.support.test.runner.AndroidJUnitRunner' + % (test_class, test_app), shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE) + else : + p = sp.Popen('adb shell am instrument -w %s/android.support.test.runner.AndroidJUnitRunner' + % (test_app), shell=True, stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE) return p.communicate() success = re.compile(r'OK \(\d+ tests\)') diff --git a/scripts/run-ci-android-instrumentation-tests.js b/scripts/run-ci-android-instrumentation-tests.js new file mode 100644 index 0000000000..cc9b4a720d --- /dev/null +++ b/scripts/run-ci-android-instrumentation-tests.js @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +'use strict'; + +/** + * This script runs instrumentation tests one by one with retries + * Instrumentation tests tend to be flaky, so rerunning them individually increases + * chances for success and reduces total average execution time. + * + * We assume that all instrumentation tests are flat in one folder + * Available arguments: + * --path - path to all .java files with tests + * --package - com.facebook.react.tests + * --retries [num] - how many times to retry possible flaky commands: npm install and running tests, default 1 + */ +/*eslint-disable no-undef */ +require('shelljs/global'); + +const argv = require('yargs').argv; +const numberOfRetries = argv.retries || 1; +const tryExecNTimes = require('./try-n-times'); +const path = require('path'); + +// ReactAndroid/src/androidTest/java/com/facebook/react/tests/ReactHorizontalScrollViewTestCase.java +const testClasses = ls(`${argv.path}/*.java`) +.map(javaFile => { + // ReactHorizontalScrollViewTestCase + return path.basename(javaFile, '.java'); +}).map(className => { + // com.facebook.react.tests.ReactHorizontalScrollViewTestCase + return argv.package + '.' + className; +}); + +let exitCode = 0; +testClasses.forEach((testClass) => { + if (tryExecNTimes( + () => { + exec('sleep 5s'); + return exec(`./scripts/run-android-instrumentation-tests.sh ${argv.package} ${testClass}`).code; + }, + numberOfRetries)) { + echo(`${testClass} failed ${numberOfRetries} times`); + exitCode = 1; + } +}); + +exit(exitCode); + +/*eslint-enable no-undef */ diff --git a/scripts/run-ci-e2e-tests.js b/scripts/run-ci-e2e-tests.js index 3f81353696..cf3534b640 100644 --- a/scripts/run-ci-e2e-tests.js +++ b/scripts/run-ci-e2e-tests.js @@ -27,6 +27,7 @@ const path = require('path'); const SCRIPTS = __dirname; const ROOT = path.normalize(path.join(__dirname, '..')); +const tryExecNTimes = require('./try-n-times'); const TEMP = exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim(); // To make sure we actually installed the local version @@ -39,33 +40,6 @@ let SERVER_PID; let APPIUM_PID; let exitCode; - -/** - * Try executing a function n times recursively. - * Return 0 the first time it succeeds - * Return code of the last failed commands if not more retries left - * @funcToRetry - function that gets retried - * @retriesLeft - number of retries to execute funcToRetry - * @onEveryError - func to execute if funcToRetry returns non 0 - */ -function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) { - const exitCode = funcToRetry(); - if (exitCode === 0) { - return exitCode; - } else { - if (onEveryError) { - onEveryError(); - } - retriesLeft--; - echo(`Command failed, ${retriesLeft} retries left`); - if (retriesLeft === 0) { - return exitCode; - } else { - return tryExecNTimes(funcToRetry, retriesLeft, onEveryError); - } - } -} - try { // install CLI cd('react-native-cli'); diff --git a/scripts/try-n-times.js b/scripts/try-n-times.js new file mode 100644 index 0000000000..1310a6d36e --- /dev/null +++ b/scripts/try-n-times.js @@ -0,0 +1,27 @@ +/** + * Try executing a function n times recursively. + * Return 0 the first time it succeeds + * Return code of the last failed commands if not more retries left + * @funcToRetry - function that gets retried + * @retriesLeft - number of retries to execute funcToRetry + * @onEveryError - func to execute if funcToRetry returns non 0 + */ +function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) { + const exitCode = funcToRetry(); + if (exitCode === 0) { + return exitCode; + } else { + if (onEveryError) { + onEveryError(); + } + retriesLeft--; + echo(`Command failed, ${retriesLeft} retries left`); + if (retriesLeft === 0) { + return exitCode; + } else { + return tryExecNTimes(funcToRetry, retriesLeft, onEveryError); + } + } +} + +module.exports = tryExecNTimes;