Bug 841101 - Add support for multi touch action chains in marionette, r=mdas, a=test-only

testing/marionette/client/marionette/__init__.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from marionette import Marionette, HTMLElement, Actions
+from marionette import Marionette, HTMLElement, Actions, MultiActions
 from marionette_test import MarionetteTestCase, CommonTestCase
 from marionette_touch import MarionetteTouchMixin
 from emulator import Emulator

testing/marionette/client/marionette/marionette.py

@@ -134,6 +134,21 @@ class Actions(object):
     def perform(self):
         return self.marionette._send_message('actionChain', 'ok', value=self.action_chain)
 
+class MultiActions(object):
+    def __init__(self, marionette):
+        self.multi_actions = []
+        self.max_length = 0
+        self.marionette = marionette
+
+    def add(self, action):
+        self.multi_actions.append(action.action_chain)
+        if len(action.action_chain) > self.max_length:
+          self.max_length = len(action.action_chain)
+        return self
+
+    def perform(self):
+        return self.marionette._send_message('multiAction', 'ok', value=self.multi_actions, max_length=self.max_length)
+
 class Marionette(object):
 
     CONTEXT_CHROME = 'chrome'

testing/marionette/client/marionette/tests/unit/test_multi_finger.py

@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import time
+from marionette_test import MarionetteTestCase
+from marionette import MultiActions, Actions
+
+class testSingleFinger(MarionetteTestCase):
+    def test_move_element(self):
+      testTouch = self.marionette.absolute_url("testAction.html")
+      self.marionette.navigate(testTouch)
+      start = self.marionette.find_element("id", "mozLink")
+      drop = self.marionette.find_element("id", "mozLinkPos")
+      ele = self.marionette.find_element("id", "mozLinkCopy")
+      multi_action = MultiActions(self.marionette)
+      action1 = Actions(self.marionette)
+      action2 = Actions(self.marionette)
+      action1.press(start).move(drop).wait(3).release()
+      action2.press(ele).wait().release()
+      multi_action.add(action1).add(action2).perform()
+      time.sleep(15)
+      self.assertEqual("Move", self.marionette.execute_script("return document.getElementById('mozLink').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkPos').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkCopy').innerHTML;"))
+
+    def test_move_offset_element(self):
+      testTouch = self.marionette.absolute_url("testAction.html")
+      self.marionette.navigate(testTouch)
+      start = self.marionette.find_element("id", "mozLink")
+      ele = self.marionette.find_element("id", "mozLinkCopy")
+      multi_action = MultiActions(self.marionette)
+      action1 = Actions(self.marionette)
+      action2 = Actions(self.marionette)
+      action1.press(start).move_by_offset(0,300).wait().release()
+      action2.press(ele).wait(5).release()
+      multi_action.add(action1).add(action2).perform()
+      time.sleep(15)
+      self.assertEqual("Move", self.marionette.execute_script("return document.getElementById('mozLink').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkPos').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkCopy').innerHTML;"))
+
+    def test_three_fingers(self):
+      testTouch = self.marionette.absolute_url("testAction.html")
+      self.marionette.navigate(testTouch)
+      start_one = self.marionette.find_element("id", "mozLink")
+      start_two = self.marionette.find_element("id", "mozLinkStart")
+      drop_two = self.marionette.find_element("id", "mozLinkEnd")
+      ele = self.marionette.find_element("id", "mozLinkCopy2")
+      multi_action = MultiActions(self.marionette)
+      action1 = Actions(self.marionette)
+      action2 = Actions(self.marionette)
+      action3 = Actions(self.marionette)
+      action1.press(start_one).move_by_offset(0,300).release()
+      action2.press(ele).wait().wait(5).release()
+      action3.press(start_two).move(drop_two).wait(2).release()
+      multi_action.add(action1).add(action2).add(action3).perform()
+      time.sleep(15)
+      self.assertEqual("Move", self.marionette.execute_script("return document.getElementById('mozLink').innerHTML;"))
+      self.assertEqual("End", self.marionette.execute_script("return document.getElementById('mozLinkPos').innerHTML;"))
+      self.assertTrue(self.marionette.execute_script("return document.getElementById('mozLinkCopy2').innerHTML >= 5000;"))
+      self.assertTrue(self.marionette.execute_script("return document.getElementById('mozLinkEnd').innerHTML >= 5000;"))

testing/marionette/client/marionette/tests/unit/unit-tests.ini

@@ -55,6 +55,10 @@ browser = false
 b2g = true
 browser = false
 
+[test_multi_finger.py]
+b2g = true
+browser = false
+
 [test_simpletest_pass.js]
 [test_simpletest_sanity.py]
 [test_simpletest_chrome.js]

testing/marionette/client/marionette/www/testAction.html

@@ -15,20 +15,34 @@
   <button id="mozLinkPos" style="position:absolute;left:0px;top:355px;" type="button" allowevents=true>Button2</button>
   <!-- "mozLinkCopy" listens for a touchdown and touchup -->
   <button id="mozLinkCopy" style="position:absolute;left:0px;top:455px;" type="button" allowevents=true>Button3</button>
+  <!-- "mozLinkStart" and "mozLinkEnd" work together to perform touchdown on mozLinkStart, horizontal move and then touchup on mozLinkEnd -->  
+  <button id="mozLinkStart" style="position:absolute;left:10px;top:200px;" type="button" allowevents=true>Press</button>
+  <button id="mozLinkEnd" style="position:absolute;left:140px;top:200px;" type="button" allowevents=true>Release</button>
+  <!-- "mozLinkCopy2" listens for a touchdown and touchup. It shows the time when it's fired-->
+  <button id="mozLinkCopy2" style="position:absolute;left:80px;top:455px;" type="button" allowevents=true>Button4</button>
   <script type="text/javascript">
     window.ready = true;
     var press = document.getElementById("mozLink");
     var second = document.getElementById("mozLinkCopy");
+    var third = document.getElementById("mozLinkStart");
+    var fourth = document.getElementById("mozLinkCopy2");
     // touchmove and touchend must be performed on the same element as touchstart
     // here is press for vertical move
-    press.addEventListener("touchstart", changePressText, false);
+    press.addEventListener("touchstart", function(){changePressText("mozLink")}, false);
     press.addEventListener("touchmove", changeMoveText, false);
     press.addEventListener("touchend", changeReleaseText, false);
     // here is second for a tap
-    second.addEventListener("touchstart", changeCopyText, false);
+    second.addEventListener("touchstart", function(){changePressText("mozLinkCopy")}, false);
     second.addEventListener("touchend", changeClickText, false);
-    function changePressText() {
+    // here is third for horizontal move
-      var press = document.getElementById("mozLink");
+    third.addEventListener("touchstart", function(){changePressText("mozLinkStart")}, false);
+    third.addEventListener("touchmove", changeHorizontalMove, false);
+    third.addEventListener("touchend", changeHorizontalRelease, false);
+    // here is fourth for touch up and down with time shown
+    fourth.addEventListener("touchstart", changeTimePress, false);
+    fourth.addEventListener("touchend", changeTimeRelease, false);
+    function changePressText(strId) {
+      var press = document.getElementById(strId);
       press.innerHTML = "Start";
     }
 
@@ -37,23 +51,40 @@
       move.innerHTML = "Move";
     }
 
-    function changeReleaseText(event) {
+    function checkPosition(event, ele) {
       var touches = event.changedTouches;
       var clientX = touches[0].clientX;
       var clientY = touches[0].clientY;
-      var release = document.getElementById("mozLinkPos");
+      var release = document.getElementById(ele);
       var boxr = release.getBoundingClientRect();
-      if (clientY >= boxr.top &&
+      return (clientY >= boxr.top &&
-          clientY <= boxr.bottom &&
+              clientY <= boxr.bottom &&
-          clientX >= boxr.left &&
+              clientX >= boxr.left &&
-          clientX <= boxr.right) {
+              clientX <= boxr.right);
-        release.innerHTML ="End";
+    }
+
+    function changeReleaseText(event) {
+      if (checkPosition(event, "mozLinkPos")) {
+        document.getElementById("mozLinkPos").innerHTML = "End";
       }
     }
 
-    function changeCopyText() {
+    function changeHorizontalMove() {
-      var second = document.getElementById("mozLinkCopy");
+      var press = document.getElementById("mozLinkStart");
-      second.innerHTML = "Start";
+      if (press.innerHTML == "Start") {
+        var d = new Date();
+        press.innerHTML = d.getTime();
+      }
+    }
+
+    function changeHorizontalRelease(event) {
+      if (checkPosition(event, "mozLinkEnd")) {
+        var press = document.getElementById("mozLinkStart");
+        var d = new Date();
+        var timeDiff = d.getTime() - press.innerHTML;
+        document.getElementById("mozLinkEnd").innerHTML = timeDiff;
+        
+      }
     }
 
     function changeClickText() {
@@ -65,6 +96,24 @@
         second.innerHTML = "Error";
       }
     }
+
+    function changeTimePress() {
+      var fourth = document.getElementById("mozLinkCopy2");
+      var d = new Date();
+      fourth.innerHTML = d.getTime();
+    }
+
+    function changeTimeRelease() {
+      var fourth = document.getElementById("mozLinkCopy2");
+      if (fourth.innerHTML != "Button4") {
+        var d = new Date();
+        var timeDiff = d.getTime() - fourth.innerHTML;
+        fourth.innerHTML = timeDiff;
+      }
+      else {
+        fourth.innerHTML = "Error";
+      }
+    }
   </script>
 </body>
 </html>

testing/marionette/client/setup.py

@@ -1,7 +1,7 @@
 import os
 from setuptools import setup, find_packages
 
-version = '0.5.20'
+version = '0.5.21'
 
 # get documentation from the README
 try:

testing/marionette/marionette-actors.js

@@ -1382,6 +1382,27 @@ MarionetteDriverActor.prototype = {
     }
   },
 
+  /**
+   * multiAction
+   *
+   * @param object aRequest
+   *        'value' represents a nested array: inner array represents each event;
+   *        middle array represents collection of events for each finger
+   *        outer array represents all the fingers
+   */
+
+  multiAction: function MDA_multiAction(aRequest) {
+    this.command_id = this.getCommandId();
+    if (this.context == "chrome") {
+       this.sendError("Not in Chrome", 500, null, this.command_id);
+    }
+    else {
+      this.sendAsync("multiAction", {value: aRequest.value,
+                                     maxlen: aRequest.max_length,
+                                     command_id: this.command_id});
+   }
+ },
+
   /**
    * Find an element using the indicated search strategy.
    *
@@ -2127,6 +2148,7 @@ MarionetteDriverActor.prototype.requestTypes = {
   "press": MarionetteDriverActor.prototype.press,
   "release": MarionetteDriverActor.prototype.release,
   "actionChain": MarionetteDriverActor.prototype.actionChain,
+  "multiAction": MarionetteDriverActor.prototype.multiAction,
   "executeAsyncScript": MarionetteDriverActor.prototype.executeWithCallback,
   "executeJSScript": MarionetteDriverActor.prototype.executeJSScript,
   "setSearchTimeout": MarionetteDriverActor.prototype.setSearchTimeout,

testing/marionette/marionette-listener.js

@@ -64,6 +64,8 @@ let touches = [];
 // For assigning unique ids to all touches
 let nextTouchId = 1000;
 let touchIds = {};
+// last touch for each fingerId
+let multiLast = {};
 // last touch for single finger
 let lastTouch = null;
 /**
@@ -109,6 +111,7 @@ function startListeners() {
   addMessageListenerId("Marionette:press", press);
   addMessageListenerId("Marionette:release", release);
   addMessageListenerId("Marionette:actionChain", actionChain);
+  addMessageListenerId("Marionette:multiAction", multiAction);
   addMessageListenerId("Marionette:setSearchTimeout", setSearchTimeout);
   addMessageListenerId("Marionette:goUrl", goUrl);
   addMessageListenerId("Marionette:getUrl", getUrl);
@@ -201,6 +204,7 @@ function deleteSession(msg) {
   removeMessageListenerId("Marionette:press", press);
   removeMessageListenerId("Marionette:release", release);
   removeMessageListenerId("Marionette:actionChain", actionChain);
+  removeMessageListenerId("Marionette:multiAction", multiAction);
   removeMessageListenerId("Marionette:setSearchTimeout", setSearchTimeout);
   removeMessageListenerId("Marionette:goUrl", goUrl);
   removeMessageListenerId("Marionette:getTitle", getTitle);
@@ -993,6 +997,185 @@ function actionChain(msg) {
   }
 }
 
+/**
+ * Function to emit touch events which allow multi touch on the screen
+ * @param type represents the type of event, touch represents the current touch,touches are all pending touches
+ */
+function emitMultiEvents(type, touch, touches) {
+  let target = touch.target;
+  let doc = target.ownerDocument;
+  let win = doc.defaultView;
+  // touches that are in the same document
+  let documentTouches = doc.createTouchList(touches.filter(function(t) {
+    return t.target.ownerDocument === doc;
+  }));
+  // touches on the same target
+  let targetTouches = doc.createTouchList(touches.filter(function(t) {
+    return t.target === target;
+  }));
+  // Create changed touches
+  let changedTouches = doc.createTouchList(touch);
+  // Create the event object
+  let event = curWindow.document.createEvent('TouchEvent');
+  event.initTouchEvent(type,
+                       true,
+                       true,
+                       win,
+                       0,
+                       false, false, false, false,
+                       documentTouches,
+                       targetTouches,
+                       changedTouches);
+  target.dispatchEvent(event);
+}
+
+/**
+ * Function to dispatch one set of actions
+ * @param touches represents all pending touches, batchIndex represents the batch we are dispatching right now
+ */
+function setDispatch(batches, touches, command_id, batchIndex) {
+  if (typeof batchIndex === "undefined") {
+    batchIndex = 0;
+  }
+  // check if all the sets have been fired
+  if (batchIndex >= batches.length) {
+    multiLast = {};
+    sendOk(command_id);
+    return;
+  }
+  // a set of actions need to be done
+  let batch = batches[batchIndex];
+  // each action for some finger
+  let pack;
+  // the touch id for the finger (pack)
+  let touchId;
+  // command for the finger
+  let command;
+  // touch that will be created for the finger
+  let el;
+  let corx;
+  let cory;
+  let touch;
+  let lastTouch;
+  let touchIndex;
+  let waitTime = 0;
+  let maxTime = 0;
+  batchIndex++;
+  // loop through the batch
+  for (let i = 0; i < batch.length; i++) {
+    pack = batch[i];
+    touchId = pack[0];
+    command = pack[1];
+    switch (command) {
+      case 'press':
+        el = elementManager.getKnownElement(pack[2], curWindow);
+        // after this block, the element will be scrolled into view
+        if (!checkVisible(el, command_id)) {
+           sendError("Element is not currently visible and may not be manipulated", 11, null, command_id);
+           return;
+        }
+        corx = pack[3];
+        cory = pack[4];
+        touch = createATouch(el, corx, cory, touchId);
+        multiLast[touchId] = touch;
+        touches.push(touch);
+        emitMultiEvents('touchstart', touch, touches);
+        break;
+      case 'release':
+        touch = multiLast[touchId];
+        // the index of the previous touch for the finger may change in the touches array
+        touchIndex = touches.indexOf(touch);
+        touches.splice(touchIndex, 1);
+        emitMultiEvents('touchend', touch, touches);
+        break;
+      case 'move':
+        el = elementManager.getKnownElement(pack[2], curWindow);
+        lastTouch = multiLast[touchId];
+        let boxTarget = el.getBoundingClientRect();
+        let startTarget = lastTouch.target;
+        let boxStart = startTarget.getBoundingClientRect();
+        // note here corx and cory are relative to the target, not the viewport
+        // we always want to touch the center of the element if the element is specified
+        corx = boxTarget.left - boxStart.left + boxTarget.width * 0.5;
+        cory = boxTarget.top - boxStart.top + boxTarget.height * 0.5;
+        touch = createATouch(startTarget, corx, cory, touchId);
+        touchIndex = touches.indexOf(lastTouch);
+        touches[touchIndex] = touch;
+        multiLast[touchId] = touch;
+        emitMultiEvents('touchmove', touch, touches);
+        break;
+      case 'moveByOffset':
+        el = multiLast[touchId].target;
+        lastTouch = multiLast[touchId];
+        touchIndex = touches.indexOf(lastTouch);
+        let doc = el.ownerDocument;
+        let win = doc.defaultView;
+        // since x and y are relative to the last touch, therefore, it's relative to the position of the last touch
+        let clientX = lastTouch.clientX + pack[2],
+            clientY = lastTouch.clientY + pack[3];
+        let pageX = clientX + win.pageXOffset,
+            pageY = clientY + win.pageYOffset;
+        let screenX = clientX + win.mozInnerScreenX,
+            screenY = clientY + win.mozInnerScreenY;
+        touch = doc.createTouch(win, el, touchId, pageX, pageY, screenX, screenY, clientX, clientY);
+        touches[touchIndex] = touch;
+        multiLast[touchId] = touch;
+        emitMultiEvents('touchmove', touch, touches);
+        break;
+      case 'wait':
+        if (pack[2] != undefined ) {
+          waitTime = pack[2]*1000;
+          if (waitTime > maxTime) {
+            maxTime = waitTime;
+          }
+        }
+        break;
+    }//end of switch block
+  }//end of for loop
+  if (maxTime != 0) {
+    checkTimer.initWithCallback(function(){setDispatch(batches, touches, command_id, batchIndex);}, maxTime, Ci.nsITimer.TYPE_ONE_SHOT);
+  }
+  else {
+    setDispatch(batches, touches, command_id, batchIndex);
+  }
+}
+
+/**
+ * Function to start multi-action
+ */
+function multiAction(msg) {
+  let command_id = msg.json.command_id;
+  let args = msg.json.value;
+  // maxlen is the longest action chain for one finger
+  let maxlen = msg.json.maxlen;
+  try {
+    // unwrap the original nested array
+    let commandArray = elementManager.convertWrappedArguments(args, curWindow);
+    let concurrentEvent = [];
+    let temp;
+    for (let i = 0; i < maxlen; i++) {
+      let row = [];
+      for (let j = 0; j < commandArray.length; j++) {
+        if (commandArray[j][i] != undefined) {
+          // add finger id to the front of each action, i.e. [finger_id, action, element]
+          temp = commandArray[j][i];
+          temp.unshift(j);
+          row.push(temp);
+        }
+      }
+      concurrentEvent.push(row);
+    }
+    // now concurrent event is made of sets where each set contain a list of actions that need to be fired.
+    // note: each action belongs to a different finger
+    // pendingTouches keeps track of current touches that's on the screen
+    let pendingTouches = [];
+    setDispatch(concurrentEvent, pendingTouches, command_id);
+  }
+  catch (e) {
+    sendError(e.message, e.code, e.stack, msg.json.command_id);
+  }
+}
+
 /**
  * Function to set the timeout period for element searching 
  */