fix for app update, buffer reuse

Needed to pass ArrayBuffers into Workers and back out, and then back down to polyfill, in order to reuse them and not constantly reallocate

examples/common.js

@@ -182,8 +182,10 @@ class XRExampleBase {
 		this.session.setVideoFrameHandler(worker)
 	}
 
-	requestVideoFrame() {
-		this.session.requestVideoFrame();
+	// request the next frame
+	// buffers is an optional parameter, suggesting buffers that could be used
+	requestVideoFrame(buffers) {
+		this.session.requestVideoFrame(buffers);
 	}
 	
 	/*

examples/simplecv/index.html

@@ -70,7 +70,8 @@
 						"size": {
 						"width": 320,
 						"height": 180,
-						"bytesPerRow": 384
+						"bytesPerRow": 320,
+						"bytesPerPixel": 1
 						},
 						"buffer": "e3x...d7d"   /// convert to Uint8 ArrayBuffer in code below
 					},
@@ -78,7 +79,8 @@
 						"size": {
 						"width": 160,
 						"height": 90,
-						"bytesPerRow": 384
+						"bytesPerRow": 320,
+						"bytesPerPixel": 2
 						},
 						"buffer": "ZZF.../fIJ7"  /// convert to Uint8 ArrayBuffer in code below
 					}
@@ -161,7 +163,15 @@
 					case "YUV420P":
 					this.averageIntensity(frame.buffers[0])
 					this.colorAtCenter(frame.buffers[1])
-					postMessage ({intensity: intensity, cr: cr, cb: cb});
+
+					// pass the buffers back or they will be garbage collected
+					var buffers = frame.buffers
+					var buffs = []
+					for (var i = 0; i < buffers.length; i++) {
+						buffs.push(buffers[i].buffer)
+					}
+	
+					postMessage ({intensity: intensity, cr: cr, cb: cb, buffers: buffs}, buffs);
 					
 				}
 			});
@@ -246,7 +256,7 @@
 							self.messageText = txt;
 						},0);
 						updateCVFPS();
-						self.requestVideoFrame();
+						self.requestVideoFrame(ev.data.buffers);
 					}
 					
 					this.worker.addEventListener('error', (e) => { 

polyfill/XRDisplay.js

@@ -59,6 +59,8 @@ export default class XRDisplay extends EventHandlerBase {
 		})
 	}
 
+	_requestVideoFrame(buffers) {}
+	
 	_requestAnimationFrame(callback){
 		return window.requestAnimationFrame(callback)
 	}

polyfill/XRSession.js

@@ -86,8 +86,8 @@ export default class XRSession extends EventHandlerBase {
 		this._display.addEventListener("videoFrame", callback)
 	}
 
-	requestVideoFrame() {
-		this._display._requestVideoFrame();
+	requestVideoFrame(buffers) {
+		this._display._requestVideoFrame(buffers);
 	}
 
 	_createPresentationFrame(){

polyfill/display/FlatDisplay.js

@@ -197,6 +197,8 @@ export default class FlatDisplay extends XRDisplay {
 
 
     _handleComputerVisionData(ev) {
+        // Do whatever is needed with the image buffers here, and then call
+		// this._arKitWrapper.requestComputerVisionData(buffers) to request a new one
 		try {
 			this.dispatchEvent(
 				new CustomEvent(
@@ -212,10 +214,8 @@ export default class FlatDisplay extends XRDisplay {
 		}
 	}
 
-	_requestVideoFrame() {
-        // Do whatever is needed with the image buffers here, and then call
-		// this._arKitWrapper.requestComputerVisionData() to request a new one
-        this._arKitWrapper.requestComputerVisionData()
+	_requestVideoFrame(buffers) {
+        this._arKitWrapper._requestComputerVisionData(buffers)
 	}
 
 	_createSession(parameters=null){

polyfill/fill/base64-binary.js

@@ -36,12 +36,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 export default class base64 {
 	
 	/* will return a  Uint8Array type */
-	static decodeArrayBuffer(input) {
+	static decodeArrayBuffer(input, buffer) {
 		var bytes = (input.length/4) * 3;
-		var ab = new ArrayBuffer(bytes);
-		this.decode(input, ab);
+		if (!buffer || buffer.byteLength != bytes) {
+			// replace the buffer with a new, appropriately sized one
+			buffer = new ArrayBuffer(bytes);
+		}
+		this.decode(input, buffer);
 		
-		return ab;
+		return buffer;
 	}
 
 	static removePaddingChars(input){

polyfill/platform/ARKitWrapper.js

@@ -98,6 +98,9 @@ export default class ARKitWrapper extends EventHandlerBase {
 			computer_vision_data: false
 		}
 			
+		// temp storage for CV arraybuffers
+		this._ab = []
+
 		// Set up some named global methods that the ARKit to JS bridge uses and send out custom events when they are called
 		let eventCallbacks = [
 			['arkitStartRecording', ARKitWrapper.RECORD_START_EVENT],
@@ -852,7 +855,8 @@ export default class ARKitWrapper extends EventHandlerBase {
 				"size": {
 				  "width": 320,
 				  "height": 180,
-				  "bytesPerRow": 384
+				  "bytesPerRow": 320,
+				  "bytesPerPixel": 1
 				},
 				"buffer": "e3x...d7d"   /// convert to Uint8 buffer in code below
 			  },
@@ -860,7 +864,8 @@ export default class ARKitWrapper extends EventHandlerBase {
 				"size": {
 				  "width": 160,
 				  "height": 90,
-				  "bytesPerRow": 384
+				  "bytesPerRow": 320,
+				  "bytesPerPixel": 2
 				},
 				"buffer": "ZZF.../fIJ7"  /// convert to Uint8 buffer in code below
 			  }
@@ -894,6 +899,11 @@ export default class ARKitWrapper extends EventHandlerBase {
 		}
 	 */
 	_onComputerVisionData(detail) {
+		// convert the arrays
+		if (!detail) {
+			console.error("detail passed to _onComputerVisionData is null")
+			return;
+		}
 		// convert the arrays
 		if (!detail.frame || !detail.frame.buffers || detail.frame.buffers.length <= 0) {
 			console.error("detail passed to _onComputerVisionData is bad, no buffers")
@@ -905,10 +915,21 @@ export default class ARKitWrapper extends EventHandlerBase {
 		} else {
 			// convert buffers in place
 			var buffers = detail.frame.buffers;
+
+			// if there are too many cached array buffers, drop the unneeded ones
+			if (this._ab.length > buffers.length) {
+				this._ab = this._ab.slice(0, buffer.length)
+			}
+
 			for (var i = 0; i < buffers.length; i++) {
+				// gradually increase the size of the ab[] array to hold the temp buffers, 
+				// and add null so it gets allocated properly
+				if (this._ab.length <= i) {
+					this._ab.push(null)
+				}
 				var bufflen = buffers[i].buffer.length;
-				buffers[i].buffer = base64.decodeArrayBuffer(buffers[i].buffer);
-				var buffersize = buffers[i].buffer.length;
+				this._ab[i] = buffers[i].buffer = base64.decodeArrayBuffer(buffers[i].buffer, this._ab[i]);
+				var buffersize = buffers[i].buffer.byteLength;
 				var imagesize = buffers[i].size.height * buffers[i].size.bytesPerRow;
 			}
 			switch(detail.frame.pixelFormatType) {
@@ -931,6 +952,27 @@ export default class ARKitWrapper extends EventHandlerBase {
 			)
 		}	
 	}
+	/*
+	Requests ARKit a new set of buffers for computer vision processing
+	 */
+    _requestComputerVisionData(buffers) {
+		if (buffers) {
+			this._ab = [];
+			// if buffers are passed in, check if they are ArrayBuffers, and if so, save
+			// them for possible use on the next frame.
+			//
+			// we do this because passing buffers down into Workers invalidates them, so we need to
+			// return them here when we get them back from the Worker, so they can be reused. 
+			for (var i=0; i< buffers.length; i++) {
+				if (buffers[i] instanceof ArrayBuffer) {
+					this._ab.push(buffers[i])
+				}
+			}
+		}
+        window.webkit.messageHandlers.requestComputerVisionData.postMessage({})
+	}
+
+
 	_buildWorkerBlob() {
 		var blobURL = URL.createObjectURL( new Blob([ '(',
 
@@ -944,12 +986,15 @@ export default class ARKitWrapper extends EventHandlerBase {
 				_keyStr: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",
 
 				/* will return a  Uint8Array type */
-				decodeArrayBuffer: function (input) {
+				decodeArrayBuffer: function(input, buffer) {
 					var bytes = (input.length/4) * 3;
-					var ab = new ArrayBuffer(bytes);
-					this.decode(input, ab);
+					if (!buffer || buffer.byteLength != bytes) {
+						// replace the buffer with a new, appropriately sized one
+						buffer = new ArrayBuffer(bytes);
+					}
+					this.decode(input, buffer);
 					
-					return ab;
+					return buffer;
 				},
 
 				removePaddingChars: function(input){
@@ -1000,6 +1045,8 @@ export default class ARKitWrapper extends EventHandlerBase {
 				}
 			}
 
+			var ab = [];
+
 			self.addEventListener('message',  function(event){
 				var frame = event.data.frame
 				var camera = event.data.camera
@@ -1007,8 +1054,17 @@ export default class ARKitWrapper extends EventHandlerBase {
 				// convert buffers in place
 				var buffers = frame.buffers;
 				var buffs = []
+				// if there are too many cached array buffers, drop the unneeded ones
+				if (ab.length > buffers.length) {
+					ab = ab.slice(0, buffer.length)
+				}
 				for (var i = 0; i < buffers.length; i++) {
-					buffers[i].buffer = b64.decodeArrayBuffer(buffers[i].buffer);
+					// gradually increase the size of the ab[] array to hold the temp buffers, 
+					// and add null so it gets allocated properly
+					if (ab.length <= i) {
+						ab.push(null)
+					}
+					ab[i] = buffers[i].buffer = b64.decodeArrayBuffer(buffers[i].buffer, ab[i]);
 					buffs.push(buffers[i].buffer)
 				}
 				switch(frame.pixelFormatType) {
@@ -1028,12 +1084,6 @@ export default class ARKitWrapper extends EventHandlerBase {
 		return( blobURL );			
 	}
 	
-	/*
-	Requests ARKit a new set of buffers for computer vision processing
-	 */
-    requestComputerVisionData() {
-        window.webkit.messageHandlers.requestComputerVisionData.postMessage({})
-	}
 }
 
 // ARKitWrapper event names:

viewer.html

@@ -231,6 +231,14 @@
 						<img src="examples/light/screenshot.jpeg" width="300" height="200"/>
 					</a>
 				</li>
+				<li>
+					<a href="examples/simplecv/">Simple CV</a>
+					<a class="source" href="https://github.com/mozilla/webxr-polyfill/blob/master/examples/simplecv/index.html">source</a>
+					<p>Show average world brightness to demonstrate simple computer vision.</p>
+					<a  class="img" href="examples/simplecv/">
+						<img src="examples/simplecv/screenshot.jpeg" width="300" height="200"/>
+					</a>
+				</li>
 			</ul>
 		</section>
 	</body>