Move toward using ARKit for anchor tracking.

2017-09-14 11:33:20 -07:00 · 2017-09-14 11:33:20 -07:00 · 5d97c75ccd
--- a/examples/ar_anchors/index.html
+++ b/examples/ar_anchors/index.html
@ -51,6 +51,10 @@

 				// Called during construction
 				initializeStageGroup(){
+					this.stageGroup.add(new THREE.AmbientLight('#FFF', 0.2))
+					let directionalLight = new THREE.DirectionalLight('#FFF', 0.6)
+					directionalLight.position.set(0, 10, 0)
+					this.stageGroup.add(directionalLight)
 				}

 				createSceneGraphNode(){
@ -60,7 +64,7 @@
 				}

 				/*
-					addAnchoredModel creates an anchor at (x,y,z) and positions the sceneGraphNode on the anchor from that point on
+					addAnchoredModel creates an anchor at (x,y,z) relative to the camera and positions the sceneGraphNode on the anchor from that point on
 				*/
 				addAnchoredModel(sceneGraphNode, x, y, z){
 					// Save this info for use during the next render frame
@ -73,15 +77,24 @@
 				// Called once per frame
 				updateStageGroup(frame, stageCoordinateSystem, stagePose){
 					let headCoordinateSystem = frame.getCoordinateSystem(XRCoordinateSystem.HEAD_MODEL)
+
 					// Create anchors for newly anchored nodes
 					for(let anchorToAdd of this.anchorsToAdd){
-						const anchor = new XRAnchor(new XRCoordinates(this.session.display, headCoordinateSystem, [anchorToAdd.x, anchorToAdd.y, anchorToAdd.z]))
+						// Create an anchor that we'd like to add, relative to the current head position
+						const anchorCoordinates = new XRCoordinates(
+							this.session.display,
+							headCoordinateSystem,
+							[anchorToAdd.x, anchorToAdd.y, anchorToAdd.z]
+						)
+						const anchor = new XRAnchor(anchorCoordinates)
+
+						// Add and store the anchor UID along with the node that will be updated with changing anchor data
 						const anchorUID = frame.addAnchor(anchor)
 						this.anchoredNodes.push({
 							anchorUID: anchorUID,
 							node: anchorToAdd.node
 						})
-						console.log("Added anchor", anchorUID)
+						this.stageGroup.add(anchorToAdd.node)
 					}
 					this.anchorsToAdd = []
 					
@ -91,8 +104,10 @@
 						if(anchor === null){
 							console.error('Unknown anchor ID', anchoredNode.anchorId)
 						} else {
-							const localCoordinates = anchor.coordinates.getTransformedCoordinates(stageCoordinateSystem)
-							anchoredNode.node.matrix.fromArray(localCoordinates.poseMatrix)
+							// TODO figure out why the stage position is not correctly calculated when creating the anchor
+							anchoredNode.node.matrixAutoUpdate = false
+							anchoredNode.node.matrix.fromArray(anchor.coordinates.getTransformedCoordinates(stageCoordinateSystem).poseMatrix)
+							anchoredNode.node.updateMatrixWorld(true)
 						}
 					}
 				}
--- a/polyfill/Reality.js
+++ b/polyfill/Reality.js
@ -46,7 +46,7 @@ export default class Reality extends EventHandlerBase {
 	/*
 	Create an anchor hung in space
 	*/
-	_addAnchor(anchor){
+	_addAnchor(anchor, display){
 		// returns DOMString anchor UID
 		throw 'Exending classes should implement _addAnchor'
 	}
@ -54,7 +54,7 @@ export default class Reality extends EventHandlerBase {
 	/*
 	Create an anchor attached to a surface, as found by a ray
 	*/
-	_findAnchor(coordinates){
+	_findAnchor(coordinates, display){
 		// returns DOMString anchor UID
 		throw 'Exending classes should implement _findAnchor'
 	}
--- a/polyfill/XRAnchor.js
+++ b/polyfill/XRAnchor.js
@ -13,6 +13,8 @@ export default class XRAnchor {

 	get coordinates(){	return this._coordinates }

+	set coordinates(value) { this._coordinates = value }
+	
 	static _generateUID(){
 		return 'anchor-' + new Date().getTime() + '-' + Math.floor((Math.random() * Number.MAX_SAFE_INTEGER))
 	}
--- a/polyfill/XRPresentationFrame.js
+++ b/polyfill/XRPresentationFrame.js
@ -35,6 +35,9 @@ export default class XRPresentationFrame {
 		return null
 	}

+	/*
+	Returns an array of known XRAnchor instances. May be empty.
+	*/
 	get anchors(){
 		//readonly attribute sequence<XRAnchor> anchors;
 		let results = []
@ -44,14 +47,18 @@ export default class XRPresentationFrame {
 		return results
 	}

+	/*
+	Create an anchor at a specific position defined by XRAnchor.coordinates
+	*/
 	addAnchor(anchor){
 		//DOMString? addAnchor(XRAnchor anchor);
-		return this._session.reality._addAnchor(anchor)
+		return this._session.reality._addAnchor(anchor, this._session.display)
 	}

 	findAnchor(coordinates){
+		throw 'This needs to change to handle ARKit x,y screen hit coordinates'
 		// XRAnchorOffset? findAnchor(XRCoordinates); // cast a ray to find or create an anchor at the first intersection in the Reality
-		return this._session.reality._findAnchor(coordinates)
+		return this._session.reality._findAnchor(coordinates, this._session.display)
 	}

 	removeAnchor(uid){
@ -59,6 +66,9 @@ export default class XRPresentationFrame {
 		return this._session.reality._removeAnchor(uid)
 	}

+	/*
+	Returns an existing XRAnchor or null if uid is unknown
+	*/
 	getAnchor(uid){
 		// XRAnchor? getAnchor(DOMString uid);
 		return this._session.reality._getAnchor(uid)
--- a/polyfill/display/FlatDisplay.js
+++ b/polyfill/display/FlatDisplay.js
@ -55,8 +55,8 @@ export default class FlatDisplay extends XRDisplay {
 			if(this._initialized === false){
 				this._initialized = true
 				this._arKitWrapper = ARKitWrapper.GetOrCreate()
-				this._arKitWrapper.addEventListener(ARKitWrapper.INIT_EVENT_NAME, this._handleARKitInit.bind(this))
-				this._arKitWrapper.addEventListener(ARKitWrapper.WATCH_EVENT_NAME, this._handleARKitUpdate.bind(this))
+				this._arKitWrapper.addEventListener(ARKitWrapper.INIT_EVENT, this._handleARKitInit.bind(this))
+				this._arKitWrapper.addEventListener(ARKitWrapper.WATCH_EVENT, this._handleARKitUpdate.bind(this))
 				this._arKitWrapper.waitForInit().then(() => {
 					this._arKitWrapper.watch()
 				})
@ -70,7 +70,7 @@ export default class FlatDisplay extends XRDisplay {
 				this._devicePosition = new Vector3()
 				this._deviceWorldMatrix = new Float32Array(16)
 				this._deviceOrientationTracker = new DeviceOrientationTracker()
-				this._deviceOrientationTracker.addEventListener(DeviceOrientationTracker.ORIENTATION_UPDATE_EVENT_NAME, this._updateFromDeviceOrientationTracker.bind(this))
+				this._deviceOrientationTracker.addEventListener(DeviceOrientationTracker.ORIENTATION_UPDATE_EVENT, this._updateFromDeviceOrientationTracker.bind(this))
 			}
 		}
 		this.running = true
--- a/polyfill/fill/DeviceOrientationTracker.js
+++ b/polyfill/fill/DeviceOrientationTracker.js
@ -18,7 +18,7 @@ export default class DeviceOrientationTracker extends EventHandlerBase {
 		}, false)
 		window.addEventListener('deviceorientation', ev => {
 			this._deviceOrientation = ev
-			this.dispatchEvent(new CustomEvent(DeviceOrientationTracker.ORIENTATION_UPDATE_EVENT_NAME, {
+			this.dispatchEvent(new CustomEvent(DeviceOrientationTracker.ORIENTATION_UPDATE_EVENT, {
 				deviceOrientation: this._deviceOrientation,
 				windowOrientation: this._windowOrientation
 			}))
@ -52,7 +52,7 @@ export default class DeviceOrientationTracker extends EventHandlerBase {
 	}
 }

-DeviceOrientationTracker.ORIENTATION_UPDATE_EVENT_NAME = 'orientation-update'
+DeviceOrientationTracker.ORIENTATION_UPDATE_EVENT = 'orientation-update'

 DeviceOrientationTracker.Z_AXIS = new Vector3(0, 0, 1)
 DeviceOrientationTracker.WORKING_EULER = new Euler()
--- a/polyfill/platform/ARKitWrapper.js
+++ b/polyfill/platform/ARKitWrapper.js
@ -8,8 +8,8 @@ ARKitWrapper is a singleton. Use ARKitWrapper.GetOrCreate() to get the instance,

 	if(ARKitWrapper.HasARKit()){
 		let arKitWrapper = ARKitWrapper.GetOrCreate()
-		arKitWrapper.addEventListener(ARKitWrapper.INIT_EVENT_NAME, ev => { console.log('ARKit initialized', ev) })
-		arKitWrapper.addEventListener(ARKitWrapper.WATCH_EVENT_NAME, ev => { console.log('ARKit update', ev) })
+		arKitWrapper.addEventListener(ARKitWrapper.INIT_EVENT, ev => { console.log('ARKit initialized', ev) })
+		arKitWrapper.addEventListener(ARKitWrapper.WATCH_EVENT, ev => { console.log('ARKit update', ev) })
 		arKitWrapper.watch({
 			location: boolean,
 			camera: boolean,
@ -36,18 +36,44 @@ export default class ARKitWrapper extends EventHandlerBase {
 		this._isInitialized = false
 		this._rawARData = null

-		this._globalCallbacksMap = {} // Used to map a window.ARCallback? method name to an ARKitWrapper.on* method name
-		// Set up the window.ARCallback? methods that the ARKit bridge depends on
-		let callbackNames = ['onInit', 'onWatch', 'onStop', 'onAddObject']
+		this._globalCallbacksMap = {} // Used to map a window.ARCallback method name to an ARKitWrapper.on* method name
+		// Set up the window.ARCallback methods that the ARKit bridge depends on
+		let callbackNames = ['onInit', 'onWatch', 'onStop', 'onHitTest', 'onAddAnchor']
 		for(let i=0; i < callbackNames.length; i++){
 			this._generateGlobalCallback(callbackNames[i], i)
 		}
+
+		// Set up some named global methods that the ARKit to JS bridge uses and send out custom events when they are called
+		let eventCallbacks = [
+			['onStartRecording', ARKitWrapper.RECORD_START_EVENT],
+			['onStopRecording', ARKitWrapper.RECORD_STOP_EVENT],
+			['didMoveBackground', ARKitWrapper.DID_MOVE_BACKGROUND_EVENT],
+			['willEnterForeground', ARKitWrapper.WILL_ENTER_FOREGROUND_EVENT],
+			['arkitInterrupted', ARKitWrapper.INTERRUPTED_EVENT],
+			['arkitInterruptionEnded', ARKitWrapper.INTERRUPTION_ENDED_EVENT], 
+			['showDebug', ARKitWrapper.SHOW_DEBUG_EVENT]
+		]
+		for(let i=0; i < eventCallbacks.length; i++){
+			window[eventCallbacks[i][0]] = (detail) => {
+				detail = detail || null
+				this.dispatchEvent(
+					new CustomEvent(
+						ARKitWrapper[eventCallbacks[i][1]],
+						{
+							source: this,
+							detail: detail
+						}
+					)
+				)
+			}
+		}
 	}

-	static GetOrCreate(){
+	static GetOrCreate(options = null){
 		if(typeof ARKitWrapper.GLOBAL_INSTANCE === 'undefined'){
 			ARKitWrapper.GLOBAL_INSTANCE = new ARKitWrapper()
-			ARKitWrapper.GLOBAL_INSTANCE._sendInit()
+			options = (options && typeof(options) == 'object') ? options : {}
+			ARKitWrapper.GLOBAL_INSTANCE._sendInit(options)
 		} 
 		return ARKitWrapper.GLOBAL_INSTANCE
 	}
@ -71,20 +97,19 @@ export default class ARKitWrapper extends EventHandlerBase {
 				return
 			}
 			let callback = () => {
-				this.removeEventListener(ARKitWrapper.INIT_EVENT_NAME, callback, false)
+				this.removeEventListener(ARKitWrapper.INIT_EVENT, callback, false)
 				resolve()
 			}
-			this.addEventListener(ARKitWrapper.INIT_EVENT_NAME, callback, false)
+			this.addEventListener(ARKitWrapper.INIT_EVENT, callback, false)
 		})
 	}

 	/*
 	getData looks into the most recent ARKit data (as received by onWatch) for a key
-	returns the key's value or null if it doesn't exist.
-	If key is null, returns the entire data structure.
+	returns the key's value or null if it doesn't exist or if a key is not specified it returns all data
 	*/
-	getData(key=null){
-		if(key === null){
+	getData(key = null){
+		if (key === null) {
 			return this._rawARData
 		}
 		if(this._rawARData && typeof this._rawARData[key] !== 'undefined'){
@ -95,19 +120,21 @@ export default class ARKitWrapper extends EventHandlerBase {

 	/*
 	returns
-
 		{
-			name: DOMString,
+			uuid: DOMString,
 			transform: [4x4 column major affine transform]
 		}

-	return null if object with `name` is not found
+	return null if object with `uuid` is not found
 	*/
-	getObject(name){
+	getObject(uuid){
+		if (!this._isInitialized) {
+			return null
+		}
 		const objects = this.getKey('objects')
 		if(objects === null) return null
 		for(const object of objects){
-			if(object.name === name){
+			if(object.uuid === uuid){
 				return object
 			}
 		}
@ -115,18 +142,36 @@ export default class ARKitWrapper extends EventHandlerBase {
 	}

 	/*
-	Sends an addObject message to ARKit
+	Sends a hitTest message to ARKit to get hit testing results
+	x, y - screen coordinates normalized to 0..1
+	types - bit mask of hit testing types
 	*/
-	addObject(name, x, y, z) {
-		window.webkit.messageHandlers.addObject.postMessage({
-			name: name,
+	hitTest(x, y, types = ARKitWrapper.HIT_TEST_TYPE_ALL) {
+		if (!this._isInitialized) {
+			return false
+		}
+		window.webkit.messageHandlers.hitTest.postMessage({
 			x: x,
 			y: y,
-			z: z,
-			callback: this._globalCallbacksMap.onAddObject
+			type: types,
+			callback: this._globalCallbacksMap.onHitTest
 		})
 	}

+	/*
+	Sends an addAnchor message to ARKit
+	*/
+	addAnchor(uuid, transform) {
+		if (!this._isInitialized) {
+			return false
+		}
+		window.webkit.messageHandlers.addAnchor.postMessage({
+			uuid: uuid,
+			transform: transform,
+			callback: this._globalCallbacksMap.onAddAnchor
+		})
+	}
+    
 	/*
 	If this instance is currently watching, send the stopAR message to ARKit to request that it stop sending data on onWatch
 	*/
@ -146,7 +191,6 @@ export default class ARKitWrapper extends EventHandlerBase {
 			location: boolean,
 			camera: boolean,
 			objects: boolean,
-			debug: boolean,
 			h_plane: boolean,
 			hit_test_result: 'hit_test_plane'
 		}
@ -165,7 +209,6 @@ export default class ARKitWrapper extends EventHandlerBase {
 				location: true,
 				camera: true,
 				objects: true,
-				debug: false,
 				h_plane: true,
 				hit_test_result: 'hit_test_plane'
 			}
@ -188,13 +231,49 @@ export default class ARKitWrapper extends EventHandlerBase {
 		})
 	}

+	/*
+	Sends a setUIOptions message to ARKit to set ui options (show or hide ui elements)
+	options: {
+		browser: true,
+		points: true,
+		focus: true,
+		rec: true,
+		rec_time: true,
+		mic: true,
+		build: true,
+		plane: true,
+		warnings: true,
+		anchors: true,
+		debug: true
+	}
+	*/
+	setUIOptions(options) {
+		window.webkit.messageHandlers.setUIOptions.postMessage(options)
+	}
+
 	/*
 	Called during instance creation to send a message to ARKit to initialize and create a device ID
 	Usually results in ARKit calling back to _onInit with a deviceId
+	options: {
+		ui: {
+			browser: true,
+			points: true,
+			focus: true,
+			rec: true,
+			rec_time: true,
+			mic: true,
+			build: true,
+			plane: true,
+			warnings: true,
+			anchors: true,
+			debug: true
+		}
+	}
 	*/
-	_sendInit(){
+	_sendInit(options){
 		// get device id
 		window.webkit.messageHandlers.initAR.postMessage({
+			options: options,
 			callback: this._globalCallbacksMap.onInit
 		})
 	}
@ -206,7 +285,7 @@ export default class ARKitWrapper extends EventHandlerBase {
 	_onInit(deviceId) {
 		this._deviceId = deviceId
 		this._isInitialized = true
-		this.dispatchEvent(new CustomEvent(ARKitWrapper.INIT_EVENT_NAME, {
+		this.dispatchEvent(new CustomEvent(ARKitWrapper.INIT_EVENT, {
 			source: this
 		}))
 	}
@ -224,7 +303,7 @@ export default class ARKitWrapper extends EventHandlerBase {
 			},
 			"objects":[
 				{
-					name: DOMString (unique UID),
+					uuid: DOMString (unique UID),
 					transform: [4x4 column major affine transform]
 				}, ...
 			]
@ -233,7 +312,7 @@ export default class ARKitWrapper extends EventHandlerBase {
 	*/
 	_onWatch(data) {
 		this._rawARData = data
-		this.dispatchEvent(new CustomEvent(ARKitWrapper.WATCH_EVENT_NAME, {
+		this.dispatchEvent(new CustomEvent(ARKitWrapper.WATCH_EVENT, {
 			source: this,
 			detail: this._rawARData
 		}))
@ -244,23 +323,46 @@ export default class ARKitWrapper extends EventHandlerBase {
 	*/
 	_onStop() {
 		this._isWatching = false
-		this.dispatchEvent(new CustomEvent(ARKitWrapper.STOP_EVENT_NAME, {
+		this.dispatchEvent(new CustomEvent(ARKitWrapper.STOP_EVENT, {
 			source: this
 		}))
 	}

 	/*
-	Callback from ARKit for when it does the work initiated by sending the addObject message from JS
-	data: { ? }
+	Callback from ARKit for when it does the work initiated by sending the addAnchor message from JS
+	data: {
+		uuid - the anchor's uuid,
+		transform - anchor transformation matrix
+	}
 	*/
-	_onAddObject(data) {
-		data = data
-		this.dispatchEvent(new CustomEvent(ARKitWrapper.ADD_OBJECT_NAME, {
+	_onAddAnchor(data) {
+		this.dispatchEvent(new CustomEvent(ARKitWrapper.ADD_ANCHOR_EVENT, {
 			source: this,
 			detail: data
 		}))
 	}
-	
+
+	/*
+	Callback from ARKit for when it does the work initiated by sending the hitTest message from JS
+	ARKit returns an array of hit results
+	data: [
+		{
+			type: hitTestType,
+			world_transform: matrix4x4 - specifies the position and orientation relative to WCS,
+			local_transform: matrix4x4 - the position and orientation of the hit test result relative to the nearest anchor or feature point,
+			anchor: {uuid, transform, ...} - the anchor representing the detected surface, if any
+		},
+		...
+	]
+	@see https://developer.apple.com/documentation/arkit/arframe/2875718-hittest
+	*/
+	_onHitTest(data) {
+		this.dispatchEvent(new CustomEvent(ARKitWrapper.HIT_TEST_EVENT, {
+			source: this,
+			detail: data
+		}))
+	}
+
 	/*
 	The ARKit iOS app depends on several callbacks on `window`. This method sets them up.
 	They end up as window.ARCallback? where ? is an integer.
@ -270,14 +372,33 @@ export default class ARKitWrapper extends EventHandlerBase {
 		const name = 'ARCallback' + num
 		this._globalCallbacksMap[callbackName] = name
 		const self = this
-		window[name] = function(deviceData) {
-			self['_' + callbackName](deviceData)
+		window[name] = function(...deviceData) {
+			self['_' + callbackName](...deviceData)
 		}
 	}
 }

 // ARKitWrapper event names:
-ARKitWrapper.INIT_EVENT_NAME = 'arkit-init'
-ARKitWrapper.WATCH_EVENT_NAME = 'arkit-watch'
-ARKitWrapper.STOP_EVENT_NAME = 'arkit-stop'
-ARKitWrapper.ADD_OBJECT_NAME = 'arkit-add-object'
+ARKitWrapper.INIT_EVENT = 'arkit-init'
+ARKitWrapper.WATCH_EVENT = 'arkit-watch'
+ARKitWrapper.STOP_EVENT = 'arkit-stop'
+ARKitWrapper.ADD_ANCHOR_EVENT = 'arkit-add-anchor'
+ARKitWrapper.RECORD_START_EVENT = 'arkit-record-start'
+ARKitWrapper.RECORD_STOP_EVENT = 'arkit-record-stop'
+ARKitWrapper.DID_MOVE_BACKGROUND_EVENT = 'arkit-did-move-background'
+ARKitWrapper.WILL_ENTER_FOREGROUND_EVENT = 'arkit-will-enter-foreground'
+ARKitWrapper.INTERRUPTED_EVENT = 'arkit-interrupted'
+ARKitWrapper.INTERRUPTION_ENDED_EVENT = 'arkit-interruption-ended'
+ARKitWrapper.HIT_TEST_EVENT = 'arkit-hit-test'
+ARKitWrapper.SHOW_DEBUG_EVENT = 'arkit-show-debug'
+
+// hit test types
+ARKitWrapper.HIT_TEST_TYPE_FEATURE_POINT = 1
+ARKitWrapper.HIT_TEST_TYPE_EXISTING_PLANE = 8
+ARKitWrapper.HIT_TEST_TYPE_ESTIMATED_HORIZONTAL_PLANE = 2
+ARKitWrapper.HIT_TEST_TYPE_EXISTING_PLANE_USING_EXTENT = 16
+
+ARKitWrapper.HIT_TEST_TYPE_ALL = ARKitWrapper.HIT_TEST_TYPE_FEATURE_POINT |
+	ARKitWrapper.HIT_TEST_TYPE_EXISTING_PLANE |
+	ARKitWrapper.HIT_TEST_TYPE_ESTIMATED_HORIZONTAL_PLANE |
+	ARKitWrapper.HIT_TEST_TYPE_EXISTING_PLANE_USING_EXTENT
--- a/polyfill/reality/CameraReality.js
+++ b/polyfill/reality/CameraReality.js
@ -80,7 +80,7 @@ export default class CameraReality extends Reality {
 			if(this._initialized === false){
 				this._initialized = true
 				this._arKitWrapper = ARKitWrapper.GetOrCreate()
-				this._arKitWrapper.addEventListener(ARKitWrapper.ADD_OBJECT_NAME, this._handleARKitAddObject.bind(this))
+				this._arKitWrapper.addEventListener(ARKitWrapper.ADD_ANCHOR_EVENT, this._handleARKitAddObject.bind(this))
 				this._arKitWrapper.waitForInit().then(() => {
 					this._arKitWrapper.watch()
 				})
@ -125,14 +125,22 @@ export default class CameraReality extends Reality {
 	}

 	_handleARKitAddObject(ev){
-		console.log('AR add object', ev)
+		const anchor = this._anchors.get(ev.detail.uuid) || null
+		if(anchor === null){
+			console.log('unknown anchor', anchor)
+			return
+		}
+		// TODO update the local anchor coordinates
 	}

-	_addAnchor(anchor){
-		console.log('reality adding anchor', anchor)
-
-		// TODO talk to ARKit or ARCore to create an anchor
-
+	_addAnchor(anchor, display){
+		// Convert coordinates to the stage coordinate system
+		anchor.coordinates = anchor.coordinates.getTransformedCoordinates(display._stageCoordinateSystem)
+		if(this._arKitWrapper !== null){
+			this._arKitWrapper.addAnchor(anchor.uid, anchor.coordinates.poseMatrix)
+		} else if(this._vrDisplay){
+			// TODO talk to ARCore to create an anchor
+		}
 		this._anchors.set(anchor.uid, anchor)
 		return anchor.uid
 	}
@ -140,7 +148,7 @@ export default class CameraReality extends Reality {
 	/*
 	Creates an anchor attached to a surface, as found by a ray
 	*/
-	_findAnchor(coordinates){
+	_findAnchor(coordinates, display){
 		// XRAnchorOffset? findAnchor(XRCoordinates); // cast a ray to find or create an anchor at the first intersection in the Reality
 		// TODO talk to ARKit to create an anchor
 		throw 'Need to implement in CameraReality'