VideoFrames now offset from anchors

XRSession creates and destroys a ongoing series of Anchors, which VideoFrames are expresses relative to.  New anchors are currently created each minute.  These anchors are not exposed.

Rather, session.getVideoFramePose will set in the provided matrix to the current tracker coordinates, by using these anchors to adjust to current time (since the videoframe was captured at some point in the past).

polyfill/XRSession.js

@@ -1,4 +1,5 @@
 import EventHandlerBase from './fill/EventHandlerBase.js'
+import MatrixMath from './fill/MatrixMath.js'
 
 /*
 A script that wishes to make use of an XRDisplay can request an XRSession.
@@ -14,6 +15,10 @@ export default class XRSession extends EventHandlerBase {
 
 		this._baseLayer = null
 		this._stageBounds = null
+
+		this._frameAnchors = []
+		this._tempMatrix = MatrixMath.mat4_generateIdentity()		
+		this._tempMatrix2 = MatrixMath.mat4_generateIdentity()		
 	}
 
 	get display(){ return this._display }
@@ -55,6 +60,8 @@ export default class XRSession extends EventHandlerBase {
 				return;
 			}
 			const frame = this._createPresentationFrame()
+			this._updateCameraAnchor(frame)
+
 			this._display._reality._handleNewFrame(frame)
 			this._display._handleNewFrame(frame)
 			callback(frame)
@@ -68,6 +75,10 @@ export default class XRSession extends EventHandlerBase {
 
 	end(){
 		if(this._ended) return
+		for (var i = 0; i< this._frameAnchors.length; i++) {
+			this._display._reality._removeAnchor(this._frameAnchors[i])			
+		}
+		this._frameAnchors = null;
 		this._ended = true
 		this._display._stop()
 		return new Promise((resolve, reject) => {
@@ -75,6 +86,45 @@ export default class XRSession extends EventHandlerBase {
 		})
 	}
 
+	_updateCameraAnchor(frame) {
+		// new anchor each minute
+		if (this._frameAnchors.length == 0 || (this._frameAnchors[0].timestamp + 60000) < frame.timestamp) {
+			const headCoordinateSystem = frame.getCoordinateSystem(XRCoordinateSystem.EYE_LEVEL)
+			const anchorUID = frame.addAnchor(headCoordinateSystem, [0,-1,0])
+			const anchor = frame.getAnchor(anchorUID)
+			anchor.timestamp = frame.timestamp;
+			this._frameAnchors.unshift(anchor)
+
+			if (this._frameAnchors.length > 10) {
+				var oldAnchor = this._frameAnchors.pop()
+				this._display._reality._removeAnchor(oldAnchor.uid)
+			}
+			return anchor;
+		} else {
+			return this._frameAnchors[0]
+		}		
+	}
+
+	_transformToCameraAnchor(camera) {
+		if (this._frameAnchors.length == 0) return camera.viewMatrix
+		
+		var matrix = camera.viewMatrix
+		camera._anchorUid = this._frameAnchors[0].uid
+
+		const anchorCoords = this._frameAnchors[0].coordinateSystem
+
+		// should only have to invert anchor coords, but we're sending in the inverse
+		// of the camera pose ...
+
+		// get world to anchor by inverting anchor to world
+		MatrixMath.mat4_invert(this._tempMatrix, anchorCoords._poseModelMatrix)
+
+		// get camera to world by inverting world to camera
+		MatrixMath.mat4_invert(this._tempMatrix2, matrix)
+
+		MatrixMath.mat4_multiply(camera.viewMatrix, this._tempMatrix, this._tempMatrix2)
+	}
+
 	setVideoFrameHandler(callback) {
 		if (callback instanceof Worker) {
 			var worker = callback;
@@ -86,13 +136,29 @@ export default class XRSession extends EventHandlerBase {
 				// 	buffs.push(buffers[i].buffer)
 				// }
 				// worker.postMessage(cv, buffs);
+				this._transformToCameraAnchor(ev.detail.camera)
 				ev.detail.postMessageToWorker(worker)
 				ev.detail.release()
 			})	
+		} else {
+			var originalCallback = callback;
+			callback = (ev => {
+				this._transformToCameraAnchor(ev.detail.camera)
+				originalCallback(ev)
+			})
 		}
 		this._display.addEventListener("videoFrame", callback)
 	}
 
+    getVideoFramePose(videoFrame, poseOut)
+    {
+        if (!videoFrame.camera._anchorUid) return 
+
+		var anchor = this.reality._getAnchor(videoFrame.camera._anchorUid)
+		var anchorPose = anchor.coordinateSystem._poseModelMatrix
+		MatrixMath.mat4_multiply(poseOut, anchorPose, videoFrame.camera.viewMatrix )
+	}
+	
 	requestVideoFrame() {
 		this._display._requestVideoFrame();
 	}