mozilla
/
webxr-polyfill
зеркало из https://github.com/mozilla/webxr-polyfill.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

cleanup of CV examples

This commit is contained in:
Blair MacIntyre 2018-04-17 14:47:21 -04:00
Родитель ff450c985f
Коммит e3067d17bc
5 изменённых файлов: 124 добавлений и 159 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

55
examples/opencv-aruco/worker.js
Просмотреть файл

@ -114,25 +114,7 @@ function createCVMat2(rotation, buffer, pixelFormat) {
cv.cvtColor(img_rgba, img_gray, cv.COLOR_RGBA2GRAY, 0);
break;
}
// switch(rotation) {
// case -90:
// cv.rotate(img_gray, rotatedImage, cv.ROTATE_90_CLOCKWISE);
// return rotatedImage;
// break;
// case 90:
// cv.rotate(img_gray, rotatedImage, cv.ROTATE_90_COUNTERCLOCKWISE);
// return rotatedImage;
// break;
// case 180:
// cv.rotate(img_gray, rotatedImage, cv.ROTATE_180);
// return rotatedImage;
// break;
// default:
// return img_gray;
// }
return img_gray;
}
///////////
@ -205,22 +187,14 @@ self.addEventListener('message', function(event){
var width = buffer.size.width;
var height = buffer.size.height;
// // let's pick a size such that the video is below 256 in size in both dimensions
// // since face detection is really expensive on large images
// while (width > 256 || height > 256) {
// width = width / 2
// height = height / 2
// scale = scale / 2;
// }
// set up a camera coefficient matrix
// set up a camera coefficient matrices, for distortion and projection
if (camDistortion == null) {
// we assume we're getting no distortion for now, since our cameras appear rectaliner
// we probably want to add distortion coefficients to the camera definition since I assume some
// cameras will have it
camDistortion = cv.matFromArray(5, 1, cv.CV_64F, [0, 0, 0, 0, 0]);
}
videoFrame.camera.cameraIntrinsics[8] = 1.0
}
//videoFrame.camera.cameraIntrinsics[8] = 1.0 // was broken in the app, should be fixed now
camIntrinsics.create(3, 3, cv.CV_64F)
camIntrinsics.data64F.set(videoFrame.camera.cameraIntrinsics);
@ -239,12 +213,16 @@ self.addEventListener('message', function(event){
}
}
// detect the aruco markers
cv.detectMarkers(image, dictionary, markerCorners, markerIds, parameter);
postMessage({type: "cvAfterDetect", time: ( performance || Date ).now()});
// did we find any?
if (markerIds.rows > 0) {
// need to adjust the corners to be around 0,0 at the center
// Need to adjust the corners to be around 0,0 at the center.
// Weirdly, the aruco detector returns the values in image coordinates,
// even though pose estimation expects them relative to the center of the
// screen.
for(let i=0; i < markerIds.rows; ++i) {
let cornervec = markerCorners.get(i).data32F
cornervec[0] -= width/2;
@ -265,6 +243,7 @@ self.addEventListener('message', function(event){
// cornervec[7] *= -1
}
// estimate the poses of the found markers
cv.estimatePoseSingleMarkers(markerCorners, 0.053, camIntrinsics, camDistortion, rvecs, tvecs);
for(let i=0; i < markerIds.rows; ++i) {
@ -281,20 +260,18 @@ self.addEventListener('message', function(event){
tm = Array.from(rotMat.data64F)
}
//let tvec = [tvecs.doublePtr(0, i)[0], tvecs.doublePtr(0, i)[1], tvecs.doublePtr(0, i)[2]];
//var returnMat = [tm[0], tm[1], tm[2], 0, tm[3], tm[4], tm[5], 0, tm[6], tm[7], tm[8], 0, -tvec[0], -tvec[1], -tvec[2], 1]
//var returnMat = [tm[0], tm[1], tm[2], 0, tm[3], tm[4], tm[5], 0, tm[6], tm[7], tm[8], 0, tvecs.doublePtr(0, i)[0], tvecs.doublePtr(0, i)[1], -tvecs.doublePtr(0, i)[2], 1]
// var returnMat = [tm[0], tm[3], tm[6], 0, tm[1], tm[4], tm[7], 0, tm[2], tm[5], tm[8], 0, tvecs.doublePtr(0, i)[0], tvecs.doublePtr(0, i)[1], -tvecs.doublePtr(0, i)[2], 1]
//var returnMat = [tm[0], tm[3], tm[6], 0, tm[1], tm[4], tm[7], 0, tm[2], tm[5], tm[8], 0, 0,0,0, 1]
// construct a pose matrix from rotation and position
var returnMat = [tm[0], tm[1], tm[2], 0, tm[3], tm[4], tm[5], 0, tm[6], tm[7], tm[8], 0,0,0,0, 1]
mat4.translate(returnMat, returnMat, [tvecs.doublePtr(0, i)[0], tvecs.doublePtr(0, i)[1], tvecs.doublePtr(0, i)[2]]);
// invert it so it's marker relative to camera, not the usual camera relative to marker
mat4.invert(returnMat,returnMat)
// account for camera rotation relative to screen, which happens in video mixed handhelds
mat4.fromZRotation(antiRotation, rotation * Math.PI/180);
mat4.multiply(returnMat, antiRotation, returnMat)
// save the marker info!
markers.push({
id: id,
corners: [{x: cornervec[0], y: cornervec[1]}, {x: cornervec[2], y: cornervec[3]}, {x: cornervec[4], y: cornervec[5]}, {x: cornervec[6], y: cornervec[7]}],
@ -303,6 +280,8 @@ self.addEventListener('message', function(event){
}
}
}
// reply, even if opencv isn't ready.
endTime = ( performance || Date ).now()
videoFrame.postReplyMessage({type: "cvFrame", markers: markers, time: endTime})

64
examples/opencv-face/index.html
Просмотреть файл

@ -75,12 +75,13 @@
<script>
var cvStatusTxt = "";
// set up for collecting different stats
var beginTime = ( performance || Date ).now(), prevTime = beginTime, frames = 0;
var beginTime = ( performance || Date ).now(), prevTime = beginTime, frames = 0;
// set up some stats, including a new pane for CV fps
var stats = new Stats();
stats.domElement.style.cssText = 'position:fixed;top:2%;right:2%;cursor:pointer;opacity:0.9;z-index:10000';
var cvPanel = stats.addPanel( new Stats.Panel( 'CV fps', '#ff8', '#221' ) );
stats.showPanel( 2 ); // 0: fps, 1: ms, 2: mb, 3+: custom
// a method to update a new panel for displaying CV FPS
var updateCVFPS = function () {
frames ++;
@ -92,6 +93,7 @@
}
beginTime = time;
}
document.body.appendChild( stats.dom );
// keep track of what we think the view size is so we can adjust the
// overlay used to render the 2D detection results
@ -122,7 +124,6 @@
var cvImageDiv = document.getElementById("video_canvas");
var cvImageCtx = cvImageDiv.getContext('2d');
document.body.appendChild( stats.dom );
class ARAnchorExample extends XRExampleBase {
constructor(domElement){
@ -136,30 +137,34 @@
this.lightEstimate = 0;
this.el.appendChild(this.textBox)
// some temp variables
this.rotation = -1;
this.rotatedImage = null;
this.face_cascade = null;
this.eye_cascade = null;
this.triggerResize = true;
// try to notice all resizes of the screen
window.addEventListener('resize', () => {
console.log("resize, trigger adjust canvas size")
this.triggerResize = true;
})
}
newSession() {
// new worker
this.worker = new Worker ("worker.js")
this.worker.onmessage = (ev) => {
switch (ev.data.type) {
case "cvFrame":
// finished with a frame!
var videoFrame = XRVideoFrame.createFromMessage(ev)
// get the face rectangles
this.faceRects = ev.data.faceRects;
// timing
cvEndTime = ev.data.time;
cvFaceTime = cvEndTime - cvAfterResizeTime;
// see if the video frame sizes have changed, since
// we'll need to adjust the 2D overlay
var rotation = videoFrame.camera.cameraOrientation;
var buffer = videoFrame.buffer(0)
@ -171,24 +176,32 @@
this.adjustRenderCanvasSize(rotation, width, height)
}
// are we showing the image (for debugging) or just the rectangles?
if (showCVImage) {
this.showVideoFrame(videoFrame)
} else {
cvImageCtx.clearRect(0, 0, cvImageDiv.width, cvImageDiv.height);
}
// show the rectangles
for (let i = 0; i < this.faceRects.length; i++) {
let rect = this.faceRects[i];
cvImageCtx.strokeRect(rect.x, rect.y, rect.width , rect.height);
}
// update CV fps, and release the image
updateCVFPS();
videoFrame.release();
break;
case "cvStart":
// request the next one when the old one finishes
// request the next frame when the worker starts working on the current
// one, to pipeline a bit
this.requestVideoFrame();
// has there been a delay since the last frame was finished processing?
// this shouldn't happen, but is a sign that the background tasks are
// being throttled
cvStartTime = ev.data.time;
if (cvEndTime > 0) {
cvIdleTime = cvStartTime - cvEndTime;
@ -206,11 +219,13 @@
break;
case "cvReady":
// opencv is loaded and ready!
console.log('OpenCV.js is ready');
openCVready = true
break;
case "cvStatus":
// opencv sends status messages sometimes
cvStatusTxt = ev.data.msg;
break;
}
@ -219,24 +234,15 @@
this.worker.addEventListener('error', (e) => {
console.log("worker error:" + e)
})
this.setVideoWorker(this.worker);
// this.setVideoWorker(ev => {
// var videoFrame = ev.detail
// if (openCVready) {
// try {
// this.handleVideoFrame(ev)
// } catch(e) {
// console.error('CV worker error', e)
// }
// updateCVFPS();
// }
// // pass the buffers back or they will be garbage collected
// videoFrame.release();
// this.requestVideoFrame();
// })
}
// new session, tell it about our worker
newSession() {
this.setVideoWorker(this.worker);
}
// when the window or video frame resizes, we need to resize and reposition
// the 2D overlay
adjustRenderCanvasSize (rotation, width, height) {
var cameraAspect;
console.log("adjustRenderCanvasSize: " + rotation + " degrees, " + width + " by " + height)
@ -325,6 +331,7 @@
this.textBox.innerHTML = this.messageText;
}
// do another check on frame size, to make sure we don't miss one
var viewport = frame.views[0].getViewport(this.session.baseLayer);
if (viewport.width != viewWidth || viewport.height != viewHeight) {
viewHeight = viewport.height;
@ -335,6 +342,7 @@
}
}
// show the video frame for debugging
showVideoFrame(videoFrame) {
var camera = videoFrame.camera
var buffer = videoFrame.buffer(0)

8
examples/opencv-face/worker.js
Просмотреть файл

@ -3,7 +3,8 @@ console.log("loaded webxr-worker.js")
var openCVready = false;
// need to load up the files for tracking the face and eyes
// need to load up the files for tracking the face and eyes. WASM module has hooks to pass in
// files to be loaded by opencv
var Module = {
preRun: [function() {
console.log("CV preRun")
@ -76,6 +77,7 @@ console.log("loaded opencv.js:" + cv);
*/
// faces and eyes
var face_cascade;
var eye_cascade;
@ -213,6 +215,10 @@ function createCVMat2(rotation, buffer, pixelFormat) {
break;
}
// face tracker only works if image is upright.
// on mobile, the camera is fixed, even though the display rotates. So, we need
// to rotate the image so it's in the right orientation (upright, relative to how the
// user sees it)
switch(rotation) {
case -90:
cv.rotate(img_gray, rotatedImage, cv.ROTATE_90_CLOCKWISE);

123
examples/simplecv/index.html
Просмотреть файл

@ -63,19 +63,23 @@
// need to figure out how to make loadScripts() work in BlobURLs
</script>
<script>
// RAINBOW Candy of a Certain Name colors
// We'll try to guess if the color matches one of the colors of a certain
// RAINBOW Candy
var colors = [
{ cr: 250, cg: 25, cb: 25, name: "RED" },
{ cr: 25, cg: 250, cb: 25, name: "GREEN" },
{ cr: 250, cg: 250, cb: 50, name: "YELLOW" }
]
// do some time stamping, just to show the performance numbers
var beginTime = ( performance || Date ).now(), prevTime = beginTime, frames = 0;
var stats = new Stats();
// create a new pane for the stats panel to show the fps of the vision processing
stats.domElement.style.cssText = 'position:fixed;top:2%;right:2%;cursor:pointer;opacity:0.9;z-index:10000';
var cvPanel = stats.addPanel( new Stats.Panel( 'CV fps', '#ff8', '#221' ) );
stats.showPanel( 2 ); // 0: fps, 1: ms, 2: mb, 3+: custom
var updateCVFPS = function () {
frames ++;
var time = ( performance || Date ).now();
@ -86,34 +90,30 @@
}
beginTime = time;
}
// flag to set true if you want to construct a texture from the UV image
var makeTexUV = false;
document.body.appendChild( stats.dom );
// flag to set true if you want to construct a texture from the UV image and show it
var makeTexUV = false;
class ARAnchorExample extends XRExampleBase {
constructor(domElement){
super(domElement, false, true, true)
this.textBox = document.createElement('span')
this.textBox.setAttribute('class', 'text-box')
this.textBox.innerText = '0.0'
this.el.appendChild(this.textBox)
// to store the returned values
this.intensity = 0;
this.cr = 0;
this.cg = 0;
this.cb = 0;
// the light estimate from WebXR, if there is one
this.lightEstimate = 0;
this.el.appendChild(this.textBox)
}
newSession() {
// var blob = new Blob([
// document.querySelector('#worker1').textContent
// ], { type: "text/javascript" })
// this.worker = new Worker(window.URL.createObjectURL(blob));
// start a background worker
this.worker = new Worker("worker.js")
var self = this;
@ -129,11 +129,12 @@
this.worker.addEventListener('error', (e) => {
console.log("worker error:" + e)
})
}
// called when there is a new session
newSession() {
// this can only be done inside the session
this.setVideoWorker(this.worker);
// use this instead to not use worker.
//this.setVideoWorker(ev => { this.handleVideoFrame(ev) })
}
// Called during construction
@ -160,7 +161,8 @@
this.camera.add( plane );
}
// Add a box at the scene origin
// Add a box at the scene origin, so that we can see if things are working when
// we look down
let box = new THREE.Mesh(
new THREE.BoxBufferGeometry(0.1, 0.1, 0.1),
new THREE.MeshPhongMaterial({ color: '#DDFFDD' })
@ -174,16 +176,10 @@
this.scene.add(directionalLight)
}
// called each frame
updateScene(frame){
this.lightEstimate = frame.lightEstimate || 0;
stats.update()
if (this.messageText != this.textBox.innerHTML) {
this.textBox.innerHTML = this.messageText;
}
}
handleVisionDone(videoFrame) {
var txt = "ARKit Light Estimate: " + this.lightEstimate.toFixed(2) + "<br>CV Average Intensity: " + this.intensity.toFixed(2)
+ "<br>Center RGB: " + this.cr.toFixed(0) + " / " + this.cg.toFixed(0) + " / " + this.cb.toFixed(0) + "<br><center>";
@ -203,7 +199,20 @@
this.messageText = txt;
// in the worker case, if we don't access the buffer in the worker, it won't be pulled back
if (this.messageText != this.textBox.innerHTML) {
this.textBox.innerHTML = this.messageText;
}
stats.update()
}
handleVisionDone(videoFrame) {
// ask for the next frame (before we construct that debugging frame, if we're doing that)
this.requestVideoFrame();
// update CV fps
updateCVFPS();
// check to make sure the 2nd plane exists; that will be UV
if (makeTexUV && videoFrame.buffer(1).buffer) {
var buffer = videoFrame.buffer(1);
var pixels = buffer.buffer;
@ -211,7 +220,8 @@
this.texSize = pixels.length /2 * 3
this.texBuff = new Uint8Array( this.texSize ); // convert each pixel from 2 to 3 bytes
}
// convert it to a simple RGB image. Super slow, but ok for debugging
var j = 0;
for ( var i = 0; i < this.texSize; i ++ ) {
this.texBuff[i] = pixels[j++];
@ -222,62 +232,11 @@
}
this.uvTexture.image = { data: this.texBuff, width: buffer.size.width, height: buffer.size.height };
this.uvTexture.needsUpdate = true;
}
updateCVFPS();
}
// must do this to clean up the internal buffer caches, let system know we're done
videoFrame.release();
this.requestVideoFrame();
}
//////
////// NOT USED with worker above, but can switch to callback model using these
//////
averageIntensity(buffer) {
var w = buffer.size.width;
var h = buffer.size.height;
var pad = buffer.size.bytesPerRow - w;
var pixels = buffer.buffer;
var intensity = 0.0;
var p = 0;
for (var r = 0; r < h; r++) {
var v = 0;
for (var i = 0; i < w; i++) {
if (p < pixels.length) {
v += pixels[p++]
} else {
console.error("overflow pixel buffer")
}
}
intensity += v / w;
p += pad;
}
this.intensity = (intensity / h) / 255.0;
}
colorAtCenter(buffer) {
var w = buffer.size.width;
var h = buffer.size.height;
var pixels = buffer.buffer;
var cx = Math.floor(w / 2) * buffer.size.bytesPerPixel
var cy = Math.floor(h / 2)
var p = cy * buffer.size.bytesPerRow + cx;
this.cb = pixels[p++];
this.cr = pixels[p];
}
handleVideoFrame(ev) {
var videoFrame = ev.detail
switch (videoFrame.pixelFormat) {
case XRVideoFrame.IMAGEFORMAT_YUV420P:
this.averageIntensity(videoFrame.buffer(0))
this.colorAtCenter(videoFrame.buffer(1))
}
this.handleVisionDone(videoFrame);
}
}
window.addEventListener('DOMContentLoaded', () => {

33
examples/simplecv/worker.js
Просмотреть файл

@ -54,11 +54,13 @@ importScripts('../../dist/webxr-worker.js')
*/
// some globals to hold the -- silly -- values we compute
var intensity = 0.0;
var cr = -1;
var cg = -1;
var cb = -1;
// a silly simply function to compute something based on 'all the pixels' in an RGBA image
averageIntensityRGBA = function (buffer) {
var w = buffer.size.width;
var h = buffer.size.height;
@ -79,6 +81,7 @@ averageIntensityRGBA = function (buffer) {
intensity = (intensity / h) / 255.0;
}
// a silly simply function to compute something based on 'all the pixels' in a grayscale image
averageIntensityLum = function (buffer) {
var w = buffer.size.width;
var h = buffer.size.height;
@ -90,11 +93,7 @@ averageIntensityLum = function (buffer) {
for (var r = 0; r < h; r++) {
var v = 0;
for (var i = 0; i < w; i++) {
if (p < pixels.length) {
v += pixels[p++]
} else {
console.error("overflow pixel buffer")
}
v += pixels[p++]
}
intensity += v / w;
p += pad;
@ -102,6 +101,7 @@ averageIntensityLum = function (buffer) {
intensity = (intensity / h) / 255.0;
}
// sample a single color, just for variety
colorAtCenterRGB = function(buffer0) {
var w = buffer0.size.width;
var h = buffer0.size.height;
@ -115,6 +115,8 @@ colorAtCenterRGB = function(buffer0) {
cb = pixels[p];
}
// Make an attempt to convert a UV color to RBG
//
// LUV == LuCbCr
//
// Y = 0.299R + 0.587G + 0.114B
@ -169,28 +171,39 @@ colorAtCenterLUV = function(buffer0, buffer1) {
// cb=y+1.772*u;
}
// The listener.
//
// We can ignore the message type field, since we are only receiving one message from
// the main thread, a new video frame
self.addEventListener('message', function(event){
try {
// a utility function to receive the message. Takes care of managing the
// internal ArrayBuffers that are being passed around
var videoFrame = XRVideoFrame.createFromMessage(event);
// The video frames will come in different formats on different platforms.
// The call to videoFrame.buffer(i) retrieves the i-th plane for the frame;
// (in the case of the WebXR Viewer, it also converts the base64 encoded message
// into an ArrayBuffer, which we don't do until the plane is used)
switch (videoFrame.pixelFormat) {
// the WebXR Viewer uses iOS native YCbCr, which is two buffers, one for Y and one for CbCr
case XRVideoFrame.IMAGEFORMAT_YUV420P:
this.averageIntensityLum(videoFrame.buffer(0))
this.colorAtCenterLUV(videoFrame.buffer(0),videoFrame.buffer(1))
break;
// WebRTC uses web-standard RGBA
case XRVideoFrame.IMAGEFORMAT_RGBA32:
this.averageIntensityRGBA(videoFrame.buffer(0))
this.colorAtCenterRGB(videoFrame.buffer(0))
break;
}
// utility function to send the video frame and additional parameters back.
// Want to use this so we pass ArrayBuffers back and forth to avoid having to
// reallocate them every frame.
videoFrame.postReplyMessage({intensity: intensity, cr: cr, cg: cg, cb: cb})
videoFrame.release();
} catch(e) {
console.error('page error', e)
}
});
// setInterval( function(){
// console.log("Help me!")
// self.postMessage (Math.random() * 255.0);
//}, 500);