Add support for separate audio source attenuation

scripts/run-local-reticulum.sh

@@ -2,12 +2,15 @@
 # Usage: run-local-reticulum.sh [IP address] to start hubs with a local reticulum instance
 
 HOST=$1
+CORS_PROXY_HOST=$1
 
 if [ -e $HOST ]; then
   HOST="hubs.local"
+  CORS_PROXY_HOST="hubs-proxy.local"
+
   export RETICULUM_SOCKET_SERVER="hubs.local"
 else
   export RETICULUM_SOCKET_SERVER="$1"
 fi
 
-CORS_PROXY_SERVER="hubs-proxy.local:4000" NON_CORS_PROXY_DOMAINS="$HOST,dev.reticulum.io" BASE_ASSETS_PATH="https://$HOST:8080/" RETICULUM_SERVER="$HOST:4000" POSTGREST_SERVER="" ITA_SERVER="" npm start
+CORS_PROXY_SERVER="$CORS_PROXY_HOST:4000" NON_CORS_PROXY_DOMAINS="$HOST,dev.reticulum.io" BASE_ASSETS_PATH="https://$HOST:8080/" RETICULUM_SERVER="$HOST:4000" POSTGREST_SERVER="" ITA_SERVER="" npm start

src/components/media-loader.js

@@ -1,5 +1,10 @@
 import { getBox, getScaleCoefficient } from "../utils/auto-box-collider";
-import { resolveUrl, injectCustomShaderChunks, addMeshScaleAnimation } from "../utils/media-utils";
+import {
+  resolveUrl,
+  getDefaultResolveQuality,
+  injectCustomShaderChunks,
+  addMeshScaleAnimation
+} from "../utils/media-utils";
 import {
   isNonCorsProxyDomain,
   guessContentType,
@@ -295,6 +300,7 @@ AFRAME.registerComponent("media-loader", {
       }
 
       let canonicalUrl = src;
+      let canonicalAudioUrl = src;
       let accessibleUrl = src;
       let contentType = this.data.contentType;
       let thumbnail;
@@ -307,12 +313,21 @@ AFRAME.registerComponent("media-loader", {
         isNonCorsProxyDomain(parsedUrl.hostname) && (guessContentType(src) || "").startsWith("model/gltf");
 
       if (this.data.resolve && !src.startsWith("data:") && !isLocalModelAsset) {
-        const result = await resolveUrl(src, version);
+        const is360 = this.data.mediaOptions.projection && this.data.mediaOptions.projection.startsWith("360");
+        const quality = getDefaultResolveQuality(is360);
+        const result = await resolveUrl(src, quality, version);
         canonicalUrl = result.origin;
+
         // handle protocol relative urls
         if (canonicalUrl.startsWith("//")) {
           canonicalUrl = location.protocol + canonicalUrl;
         }
+
+        canonicalAudioUrl = result.origin_audio;
+        if (canonicalAudioUrl && canonicalAudioUrl.startsWith("//")) {
+          canonicalAudioUrl = location.protocol + canonicalAudioUrl;
+        }
+
         contentType = (result.meta && result.meta.expected_content_type) || contentType;
         thumbnail = result.meta && result.meta.thumbnail && proxiedUrlFor(result.meta.thumbnail);
       }
@@ -352,7 +367,12 @@ AFRAME.registerComponent("media-loader", {
         );
         this.el.setAttribute(
           "media-video",
-          Object.assign({}, this.data.mediaOptions, { src: accessibleUrl, time: startTime, contentType })
+          Object.assign({}, this.data.mediaOptions, {
+            src: accessibleUrl,
+            audioSrc: canonicalAudioUrl ? proxiedUrlFor(canonicalAudioUrl) : null,
+            time: startTime,
+            contentType
+          })
         );
         if (this.el.components["position-at-box-shape-border__freeze"]) {
           this.el.setAttribute("position-at-box-shape-border__freeze", { dirs: ["forward", "back"] });

src/components/media-views.js

@@ -108,19 +108,19 @@ async function createGIFTexture(url) {
  * @param {string} src - Url to a video file.
  * @returns {Element} Video element.
  */
-async function createVideoEl() {
-  const videoEl = document.createElement("video");
-  videoEl.setAttribute("playsinline", "");
-  videoEl.setAttribute("webkit-playsinline", "");
+async function createVideoOrAudioEl(type) {
+  const el = document.createElement(type);
+  el.setAttribute("playsinline", "");
+  el.setAttribute("webkit-playsinline", "");
   // iOS Safari requires the autoplay attribute, or it won't play the video at all.
-  videoEl.autoplay = true;
+  el.autoplay = true;
   // iOS Safari will not play videos without user interaction. We mute the video so that it can autoplay and then
   // allow the user to unmute it with an interaction in the unmute-video-button component.
-  videoEl.muted = isIOS;
-  videoEl.preload = "auto";
-  videoEl.crossOrigin = "anonymous";
+  el.muted = isIOS;
+  el.preload = "auto";
+  el.crossOrigin = "anonymous";
 
-  return videoEl;
+  return el;
 }
 
 function scaleToAspectRatio(el, ratio) {
@@ -222,6 +222,7 @@ function timeFmt(t) {
 AFRAME.registerComponent("media-video", {
   schema: {
     src: { type: "string" },
+    audioSrc: { type: "string" },
     contentType: { type: "string" },
     volume: { type: "number", default: 0.5 },
     loop: { type: "boolean", default: true },
@@ -480,9 +481,9 @@ AFRAME.registerComponent("media-video", {
       this.mesh.material.needsUpdate = true;
     }
 
-    let texture;
+    let texture, audioSource;
     try {
-      texture = await this.createVideoTexture();
+      ({ texture, audioSource } = await this.createVideoTextureAndAudioSource());
 
       // No way to cancel promises, so if src has changed while we were creating the texture just throw it away.
       if (this.data.src !== src) {
@@ -494,7 +495,7 @@ AFRAME.registerComponent("media-video", {
         // iOS video audio is broken, see: https://github.com/mozilla/hubs/issues/1797
         if (!isIOS) {
           // TODO FF error here if binding mediastream: The captured HTMLMediaElement is playing a MediaStream. Applying volume or mute status is not currently supported -- not an issue since we have no audio atm in shared video.
-          texture.audioSource = this.el.sceneEl.audioListener.context.createMediaElementSource(texture.image);
+          const mediaAudioSource = this.el.sceneEl.audioListener.context.createMediaElementSource(audioSource);
 
           if (this.data.audioType === "pannernode") {
             this.audio = new THREE.PositionalAudio(this.el.sceneEl.audioListener);
@@ -509,7 +510,7 @@ AFRAME.registerComponent("media-video", {
             this.audio = new THREE.Audio(this.el.sceneEl.audioListener);
           }
 
-          this.audio.setNodeSource(texture.audioSource);
+          this.audio.setNodeSource(mediaAudioSource);
           this.el.setObject3D("sound", this.audio);
         }
       }
@@ -599,14 +600,19 @@ AFRAME.registerComponent("media-video", {
     this.el.emit("video-loaded", { projection: projection });
   },
 
-  async createVideoTexture() {
+  async createVideoTextureAndAudioSource() {
     const url = this.data.src;
     const contentType = this.data.contentType;
 
     return new Promise(async (resolve, reject) => {
-      const videoEl = await createVideoEl();
+      if (this.audioEl) {
+        clearInterval(this._audioSyncInterval);
+        this.audioEl = null;
+      }
 
-      let texture, isReady;
+      const videoEl = await createVideoOrAudioEl("video");
+
+      let texture, audioEl, isReady;
       if (contentType.startsWith("audio/")) {
         // We want to treat audio almost exactly like video, so we mock a video texture with an image property.
         texture = new THREE.Texture();
@@ -700,13 +706,31 @@ AFRAME.registerComponent("media-video", {
       } else {
         videoEl.src = url;
         videoEl.onerror = reject;
+
+        if (this.data.audioSrc) {
+          audioEl = await createVideoOrAudioEl("audio");
+          audioEl.src = this.data.audioSrc;
+          audioEl.onerror = reject;
+
+          this._audioSyncInterval = setInterval(() => {
+            if (Math.abs(audioEl.currentTime - videoEl.currentTime) >= 0.33) {
+              // In Chrome, drift of a few frames seems persistent
+              audioEl.currentTime = videoEl.currentTime;
+            }
+
+            if (videoEl.paused !== audioEl.paused) {
+              videoEl.paused ? audioEl.pause() : audioEl.play();
+              audioEl.currentTime = videoEl.currentTime;
+            }
+          }, 1000);
+        }
       }
 
       // NOTE: We used to use the canplay event here to yield the texture, but that fails to fire on iOS Safari
       // and also sometimes in Chrome it seems.
       const poll = () => {
         if (isReady()) {
-          resolve(texture);
+          resolve({ texture, audioSource: audioEl || texture.image });
         } else {
           setTimeout(poll, 500);
         }

src/utils/media-utils.js

@@ -9,17 +9,24 @@ import { proxiedUrlFor } from "../utils/media-url-utils";
 import anime from "animejs";
 
 const mediaAPIEndpoint = getReticulumFetchUrl("/api/v1/media");
+const isMobile = AFRAME.utils.device.isMobile();
+const isMobileVR = AFRAME.utils.device.isMobile();
 
 // Map<String, Promise<Object>
 const resolveUrlCache = new Map();
-export const resolveUrl = async (url, version = 1) => {
+export const getDefaultResolveQuality = (is360 = false) => {
+  const useLowerQuality = isMobile || isMobileVR;
+  return !is360 ? (useLowerQuality ? "low" : "high") : useLowerQuality ? "low_360" : "high_360";
+};
+
+export const resolveUrl = async (url, quality = null, version = 1) => {
   const key = `${url}_${version}`;
   if (resolveUrlCache.has(key)) return resolveUrlCache.get(key);
 
   const resultPromise = fetch(mediaAPIEndpoint, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ media: { url }, version })
+    body: JSON.stringify({ media: { url, quality: quality || getDefaultResolveQuality() }, version })
   }).then(async response => {
     if (!response.ok) {
       const message = `Error resolving url "${url}":`;