diff --git a/src/components/image-plus.js b/src/components/image-plus.js
index 6f0163d23bb52ac1974077573d99f2408ab72640..a7efa5508fdce28031d1f1171e7c5b19b5944538 100644
--- a/src/components/image-plus.js
+++ b/src/components/image-plus.js
@@ -32,8 +32,40 @@ class GIFTexture extends THREE.Texture {
}
}
+/**
+ * Create video element to be used as a texture.
+ *
+ * @param {string} src - Url to a video file.
+ * @returns {Element} Video element.
+ */
+function createVideoEl(src) {
+ const videoEl = document.createElement("video");
+ videoEl.setAttribute("playsinline", "");
+ videoEl.setAttribute("webkit-playsinline", "");
+ videoEl.autoplay = true;
+ videoEl.loop = true;
+ videoEl.crossOrigin = "anonymous";
+ videoEl.src = src;
+ return videoEl;
+}
+
+const textureLoader = new THREE.TextureLoader();
+textureLoader.setCrossOrigin("anonymous");
+
+const textureCache = new Map();
+
+const noop = function() {};
+
+const errorImage = new Image();
+errorImage.src =
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALIAAACyCAYAAADmipVoAAAAIHRFWHRTUFJEAGltYWdlLW5hMDUucHJ2LmRmdy5zcHJkLm5ldAUwrwIAAAWkSURBVHja7d1RaFV1HMDx81ThaG8iIVlULz2ITi3yZjliaSn1UAOhh0kSNAMrIwK1mZPY1ppjKFlpuFCIrB5q6qy5O0VX+LCnwJceAt8tyYecmP47V0cP+dD/uHvavWefH3zfLpfDOR/+L/d/zz9Jsk1QfbZ27drQ3d0dVU9Pz/ak4AMFyCALZJAFMsgCGWSQQRbIIAtkkAUyyCCDLJCLBnlNWqNqo87Ozt2xkNMup5gvxpR+9v2iQ25OTM1MCq4vA+Qsq/cukA3IIBuQQQYZZJBBBtmADLIBGWSQQQYZZJANyLMF8sjI6fZyebys/Nq//8D5DDgP9Pb2rowp/ew7aWMxpd/dX2jIo6Nn+tKC8mtw8GAum4a6urrezLCCnwRZIIMskEEGGWSQQQZZIIMskEEGGWSQQQZZIIM8vQ4e+ya0ft6eS0WGnH6+KW1zTJWfs6f2fEQF8m3U/+2+kHywIJeKDDnLpN/7ZJYNSSCDDDLIIIMMMsgggwwyyCCDDDLIIIMMMsgggwwyyCDPFsi7vzsQ2g5tjmr14ZdDMrY0rhNLMkFua9sQ3fHjY7lA3rKlI7S0tES1cOHCyp6IHdVu/vz5g7HXUAnkqTZ+sTUe3KGHQ3Lpmbh+W5VtVc5w34aGfsgFcmvruno8MgJkkEEGGWSQQQYZZJBBBhlkkEEGGWSQQQYZZJBBBhlkkP/dJw+F5MyjcZ1aVneQd+7svoG52pVKKybS6+6P7CuQ84acZzUAOcey/G2/GWSQQQYZZJBBBhlkkEEGGWSQQQYZZJBBBhlkkEEGGWSQb63j676waO/qqB7sfTwkr87LpUWLmqI7evQEyCBP470W/Xty2/xS8GMdQAYZZJBBBhlkkEEGGWSQQQYZZJBBBhlkkEEGGWSQQQb5f2xg4KMwZ05DLmW5jiPf/xSGZrpjJ2/s+YjpyJGRLJBXpF3KEMj1WsPGyZC8cnVmW7I9y3PuT+pwQAYZZJBBBhlkkEEGGWSQQQYZZJBBBhlkkEEGGWSQQQYZZJAj+nXbhhCWJ1Gdf3tdsSGfuxaSq9fj+vIayCCDDDLIIIMMMsgggwwyyCCDDDLIIIMMMsgggwwyyCCDDDLI2SD/sm0iTJbORXXliW3hr+aGqC6sf6z+IL+Wtimy538MySOfxtU00J0+lsbI7gL5tiD/HC6Xzkd1tdQRvXpffKmp2JuGnj2drrR7YtuRFHxABhlkkEEGGWSQQQYZZJBBBhlkkEEGGWSQQQYZZJBBriXIk6Wz4UrpcFR/PvVh+D3FHNOF9cvzg/zAqpDcs7L63ffi3mTZnuaoFn98f05+FqedzBDIWatgjl29K3szcoPc0JDXWSZv1cBC2Jw4QwRkkEEGGWSQQQYZZJBBBhlkkEEGGWSQQQYZZJBBBhnkaVX5ObuyNyOmyr+zK68aiC0L5Llz522dQlftlsY/j/Gnp57Jf1Yun3kO5BqCnA39uejVu1IGyNeHh882zvSymULuyHDNfSCDDDLIIIMMMsgggwwyyCCDDDLIIIMMMsgggwwyyCCDfGufbdoX3r1384z23oLXJ0Mp2RHbTRgxne4YHh6+E+RZALmtbUNem2qydCkp8IAMMsgggwwyyCCDDDLIIIMMMsgggwwyyCCDDDLIIIMM8vQh9/TsCq2t66IqlVZMJDcP/a52XbUAbmxs/IX0nrxR7crl8WMg5ww5Y31FXjlTcOUaOC4CZJBBBhlkkEEGGWSQQQYZZJBBBhlkkEEGGWSQQQYZ5FkDeWRkvH10dHys2qUPur3IkFNEu/K4bzne42JDNrNmQDYgg2xABtmAbAzIBmSQDcggG5BBNiAbkGsR8pq0xsju8Iz/mbsz3Ld6a009Qq63k+trZf5IZv69HbUSyCCDDDLIIIMMMsgGZJBBBhlkkEEGGWSQQQYZZJBBBhnkQkH+Gygybr0mifUnAAAAAElFTkSuQmCC";
+const errorTexture = new THREE.Texture(errorImage);
+errorImage.onload = () => {
+ errorTexture.needsUpdate = true;
+};
+
AFRAME.registerComponent("image-plus", {
- dependencies: ["geometry", "material"],
+ dependencies: ["geometry"],
schema: {
src: { type: "string" },
@@ -68,15 +100,6 @@ AFRAME.registerComponent("image-plus", {
});
},
- _onMaterialLoaded(e) {
- const src = e.detail.src;
- const w = src.videoWidth || src.width;
- const h = src.videoHeight || src.height;
- if (w || h) {
- this._fit(w, h);
- }
- },
-
_onGrab: (function() {
const q = new THREE.Quaternion();
return function() {
@@ -88,13 +111,15 @@ AFRAME.registerComponent("image-plus", {
})(),
init() {
- this._onMaterialLoaded = this._onMaterialLoaded.bind(this);
this._onGrab = this._onGrab.bind(this);
- this.el.addEventListener("materialtextureloaded", this._onMaterialLoaded);
- this.el.addEventListener("materialvideoloadeddata", this._onMaterialLoaded);
this.el.addEventListener("grab-start", this._onGrab);
+ const material = new THREE.MeshBasicMaterial();
+ material.side = THREE.DoubleSide;
+ material.transparent = true;
+ this.el.getObject3D("mesh").material = material;
+
const worldPos = new THREE.Vector3().copy(this.data.initialOffset);
this.billboardTarget = document.querySelector("#player-camera").object3D;
this.billboardTarget.localToWorld(worldPos);
@@ -102,54 +127,151 @@ AFRAME.registerComponent("image-plus", {
this.billboardTarget.getWorldQuaternion(this.el.object3D.quaternion);
},
- async loadGIF(url) {
- const worker = new GIFWorker();
- worker.onmessage = e => {
- const [success, frames, delays, disposals] = e.data;
- if (!success) {
- console.error("error loading gif", e.data[1]);
- return;
+ remove() {
+ const material = this.el.getObject3D("mesh").material;
+ const texture = material.map;
+
+ if (texture === errorTexture) return;
+
+ const url = texture.image.src;
+ const cacheItem = textureCache.get(url);
+ cacheItem.count--;
+ if (cacheItem.count <= 0) {
+ console.log("removing from cache");
+ // Unload the video element to prevent it from continuing to play in the background
+ if (texture.image instanceof HTMLVideoElement) {
+ const video = texture.image;
+ video.pause();
+ video.src = "";
+ video.load();
}
- let loadCnt = 0;
- for (let i = 0; i < frames.length; i++) {
- const img = new Image();
- img.onload = e => {
- loadCnt++;
- frames[i] = e.target;
- if (loadCnt === frames.length) {
- const material = this.el.components.material.material;
- material.map = new GIFTexture(frames, delays, disposals);
- material.needsUpdate = true;
- this._fit(frames[0].width, frames[0].height);
- }
- };
- img.src = frames[i];
+ texture.dispose();
+
+ // THREE never lets go of material refs, long running PR HERE https://github.com/mrdoob/three.js/pull/12464
+ // Mitigate the damage a bit by at least breaking the image ref so Image/Video elements can be freed
+ // TODO: If/when THREE gets fixed, we should be able to safely remove this
+ delete texture.image;
+
+ textureCache.delete(url);
+ }
+ },
+
+ async loadGIF(url) {
+ return new Promise((resolve, reject) => {
+ // TODO: pool workers
+ const worker = new GIFWorker();
+ worker.onmessage = e => {
+ const [success, frames, delays, disposals] = e.data;
+ if (!success) {
+ reject(`error loading gif: ${e.data[1]}`);
+ return;
+ }
+
+ let loadCnt = 0;
+ for (let i = 0; i < frames.length; i++) {
+ const img = new Image();
+ img.onload = e => {
+ loadCnt++;
+ frames[i] = e.target;
+ if (loadCnt === frames.length) {
+ const texture = new GIFTexture(frames, delays, disposals);
+ texture.image.src = url;
+ resolve(texture);
+ }
+ };
+ img.src = frames[i];
+ }
+ };
+ fetch(url, { mode: "cors" })
+ .then(r => r.arrayBuffer())
+ .then(rawImageData => {
+ worker.postMessage(rawImageData, [rawImageData]);
+ })
+ .catch(reject);
+ });
+ },
+
+ loadVideo(url) {
+ return new Promise((resolve, reject) => {
+ const videoEl = createVideoEl(url);
+
+ const texture = new THREE.VideoTexture(videoEl);
+ texture.minFilter = THREE.LinearFilter;
+ videoEl.addEventListener("loadedmetadata", () => resolve(texture));
+ videoEl.onerror = reject;
+
+ // If iOS and video is HLS, do some hacks.
+ if (
+ this.el.sceneEl.isIOS &&
+ AFRAME.utils.material.isHLS(
+ videoEl.src || videoEl.getAttribute("src"),
+ videoEl.type || videoEl.getAttribute("type")
+ )
+ ) {
+ // Actually BGRA. Tell shader to correct later.
+ texture.format = THREE.RGBAFormat;
+ texture.needsCorrectionBGRA = true;
+ // Apparently needed for HLS. Tell shader to correct later.
+ texture.flipY = false;
+ texture.needsCorrectionFlipY = true;
}
- };
- const rawImageData = await fetch(url, { mode: "cors" }).then(r => r.arrayBuffer());
- worker.postMessage(rawImageData, [rawImageData]);
+ });
+ },
+
+ loadImage(url) {
+ return new Promise((resolve, reject) => {
+ textureLoader.load(url, resolve, noop, function(xhr) {
+ reject(`'${url}' could not be fetched (Error code: ${xhr.status}; Response: ${xhr.statusText})`);
+ });
+ });
},
async update() {
- const mediaJson = await fetch("https://smoke-dev.reticulum.io/api/v1/media", {
- method: "POST",
- headers: {
- "Content-Type": "application/json"
- },
- body: JSON.stringify({
- media: {
- url: this.data.src
+ let texture;
+ try {
+ const mediaJson = await fetch("https://smoke-dev.reticulum.io/api/v1/media", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json"
+ },
+ body: JSON.stringify({
+ media: {
+ url: this.data.src
+ }
+ })
+ }).then(r => r.json());
+ const url = mediaJson.images.raw;
+
+ if (textureCache.has(url)) {
+ const cacheItem = textureCache.get(url);
+ texture = cacheItem.texture;
+ cacheItem.count++;
+ } else {
+ const contentType = await fetch(url, { method: "HEAD" }).then(r => r.headers.get("content-type"));
+ if (contentType === "image/gif") {
+ console.log("load gif", contentType);
+ texture = await this.loadGIF(url);
+ } else if (contentType.startsWith("image/")) {
+ console.log("load image", contentType);
+ texture = await this.loadImage(url);
+ } else if (contentType.startsWith("video")) {
+ console.log("load video", contentType);
+ texture = await this.loadVideo(url);
+ } else {
+ throw new Error(`Unknown centent type: ${contentType}`);
}
- })
- }).then(r => r.json());
- const imageUrl = mediaJson.images.raw;
- const contentType = await fetch(imageUrl, { method: "HEAD" }).then(r => r.headers.get("content-type"));
- if (contentType === "image/gif") {
- return this.loadGIF(imageUrl);
- } else {
- this.el.setAttribute("material", "src", `url(${imageUrl})`);
- return Promise.resolve();
+
+ textureCache.set(url, { count: 1, texture });
+ }
+ } catch (e) {
+ console.error("Error loading media", this.data.src, e);
+ texture = errorTexture;
}
+
+ const material = this.el.getObject3D("mesh").material;
+ material.map = texture;
+ material.needsUpdate = true;
+ this._fit(texture.image.videoWidth || texture.image.width, texture.image.videoHeight || texture.image.height);
}
});
diff --git a/src/components/sticky-object.js b/src/components/sticky-object.js
index 9e9d8bff32b729edc341d8d7fb424abbfc7e166f..084004b8831c2626e0a8fd278c11767dc5008c62 100644
--- a/src/components/sticky-object.js
+++ b/src/components/sticky-object.js
@@ -12,11 +12,20 @@ AFRAME.registerComponent("sticky-object", {
this._onGrab = this._onGrab.bind(this);
this._onRelease = this._onRelease.bind(this);
this._onBodyLoaded = this._onBodyLoaded.bind(this);
+ },
+
+ play() {
this.el.addEventListener("grab-start", this._onGrab);
this.el.addEventListener("grab-end", this._onRelease);
this.el.addEventListener("body-loaded", this._onBodyLoaded);
},
+ pause() {
+ this.el.removeEventListener("grab-start", this._onGrab);
+ this.el.removeEventListener("grab-end", this._onRelease);
+ this.el.removeEventListener("body-loaded", this._onBodyLoaded);
+ },
+
setLocked(locked) {
if (!NAF.utils.isMine(this.el)) return;
diff --git a/src/hub.html b/src/hub.html
index 1581794db7e7a26007a0ce9ee1c99daae1452835..a75e5fa915dd613e33c9a3f50be78f248326732b 100644
--- a/src/hub.html
+++ b/src/hub.html
@@ -202,7 +202,6 @@
stretchable="useWorldPosition: true; usePhysics: never"
hoverable
geometry="primitive: plane"
- material="shader: flat; side: double; transparent: true;"
image-plus
sticky-object="autoLockOnLoad: true; autoLockOnRelease: true;"
position-at-box-shape-border="target:.delete-button;dirs:forward,back"