From 53db386731422c88d4e24e299e9e56a150dad3cf Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 10 Aug 2020 22:06:30 -0600 Subject: [PATCH 01/15] Add support for blurhash (MSC2448) MSC: https://github.com/matrix-org/matrix-doc/pull/2448 While the image loads, we can show a blurred version of it (calculated at upload time) so we don't have a blank space in the timeline. --- package.json | 1 + src/ContentMessages.tsx | 10 +++- .../views/elements/BlurhashPlaceholder.tsx | 56 +++++++++++++++++++ src/components/views/messages/MImageBody.js | 18 +++--- yarn.lock | 5 ++ 5 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 src/components/views/elements/BlurhashPlaceholder.tsx diff --git a/package.json b/package.json index 548b33f353e..989672d414f 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@babel/runtime": "^7.10.5", "await-lock": "^2.0.1", "blueimp-canvas-to-blob": "^3.27.0", + "blurhash": "^1.1.3", "browser-encrypt-attachment": "^0.3.0", "browser-request": "^0.3.3", "classnames": "^2.2.6", diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index 6f55a75d0cf..7e57b34ff7f 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -334,6 +334,7 @@ function uploadFile(matrixClient: MatrixClient, roomId: string, file: File | Blo if (file.type) { encryptInfo.mimetype = file.type; } + // TODO: Blurhash for encrypted media? return {"file": encryptInfo}; }); (prom as IAbortablePromise).abort = () => { @@ -344,11 +345,15 @@ function uploadFile(matrixClient: MatrixClient, roomId: string, file: File | Blo } else { const basePromise = matrixClient.uploadContent(file, { progressHandler: progressHandler, + onlyContentUri: false, }); - const promise1 = basePromise.then(function(url) { + const promise1 = basePromise.then(function(body) { if (canceled) throw new UploadCanceledError(); // If the attachment isn't encrypted then include the URL directly. - return {"url": url}; + return { + "url": body.content_uri, + "blurhash": body["xyz.amorgan.blurhash"], // TODO: Use `body.blurhash` when MSC2448 lands + }; }); promise1.abort = () => { canceled = true; @@ -550,6 +555,7 @@ export default class ContentMessages { return upload.promise.then(function(result) { content.file = result.file; content.url = result.url; + content.info['xyz.amorgan.blurhash'] = result.blurhash; // TODO: Use `blurhash` when MSC2448 lands }); }).then(() => { // Await previous message being sent into the room diff --git a/src/components/views/elements/BlurhashPlaceholder.tsx b/src/components/views/elements/BlurhashPlaceholder.tsx new file mode 100644 index 00000000000..bed45bfe5ad --- /dev/null +++ b/src/components/views/elements/BlurhashPlaceholder.tsx @@ -0,0 +1,56 @@ +/* + Copyright 2020 The Matrix.org Foundation C.I.C. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import React from 'react'; +import {decode} from "blurhash"; + +interface IProps { + blurhash: string; + width: number; + height: number; +} + +export default class BlurhashPlaceholder extends React.PureComponent { + private canvas: React.RefObject = React.createRef(); + + public componentDidMount() { + this.draw(); + } + + public componentDidUpdate() { + this.draw(); + } + + private draw() { + if (!this.canvas.current) return; + + try { + const {width, height} = this.props; + + const pixels = decode(this.props.blurhash, Math.ceil(width), Math.ceil(height)); + const ctx = this.canvas.current.getContext("2d"); + const imgData = ctx.createImageData(width, height); + imgData.data.set(pixels); + ctx.putImageData(imgData, 0, 0); + } catch (e) { + console.error("Error rendering blurhash: ", e); + } + } + + public render() { + return ; + } +} diff --git a/src/components/views/messages/MImageBody.js b/src/components/views/messages/MImageBody.js index c92ae475bf1..e0ac24c641e 100644 --- a/src/components/views/messages/MImageBody.js +++ b/src/components/views/messages/MImageBody.js @@ -27,6 +27,7 @@ import { _t } from '../../../languageHandler'; import SettingsStore from "../../../settings/SettingsStore"; import MatrixClientContext from "../../../contexts/MatrixClientContext"; import InlineSpinner from '../elements/InlineSpinner'; +import BlurhashPlaceholder from "../elements/BlurhashPlaceholder"; export default class MImageBody extends React.Component { static propTypes = { @@ -53,6 +54,8 @@ export default class MImageBody extends React.Component { this.onClick = this.onClick.bind(this); this._isGif = this._isGif.bind(this); + const imageInfo = this.props.mxEvent.getContent().info; + this.state = { decryptedUrl: null, decryptedThumbnailUrl: null, @@ -63,6 +66,7 @@ export default class MImageBody extends React.Component { loadedImageDimensions: null, hover: false, showImage: SettingsStore.getValue("showImages"), + blurhash: imageInfo ? imageInfo['xyz.amorgan.blurhash'] : null, // TODO: Use `blurhash` when MSC2448 lands. }; this._image = createRef(); @@ -329,7 +333,8 @@ export default class MImageBody extends React.Component { infoWidth = content.info.w; infoHeight = content.info.h; } else { - // Whilst the image loads, display nothing. + // Whilst the image loads, display nothing. We also don't display a blurhash image + // because we don't really know what size of image we'll end up with. // // Once loaded, use the loaded image dimensions stored in `loadedImageDimensions`. // @@ -368,8 +373,7 @@ export default class MImageBody extends React.Component { if (content.file !== undefined && this.state.decryptedUrl === null) { placeholder = ; } else if (!this.state.imgLoaded) { - // Deliberately, getSpinner is left unimplemented here, MStickerBody overides - placeholder = this.getPlaceholder(); + placeholder = this.getPlaceholder(maxWidth, maxHeight); } let showPlaceholder = Boolean(placeholder); @@ -391,7 +395,7 @@ export default class MImageBody extends React.Component { if (!this.state.showImage) { img = ; - showPlaceholder = false; // because we're hiding the image, so don't show the sticker icon. + showPlaceholder = false; // because we're hiding the image, so don't show the placeholder. } if (this._isGif() && !SettingsStore.getValue("autoplayGifsAndVideos") && !this.state.hover) { @@ -433,9 +437,9 @@ export default class MImageBody extends React.Component { } // Overidden by MStickerBody - getPlaceholder() { - // MImageBody doesn't show a placeholder whilst the image loads, (but it could do) - return null; + getPlaceholder(width, height) { + if (!this.state.blurhash) return null; + return ; } // Overidden by MStickerBody diff --git a/yarn.lock b/yarn.lock index 98fe42ef135..f1cace67a16 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2493,6 +2493,11 @@ blueimp-canvas-to-blob@^3.27.0: resolved "https://registry.yarnpkg.com/blueimp-canvas-to-blob/-/blueimp-canvas-to-blob-3.27.0.tgz#a2bd5c43587b95dedf0f6998603452d1bfcc9b9e" integrity sha512-AcIj+hCw6WquxzJuzC6KzgYmqxLFeTWe88KuY2BEIsW1zbEOfoinDAGlhyvFNGt+U3JElkVSK7anA1FaSdmmfA== +blurhash@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.3.tgz#dc325af7da836d07a0861d830bdd63694382483e" + integrity sha512-yUhPJvXexbqbyijCIE/T2NCXcj9iNPhWmOKbPTuR/cm7Q5snXYIfnVnz6m7MWOXxODMz/Cr3UcVkRdHiuDVRDw== + bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.4.0: version "4.11.9" resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.9.tgz#26d556829458f9d1e81fc48952493d0ba3507828" From 00d093b4ffb83e4d100b706757e0e44a8f1fb376 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 19 May 2021 08:56:14 +0100 Subject: [PATCH 02/15] yarn lock --- yarn.lock | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/yarn.lock b/yarn.lock index 19c0646d328..57b1198019d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2180,6 +2180,11 @@ blueimp-canvas-to-blob@^3.28.0: resolved "https://registry.yarnpkg.com/blueimp-canvas-to-blob/-/blueimp-canvas-to-blob-3.28.0.tgz#c8ab4dc6bb08774a7f273798cdf94b0776adf6c8" integrity sha512-5q+YHzgGsuHQ01iouGgJaPJXod2AzTxJXmVv90PpGrRxU7G7IqgPqWXz+PBmt3520jKKi6irWbNV87DicEa7wg== +blurhash@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-1.1.3.tgz#dc325af7da836d07a0861d830bdd63694382483e" + integrity sha512-yUhPJvXexbqbyijCIE/T2NCXcj9iNPhWmOKbPTuR/cm7Q5snXYIfnVnz6m7MWOXxODMz/Cr3UcVkRdHiuDVRDw== + boolbase@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" From c21445c406fd077e17fd17661590739069a82995 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 20 May 2021 13:23:17 +0100 Subject: [PATCH 03/15] switch from MatrixClientPeg in ContentMessages for consistency --- src/ContentMessages.tsx | 19 +++++++++---------- src/components/structures/UploadBar.tsx | 5 ++++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index 65b6f1aba44..ba84b307332 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -18,7 +18,6 @@ limitations under the License. import React from "react"; import dis from './dispatcher/dispatcher'; -import {MatrixClientPeg} from './MatrixClientPeg'; import {MatrixClient} from "matrix-js-sdk/src/client"; import * as sdk from './index'; import { _t } from './languageHandler'; @@ -344,7 +343,7 @@ function uploadFile(matrixClient: MatrixClient, roomId: string, file: File | Blo }); (prom as IAbortablePromise).abort = () => { canceled = true; - if (uploadPromise) MatrixClientPeg.get().cancelUpload(uploadPromise); + if (uploadPromise) matrixClient.cancelUpload(uploadPromise); }; return prom; } else { @@ -362,7 +361,7 @@ function uploadFile(matrixClient: MatrixClient, roomId: string, file: File | Blo }); promise1.abort = () => { canceled = true; - MatrixClientPeg.get().cancelUpload(basePromise); + matrixClient.cancelUpload(basePromise); }; return promise1; } @@ -372,9 +371,9 @@ export default class ContentMessages { private inprogress: IUpload[] = []; private mediaConfig: IMediaConfig = null; - sendStickerContentToRoom(url: string, roomId: string, info: string, text: string, matrixClient: MatrixClient) { + sendStickerContentToRoom(url: string, roomId: string, info: object, text: string, matrixClient: MatrixClient) { const startTime = CountlyAnalytics.getTimestamp(); - const prom = MatrixClientPeg.get().sendStickerMessage(roomId, url, info, text).catch((e) => { + const prom = matrixClient.sendStickerMessage(roomId, url, info, text).catch((e) => { console.warn(`Failed to send content with URL ${url} to room ${roomId}`, e); throw e; }); @@ -416,7 +415,7 @@ export default class ContentMessages { if (!this.mediaConfig) { // hot-path optimization to not flash a spinner if we don't need to const modal = Modal.createDialog(Spinner, null, 'mx_Dialog_spinner'); - await this.ensureMediaConfigFetched(); + await this.ensureMediaConfigFetched(matrixClient); modal.close(); } @@ -471,7 +470,7 @@ export default class ContentMessages { return this.inprogress.filter(u => !u.canceled); } - cancelUpload(promise: Promise) { + cancelUpload(promise: Promise, matrixClient: MatrixClient) { let upload: IUpload; for (let i = 0; i < this.inprogress.length; ++i) { if (this.inprogress[i].promise === promise) { @@ -481,7 +480,7 @@ export default class ContentMessages { } if (upload) { upload.canceled = true; - MatrixClientPeg.get().cancelUpload(upload.promise); + matrixClient.cancelUpload(upload.promise); dis.dispatch({action: Action.UploadCanceled, upload}); } } @@ -623,11 +622,11 @@ export default class ContentMessages { return true; } - private ensureMediaConfigFetched() { + private ensureMediaConfigFetched(matrixClient: MatrixClient) { if (this.mediaConfig !== null) return; console.log("[Media Config] Fetching"); - return MatrixClientPeg.get().getMediaConfig().then((config) => { + return matrixClient.getMediaConfig().then((config) => { console.log("[Media Config] Fetched config:", config); return config; }).catch(() => { diff --git a/src/components/structures/UploadBar.tsx b/src/components/structures/UploadBar.tsx index e19e312f584..269c615698a 100644 --- a/src/components/structures/UploadBar.tsx +++ b/src/components/structures/UploadBar.tsx @@ -26,6 +26,7 @@ import ProgressBar from "../views/elements/ProgressBar"; import AccessibleButton from "../views/elements/AccessibleButton"; import { IUpload } from "../../models/IUpload"; import {replaceableComponent} from "../../utils/replaceableComponent"; +import MatrixClientContext from "../../contexts/MatrixClientContext"; interface IProps { room: Room; @@ -38,6 +39,8 @@ interface IState { @replaceableComponent("structures.UploadBar") export default class UploadBar extends React.Component { + static contextType = MatrixClientContext; + private dispatcherRef: string; private mounted: boolean; @@ -82,7 +85,7 @@ export default class UploadBar extends React.Component { private onCancelClick = (ev) => { ev.preventDefault(); - ContentMessages.sharedInstance().cancelUpload(this.state.currentUpload.promise); + ContentMessages.sharedInstance().cancelUpload(this.state.currentUpload.promise, this.context); }; render() { From ba7604fd44679d020335160ed5f30f20f294184b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 20 May 2021 13:24:19 +0100 Subject: [PATCH 04/15] fix types around sending stickers --- src/components/structures/RoomView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/structures/RoomView.tsx b/src/components/structures/RoomView.tsx index c0f3c594576..1595bc6c534 100644 --- a/src/components/structures/RoomView.tsx +++ b/src/components/structures/RoomView.tsx @@ -1196,7 +1196,7 @@ export default class RoomView extends React.Component { }); }; - private injectSticker(url, info, text) { + private injectSticker(url: string, info: object, text: string) { if (this.context.isGuest()) { dis.dispatch({action: 'require_registration'}); return; From 563e6433b9eec30ec404abed1cbfcf2bb2279119 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 20 May 2021 15:47:11 +0100 Subject: [PATCH 05/15] Don't store blurhash in state, its immutable and pointless --- src/components/views/messages/MImageBody.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/components/views/messages/MImageBody.js b/src/components/views/messages/MImageBody.js index 14043b95f3f..34320f1b318 100644 --- a/src/components/views/messages/MImageBody.js +++ b/src/components/views/messages/MImageBody.js @@ -60,7 +60,6 @@ export default class MImageBody extends React.Component { this.onClick = this.onClick.bind(this); this._isGif = this._isGif.bind(this); - const imageInfo = this.props.mxEvent.getContent().info; this.state = { decryptedUrl: null, @@ -72,7 +71,6 @@ export default class MImageBody extends React.Component { loadedImageDimensions: null, hover: false, showImage: SettingsStore.getValue("showImages"), - blurhash: imageInfo ? imageInfo['xyz.amorgan.blurhash'] : null, // TODO: Use `blurhash` when MSC2448 lands. }; this._image = createRef(); @@ -442,8 +440,9 @@ export default class MImageBody extends React.Component { // Overidden by MStickerBody getPlaceholder(width, height) { - if (!this.state.blurhash) return null; - return ; + const blurhash = this.props.mxEvent.getContent().info['xyz.amorgan.blurhash']; + if (!blurhash) return null; + return ; } // Overidden by MStickerBody From 8368b864406453d35c3d47b096a2febd9a83273c Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 May 2021 14:52:27 +0100 Subject: [PATCH 06/15] Wire up local blurhash encoding --- src/ContentMessages.tsx | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index ba84b307332..9d9709a04df 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -17,6 +17,8 @@ limitations under the License. */ import React from "react"; +import { encode } from "blurhash"; + import dis from './dispatcher/dispatcher'; import {MatrixClient} from "matrix-js-sdk/src/client"; import * as sdk from './index'; @@ -47,6 +49,10 @@ const MAX_HEIGHT = 600; // 5669 px (x-axis) , 5669 px (y-axis) , per metre const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01]; +const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC 2448 +const BLURHASH_X_COMPONENTS = 8; +const BLURHASH_Y_COMPONENTS = 8; + export class UploadCanceledError extends Error {} type ThumbnailableElement = HTMLImageElement | HTMLVideoElement; @@ -77,6 +83,7 @@ interface IThumbnail { }; w: number; h: number; + [BLURHASH_FIELD]: string; }; thumbnail: Blob; } @@ -124,7 +131,16 @@ function createThumbnail( const canvas = document.createElement("canvas"); canvas.width = targetWidth; canvas.height = targetHeight; - canvas.getContext("2d").drawImage(element, 0, 0, targetWidth, targetHeight); + const context = canvas.getContext("2d"); + context.drawImage(element, 0, 0, targetWidth, targetHeight); + const imageData = context.getImageData(0, 0, targetWidth, targetHeight); + const blurhash = encode( + imageData.data, + imageData.width, + imageData.height, + BLURHASH_X_COMPONENTS, + BLURHASH_Y_COMPONENTS, + ); canvas.toBlob(function(thumbnail) { resolve({ info: { @@ -136,8 +152,9 @@ function createThumbnail( }, w: inputWidth, h: inputHeight, + [BLURHASH_FIELD]: blurhash, }, - thumbnail: thumbnail, + thumbnail, }); }, mimeType); }); @@ -338,7 +355,6 @@ function uploadFile(matrixClient: MatrixClient, roomId: string, file: File | Blo if (file.type) { encryptInfo.mimetype = file.type; } - // TODO: Blurhash for encrypted media? return {"file": encryptInfo}; }); (prom as IAbortablePromise).abort = () => { @@ -349,15 +365,11 @@ function uploadFile(matrixClient: MatrixClient, roomId: string, file: File | Blo } else { const basePromise = matrixClient.uploadContent(file, { progressHandler: progressHandler, - onlyContentUri: false, }); - const promise1 = basePromise.then(function(body) { + const promise1 = basePromise.then(function(url) { if (canceled) throw new UploadCanceledError(); // If the attachment isn't encrypted then include the URL directly. - return { - "url": body.content_uri, - "blurhash": body["xyz.amorgan.blurhash"], // TODO: Use `body.blurhash` when MSC2448 lands - }; + return { url }; }); promise1.abort = () => { canceled = true; @@ -565,7 +577,6 @@ export default class ContentMessages { return upload.promise.then(function(result) { content.file = result.file; content.url = result.url; - content.info['xyz.amorgan.blurhash'] = result.blurhash; // TODO: Use `blurhash` when MSC2448 lands }); }).then(() => { // Await previous message being sent into the room From 6672cbcf8029f93a3e5b89724ec4346404fb5de0 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 May 2021 15:29:17 +0100 Subject: [PATCH 07/15] fix typo --- src/ContentMessages.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index 9d9709a04df..626e5cbf697 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -49,7 +49,7 @@ const MAX_HEIGHT = 600; // 5669 px (x-axis) , 5669 px (y-axis) , per metre const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01]; -const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC 2448 +const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448 const BLURHASH_X_COMPONENTS = 8; const BLURHASH_Y_COMPONENTS = 8; From 3ed421dacf5e9df2f346dcfd549b69555bf6707b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 May 2021 21:04:22 +0100 Subject: [PATCH 08/15] Tweak blurhash size --- src/ContentMessages.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index 626e5cbf697..0914bff41a8 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -50,8 +50,8 @@ const MAX_HEIGHT = 600; const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01]; const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448 -const BLURHASH_X_COMPONENTS = 8; -const BLURHASH_Y_COMPONENTS = 8; +const BLURHASH_X_COMPONENTS = 6; +const BLURHASH_Y_COMPONENTS = 6; export class UploadCanceledError extends Error {} From 254697644e956e032451b11c500de46d49410564 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 May 2021 21:04:36 +0100 Subject: [PATCH 09/15] Fix video thumbnailer --- src/ContentMessages.tsx | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index 0914bff41a8..98a4554a07c 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -237,7 +237,8 @@ function infoForImageFile(matrixClient, roomId, imageFile) { } /** - * Load a file into a newly created video element. + * Load a file into a newly created video element and pull some strings + * in an attempt to guarantee the first frame will be showing. * * @param {File} videoFile The file to load in an video element. * @return {Promise} A promise that resolves with the video image element. @@ -246,20 +247,25 @@ function loadVideoElement(videoFile): Promise { return new Promise((resolve, reject) => { // Load the file into an html element const video = document.createElement("video"); + video.preload = "metadata"; + video.playsInline = true; + video.muted = true; const reader = new FileReader(); reader.onload = function(ev) { - video.src = ev.target.result as string; - - // Once ready, returns its size // Wait until we have enough data to thumbnail the first frame. - video.onloadeddata = function() { + video.onloadeddata = async function() { resolve(video); + video.pause(); }; video.onerror = function(e) { reject(e); }; + + video.src = ev.target.result as string; + video.load(); + video.play(); }; reader.onerror = function(e) { reject(e); From dbca9b4625fbde6c32edef7522f2a2668b484829 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 May 2021 21:19:41 +0100 Subject: [PATCH 10/15] use const for blurhash field --- src/ContentMessages.tsx | 2 +- src/components/views/messages/MImageBody.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ContentMessages.tsx b/src/ContentMessages.tsx index 98a4554a07c..44bb3edca9f 100644 --- a/src/ContentMessages.tsx +++ b/src/ContentMessages.tsx @@ -49,7 +49,7 @@ const MAX_HEIGHT = 600; // 5669 px (x-axis) , 5669 px (y-axis) , per metre const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01]; -const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448 +export const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448 const BLURHASH_X_COMPONENTS = 6; const BLURHASH_Y_COMPONENTS = 6; diff --git a/src/components/views/messages/MImageBody.js b/src/components/views/messages/MImageBody.js index 34320f1b318..60623752439 100644 --- a/src/components/views/messages/MImageBody.js +++ b/src/components/views/messages/MImageBody.js @@ -30,6 +30,7 @@ import InlineSpinner from '../elements/InlineSpinner'; import {replaceableComponent} from "../../../utils/replaceableComponent"; import {mediaFromContent} from "../../../customisations/Media"; import BlurhashPlaceholder from "../elements/BlurhashPlaceholder"; +import {BLURHASH_FIELD} from "../../../ContentMessages"; @replaceableComponent("views.messages.MImageBody") export default class MImageBody extends React.Component { @@ -440,7 +441,7 @@ export default class MImageBody extends React.Component { // Overidden by MStickerBody getPlaceholder(width, height) { - const blurhash = this.props.mxEvent.getContent().info['xyz.amorgan.blurhash']; + const blurhash = this.props.mxEvent.getContent().info[BLURHASH_FIELD]; if (!blurhash) return null; return ; } From 3a2e5389f6ccad7fc1ed3e039544d3fe596a104a Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 May 2021 21:20:00 +0100 Subject: [PATCH 11/15] Support blurhash for video posters --- src/components/views/messages/MVideoBody.tsx | 67 +++++++++++++++++--- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/src/components/views/messages/MVideoBody.tsx b/src/components/views/messages/MVideoBody.tsx index 2efdce506e7..44b246f0ade 100644 --- a/src/components/views/messages/MVideoBody.tsx +++ b/src/components/views/messages/MVideoBody.tsx @@ -16,6 +16,8 @@ limitations under the License. */ import React from 'react'; +import { decode } from "blurhash"; + import MFileBody from './MFileBody'; import { decryptFile } from '../../../utils/DecryptFile'; import { _t } from '../../../languageHandler'; @@ -23,6 +25,7 @@ import SettingsStore from "../../../settings/SettingsStore"; import InlineSpinner from '../elements/InlineSpinner'; import {replaceableComponent} from "../../../utils/replaceableComponent"; import {mediaFromContent} from "../../../customisations/Media"; +import {BLURHASH_FIELD} from "../../../ContentMessages"; interface IProps { /* the MatrixEvent to show */ @@ -37,6 +40,8 @@ interface IState { decryptedBlob: Blob|null, error: any|null, fetchingData: boolean, + posterLoading: boolean; + blurhashUrl: string; } @replaceableComponent("views.messages.MVideoBody") @@ -51,10 +56,12 @@ export default class MVideoBody extends React.PureComponent { decryptedThumbnailUrl: null, decryptedBlob: null, error: null, + posterLoading: false, + blurhashUrl: null, } } - thumbScale(fullWidth: number, fullHeight: number, thumbWidth: number, thumbHeight: number) { + thumbScale(fullWidth: number, fullHeight: number, thumbWidth = 480, thumbHeight = 360) { if (!fullWidth || !fullHeight) { // Cannot calculate thumbnail height for image: missing w/h in metadata. We can't even // log this because it's spammy @@ -92,8 +99,11 @@ export default class MVideoBody extends React.PureComponent { private getThumbUrl(): string|null { const content = this.props.mxEvent.getContent(); const media = mediaFromContent(content); - if (media.isEncrypted) { + + if (media.isEncrypted && this.state.decryptedThumbnailUrl) { return this.state.decryptedThumbnailUrl; + } else if (this.state.posterLoading) { + return this.state.blurhashUrl; } else if (media.hasThumbnail) { return media.thumbnailHttp; } else { @@ -101,18 +111,57 @@ export default class MVideoBody extends React.PureComponent { } } + private loadBlurhash() { + const info = this.props.mxEvent.getContent()?.info; + if (!info[BLURHASH_FIELD]) return; + + const canvas = document.createElement("canvas"); + + let width = info.w; + let height = info.h; + const scale = this.thumbScale(info.w, info.h); + if (scale) { + width = Math.floor(info.w * scale); + height = Math.floor(info.h * scale); + } + + canvas.width = width; + canvas.height = height; + + const pixels = decode(info[BLURHASH_FIELD], width, height); + const ctx = canvas.getContext("2d"); + const imgData = ctx.createImageData(width, height); + imgData.data.set(pixels); + ctx.putImageData(imgData, 0, 0); + + this.setState({ + blurhashUrl: canvas.toDataURL(), + posterLoading: true, + }); + + const content = this.props.mxEvent.getContent(); + const media = mediaFromContent(content); + if (media.hasThumbnail) { + const image = new Image(); + image.onload = () => { + this.setState({ posterLoading: false }); + }; + image.src = media.thumbnailHttp; + } + } + async componentDidMount() { const autoplay = SettingsStore.getValue("autoplayGifsAndVideos") as boolean; const content = this.props.mxEvent.getContent(); + this.loadBlurhash(); + if (content.file !== undefined && this.state.decryptedUrl === null) { let thumbnailPromise = Promise.resolve(null); - if (content.info && content.info.thumbnail_file) { - thumbnailPromise = decryptFile( - content.info.thumbnail_file, - ).then(function(blob) { - return URL.createObjectURL(blob); - }); + if (content?.info?.thumbnail_file) { + thumbnailPromise = decryptFile(content.info.thumbnail_file) + .then(blob => URL.createObjectURL(blob)); } + try { const thumbnailUrl = await thumbnailPromise; if (autoplay) { @@ -218,7 +267,7 @@ export default class MVideoBody extends React.PureComponent { let poster = null; let preload = "metadata"; if (content.info) { - const scale = this.thumbScale(content.info.w, content.info.h, 480, 360); + const scale = this.thumbScale(content.info.w, content.info.h); if (scale) { width = Math.floor(content.info.w * scale); height = Math.floor(content.info.h * scale); From 9c599b567df828ed5e888c846aa080b93c535ae2 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 May 2021 21:39:38 +0100 Subject: [PATCH 12/15] Fix alignment of image blurhashes and interplay with spinners --- src/components/views/messages/MImageBody.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/components/views/messages/MImageBody.js b/src/components/views/messages/MImageBody.js index 60623752439..8d3f1c4c10c 100644 --- a/src/components/views/messages/MImageBody.js +++ b/src/components/views/messages/MImageBody.js @@ -372,10 +372,7 @@ export default class MImageBody extends React.Component { let placeholder = null; let gifLabel = null; - // e2e image hasn't been decrypted yet - if (content.file !== undefined && this.state.decryptedUrl === null) { - placeholder = ; - } else if (!this.state.imgLoaded) { + if (!this.state.imgLoaded) { placeholder = this.getPlaceholder(maxWidth, maxHeight); } @@ -414,9 +411,7 @@ export default class MImageBody extends React.Component { // Constrain width here so that spinner appears central to the loaded thumbnail maxWidth: infoWidth + "px", }}> -
- { placeholder } -
+ { placeholder } } @@ -442,8 +437,10 @@ export default class MImageBody extends React.Component { // Overidden by MStickerBody getPlaceholder(width, height) { const blurhash = this.props.mxEvent.getContent().info[BLURHASH_FIELD]; - if (!blurhash) return null; - return ; + if (blurhash) return ; + return
+ +
; } // Overidden by MStickerBody From 3f9b6caacd76608cb1d22403442460223ab7090a Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Fri, 21 May 2021 21:46:18 +0100 Subject: [PATCH 13/15] fix missing rounded corners on blurhash placeholder --- res/css/views/messages/_MImageBody.scss | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/res/css/views/messages/_MImageBody.scss b/res/css/views/messages/_MImageBody.scss index 1c773c2f065..d04b3d5c938 100644 --- a/res/css/views/messages/_MImageBody.scss +++ b/res/css/views/messages/_MImageBody.scss @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +$timelineImageBorderRadius: 4px; + .mx_MImageBody { display: block; margin-right: 34px; @@ -25,7 +27,11 @@ limitations under the License. height: 100%; left: 0; top: 0; - border-radius: 4px; + border-radius: $timelineImageBorderRadius; + + > canvas { + border-radius: $timelineImageBorderRadius; + } } .mx_MImageBody_thumbnail_container { From 44e14a7d4305639b928e3a78a0b6f368f33f8424 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Mon, 24 May 2021 10:06:17 +0100 Subject: [PATCH 14/15] Support blurhashes on stickers --- src/components/views/messages/MStickerBody.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/views/messages/MStickerBody.js b/src/components/views/messages/MStickerBody.js index 54eb7649b42..d3dea2bfa48 100644 --- a/src/components/views/messages/MStickerBody.js +++ b/src/components/views/messages/MStickerBody.js @@ -18,6 +18,7 @@ import React from 'react'; import MImageBody from './MImageBody'; import * as sdk from '../../../index'; import {replaceableComponent} from "../../../utils/replaceableComponent"; +import {BLURHASH_FIELD} from "../../../ContentMessages"; @replaceableComponent("views.messages.MStickerBody") export default class MStickerBody extends MImageBody { @@ -41,7 +42,8 @@ export default class MStickerBody extends MImageBody { // Placeholder to show in place of the sticker image if // img onLoad hasn't fired yet. - getPlaceholder() { + getPlaceholder(width, height) { + if (this.props.mxEvent.getContent().info[BLURHASH_FIELD]) return super.getPlaceholder(width, height); const TintableSVG = sdk.getComponent('elements.TintableSvg'); return ; } From 12cce2a93884046d39eae98df1b9fc214a2b3e58 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 1 Jul 2021 20:51:13 +0100 Subject: [PATCH 15/15] delint --- src/components/views/elements/BlurhashPlaceholder.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/views/elements/BlurhashPlaceholder.tsx b/src/components/views/elements/BlurhashPlaceholder.tsx index bed45bfe5ad..0e59253fe86 100644 --- a/src/components/views/elements/BlurhashPlaceholder.tsx +++ b/src/components/views/elements/BlurhashPlaceholder.tsx @@ -15,7 +15,7 @@ */ import React from 'react'; -import {decode} from "blurhash"; +import { decode } from "blurhash"; interface IProps { blurhash: string; @@ -38,7 +38,7 @@ export default class BlurhashPlaceholder extends React.PureComponent { if (!this.canvas.current) return; try { - const {width, height} = this.props; + const { width, height } = this.props; const pixels = decode(this.props.blurhash, Math.ceil(width), Math.ceil(height)); const ctx = this.canvas.current.getContext("2d");