Add support for blurhash (MSC2448)

package.json

@@ -55,6 +55,7 @@
   "dependencies": {
     "@babel/runtime": "^7.12.5",
     "await-lock": "^2.1.0",
+    "blurhash": "^1.1.3",
     "browser-encrypt-attachment": "^0.3.0",
     "browser-request": "^0.3.3",
     "cheerio": "^1.0.0-rc.9",

res/css/views/messages/_MImageBody.scss

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+$timelineImageBorderRadius: 4px;
+
 .mx_MImageBody {
     display: block;
     margin-right: 34px;
@@ -25,7 +27,11 @@ limitations under the License.
     height: 100%;
     left: 0;
     top: 0;
-    border-radius: 4px;
+    border-radius: $timelineImageBorderRadius;
+
+    > canvas {
+        border-radius: $timelineImageBorderRadius;
+    }
 }
 
 .mx_MImageBody_thumbnail_container {

src/ContentMessages.tsx

@@ -17,9 +17,10 @@ limitations under the License.
 */
 
 import React from "react";
-import dis from './dispatcher/dispatcher';
-import { MatrixClientPeg } from './MatrixClientPeg';
+import { encode } from "blurhash";
 import { MatrixClient } from "matrix-js-sdk/src/client";
+
+import dis from './dispatcher/dispatcher';
 import * as sdk from './index';
 import { _t } from './languageHandler';
 import Modal from './Modal';
@@ -47,6 +48,10 @@ const MAX_HEIGHT = 600;
 //                  5669 px (x-axis)      , 5669 px (y-axis)      , per metre
 const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01];
 
+export const BLURHASH_FIELD = "xyz.amorgan.blurhash"; // MSC2448
+const BLURHASH_X_COMPONENTS = 6;
+const BLURHASH_Y_COMPONENTS = 6;
+
 export class UploadCanceledError extends Error {}
 
 type ThumbnailableElement = HTMLImageElement | HTMLVideoElement;
@@ -77,6 +82,7 @@ interface IThumbnail {
         };
         w: number;
         h: number;
+        [BLURHASH_FIELD]: string;
     };
     thumbnail: Blob;
 }
@@ -124,7 +130,16 @@ function createThumbnail(
         const canvas = document.createElement("canvas");
         canvas.width = targetWidth;
         canvas.height = targetHeight;
-        canvas.getContext("2d").drawImage(element, 0, 0, targetWidth, targetHeight);
+        const context = canvas.getContext("2d");
+        context.drawImage(element, 0, 0, targetWidth, targetHeight);
+        const imageData = context.getImageData(0, 0, targetWidth, targetHeight);
+        const blurhash = encode(
+            imageData.data,
+            imageData.width,
+            imageData.height,
+            BLURHASH_X_COMPONENTS,
+            BLURHASH_Y_COMPONENTS,
+        );
         canvas.toBlob(function(thumbnail) {
             resolve({
                 info: {
@@ -136,8 +151,9 @@ function createThumbnail(
                     },
                     w: inputWidth,
                     h: inputHeight,
+                    [BLURHASH_FIELD]: blurhash,
                 },
-                thumbnail: thumbnail,
+                thumbnail,
             });
         }, mimeType);
     });
@@ -220,7 +236,8 @@ function infoForImageFile(matrixClient, roomId, imageFile) {
 }
 
 /**
- * Load a file into a newly created video element.
+ * Load a file into a newly created video element and pull some strings
+ * in an attempt to guarantee the first frame will be showing.
  *
  * @param {File} videoFile The file to load in an video element.
  * @return {Promise} A promise that resolves with the video image element.
@@ -229,20 +246,25 @@ function loadVideoElement(videoFile): Promise<HTMLVideoElement> {
     return new Promise((resolve, reject) => {
         // Load the file into an html element
         const video = document.createElement("video");
+        video.preload = "metadata";
+        video.playsInline = true;
+        video.muted = true;
 
         const reader = new FileReader();
 
         reader.onload = function(ev) {
-            video.src = ev.target.result as string;
-
-            // Once ready, returns its size
             // Wait until we have enough data to thumbnail the first frame.
-            video.onloadeddata = function() {
+            video.onloadeddata = async function() {
                 resolve(video);
+                video.pause();
             };
             video.onerror = function(e) {
                 reject(e);
             };
+
+            video.src = ev.target.result as string;
+            video.load();
+            video.play();
         };
         reader.onerror = function(e) {
             reject(e);
@@ -347,7 +369,7 @@ export function uploadFile(
         });
         (prom as IAbortablePromise<any>).abort = () => {
             canceled = true;
-            if (uploadPromise) MatrixClientPeg.get().cancelUpload(uploadPromise);
+            if (uploadPromise) matrixClient.cancelUpload(uploadPromise);
         };
         return prom;
     } else {
@@ -357,11 +379,11 @@ export function uploadFile(
         const promise1 = basePromise.then(function(url) {
             if (canceled) throw new UploadCanceledError();
             // If the attachment isn't encrypted then include the URL directly.
-            return { "url": url };
+            return { url };
         });
         (promise1 as any).abort = () => {
             canceled = true;
-            MatrixClientPeg.get().cancelUpload(basePromise);
+            matrixClient.cancelUpload(basePromise);
         };
         return promise1;
     }
@@ -373,7 +395,7 @@ export default class ContentMessages {
 
     sendStickerContentToRoom(url: string, roomId: string, info: IImageInfo, text: string, matrixClient: MatrixClient) {
         const startTime = CountlyAnalytics.getTimestamp();
-        const prom = MatrixClientPeg.get().sendStickerMessage(roomId, url, info, text).catch((e) => {
+        const prom = matrixClient.sendStickerMessage(roomId, url, info, text).catch((e) => {
             console.warn(`Failed to send content with URL ${url} to room ${roomId}`, e);
             throw e;
         });
@@ -415,7 +437,7 @@ export default class ContentMessages {
 
         if (!this.mediaConfig) { // hot-path optimization to not flash a spinner if we don't need to
             const modal = Modal.createDialog(Spinner, null, 'mx_Dialog_spinner');
-            await this.ensureMediaConfigFetched();
+            await this.ensureMediaConfigFetched(matrixClient);
             modal.close();
         }
 
@@ -470,7 +492,7 @@ export default class ContentMessages {
         return this.inprogress.filter(u => !u.canceled);
     }
 
-    cancelUpload(promise: Promise<any>) {
+    cancelUpload(promise: Promise<any>, matrixClient: MatrixClient) {
         let upload: IUpload;
         for (let i = 0; i < this.inprogress.length; ++i) {
             if (this.inprogress[i].promise === promise) {
@@ -480,7 +502,7 @@ export default class ContentMessages {
         }
         if (upload) {
             upload.canceled = true;
-            MatrixClientPeg.get().cancelUpload(upload.promise);
+            matrixClient.cancelUpload(upload.promise);
             dis.dispatch<UploadCanceledPayload>({ action: Action.UploadCanceled, upload });
         }
     }
@@ -621,11 +643,11 @@ export default class ContentMessages {
         return true;
     }
 
-    private ensureMediaConfigFetched() {
+    private ensureMediaConfigFetched(matrixClient: MatrixClient) {
         if (this.mediaConfig !== null) return;
 
         console.log("[Media Config] Fetching");
-        return MatrixClientPeg.get().getMediaConfig().then((config) => {
+        return matrixClient.getMediaConfig().then((config) => {
             console.log("[Media Config] Fetched config:", config);
             return config;
         }).catch(() => {

src/components/structures/RoomView.tsx

@@ -1261,7 +1261,7 @@ export default class RoomView extends React.Component<IProps, IState> {
         });
     };
 
-    private injectSticker(url, info, text) {
+    private injectSticker(url: string, info: object, text: string) {
         if (this.context.isGuest()) {
             dis.dispatch({ action: 'require_registration' });
             return;

src/components/structures/UploadBar.tsx

@@ -26,6 +26,7 @@ import ProgressBar from "../views/elements/ProgressBar";
 import AccessibleButton from "../views/elements/AccessibleButton";
 import { IUpload } from "../../models/IUpload";
 import { replaceableComponent } from "../../utils/replaceableComponent";
+import MatrixClientContext from "../../contexts/MatrixClientContext";
 
 interface IProps {
     room: Room;
@@ -38,6 +39,8 @@ interface IState {
 
 @replaceableComponent("structures.UploadBar")
 export default class UploadBar extends React.Component<IProps, IState> {
+    static contextType = MatrixClientContext;
+
     private dispatcherRef: string;
     private mounted: boolean;
 
@@ -82,7 +85,7 @@ export default class UploadBar extends React.Component<IProps, IState> {
 
     private onCancelClick = (ev) => {
         ev.preventDefault();
-        ContentMessages.sharedInstance().cancelUpload(this.state.currentUpload.promise);
+        ContentMessages.sharedInstance().cancelUpload(this.state.currentUpload.promise, this.context);
     };
 
     render() {

src/components/views/elements/BlurhashPlaceholder.tsx

@@ -0,0 +1,56 @@
+/*
+ Copyright 2020 The Matrix.org Foundation C.I.C.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+import React from 'react';
+import { decode } from "blurhash";
+
+interface IProps {
+    blurhash: string;
+    width: number;
+    height: number;
+}
+
+export default class BlurhashPlaceholder extends React.PureComponent<IProps> {
+    private canvas: React.RefObject<HTMLCanvasElement> = React.createRef();
+
+    public componentDidMount() {
+        this.draw();
+    }
+
+    public componentDidUpdate() {
+        this.draw();
+    }
+
+    private draw() {
+        if (!this.canvas.current) return;
+
+        try {
+            const { width, height } = this.props;
+
+            const pixels = decode(this.props.blurhash, Math.ceil(width), Math.ceil(height));
+            const ctx = this.canvas.current.getContext("2d");
+            const imgData = ctx.createImageData(width, height);
+            imgData.data.set(pixels);
+            ctx.putImageData(imgData, 0, 0);
+        } catch (e) {
+            console.error("Error rendering blurhash: ", e);
+        }
+    }
+
+    public render() {
+        return <canvas height={this.props.height} width={this.props.width} ref={this.canvas} />;
+    }
+}

src/components/views/messages/MImageBody.js

@@ -29,6 +29,8 @@ import MatrixClientContext from "../../../contexts/MatrixClientContext";
 import InlineSpinner from '../elements/InlineSpinner';
 import { replaceableComponent } from "../../../utils/replaceableComponent";
 import { mediaFromContent } from "../../../customisations/Media";
+import BlurhashPlaceholder from "../elements/BlurhashPlaceholder";
+import { BLURHASH_FIELD } from "../../../ContentMessages";
 
 @replaceableComponent("views.messages.MImageBody")
 export default class MImageBody extends React.Component {
@@ -333,7 +335,8 @@ export default class MImageBody extends React.Component {
             infoWidth = content.info.w;
             infoHeight = content.info.h;
         } else {
-            // Whilst the image loads, display nothing.
+            // Whilst the image loads, display nothing. We also don't display a blurhash image
+            // because we don't really know what size of image we'll end up with.
             //
             // Once loaded, use the loaded image dimensions stored in `loadedImageDimensions`.
             //
@@ -368,12 +371,8 @@ export default class MImageBody extends React.Component {
         let placeholder = null;
         let gifLabel = null;
 
-        // e2e image hasn't been decrypted yet
-        if (content.file !== undefined && this.state.decryptedUrl === null) {
-            placeholder = <InlineSpinner w={32} h={32} />;
-        } else if (!this.state.imgLoaded) {
-            // Deliberately, getSpinner is left unimplemented here, MStickerBody overides
-            placeholder = this.getPlaceholder();
+        if (!this.state.imgLoaded) {
+            placeholder = this.getPlaceholder(maxWidth, maxHeight);
         }
 
         let showPlaceholder = Boolean(placeholder);
@@ -395,7 +394,7 @@ export default class MImageBody extends React.Component {
 
         if (!this.state.showImage) {
             img = <HiddenImagePlaceholder style={{ maxWidth: maxWidth + "px" }} />;
-            showPlaceholder = false; // because we're hiding the image, so don't show the sticker icon.
+            showPlaceholder = false; // because we're hiding the image, so don't show the placeholder.
         }
 
         if (this._isGif() && !SettingsStore.getValue("autoplayGifsAndVideos") && !this.state.hover) {
@@ -411,9 +410,7 @@ export default class MImageBody extends React.Component {
                         // Constrain width here so that spinner appears central to the loaded thumbnail
                         maxWidth: infoWidth + "px",
                     }}>
-                        <div className="mx_MImageBody_thumbnail_spinner">
-                            { placeholder }
-                        </div>
+                        { placeholder }
                     </div>
                 }
 
@@ -437,9 +434,12 @@ export default class MImageBody extends React.Component {
     }
 
     // Overidden by MStickerBody
-    getPlaceholder() {
-        // MImageBody doesn't show a placeholder whilst the image loads, (but it could do)
-        return null;
+    getPlaceholder(width, height) {
+        const blurhash = this.props.mxEvent.getContent().info[BLURHASH_FIELD];
+        if (blurhash) return <BlurhashPlaceholder blurhash={blurhash} width={width} height={height} />;
+        return <div className="mx_MImageBody_thumbnail_spinner">
+            <InlineSpinner w={32} h={32} />
+        </div>;
     }
 
     // Overidden by MStickerBody

src/components/views/messages/MStickerBody.js

@@ -18,6 +18,7 @@ import React from 'react';
 import MImageBody from './MImageBody';
 import * as sdk from '../../../index';
 import { replaceableComponent } from "../../../utils/replaceableComponent";
+import { BLURHASH_FIELD } from "../../../ContentMessages";
 
 @replaceableComponent("views.messages.MStickerBody")
 export default class MStickerBody extends MImageBody {
@@ -41,7 +42,8 @@ export default class MStickerBody extends MImageBody {
 
     // Placeholder to show in place of the sticker image if
     // img onLoad hasn't fired yet.
-    getPlaceholder() {
+    getPlaceholder(width, height) {
+        if (this.props.mxEvent.getContent().info[BLURHASH_FIELD]) return super.getPlaceholder(width, height);
         return <img src={require("../../../../res/img/icons-show-stickers.svg")} width="75" height="75" />;
     }