From 0391543bbc7de6b7fa1713b59d0eb0a086c5d733 Mon Sep 17 00:00:00 2001 From: Zack Date: Wed, 1 Apr 2026 11:48:22 +0200 Subject: [PATCH] Refactor and move MVideoBody to shared components (#32849) * init MVideoBody to shared components, including test, stories and view * fix prettier and other warnings * move video message body to shared view + app viewmodel * Fix prettier warnings and masking spinner for tests * stabilize VideoBodyView screenshots with local media asset * Disable spinner from changing image all the time * Added mask over video spinner to prevent issues with new generated images on playwright tests * Update prettier fix * Update snapshot * Add tests to cover different states of Video * Update code to prevent the previous component Hack fix regarding jumps on the timeline. * Update snapshot * Update code to improve code quality for Sonar + update snapshot * adde documentation snippets * refactor: move m.video rendering into body factory * docs: add tsdoc for video body view model * docs: add thumbnail tsdoc for video body view model * docs: add content-url tsdoc for video body view model * docs: add dimensions tsdoc for video body view model * docs: add aspect-ratio tsdoc for video body view model * docs: add tsdoc for video body view state * refactor: replace video body view state enum * refactor: remove duplicate video body state init * refactor: drop unused video body view state attribute * Fix Prettier * Update snapshot screenshot * test: restore video story screenshot mask * chore: refresh PR head * Add mask to screenshot to pass CI tests * test: narrow video story mask hook * Fix easy Sonar warnings in video body components * Move shared message body views into event-tile layout * Move shared message body visual baselines * Revert unrelated shared message body moves --- apps/web/res/css/_components.pcss | 1 - .../res/css/views/messages/_MVideoBody.pcss | 21 - .../views/messages/MBodyFactory.tsx | 85 ++- .../components/views/messages/MVideoBody.tsx | 353 ------------ .../views/messages/MessageEvent.tsx | 14 +- .../message-body/VideoBodyViewModel.ts | 543 ++++++++++++++++++ .../views/messages/MBodyFactory-test.tsx | 14 +- .../views/messages/MVideoBody-test.tsx | 31 +- .../views/messages/MessageEvent-test.tsx | 15 +- .../__snapshots__/MBodyFactory-test.tsx.snap | 38 -- .../__snapshots__/MVideoBody-test.tsx.snap | 78 --- .../message-body/VideoBodyViewModel-test.tsx | 464 +++++++++++++++ .../.storybook/vitest.setup.ts | 18 + .../VideoBodyView.stories.tsx/error-auto.png | Bin 0 -> 6346 bytes .../error-state-auto.png | Bin 0 -> 6346 bytes .../VideoBodyView.stories.tsx/hidden-auto.png | Bin 0 -> 4854 bytes .../loading-auto.png | Bin 0 -> 4015 bytes .../VideoBodyView.stories.tsx/ready-auto.png | Bin 0 -> 30019 bytes packages/shared-components/src/index.ts | 1 + .../event-tile/body/MVideoBodyView/.gitkeep | 1 - .../MVideoBodyView/VideoBodyView.module.css | 51 ++ .../MVideoBodyView/VideoBodyView.stories.tsx | 100 ++++ .../MVideoBodyView/VideoBodyView.test.tsx | 128 +++++ .../body/MVideoBodyView/VideoBodyView.tsx | 227 ++++++++ .../__snapshots__/VideoBodyView.test.tsx.snap | 90 +++ .../event-tile/body/MVideoBodyView/index.tsx | 14 + .../static/videoBodyDemo.webm | Bin 0 -> 646144 bytes 27 files changed, 1763 insertions(+), 524 deletions(-) delete mode 100644 apps/web/res/css/views/messages/_MVideoBody.pcss delete mode 100644 apps/web/src/components/views/messages/MVideoBody.tsx create mode 100644 apps/web/src/viewmodels/message-body/VideoBodyViewModel.ts delete mode 100644 apps/web/test/unit-tests/components/views/messages/__snapshots__/MVideoBody-test.tsx.snap create mode 100644 apps/web/test/viewmodels/message-body/VideoBodyViewModel-test.tsx create mode 100644 packages/shared-components/__vis__/linux/__baselines__/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.stories.tsx/error-auto.png create mode 100644 packages/shared-components/__vis__/linux/__baselines__/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.stories.tsx/error-state-auto.png create mode 100644 packages/shared-components/__vis__/linux/__baselines__/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.stories.tsx/hidden-auto.png create mode 100644 packages/shared-components/__vis__/linux/__baselines__/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.stories.tsx/loading-auto.png create mode 100644 packages/shared-components/__vis__/linux/__baselines__/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.stories.tsx/ready-auto.png delete mode 100644 packages/shared-components/src/room/timeline/event-tile/body/MVideoBodyView/.gitkeep create mode 100644 packages/shared-components/src/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.module.css create mode 100644 packages/shared-components/src/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.stories.tsx create mode 100644 packages/shared-components/src/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.test.tsx create mode 100644 packages/shared-components/src/room/timeline/event-tile/body/MVideoBodyView/VideoBodyView.tsx create mode 100644 packages/shared-components/src/room/timeline/event-tile/body/MVideoBodyView/__snapshots__/VideoBodyView.test.tsx.snap create mode 100644 packages/shared-components/src/room/timeline/event-tile/body/MVideoBodyView/index.tsx create mode 100644 packages/shared-components/static/videoBodyDemo.webm diff --git a/apps/web/res/css/_components.pcss b/apps/web/res/css/_components.pcss index 9435e53b6f..ad1eef3644 100644 --- a/apps/web/res/css/_components.pcss +++ b/apps/web/res/css/_components.pcss @@ -232,7 +232,6 @@ @import "./views/messages/_MPollBody.pcss"; @import "./views/messages/_MStickerBody.pcss"; @import "./views/messages/_MTextBody.pcss"; -@import "./views/messages/_MVideoBody.pcss"; @import "./views/messages/_MediaBody.pcss"; @import "./views/messages/_MessageActionBar.pcss"; @import "./views/messages/_MjolnirBody.pcss"; diff --git a/apps/web/res/css/views/messages/_MVideoBody.pcss b/apps/web/res/css/views/messages/_MVideoBody.pcss deleted file mode 100644 index 0727a8dc44..0000000000 --- a/apps/web/res/css/views/messages/_MVideoBody.pcss +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2024 New Vector Ltd. -Copyright 2020, 2021 The Matrix.org Foundation C.I.C. - -SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE files in the repository root for full details. -*/ - -span.mx_MVideoBody { - overflow: hidden; - - .mx_MVideoBody_container { - border-radius: var(--MBody-border-radius); - overflow: hidden; - - video { - height: 100%; - width: 100%; - } - } -} diff --git a/apps/web/src/components/views/messages/MBodyFactory.tsx b/apps/web/src/components/views/messages/MBodyFactory.tsx index 3e79bc6e66..deb10cd822 100644 --- a/apps/web/src/components/views/messages/MBodyFactory.tsx +++ b/apps/web/src/components/views/messages/MBodyFactory.tsx @@ -11,15 +11,18 @@ import { DecryptionFailureBodyView, FileBodyView, RedactedBodyView, + VideoBodyView, useCreateAutoDisposedViewModel, } from "@element-hq/web-shared-components"; import { type IBodyProps } from "./IBodyProps"; -import RoomContext from "../../../contexts/RoomContext"; +import RoomContext, { TimelineRenderingType } from "../../../contexts/RoomContext"; import { LocalDeviceVerificationStateContext } from "../../../contexts/LocalDeviceVerificationStateContext"; +import { useMediaVisible } from "../../../hooks/useMediaVisible"; import { DecryptionFailureBodyViewModel } from "../../../viewmodels/room/timeline/event-tile/body/DecryptionFailureBodyViewModel"; import { FileBodyViewModel } from "../../../viewmodels/message-body/FileBodyViewModel"; import { RedactedBodyViewModel } from "../../../viewmodels/message-body/RedactedBodyViewModel"; +import { VideoBodyViewModel } from "../../../viewmodels/message-body/VideoBodyViewModel"; type MBodyComponent = React.ComponentType; @@ -59,6 +62,78 @@ export function FileBodyFactory({ return ; } +export function VideoBodyFactory({ + mxEvent, + mediaEventHelper, + forExport, + inhibitInteraction, +}: Readonly>): JSX.Element { + const { timelineRenderingType } = useContext(RoomContext); + const [mediaVisible, setMediaVisible] = useMediaVisible(mxEvent); + const videoRef = useRef(null); + + const vm = useCreateAutoDisposedViewModel( + () => + new VideoBodyViewModel({ + mxEvent, + mediaEventHelper, + forExport, + inhibitInteraction, + mediaVisible, + onPreviewClick: (): void => setMediaVisible(true), + videoRef, + }), + ); + + useEffect(() => { + vm.loadInitialMediaIfVisible(); + }, [vm]); + + useEffect(() => { + vm.setEvent(mxEvent, mediaEventHelper); + }, [mxEvent, mediaEventHelper, vm]); + + useEffect(() => { + vm.setForExport(forExport); + }, [forExport, vm]); + + useEffect(() => { + vm.setInhibitInteraction(inhibitInteraction); + }, [inhibitInteraction, vm]); + + useEffect(() => { + vm.setMediaVisible(mediaVisible); + }, [mediaVisible, vm]); + + useEffect(() => { + vm.setOnPreviewClick((): void => setMediaVisible(true)); + }, [setMediaVisible, vm]); + + const showFileBody = + !forExport && + timelineRenderingType !== TimelineRenderingType.Room && + timelineRenderingType !== TimelineRenderingType.Pinned && + timelineRenderingType !== TimelineRenderingType.Search; + + return ( + + {showFileBody ? ( + + ) : null} + + ); +} + export function RedactedBodyFactory({ mxEvent, ref }: Pick): JSX.Element { const vm = useCreateAutoDisposedViewModel(() => new RedactedBodyViewModel({ mxEvent })); @@ -87,9 +162,11 @@ export function DecryptionFailureBodyFactory({ mxEvent, ref }: Pick; } -// Message body factory registry. -// Start small: only m.file currently routes to the new FileBodyView path. -const MESSAGE_BODY_TYPES = new Map([[MsgType.File, FileBodyFactory]]); +// Message body factory registry for bodies that already route through view-model-backed wrappers. +const MESSAGE_BODY_TYPES = new Map([ + [MsgType.File, FileBodyFactory], + [MsgType.Video, VideoBodyFactory], +]); // Render a body using the picked factory. // Falls back to the provided factory when msgtype has no specific handler. diff --git a/apps/web/src/components/views/messages/MVideoBody.tsx b/apps/web/src/components/views/messages/MVideoBody.tsx deleted file mode 100644 index 848dc7a32e..0000000000 --- a/apps/web/src/components/views/messages/MVideoBody.tsx +++ /dev/null @@ -1,353 +0,0 @@ -/* -Copyright 2024 New Vector Ltd. -Copyright 2015-2021 The Matrix.org Foundation C.I.C. - -SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE files in the repository root for full details. -*/ - -import React, { type ReactNode } from "react"; -import { decode } from "blurhash"; -import { type MediaEventContent } from "matrix-js-sdk/src/types"; -import { logger } from "matrix-js-sdk/src/logger"; - -import { _t } from "../../../languageHandler"; -import SettingsStore from "../../../settings/SettingsStore"; -import InlineSpinner from "../elements/InlineSpinner"; -import { mediaFromContent } from "../../../customisations/Media"; -import { BLURHASH_FIELD } from "../../../utils/image-media"; -import { type IBodyProps } from "./IBodyProps"; -import { type ImageSize, suggestedSize as suggestedVideoSize } from "../../../settings/enums/ImageSize"; -import RoomContext, { TimelineRenderingType } from "../../../contexts/RoomContext"; -import MediaProcessingError from "./shared/MediaProcessingError"; -import { HiddenMediaPlaceholder } from "./HiddenMediaPlaceholder"; -import { useMediaVisible } from "../../../hooks/useMediaVisible"; -import { FileBodyFactory, renderMBody } from "./MBodyFactory"; - -interface IState { - decryptedUrl: string | null; - decryptedThumbnailUrl: string | null; - decryptedBlob: Blob | null; - error?: any; - fetchingData: boolean; - posterLoading: boolean; - blurhashUrl: string | null; -} - -interface IProps extends IBodyProps { - /** - * Should the media be behind a preview. - */ - mediaVisible: boolean; - /** - * Set the visibility of the media event. - * @param visible Should the event be visible. - */ - setMediaVisible: (visible: boolean) => void; -} - -class MVideoBodyInner extends React.PureComponent { - public static contextType = RoomContext; - declare public context: React.ContextType; - - private videoRef = React.createRef(); - private sizeWatcher?: string; - - public state = { - fetchingData: false, - decryptedUrl: null, - decryptedThumbnailUrl: null, - decryptedBlob: null, - error: null, - posterLoading: false, - blurhashUrl: null, - }; - - private onClick = (): void => { - this.props.setMediaVisible(true); - }; - - private getContentUrl(): string | undefined { - const content = this.props.mxEvent.getContent(); - // During export, the content url will point to the MSC, which will later point to a local url - if (this.props.forExport) return content.file?.url ?? content.url; - const media = mediaFromContent(content); - if (media.isEncrypted) { - return this.state.decryptedUrl ?? undefined; - } else { - return media.srcHttp ?? undefined; - } - } - - private hasContentUrl(): boolean { - const url = this.getContentUrl(); - return !!url && !url.startsWith("data:"); - } - - private getThumbUrl(): string | null { - // there's no need of thumbnail when the content is local - if (this.props.forExport) return null; - - const content = this.props.mxEvent.getContent(); - const media = mediaFromContent(content); - - if (media.isEncrypted && this.state.decryptedThumbnailUrl) { - return this.state.decryptedThumbnailUrl; - } else if (this.state.posterLoading) { - return this.state.blurhashUrl; - } else if (media.hasThumbnail) { - return media.thumbnailHttp; - } else { - return null; - } - } - - private loadBlurhash(): void { - const info = this.props.mxEvent.getContent()?.info; - if (!info[BLURHASH_FIELD]) return; - - const canvas = document.createElement("canvas"); - - const { w: width, h: height } = suggestedVideoSize(SettingsStore.getValue("Images.size") as ImageSize, { - w: info.w, - h: info.h, - }); - - canvas.width = width; - canvas.height = height; - - const pixels = decode(info[BLURHASH_FIELD], width, height); - const ctx = canvas.getContext("2d")!; - const imgData = ctx.createImageData(width, height); - imgData.data.set(pixels); - ctx.putImageData(imgData, 0, 0); - - this.setState({ - blurhashUrl: canvas.toDataURL(), - posterLoading: true, - }); - - const content = this.props.mxEvent.getContent(); - const media = mediaFromContent(content); - if (media.hasThumbnail) { - const image = new Image(); - image.onload = () => { - this.setState({ posterLoading: false }); - }; - image.src = media.thumbnailHttp!; - } - } - - private async downloadVideo(): Promise { - try { - this.loadBlurhash(); - } catch (e) { - logger.error("Failed to load blurhash", e); - } - - if (this.props.mediaEventHelper?.media.isEncrypted && this.state.decryptedUrl === null) { - try { - const autoplay = SettingsStore.getValue("autoplayVideo") as boolean; - const thumbnailUrl = await this.props.mediaEventHelper.thumbnailUrl.value; - if (autoplay) { - logger.log("Preloading video"); - this.setState({ - decryptedUrl: await this.props.mediaEventHelper.sourceUrl.value, - decryptedThumbnailUrl: thumbnailUrl, - decryptedBlob: await this.props.mediaEventHelper.sourceBlob.value, - }); - } else { - logger.log("NOT preloading video"); - const content = this.props.mxEvent.getContent(); - - let mimetype = content?.info?.mimetype; - - // clobber quicktime muxed files to be considered MP4 so browsers - // are willing to play them - if (mimetype == "video/quicktime") { - mimetype = "video/mp4"; - } - - this.setState({ - // For Chrome and Electron, we need to set some non-empty `src` to - // enable the play button. Firefox does not seem to care either - // way, so it's fine to do for all browsers. - decryptedUrl: `data:${mimetype},`, - decryptedThumbnailUrl: thumbnailUrl || `data:${mimetype},`, - decryptedBlob: null, - }); - } - } catch (err) { - logger.warn("Unable to decrypt attachment: ", err); - // Set a placeholder image when we can't decrypt the image. - this.setState({ - error: err, - }); - } - } - } - - public async componentDidMount(): Promise { - this.sizeWatcher = SettingsStore.watchSetting("Images.size", null, () => { - this.forceUpdate(); // we don't really have a reliable thing to update, so just update the whole thing - }); - - // Do not attempt to load the media if we do not want to show previews here. - if (this.props.mediaVisible) { - await this.downloadVideo(); - } - } - - public async componentDidUpdate(prevProps: Readonly): Promise { - if (!prevProps.mediaVisible && this.props.mediaVisible) { - await this.downloadVideo(); - } - } - - public componentWillUnmount(): void { - SettingsStore.unwatchSetting(this.sizeWatcher); - } - - private videoOnPlay = async (): Promise => { - if (this.hasContentUrl() || this.state.fetchingData || this.state.error) { - // We have the file, we are fetching the file, or there is an error. - return; - } - this.setState({ - // To stop subsequent download attempts - fetchingData: true, - }); - if (!this.props.mediaEventHelper!.media.isEncrypted) { - this.setState({ - error: "No file given in content", - }); - return; - } - this.setState( - { - decryptedUrl: await this.props.mediaEventHelper!.sourceUrl.value, - decryptedBlob: await this.props.mediaEventHelper!.sourceBlob.value, - fetchingData: false, - }, - () => { - if (!this.videoRef.current) return; - this.videoRef.current.play(); - }, - ); - }; - - protected get showFileBody(): boolean { - return ( - this.context.timelineRenderingType !== TimelineRenderingType.Room && - this.context.timelineRenderingType !== TimelineRenderingType.Pinned && - this.context.timelineRenderingType !== TimelineRenderingType.Search - ); - } - - private getFileBody = (): ReactNode => { - if (this.props.forExport) return null; - return this.showFileBody && renderMBody({ ...this.props, showFileInfo: false }, FileBodyFactory); - }; - - public render(): React.ReactNode { - const content = this.props.mxEvent.getContent(); - const autoplay = !this.props.inhibitInteraction && SettingsStore.getValue("autoplayVideo"); - - let aspectRatio; - if (content.info?.w && content.info?.h) { - aspectRatio = `${content.info.w}/${content.info.h}`; - } - const { w: maxWidth, h: maxHeight } = suggestedVideoSize(SettingsStore.getValue("Images.size") as ImageSize, { - w: content.info?.w, - h: content.info?.h, - }); - - // HACK: This div fills out space while the video loads, to prevent scroll jumps - const spaceFiller =
; - - if (this.state.error !== null) { - return ( - - {_t("timeline|m.video|error_decrypting")} - - ); - } - - // Users may not even want to show a poster, so instead show a preview button. - if (!this.props.mediaVisible) { - return ( - -
- - {_t("timeline|m.video|show_video")} - -
- - ); - } - - // Important: If we aren't autoplaying and we haven't decrypted it yet, show a video with a poster. - if (!this.props.forExport && content.file !== undefined && this.state.decryptedUrl === null && autoplay) { - // Need to decrypt the attachment - // The attachment is decrypted in componentDidMount. - // For now show a spinner. - return ( - -
- -
- {spaceFiller} - - ); - } - - const contentUrl = this.getContentUrl(); - const thumbUrl = this.getThumbUrl(); - let poster: string | undefined; - let preload = "metadata"; - if (content.info && thumbUrl) { - poster = thumbUrl; - preload = "none"; - } - - const fileBody = this.getFileBody(); - return ( - -
-
- {fileBody} - - ); - } -} - -// Wrap MVideoBody component so we can use a hook here. -const MVideoBody: React.FC = (props) => { - const [mediaVisible, setVisible] = useMediaVisible(props.mxEvent); - return ; -}; - -export default MVideoBody; diff --git a/apps/web/src/components/views/messages/MessageEvent.tsx b/apps/web/src/components/views/messages/MessageEvent.tsx index d8e0b656bb..549f38226c 100644 --- a/apps/web/src/components/views/messages/MessageEvent.tsx +++ b/apps/web/src/components/views/messages/MessageEvent.tsx @@ -28,14 +28,19 @@ import { type IBodyProps } from "./IBodyProps"; import TextualBody from "./TextualBody"; import MImageBody from "./MImageBody"; import MVoiceOrAudioBody from "./MVoiceOrAudioBody"; -import MVideoBody from "./MVideoBody"; import MStickerBody from "./MStickerBody"; import MPollBody from "./MPollBody"; import MLocationBody from "./MLocationBody"; import MjolnirBody from "./MjolnirBody"; import MBeaconBody from "./MBeaconBody"; import { type GetRelationsForEvent, type IEventTileOps } from "../rooms/EventTile"; -import { DecryptionFailureBodyFactory, FileBodyFactory, RedactedBodyFactory, renderMBody } from "./MBodyFactory"; +import { + DecryptionFailureBodyFactory, + FileBodyFactory, + RedactedBodyFactory, + VideoBodyFactory, + renderMBody, +} from "./MBodyFactory"; // onMessageAllowed is handled internally interface IProps extends Omit { @@ -65,7 +70,7 @@ const baseBodyTypes = new Map>([ [MsgType.Image, MImageBody], [MsgType.File, (props: IBodyProps) => renderMBody(props, FileBodyFactory)!], [MsgType.Audio, MVoiceOrAudioBody], - [MsgType.Video, MVideoBody], + [MsgType.Video, VideoBodyFactory], ]); const baseEvTypes = new Map>([ [EventType.Sticker, MStickerBody], @@ -260,7 +265,8 @@ export default class MessageEvent extends React.Component implements IMe } if ( - ((BodyType === MImageBody || BodyType == MVideoBody) && !this.validateImageOrVideoMimetype(content)) || + ((BodyType === MImageBody || BodyType === VideoBodyFactory) && + !this.validateImageOrVideoMimetype(content)) || (BodyType === MStickerBody && !this.validateStickerMimetype(content)) ) { BodyType = this.bodyTypes.get(MsgType.File)!; diff --git a/apps/web/src/viewmodels/message-body/VideoBodyViewModel.ts b/apps/web/src/viewmodels/message-body/VideoBodyViewModel.ts new file mode 100644 index 0000000000..95b0226bb6 --- /dev/null +++ b/apps/web/src/viewmodels/message-body/VideoBodyViewModel.ts @@ -0,0 +1,543 @@ +/* + * Copyright 2026 Element Creations Ltd. + * + * SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial + * Please see LICENSE files in the repository root for full details. + */ + +import { decode } from "blurhash"; +import { type RefObject } from "react"; +import { logger } from "matrix-js-sdk/src/logger"; +import { type MatrixEvent } from "matrix-js-sdk/src/matrix"; +import { type MediaEventContent, type VideoInfo } from "matrix-js-sdk/src/types"; +import { + BaseViewModel, + VideoBodyViewState, + type VideoBodyViewModel as VideoBodyViewModelInterface, + type VideoBodyViewSnapshot, +} from "@element-hq/web-shared-components"; + +import { _t } from "../../languageHandler"; +import SettingsStore from "../../settings/SettingsStore"; +import { mediaFromContent } from "../../customisations/Media"; +import { BLURHASH_FIELD } from "../../utils/image-media"; +import { type ImageSize, suggestedSize as suggestedVideoSize } from "../../settings/enums/ImageSize"; +import { type MediaEventHelper } from "../../utils/MediaEventHelper"; + +export interface VideoBodyViewModelProps { + /** + * Video event being rendered. + */ + mxEvent: MatrixEvent; + /** + * Helper for resolving encrypted and unencrypted media sources. + */ + mediaEventHelper?: MediaEventHelper; + /** + * Whether the video is being rendered for export instead of live playback. + */ + forExport?: boolean; + /** + * Whether playback controls and autoplay should be disabled. + */ + inhibitInteraction?: boolean; + /** + * Whether the media should currently be shown instead of the preview button. + */ + mediaVisible: boolean; + /** + * Callback invoked when the hidden-media preview is revealed. + */ + onPreviewClick?: () => void; + /** + * Ref to the underlying video element used for replay after lazy decryption. + */ + videoRef: RefObject; +} + +interface InternalState { + /** + * Decrypted playable media URL for encrypted videos. + */ + decryptedUrl: string | null; + /** + * Decrypted thumbnail URL for encrypted videos. + */ + decryptedThumbnailUrl: string | null; + /** + * Decrypted media blob cached for download or replay. + */ + decryptedBlob: Blob | null; + /** + * Last media-processing error, if any. + */ + error: unknown | null; + /** + * Whether an on-demand media fetch is in progress. + */ + fetchingData: boolean; + /** + * Whether the blurhash poster is being shown while the real poster loads. + */ + posterLoading: boolean; + /** + * Data URL generated from the blurhash placeholder. + */ + blurhashUrl: string | null; + /** + * Current media sizing preference from settings. + */ + imageSize: ImageSize; +} + +type VideoInfoWithBlurhash = VideoInfo & { + [BLURHASH_FIELD]?: string; +}; + +/** + * View model for the video message body, encapsulating media-loading and playback state. + */ +export class VideoBodyViewModel + extends BaseViewModel + implements VideoBodyViewModelInterface +{ + private state: InternalState; + + public constructor(props: VideoBodyViewModelProps) { + const initialState = VideoBodyViewModel.createInitialState(); + super(props, VideoBodyViewModel.computeSnapshot(props, initialState)); + this.state = initialState; + + const imageSizeWatcherRef = SettingsStore.watchSetting("Images.size", null, (_s, _r, _l, _nvl, value) => { + this.setImageSize(value as ImageSize); + }); + this.disposables.track(() => SettingsStore.unwatchSetting(imageSizeWatcherRef)); + } + + public loadInitialMediaIfVisible(): void { + if (this.props.mediaVisible) { + void this.downloadVideo(); + } + } + + private static createInitialState(): InternalState { + return { + fetchingData: false, + decryptedUrl: null, + decryptedThumbnailUrl: null, + decryptedBlob: null, + error: null, + posterLoading: false, + blurhashUrl: null, + imageSize: SettingsStore.getValue("Images.size") as ImageSize, + }; + } + + /** + * Derive the aspect ratio for the video frame from the event metadata, when available. + */ + private static getAspectRatio(mxEvent: MatrixEvent): string | undefined { + const { w, h } = (mxEvent.getContent().info as VideoInfoWithBlurhash | undefined) ?? {}; + if (!w || !h) { + return undefined; + } + + return `${w}/${h}`; + } + + /** + * Compute the rendered video dimensions from the event metadata and current image-size setting. + */ + private static getDimensions(mxEvent: MatrixEvent, imageSize: ImageSize): Required<{ w?: number; h?: number }> { + const { w, h } = (mxEvent.getContent().info as VideoInfoWithBlurhash | undefined) ?? {}; + return suggestedVideoSize(imageSize, { w, h }); + } + + /** + * Resolve the current playable video source URL for the event. + */ + private static getContentUrl(props: VideoBodyViewModelProps, state: InternalState): string | undefined { + const content = props.mxEvent.getContent(); + if (props.forExport) { + return content.file?.url ?? content.url; + } + + const media = mediaFromContent(content); + if (media.isEncrypted) { + return state.decryptedUrl ?? undefined; + } + + return media.srcHttp ?? undefined; + } + + /** + * Resolve the best thumbnail or poster URL for the current video state. + */ + private static getThumbnailUrl(props: VideoBodyViewModelProps, state: InternalState): string | null { + if (props.forExport) { + return null; + } + + const content = props.mxEvent.getContent(); + const media = mediaFromContent(content); + + if (media.isEncrypted && state.decryptedThumbnailUrl) { + return state.decryptedThumbnailUrl; + } + if (state.posterLoading) { + return state.blurhashUrl; + } + if (media.hasThumbnail) { + return media.thumbnailHttp; + } + + return null; + } + + private static computeSnapshot(props: VideoBodyViewModelProps, state: InternalState): VideoBodyViewSnapshot { + const content = props.mxEvent.getContent(); + const autoplay = !props.inhibitInteraction && (SettingsStore.getValue("autoplayVideo") as boolean); + const aspectRatio = VideoBodyViewModel.getAspectRatio(props.mxEvent); + const { w: maxWidth, h: maxHeight } = VideoBodyViewModel.getDimensions(props.mxEvent, state.imageSize); + + if (state.error !== null) { + return { + state: VideoBodyViewState.ERROR, + errorLabel: _t("timeline|m.video|error_decrypting"), + maxWidth, + maxHeight, + aspectRatio, + }; + } + + if (!props.mediaVisible) { + return { + state: VideoBodyViewState.HIDDEN, + hiddenButtonLabel: _t("timeline|m.video|show_video"), + maxWidth, + maxHeight, + aspectRatio, + }; + } + + if (!props.forExport && content.file !== undefined && state.decryptedUrl === null && autoplay) { + return { + state: VideoBodyViewState.LOADING, + maxWidth, + maxHeight, + aspectRatio, + }; + } + + const thumbnailUrl = VideoBodyViewModel.getThumbnailUrl(props, state); + let preload: VideoBodyViewSnapshot["preload"] = "metadata"; + let poster: string | undefined; + if (content.info && thumbnailUrl) { + preload = "none"; + poster = thumbnailUrl; + } + + return { + state: VideoBodyViewState.READY, + videoLabel: content.body, + videoTitle: content.body, + maxWidth, + maxHeight, + aspectRatio, + src: VideoBodyViewModel.getContentUrl(props, state), + poster, + preload, + controls: !props.inhibitInteraction, + muted: autoplay, + autoPlay: autoplay, + }; + } + + private updateSnapshotFromState(): void { + this.snapshot.set(VideoBodyViewModel.computeSnapshot(this.props, this.state)); + } + + private hasContentUrl(): boolean { + const url = VideoBodyViewModel.getContentUrl(this.props, this.state); + return !!url && !url.startsWith("data:"); + } + + private setImageSize(imageSize: ImageSize): void { + if (this.state.imageSize === imageSize) { + return; + } + + this.state = { + ...this.state, + imageSize, + }; + this.updateSnapshotFromState(); + } + + private resetMediaState(): void { + this.state = { + ...this.state, + decryptedUrl: null, + decryptedThumbnailUrl: null, + decryptedBlob: null, + error: null, + fetchingData: false, + posterLoading: false, + blurhashUrl: null, + }; + } + + private loadBlurhash(): void { + const info = this.props.mxEvent.getContent().info as VideoInfoWithBlurhash | undefined; + const blurhash = info?.[BLURHASH_FIELD]; + if (!blurhash) { + return; + } + + const canvas = document.createElement("canvas"); + const { w: width, h: height } = VideoBodyViewModel.getDimensions(this.props.mxEvent, this.state.imageSize); + + canvas.width = width; + canvas.height = height; + + const pixels = decode(blurhash, width, height); + const ctx = canvas.getContext("2d"); + if (!ctx) { + return; + } + + const imgData = ctx.createImageData(width, height); + imgData.data.set(pixels); + ctx.putImageData(imgData, 0, 0); + + this.state = { + ...this.state, + blurhashUrl: canvas.toDataURL(), + posterLoading: true, + }; + this.updateSnapshotFromState(); + + const media = mediaFromContent(this.props.mxEvent.getContent()); + if (!media.hasThumbnail || !media.thumbnailHttp) { + return; + } + + const currentEvent = this.props.mxEvent; + const image = new Image(); + image.onload = (): void => { + if (this.isDisposed || currentEvent !== this.props.mxEvent || !this.state.posterLoading) { + return; + } + + this.state = { + ...this.state, + posterLoading: false, + }; + this.updateSnapshotFromState(); + }; + image.src = media.thumbnailHttp; + } + + private async downloadVideo(): Promise { + try { + this.loadBlurhash(); + } catch (error) { + logger.error("Failed to load blurhash", error); + } + + if (!this.props.mediaEventHelper?.media.isEncrypted || this.state.decryptedUrl !== null) { + return; + } + + const currentEvent = this.props.mxEvent; + const currentHelper = this.props.mediaEventHelper; + try { + const autoplay = !this.props.inhibitInteraction && (SettingsStore.getValue("autoplayVideo") as boolean); + const thumbnailUrl = await currentHelper.thumbnailUrl.value; + + if ( + this.isDisposed || + currentEvent !== this.props.mxEvent || + currentHelper !== this.props.mediaEventHelper + ) { + return; + } + + if (autoplay) { + logger.log("Preloading video"); + this.state = { + ...this.state, + decryptedUrl: await currentHelper.sourceUrl.value, + decryptedThumbnailUrl: thumbnailUrl, + decryptedBlob: await currentHelper.sourceBlob.value, + }; + } else { + logger.log("NOT preloading video"); + const content = currentEvent.getContent(); + let mimetype = content.info?.mimetype ?? "application/octet-stream"; + if (mimetype === "video/quicktime") { + mimetype = "video/mp4"; + } + + this.state = { + ...this.state, + decryptedUrl: `data:${mimetype},`, + decryptedThumbnailUrl: thumbnailUrl || `data:${mimetype},`, + decryptedBlob: null, + }; + } + + this.updateSnapshotFromState(); + } catch (error) { + if ( + this.isDisposed || + currentEvent !== this.props.mxEvent || + currentHelper !== this.props.mediaEventHelper + ) { + return; + } + + logger.warn("Unable to decrypt attachment: ", error); + this.state = { + ...this.state, + error, + }; + this.updateSnapshotFromState(); + } + } + + public setEvent(mxEvent: MatrixEvent, mediaEventHelper?: MediaEventHelper): void { + if (this.props.mxEvent === mxEvent && this.props.mediaEventHelper === mediaEventHelper) { + return; + } + + this.props = { + ...this.props, + mxEvent, + mediaEventHelper, + }; + this.resetMediaState(); + this.updateSnapshotFromState(); + + if (this.props.mediaVisible) { + void this.downloadVideo(); + } + } + + public setForExport(forExport?: boolean): void { + if (this.props.forExport === forExport) { + return; + } + + this.props = { + ...this.props, + forExport, + }; + this.updateSnapshotFromState(); + } + + public setInhibitInteraction(inhibitInteraction?: boolean): void { + if (this.props.inhibitInteraction === inhibitInteraction) { + return; + } + + this.props = { + ...this.props, + inhibitInteraction, + }; + this.updateSnapshotFromState(); + } + + public setMediaVisible(mediaVisible: boolean): void { + if (this.props.mediaVisible === mediaVisible) { + return; + } + + this.props = { + ...this.props, + mediaVisible, + }; + this.updateSnapshotFromState(); + + if (mediaVisible) { + void this.downloadVideo(); + } + } + + public setOnPreviewClick(onPreviewClick?: () => void): void { + if (this.props.onPreviewClick === onPreviewClick) { + return; + } + + this.props = { + ...this.props, + onPreviewClick, + }; + } + + public onPreviewClick = (): void => { + this.props.onPreviewClick?.(); + }; + + public onPlay = async (): Promise => { + if (this.hasContentUrl() || this.state.fetchingData || this.state.error !== null) { + return; + } + + this.state = { + ...this.state, + fetchingData: true, + }; + + if (!this.props.mediaEventHelper?.media.isEncrypted) { + this.state = { + ...this.state, + error: "No file given in content", + fetchingData: false, + }; + this.updateSnapshotFromState(); + return; + } + + const currentEvent = this.props.mxEvent; + const currentHelper = this.props.mediaEventHelper; + + try { + const decryptedUrl = await currentHelper.sourceUrl.value; + const decryptedBlob = await currentHelper.sourceBlob.value; + + if ( + this.isDisposed || + currentEvent !== this.props.mxEvent || + currentHelper !== this.props.mediaEventHelper + ) { + return; + } + + this.state = { + ...this.state, + decryptedUrl, + decryptedBlob, + fetchingData: false, + }; + this.updateSnapshotFromState(); + this.props.videoRef.current?.play(); + } catch (error) { + if ( + this.isDisposed || + currentEvent !== this.props.mxEvent || + currentHelper !== this.props.mediaEventHelper + ) { + return; + } + + logger.warn("Unable to decrypt attachment: ", error); + this.state = { + ...this.state, + error, + fetchingData: false, + }; + this.updateSnapshotFromState(); + } + }; +} diff --git a/apps/web/test/unit-tests/components/views/messages/MBodyFactory-test.tsx b/apps/web/test/unit-tests/components/views/messages/MBodyFactory-test.tsx index e20c8791e8..4a25bcb57f 100644 --- a/apps/web/test/unit-tests/components/views/messages/MBodyFactory-test.tsx +++ b/apps/web/test/unit-tests/components/views/messages/MBodyFactory-test.tsx @@ -19,7 +19,11 @@ import { } from "../../../../test-utils"; import { MediaEventHelper } from "../../../../../src/utils/MediaEventHelper"; import SettingsStore from "../../../../../src/settings/SettingsStore"; -import { FileBodyFactory, renderMBody } from "../../../../../src/components/views/messages/MBodyFactory"; +import { + FileBodyFactory, + VideoBodyFactory, + renderMBody, +} from "../../../../../src/components/views/messages/MBodyFactory"; import { TimelineRenderingType } from "../../../../../src/contexts/RoomContext.ts"; import { ScopedRoomContextProvider } from "../../../../../src/contexts/ScopedRoomContext.tsx"; @@ -90,10 +94,14 @@ describe("MBodyFactory", () => { expect(container).toMatchSnapshot(); }); - it.each(["m.audio", "m.video", "m.text"])("returns null for unsupported msgtype %s", (msgtype) => { + it.each(["m.audio", "m.text"])("returns null for unsupported msgtype %s", (msgtype) => { expect(renderMBody({ ...props, mxEvent: mkEvent(msgtype) })).toBeNull(); }); + it("returns the video body factory for m.video", () => { + expect(renderMBody({ ...props, mxEvent: mkEvent("m.video") })?.type).toBe(VideoBodyFactory); + }); + it("returns null when msgtype is missing", () => { expect(renderMBody({ ...props, mxEvent: mkEvent() })).toBeNull(); }); @@ -116,7 +124,7 @@ describe("MBodyFactory", () => { }); }); - it.each(["m.file", "m.audio", "m.video"])( + it.each(["m.file", "m.audio"])( "renderMBody fallback shows %s generic placeholder when showFileInfo is true", async (msgtype) => { const mediaEvent = new MatrixEvent({ diff --git a/apps/web/test/unit-tests/components/views/messages/MVideoBody-test.tsx b/apps/web/test/unit-tests/components/views/messages/MVideoBody-test.tsx index 51390f2136..9348a5b0b7 100644 --- a/apps/web/test/unit-tests/components/views/messages/MVideoBody-test.tsx +++ b/apps/web/test/unit-tests/components/views/messages/MVideoBody-test.tsx @@ -23,7 +23,7 @@ import { mockClientMethodsUser, withClientContextRenderOptions, } from "../../../../test-utils"; -import MVideoBody from "../../../../../src/components/views/messages/MVideoBody"; +import { VideoBodyFactory } from "../../../../../src/components/views/messages/MBodyFactory"; import type { IBodyProps } from "../../../../../src/components/views/messages/IBodyProps"; import SettingsStore from "../../../../../src/settings/SettingsStore"; import { MediaPreviewValue } from "../../../../../src/@types/media_preview"; @@ -33,7 +33,7 @@ jest.mock("matrix-encrypt-attachment", () => ({ decryptAttachment: jest.fn(), })); -describe("MVideoBody", () => { +describe("VideoBodyFactory", () => { const ourUserId = "@user:server"; const senderUserId = "@other_use:server"; const deviceId = "DEADB33F"; @@ -122,23 +122,25 @@ describe("MVideoBody", () => { mediaEventHelper: { media: { isEncrypted: false } } as MediaEventHelper, }; - const { asFragment } = render( + const { container } = render( - + , withClientContextRenderOptions(cli), ); - expect(asFragment()).toMatchSnapshot(); - // If we get here, we did not crash. + expect(container.querySelector("video")).not.toBeNull(); }); it("should show poster for encrypted media before downloading it", async () => { fetchMock.getOnce(thumbUrl, { status: 200 }); - const { asFragment } = render( - , + render( + , withClientContextRenderOptions(cli), ); - expect(asFragment()).toMatchSnapshot(); + expect(await screen.findByLabelText("alt for a test video")).toHaveAttribute("poster"); }); describe("with video previews/thumbnails disabled", () => { @@ -161,7 +163,7 @@ describe("MVideoBody", () => { fetchMock.getOnce(thumbUrl, { status: 200 }); render( - , @@ -177,7 +179,7 @@ describe("MVideoBody", () => { fetchMock.getOnce(thumbUrl, { status: 200 }); render( - , @@ -189,6 +191,7 @@ describe("MVideoBody", () => { expect(placeholderButton).toBeInTheDocument(); fireEvent.click(placeholderButton); + await screen.findByLabelText("alt for a test video"); expect(fetchMock).toHaveFetched(thumbUrl); }); @@ -214,16 +217,16 @@ describe("MVideoBody", () => { }, }, }); - const { asFragment } = render( - , withClientContextRenderOptions(cli), ); + expect(await screen.findByLabelText("alt for a test video")).toBeInTheDocument(); expect(fetchMock).toHaveFetched(thumbUrl); - expect(asFragment()).toMatchSnapshot(); }); }); }); diff --git a/apps/web/test/unit-tests/components/views/messages/MessageEvent-test.tsx b/apps/web/test/unit-tests/components/views/messages/MessageEvent-test.tsx index ce79f18efe..5559647bd3 100644 --- a/apps/web/test/unit-tests/components/views/messages/MessageEvent-test.tsx +++ b/apps/web/test/unit-tests/components/views/messages/MessageEvent-test.tsx @@ -29,16 +29,12 @@ jest.mock("../../../../../src/components/views/messages/MImageBody", () => ({ default: () =>
, })); -jest.mock("../../../../../src/components/views/messages/MVideoBody", () => ({ - __esModule: true, - default: () =>
, -})); - jest.mock("../../../../../src/components/views/messages/MBodyFactory", () => ({ __esModule: true, DecryptionFailureBodyFactory: () =>
, FileBodyFactory: () =>
, RedactedBodyFactory: () =>
Message deleted by Moderator
, + VideoBodyFactory: () =>