Gallery: Add video uploads to gallery (#4796) * Gallery: Add video uploads to gallery * Improve thumbnailing logic * Add error handling to thumnailing process * Improve formatting in utils.js

a8004a9a9e84e1ff78d96e0aaa56728d31f08654

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 269c694bc71395addcea84b9468153c64f194989
9 files changed, +181 -54Showing whitespace changes
package-lock.json+9 -5
@@ -73,7 +73,7 @@
7373 "is-docker": "^3.0.0",
7474 "localforage": "^1.10.0",
7575 "lodash": "^4.17.21",
7676 "mime-types": "^3.0.12",
7777 "moment": "^2.30.1",
7878 "morphdom": "^2.7.7",
7979 "multer": "^2.0.2",
@@ -6432,15 +6432,19 @@
64326432 }
64336433 },
64346434 "node_modules/mime-types": {
64356435 "version": "3.0.12",
64366436 "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.12.tgz",
64376437 "integrity": "sha512-xRc4oEhT6eaBpU1XF7AjpOFDLbgzdk0h4juoQ9fCKXW4by0UJqj+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWAnOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==",
64386438 "license": "MIT",
64396439 "dependencies": {
64406440 "mime-db": "^1.54.0"
64416441 },
64426442 "engines": {
64436443 "node": ">= 0.618"
6444+ },
6445+ "funding": {
6446+ "type": "opencollective",
6447+ "url": "https://opencollective.com/express"
64446448 }
64456449 },
64466450 "node_modules/mime-types/node_modules/mime-db": {
package.json+3 -3
@@ -63,7 +63,7 @@
6363 "is-docker": "^3.0.0",
6464 "localforage": "^1.10.0",
6565 "lodash": "^4.17.21",
6666 "mime-types": "^3.0.12",
6767 "moment": "^2.30.1",
6868 "morphdom": "^2.7.7",
6969 "multer": "^2.0.2",
@@ -164,7 +164,7 @@
164164 "@types/yargs": "^17.0.33",
165165 "@types/yauzl": "^2.10.3",
166166 "eslint": "^8.57.1",
167167 "eslint-plugin-jsdocjest": "^4827.109.0",
168168 "eslint-plugin-jestjsdoc": "^2748.910.0"
169169 }
170170}
public/scripts/constants.js+11 -0
@@ -118,6 +118,17 @@ export const MEDIA_TYPE = {
118118};
119119
120120/**
121+ * Bitwise flag-style media request types.
122+ * @enum {number}
123+ * @readonly
124+ */
125+export const MEDIA_REQUEST_TYPE = {
126+ IMAGE: 0b001,
127+ VIDEO: 0b010,
128+ AUDIO: 0b100,
129+};
130+
131+/**
121132 * Scroll behavior options when appending media to messages.
122133 * @enum {string}
123134 * @readonly
public/scripts/extensions/gallery/index.js+40 -12
@@ -8,7 +8,7 @@ import {
88 animation_easing,
99} from '../../../script.js';
1010import { groups, selected_group } from '../../group-chats.js';
1111import { loadFileToDocument, delay, getBase64Async, getSanitizedFilename, saveBase64AsFile, getFileExtension, getVideoThumbnail, clamp } from '../../utils.js';
1212import { loadMovingUIState } from '../../power-user.js';
1313import { dragElement } from '../../RossAscends-mods.js';
1414import { SlashCommandParser } from '../../slash-commands/SlashCommandParser.js';
@@ -19,17 +19,14 @@ import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnum
1919import { t, translate } from '../../i18n.js';
2020import { Popup } from '../../popup.js';
2121import { deleteMediaFromServer } from '../../chats.js';
22+import { MEDIA_REQUEST_TYPE, VIDEO_EXTENSIONS } from '../../constants.js';
2223
24+const isVideo = (/** @type {string} */ url) => VIDEO_EXTENSIONS.some(ext => new RegExp(`.${ext}$`, 'i').test(url));
2325const extensionName = 'gallery';
2426const extensionFolderPath = `scripts/extensions/${extensionName}/`;
2527let firstTime = true;
2628let deleteModeActive = false;
2729
28-// Exposed defaults for future tweaking
29-let thumbnailHeight = 150;
30-let paginationVisiblePages = 10;
31-let paginationMaxLinesPerPage = 2;
32-let galleryMaxRows = 3;
3330
3431// Remove all draggables associated with the gallery
3532$('#movingDivs').on('click', '.dragClose', function () {
@@ -122,17 +119,34 @@ async function getGalleryItems(url) {
122119 folder: url,
123120 sortField: sortObj.field,
124121 sortOrder: sortObj.order,
122+ type: MEDIA_REQUEST_TYPE.IMAGE | MEDIA_REQUEST_TYPE.VIDEO,
125123 }),
126124 });
127125
128126 url = await getSanitizedFilename(url);
129127
130128 const data = await response.json();
131- const items = data.map((file) => ({
129+ const items = [];
130+
131+ for (const file of data) {
132+ const item = {
132133 src: `user/images/${url}/${file}`,
133134 srct: `user/images/${url}/${file}`,
134135 title: '', // Optional title for each item
135136 }));
137+
138+ if (isVideo(file)) {
139+ try {
140+ // 150px of max height with some allowance for various aspect ratios
141+ const maxSide = Math.round(150 * 1.5);
142+ item.srct = await getVideoThumbnail(item.src, maxSide, maxSide);
143+ } catch (error) {
144+ console.error('Failed to generate video thumbnail for gallery:', error);
145+ }
146+ }
147+
148+ items.push(item);
149+ }
136150
137151 return items;
138152}
@@ -198,6 +212,12 @@ function getSortOrder() {
198212 * @returns {Promise<void>} - Promise representing the completion of the gallery initialization.
199213 */
200214async function initGallery(items, url) {
215+ // Exposed defaults for future tweaking
216+ const thumbnailHeight = 150;
217+ const paginationVisiblePages = 5;
218+ const paginationMaxLinesPerPage = 2;
219+ const galleryMaxRows = clamp(Math.floor((window.innerHeight * 0.9 - 75) / thumbnailHeight), 1, 10);
220+
201221 const nonce = `nonce-${Math.random().toString(36).substring(2, 15)}`;
202222 const gallery = $('#dragGallery');
203223 gallery.addClass(nonce);
@@ -210,6 +230,7 @@ async function initGallery(items, url) {
210230 galleryMaxRows: galleryMaxRows,
211231 galleryPaginationTopButtons: false,
212232 galleryNavigationOverlayButtons: true,
233+ galleryPaginationMode: 'rectangles',
213234 galleryTheme: {
214235 navigationBar: { background: 'none', borderTop: '', borderBottom: '', borderRight: '', borderLeft: '' },
215236 navigationBreadcrumb: { background: '#111', color: '#fff', colorHover: '#ccc', borderRadius: '4px' },
@@ -399,7 +420,7 @@ async function makeMovable(url) {
399420 // Create a hidden file input
400421 const fileInput = document.createElement('input');
401422 fileInput.type = 'file';
402423 fileInput.accept = 'image/*,video/*';
403424 fileInput.multiple = true;
404425 fileInput.style.display = 'none';
405426
@@ -617,12 +638,19 @@ function makeDragImg(id, url) {
617638 const newElement = document.importNode(template.content, true);
618639
619640 // Step 2: Append the given image
620641 const imgElemmediaElement = document.createElementisVideo('img'url);
621- imgElem.src = url;
642+ ? document.createElement('video')
643+ : document.createElement('img');
644+ mediaElement.src = url;
645+ if (mediaElement instanceof HTMLVideoElement) {
646+ mediaElement.controls = true;
647+ mediaElement.autoplay = true;
648+ }
649+
622650 let uniqueId = `draggable_${id}`;
623651 const draggableElem = /** @type {HTMLElement} */ (newElement.querySelector('.draggable'));
624652 if (draggableElem) {
625653 draggableElem.appendChild(imgElemmediaElement);
626654
627655 // Find a unique id for the draggable element
628656
public/scripts/extensions/gallery/style.css+4 -0
@@ -49,3 +49,7 @@
4949 opacity: 1;
5050 filter: unset;
5151}
52+
53+.galleryImageDraggable video {
54+ width: 100%;
55+}
public/scripts/utils.js+81 -27
@@ -1207,6 +1207,84 @@ export function getVideoDurationFromDataURL(dataUrl) {
12071207}
12081208
12091209/**
1210+ * Gets a thumbnail image from a video URL.
1211+ * @param {string} videoUrl URL of the video
1212+ * @param {number|null} [maxWidth=null] Maximum width of the thumbnail
1213+ * @param {number|null} [maxHeight=null] Maximum height of the thumbnail
1214+ * @param {string} [type='image/jpeg'] MIME type of the thumbnail
1215+ * @returns {Promise<string>} Promise that resolves to a data URL of the video thumbnail
1216+ */
1217+export function getVideoThumbnail(videoUrl, maxWidth = null, maxHeight = null, type = 'image/jpeg') {
1218+ const video = document.createElement('video');
1219+ video.src = videoUrl;
1220+ return new Promise((resolve, reject) => {
1221+ video.onloadeddata = function () {
1222+ // Set the time to capture the thumbnail at the middle of the video
1223+ video.currentTime = video.duration / 2;
1224+ };
1225+ video.onseeked = function () {
1226+ // Create a canvas to draw the thumbnail
1227+ const canvas = document.createElement('canvas');
1228+ const ctx = canvas.getContext('2d');
1229+ const { thumbnailWidth, thumbnailHeight } = calculateThumbnailSize(video.videoWidth, video.videoHeight, maxWidth, maxHeight);
1230+
1231+ canvas.width = thumbnailWidth;
1232+ canvas.height = thumbnailHeight;
1233+ ctx.imageSmoothingEnabled = true;
1234+ ctx.imageSmoothingQuality = 'high';
1235+ ctx.fillStyle = 'black';
1236+ ctx.fillRect(0, 0, thumbnailWidth, thumbnailHeight);
1237+ ctx.drawImage(video, 0, 0, thumbnailWidth, thumbnailHeight);
1238+ // Get the data URL of the thumbnail
1239+ const dataUrl = canvas.toDataURL(type);
1240+ resolve(dataUrl);
1241+ };
1242+ video.onerror = function () {
1243+ reject(new Error('Failed to load video'));
1244+ };
1245+ });
1246+}
1247+
1248+/**
1249+ * Calculates the thumbnail size for a media element while maintaining aspect ratio.
1250+ * @param {number} width Media width
1251+ * @param {number} height Media height
1252+ * @param {number?} maxWidth Max width (null = no limit)
1253+ * @param {number?} maxHeight Max height (null = no limit)
1254+ * @returns {{ thumbnailWidth: number, thumbnailHeight: number }} Thumbnail size
1255+ */
1256+export function calculateThumbnailSize(width, height, maxWidth, maxHeight) {
1257+ // Calculate the thumbnail dimensions while maintaining the aspect ratio
1258+ const aspectRatio = width / height;
1259+ let thumbnailWidth = maxWidth;
1260+ let thumbnailHeight = maxHeight;
1261+
1262+ if (maxWidth === null) {
1263+ thumbnailWidth = width;
1264+ maxWidth = width;
1265+ }
1266+
1267+ if (maxHeight === null) {
1268+ thumbnailHeight = height;
1269+ maxHeight = height;
1270+ }
1271+
1272+ // Do not upscale if image is already smaller than max dimensions
1273+ if (width <= maxWidth && height <= maxHeight) {
1274+ thumbnailWidth = width;
1275+ thumbnailHeight = height;
1276+ } else {
1277+ if (width > height) {
1278+ thumbnailHeight = maxWidth / aspectRatio;
1279+ } else {
1280+ thumbnailWidth = maxHeight * aspectRatio;
1281+ }
1282+ }
1283+
1284+ return { thumbnailWidth: Math.round(thumbnailWidth), thumbnailHeight: Math.round(thumbnailHeight) };
1285+}
1286+
1287+/**
12101288 * Gets the duration of an audio from a data URL.
12111289 * @param {string} dataUrl Audio data URL
12121290 * @returns {Promise<number>} Duration in seconds
@@ -1690,33 +1768,7 @@ export function createThumbnail(dataUrl, maxWidth = null, maxHeight = null, type
16901768 img.onload = () => {
16911769 const canvas = document.createElement('canvas');
16921770 const ctx = canvas.getContext('2d');
1693-
1771+ const { thumbnailWidth, thumbnailHeight } = calculateThumbnailSize(img.width, img.height, maxWidth, maxHeight);
1694- // Calculate the thumbnail dimensions while maintaining the aspect ratio
1695- const aspectRatio = img.width / img.height;
1696- let thumbnailWidth = maxWidth;
1697- let thumbnailHeight = maxHeight;
1698-
1699- if (maxWidth === null) {
1700- thumbnailWidth = img.width;
1701- maxWidth = img.width;
1702- }
1703-
1704- if (maxHeight === null) {
1705- thumbnailHeight = img.height;
1706- maxHeight = img.height;
1707- }
1708-
1709- // Do not upscale if image is already smaller than max dimensions
1710- if (img.width <= maxWidth && img.height <= maxHeight) {
1711- thumbnailWidth = img.width;
1712- thumbnailHeight = img.height;
1713- } else {
1714- if (img.width > img.height) {
1715- thumbnailHeight = maxWidth / aspectRatio;
1716- } else {
1717- thumbnailWidth = maxHeight * aspectRatio;
1718- }
1719- }
17201772
17211773 // Set the canvas dimensions and draw the resized image
17221774 canvas.width = thumbnailWidth;
@@ -2386,6 +2438,7 @@ export async function fetchFaFile(name) {
23862438 .map(rule => rule.selectorText.split(/,\s*/).map(selector => selector.split('::').shift().slice(1)))
23872439 ;
23882440}
2441+
23892442export async function fetchFa() {
23902443 return [...new Set((await Promise.all([
23912444 fetchFaFile('fontawesome.min.css'),
@@ -2782,4 +2835,5 @@ export async function importFromExternalUrl(url, { preserveFileName = null } = {
27822835 break;
27832836 }
27842837}
2838+
27852839export const clamp = (value, min, max) => Math.min(Math.max(value, min), max);
src/constants.js+10 -0
@@ -513,6 +513,16 @@ export const MEDIA_EXTENSIONS = [
513513 'aiff',
514514];
515515
516+/**
517+ * Bitwise flag-style media request types.
518+ */
519+export const MEDIA_REQUEST_TYPE = {
520+ IMAGE: 0b001,
521+ VIDEO: 0b010,
522+ AUDIO: 0b100,
523+};
524+
525+
516526export const ZAI_ENDPOINT = {
517527 COMMON: 'common',
518528 CODING: 'coding',
src/endpoints/images.js+3 -2
@@ -6,7 +6,7 @@ import express from 'express';
66import sanitize from 'sanitize-filename';
77
88import { clientRelativePath, removeFileExtension, getImages, isPathUnderParent } from '../util.js';
99import { MEDIA_EXTENSIONS, MEDIA_REQUEST_TYPE } from '../constants.js';
1010
1111/**
1212 * Ensure the directory for the provided file path exists.
@@ -93,6 +93,7 @@ router.post('/list/:folder?', (request, response) => {
9393 }
9494
9595 const directoryPath = path.join(request.user.directories.userImages, sanitize(request.body.folder));
96+ const type = Number(request.body.type ?? MEDIA_REQUEST_TYPE.IMAGE);
9697 const sort = request.body.sortField || 'date';
9798 const order = request.body.sortOrder || 'asc';
9899
@@ -100,7 +101,7 @@ router.post('/list/:folder?', (request, response) => {
100101 fs.mkdirSync(directoryPath, { recursive: true });
101102 }
102103
103104 const images = getImages(directoryPath, sort, type);
104105 if (order === 'desc') {
105106 images.reverse();
106107 }
src/util.js+20 -5
@@ -17,7 +17,7 @@ import mime from 'mime-types';
1717import { default as simpleGit } from 'simple-git';
1818import chalk from 'chalk';
1919import bytes from 'bytes';
2020import { LOG_LEVELS, CHAT_COMPLETION_SOURCES, MEDIA_REQUEST_TYPE } from './constants.js';
2121import { serverDirectory } from './server-directory.js';
2222import { isFirefox } from './express-common.js';
2323
@@ -500,9 +500,10 @@ export function removeOldBackups(directory, prefix, limit = null) {
500500 * Get a list of images in a directory.
501501 * @param {string} directoryPath Path to the directory containing the images
502502 * @param {'name' | 'date'} sortBy Sort images by name or date
503+ * @param {number} type Bitwise flag representing media types to include
503504 * @returns {string[]} List of image file names
504505 */
505506export function getImages(directoryPath, sortBy = 'name', type = MEDIA_REQUEST_TYPE.IMAGE) {
506507 function getSortFunction() {
507508 switch (sortBy) {
508509 case 'name':
@@ -515,10 +516,24 @@ export function getImages(directoryPath, sortBy = 'name') {
515516 }
516517
517518 return fs
518519 .readdirSync(directoryPath, { withFileTypes: true })
520+ .filter(dirent => dirent.isFile())
521+ .map(dirent => dirent.name)
519522 .filter(file => {
520523 const typefileType = mime.lookup(file);
521- return type && type.startsWith('image/');
524+ if (!fileType) {
525+ return false;
526+ }
527+ if ((type & MEDIA_REQUEST_TYPE.IMAGE) && fileType.startsWith('image/')) {
528+ return true;
529+ }
530+ if ((type & MEDIA_REQUEST_TYPE.VIDEO) && fileType.startsWith('video/')) {
531+ return true;
532+ }
533+ if ((type & MEDIA_REQUEST_TYPE.AUDIO) && fileType.startsWith('audio/')) {
534+ return true;
535+ }
536+ return false;
522537 })
523538 .sort(getSortFunction());
524539}