Audio media attachments (#4741) * Add audio-player component * Enhance audio player functionality and styles - Adjust audio player layout and styles in CSS for better spacing and alignment. - Add audio element to message template in HTML. - Implement audio attachment handling in JavaScript, including a new AudioPlayer class. - Update media attachment structure to include titles for audio files. - Extend supported media file types in constants. * Add audio inlining control * Fix file formats list * Simplify hints * Add to prompt * Add progress title hint * Add to scrollOnMediaLoad * Add audio size hint * Add gallery controls * Delete removed media attachments from server * Adjust min width * No-op when swiping a singular media * Fix clean-up * Fix silent deletion logic * More accurate media prompt estimations * Round durations with ceiling function * Adjust rounding logic * you don't have to say it twice * Export deleteFileFromServer * Don't reference module from HTML * Clean-up styles * Move formatTime to utils * Add fa-fw to delete

2577e35c0bc162774745970fdd0057e055d4399f

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed
Parents: 447c271b36be10a4f117331ec35354bcdc2a0194
27 files changed, +1058 -110Showing whitespace changes
public/css/file-form.css+93 -0
@@ -65,3 +65,96 @@
6565 display: flex;
6666 text-align: left;
6767}
68+
69+/* Audio Player Styles */
70+.mes_audio_container {
71+ cursor: default;
72+ display: flex;
73+ width: fit-content;
74+ min-width: min(350px, 100%);
75+ max-width: 100%;
76+ background-color: var(--white20a);
77+ border: 2px solid var(--SmartThemeBorderColor);
78+ padding: 0.5em 1em;
79+ border-radius: 15px;
80+}
81+
82+.mes_audio_container .mes_img_swipes {
83+ position: unset;
84+ opacity: unset;
85+ background: none;
86+ padding: 0;
87+}
88+
89+.mes_audio_container .mes_img_swipes .right_menu_button {
90+ filter: brightness(75%);
91+ text-shadow: none;
92+}
93+
94+.mes_audio_container .mes_img_swipes .mes_img_swipe_counter {
95+ filter: none;
96+ text-shadow: none;
97+}
98+
99+.audio-player {
100+ display: flex;
101+ flex-direction: column;
102+ gap: 5px;
103+}
104+
105+.audio-player-header {
106+ display: flex;
107+ flex-direction: row;
108+ gap: 15px;
109+ align-items: center;
110+ justify-content: space-between;
111+}
112+
113+.audio-player-title {
114+ overflow: hidden;
115+ text-overflow: ellipsis;
116+ white-space: nowrap;
117+}
118+
119+.audio-player-controls {
120+ display: flex;
121+ align-items: center;
122+ gap: 10px;
123+}
124+
125+.audio-player-play-pause,
126+.audio-player-volume {
127+ background: none;
128+ border: none;
129+}
130+
131+.audio-player-time-separator {
132+ font-size: 0.9em;
133+ text-align: center;
134+}
135+
136+.audio-player-current-time,
137+.audio-player-total-time {
138+ font-size: 0.9em;
139+ color: var(--SmartThemeQuoteColor);
140+ min-width: 40px;
141+ text-align: center;
142+}
143+
144+.audio-player-progress {
145+ flex: 1;
146+ height: 6px;
147+ background-color: var(--black30a);
148+ border-radius: 3px;
149+ cursor: pointer;
150+ position: relative;
151+ overflow: hidden;
152+}
153+
154+.audio-player-progress-bar {
155+ height: 100%;
156+ background-color: var(--SmartThemeEmColor);
157+ border-radius: 3px;
158+ transition: width 0.1s linear;
159+ width: 0%;
160+}
public/index.html+40 -10
@@ -1990,11 +1990,7 @@
19901990 </div>
19911991 </label>
19921992 <div id="image_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
19931993 <span data-i18n="image_inlining_hint_1">Sends images in prompts if the model supports it. Use the</span>
1994- <code><i class="fa-solid fa-paperclip"></i></code>
1995- <span data-i18n="image_inlining_hint_2">action on any message or the</span>
1996- <code><i class="fa-solid fa-wand-magic-sparkles"></i></code>
1997- <span data-i18n="image_inlining_hint_3">menu to attach an image file to the chat.</span>
19981994 </div>
19991995 <div class="flex-container flexFlowColumn wide100p textAlignCenter marginTop10" data-source="openai,custom,xai,pollinations,cohere,cometapi,nanogpt,moonshot,aimlapi,openrouter,mistralai,electronhub,azure_openai,zai">
20001996 <div class="flex-container oneline-dropdown">
@@ -2019,15 +2015,25 @@
20192015 </div>
20202016 </label>
20212017 <div id="video_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
20222018 <span data-i18n="video_inlining_hint_1">Sends videos in prompts if the model supports it. Use the</span>
2023- <code><i class="fa-solid fa-paperclip"></i></code>
2024- <span data-i18n="video_inlining_hint_2">action on any message or the</span>
2025- <code><i class="fa-solid fa-wand-magic-sparkles"></i></code>
2026- <span data-i18n="video_inlining_hint_3">menu to attach a video file to the chat.</span>
20272019 <strong data-i18n="video_inlining_hint_4">Videos must be less than 20 MB and under 1 minute long</strong>
20282020 </div>
20292021 </div>
20302022 <div class="range-block" data-source="makersuite,vertexai">
2023+ <label for="openai_audio_inlining" class="checkbox_label flexWrap widthFreeExpand">
2024+ <input id="openai_audio_inlining" type="checkbox" />
2025+ <span data-i18n="Send inline audio">Send inline audio</span>
2026+ <div id="openai_audio_inlining_supported" data-cc-toggle="false">
2027+ <i class="icon-supported fa-solid fa-circle-check" title="Supported by the current model" data-i18n="[title]Supported by the current model"></i>
2028+ <i class="icon-unsupported fa-solid fa-circle-exclamation" title="Unsupported by the current model" data-i18n="[title]Unsupported by the current model"></i>
2029+ </div>
2030+ </label>
2031+ <div id="audio_inlining_hint" class="flexBasis100p toggle-description justifyLeft">
2032+ <span data-i18n="audio_inlining_hint_1">Sends audio in prompts if the model supports it.</span>
2033+ <strong data-i18n="audio_inlining_hint_2">Audio must be less than 20 MB</strong>
2034+ </div>
2035+ </div>
2036+ <div class="range-block" data-source="makersuite,vertexai">
20312037 <label for="openai_request_images" class="checkbox_label widthFreeExpand">
20322038 <input id="openai_request_images" type="checkbox" />
20332039 <span>
@@ -7302,6 +7308,30 @@
73027308 <div title="Swipe right" class="right_menu_button fa-lg fa-solid fa-chevron-right mes_img_swipe_right" data-i18n="[title]Swipe right"></div>
73037309 </div>
73047310 </div>
7311+
7312+ <div id="message_audio_template" class="template_element">
7313+ <div class="mes_media_container mes_audio_container audio-player">
7314+ <audio class="mes_audio" preload="auto" hidden></audio>
7315+ <div class="audio-player-header">
7316+ <div class="audio-player-title">Audio</div>
7317+ <div class="right_menu_button mes_media_delete fa-fw fa-solid fa-trash-can" title="Delete" data-i18n="[title]Delete"></div>
7318+ </div>
7319+ <div class="audio-player-controls">
7320+ <button class="audio-player-play-pause right_menu_button fa-fw fa-solid fa-play" title="Play" data-i18n="[title]Play"></button>
7321+ <div class="audio-player-time">
7322+ <span class="audio-player-current-time">0:00</span>
7323+ <span class="audio-player-time-separator">/</span>
7324+ <span class="audio-player-total-time">0:00</span>
7325+ </div>
7326+ <div class="audio-player-progress">
7327+ <div class="audio-player-progress-bar"></div>
7328+ </div>
7329+ <div class="audio-player-volume-control">
7330+ <button class="audio-player-volume right_menu_button fa-fw fa-solid fa-volume-high" title="Mute" data-i18n="[title]Mute"></button>
7331+ </div>
7332+ </div>
7333+ </div>
7334+ </div>
73057335 </div>
73067336 <div id="movingDivs">
73077337 <div id="floatingPrompt" class="drawer-content flexGap5">
public/locales/ar-sa.json+1 -1
@@ -235,7 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "يجمع الرسائل المتتالية للنظام في رسالة واحدة (باستثناء الحوارات المثالية). قد يحسن التتابع لبعض النماذج.",
236236 "Enable function calling": "تمكين استدعاء الوظيفة",
237237 "Send inline images": "إرسال الصور المضمنة",
238238 "image_inlining_hint_1": "يرسل الصور في المطالبات إذا كان النموذج يدعمها .\n استخدم ال",
239239 "image_inlining_hint_2": "الإجراء على أي رسالة أو",
240240 "image_inlining_hint_3": "القائمة لإرفاق ملف صورة للدردشة.",
241241 "Inline Image Quality": "جودة الصورة المضمنة",
public/locales/de-de.json+1 -1
@@ -235,7 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Kombiniert aufeinanderfolgende Systemnachrichten zu einer (ausschließlich Beispiel-Dialoge ausgeschlossen). Kann die Kohärenz für einige Modelle verbessern.",
236236 "Enable function calling": "Funktionsaufruf aktivieren",
237237 "Send inline images": "Inline-Bilder senden",
238238 "image_inlining_hint_1": "Sendet Bilder in Eingabeaufforderungen, wenn das Modell dies unterstützt.\nVerwenden Sie die",
239239 "image_inlining_hint_2": "Aktion auf eine Nachricht oder die",
240240 "image_inlining_hint_3": "Menü, um eine Bilddatei an den Chat anzuhängen.",
241241 "Inline Image Quality": "Inline-Bildqualität",
public/locales/es-es.json+1 -1
@@ -235,7 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Combina mensajes del sistema consecutivos en uno solo (excluyendo diálogos de ejemplo). Puede mejorar la coherencia para algunos modelos.",
236236 "Enable function calling": "Habilitar llamada a función",
237237 "Send inline images": "Enviar imágenes en línea",
238238 "image_inlining_hint_1": "Envía imágenes en mensajes si el modelo lo admite.\n Utilizar el",
239239 "image_inlining_hint_2": "acción sobre cualquier mensaje o el",
240240 "image_inlining_hint_3": "menú para adjuntar un archivo de imagen al chat.",
241241 "Inline Image Quality": "Calidad de imagen en línea",
public/locales/fr-fr.json+1 -1
@@ -227,7 +227,7 @@
227227 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Combine les messages système consécutifs en un seul (à l'exclusion des dialogues d'exemple). Peut améliorer la cohérence pour certains modèles.",
228228 "Enable function calling": "Activer l'appel de fonction",
229229 "Send inline images": "Envoyer des images en ligne",
230230 "image_inlining_hint_1": "Envoie des images dans les prompts si le modèle le prend en charge.\nUtilisez le",
231231 "image_inlining_hint_2": "action sur n'importe quel message ou le",
232232 "image_inlining_hint_3": "menu pour joindre un fichier image au chat.",
233233 "Inline Image Quality": "Qualité d'image en ligne",
public/locales/is-is.json+1 -1
@@ -235,7 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Sameinar samhliða kerfisskilaboð í eitt (sem er utan umsagna dæmum). Getur bætt samfelldni fyrir sumar módel.",
236236 "Enable function calling": "Virkja aðgerðarkall",
237237 "Send inline images": "Senda myndir í línu",
238238 "image_inlining_hint_1": "Sendir myndir í skilaboðum ef líkanið styður það.\n Nota",
239239 "image_inlining_hint_2": "aðgerð á hvaða skilaboðum sem er eða",
240240 "image_inlining_hint_3": "valmynd til að hengja myndskrá við spjallið.",
241241 "Inline Image Quality": "Innbyggð myndgæði",
public/locales/it-it.json+1 -1
@@ -235,7 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Combina i messaggi di sistema consecutivi in uno solo (escludendo i dialoghi di esempio). Potrebbe migliorare la coerenza per alcuni modelli.",
236236 "Enable function calling": "Abilita la chiamata alla funzione",
237237 "Send inline images": "Invia immagini inline",
238238 "image_inlining_hint_1": "Invia immagini nei prompt se il modello lo supporta.\n Usa il",
239239 "image_inlining_hint_2": "azione su qualsiasi messaggio o il",
240240 "image_inlining_hint_3": "menu per allegare un file immagine alla chat.",
241241 "Inline Image Quality": "Qualità dell'immagine in linea",
public/locales/ko-kr.json+1 -3
@@ -237,9 +237,7 @@
237237 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "연속된 시스템 메시지를 하나로 결합합니다(예제 대화 제외). 일부 모델의 일관성을 향상시킬 수 있습니다.",
238238 "Enable function calling": "함수 호출 활성화",
239239 "Send inline images": "인라인 이미지 전송",
240240 "image_inlining_hint_1": "모델이 지원하는 경우 메시지로 이미지를 보냅니다.\n 사용",
241- "image_inlining_hint_2": "메시지에 대한 조치 또는",
242- "image_inlining_hint_3": "채팅에 이미지 파일을 첨부하는 메뉴입니다.",
243241 "Inline Image Quality": "인라인 이미지 품질",
244242 "openai_inline_image_quality_auto": "자동",
245243 "openai_inline_image_quality_low": "낮은",
public/locales/nl-nl.json+1 -3
@@ -235,9 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Combineert opeenvolgende systeemberichten tot één (exclusief voorbeeld dialogen). Kan de coherentie verbeteren voor sommige modellen.",
236236 "Enable function calling": "Schakel functieaanroepen in",
237237 "Send inline images": "Inline afbeeldingen verzenden",
238238 "image_inlining_hint_1": "Verzendt afbeeldingen in prompts als het model dit ondersteunt.\n Gebruik de",
239- "image_inlining_hint_2": "actie op elk bericht of de",
240- "image_inlining_hint_3": "menu om een ​​afbeeldingsbestand aan de chat toe te voegen.",
241239 "Inline Image Quality": "Inline-beeldkwaliteit",
242240 "openai_inline_image_quality_auto": "Auto",
243241 "openai_inline_image_quality_low": "Laag",
public/locales/pt-pt.json+1 -3
@@ -235,9 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Combina mensagens do sistema consecutivas em uma (excluindo diálogos de exemplo). Pode melhorar a coerência para alguns modelos.",
236236 "Enable function calling": "Habilitar chamada de função",
237237 "Send inline images": "Enviar imagens inline",
238238 "image_inlining_hint_1": "Envia imagens em prompts se o modelo suportar.\n Use o",
239- "image_inlining_hint_2": "ação em qualquer mensagem ou",
240- "image_inlining_hint_3": "menu para anexar um arquivo de imagem ao chat.",
241239 "Inline Image Quality": "Qualidade de imagem embutida",
242240 "openai_inline_image_quality_auto": "Auto",
243241 "openai_inline_image_quality_low": "Baixo",
public/locales/ru-ru.json+2 -6
@@ -1020,9 +1020,7 @@
10201020 "The prompt to be sent.": "Текст промпта.",
10211021 "prompt_manager_forbid_overrides": "Запретить перезапись",
10221022 "This prompt cannot be overridden by character cards, even if overrides are preferred.": "Карточка персонажа не сможет перезаписать этот промпт, даже если настройки отдают приоритет именно ей.",
10231023 "image_inlining_hint_1": "Отправлять картинки как часть промпта, если позволяет модель. Чтобы добавить в чат изображение, используйте на нужном сообщении действие",
1024- "image_inlining_hint_2": ". Также это можно сделать через меню",
1025- "image_inlining_hint_3": ".",
10261024 "Contest Winners": "Победители конкурса",
10271025 "Rename Background": "Переименовать фон",
10281026 "Lock": "Закрепить",
@@ -2445,9 +2443,7 @@
24452443 "Allocates a portion of the response length for thinking (min: 1024 tokens, low: 10%, medium: 25%, high: 50%, max: 95%), but minimum 1024 tokens. Auto does not request thinking.": "Отводит часть ответа под рассуждения (Минимальные: 10%, Обычные: 25%, Подробные: 50%, Максимальные: 95%), но минимум 1024 токена. При выборе значение Авто рассуждения не запрашиваются.",
24462444 "Use system prompt": "Включить системный промпт",
24472445 "Send inline videos": "Отправлять inline-видео",
24482446 "video_inlining_hint_1": "Отправляет модели видео, если она поддерживает такую возможность. Чтобы прикрепить видео к чату, используйте на любом сообщении кнопку",
2449- "video_inlining_hint_2": "либо меню",
2450- "video_inlining_hint_3": ".",
24512447 "video_inlining_hint_4": "Видео должно весить не более 20 Мб и длиться не дольше 1 минуты.",
24522448 "Allocates a portion of the response length for thinking (Flash 2.5/Pro 2.5) (min: 0/128 tokens, low: 10%, medium: 25%, high: 50%, max: 24576/32768 tokens). Auto lets the model decide.": "Отводит часть ответа под рассуждения (Flash 2.5/Pro 2.5) (Минимальные: 0/128 токенов, Поверхностные: 10%, Обычные: 25%, Подробные: 50%, Максимальные: 24576/32768 токенов). При выборе значения Авто модель определяет объём самостоятельно.",
24532449 "Google Vertex AI Configuration": "Настройки Google Vertex AI",
public/locales/th-th.json+1 -3
@@ -241,9 +241,7 @@
241241 "Allows the model to return its thinking process.": "อนุญาตให้โมเดลส่งคืนกระบวนการคิดของตัวเอง",
242242 "This setting affects visibility only.": "การตั้งค่านี้มีผลต่อการมองเห็นเท่านั้น",
243243 "Send inline images": "ส่งรูปภาพแบบ inline",
244244 "image_inlining_hint_1": "ส่งรูปภาพในพรอมต์หากโมเดลรองรับ โดยใช้",
245- "image_inlining_hint_2": "เพื่อดำเนินการในข้อความนั้นๆ หรือใช้ตัวเลือก",
246- "image_inlining_hint_3": "เพื่อแนบไฟล์รูปภาพลงในแชท",
247245 "Inline Image Quality": "คุณภาพรูปภาพ Inline",
248246 "openai_inline_image_quality_auto": "อัตโนมัติ",
249247 "openai_inline_image_quality_low": "ต่ำ",
public/locales/uk-ua.json+1 -3
@@ -235,9 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Об'єднує послідовні системні повідомлення в одне (крім прикладів діалогів). Може покращити співпрацю для деяких моделей.",
236236 "Enable function calling": "Увімкнути виклик функцій",
237237 "Send inline images": "Надсилати вбудовані зображення",
238238 "image_inlining_hint_1": "Надсилає зображення у підказках, якщо модель це підтримує.\n Використовувати",
239- "image_inlining_hint_2": "дії з будь-яким повідомленням або",
240- "image_inlining_hint_3": "меню, щоб прикріпити файл зображення до чату.",
241239 "Inline Image Quality": "Якість вбудованого зображення",
242240 "openai_inline_image_quality_auto": "Авто",
243241 "openai_inline_image_quality_low": "Низький",
public/locales/vi-vn.json+1 -3
@@ -235,9 +235,7 @@
235235 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "Kết hợp các tin nhắn hệ thống liên tiếp thành một (loại bỏ các đoạn hội thoại mẫu). Có thể cải thiện tính nhất quán cho một số model.",
236236 "Enable function calling": "Sử dụng tính năng gọi hàm (function calling)",
237237 "Send inline images": "Gửi hình ảnh nội bộ",
238238 "image_inlining_hint_1": "Gửi hình ảnh theo Prompt nếu kiểu máy hỗ trợ.\n Sử dụng",
239- "image_inlining_hint_2": "hành động đối với bất kỳ tin nhắn nào hoặc",
240- "image_inlining_hint_3": "menu để đính kèm tệp hình ảnh vào cuộc trò chuyện.",
241239 "Inline Image Quality": "Chất lượng hình ảnh nội tuyến",
242240 "openai_inline_image_quality_auto": "Tự động",
243241 "openai_inline_image_quality_low": "Thấp",
public/locales/zh-cn.json+2 -6
@@ -269,17 +269,13 @@
269269 "enable_functions_desc_3": "可以被各种扩展利用来提供附加功能。",
270270 "enable_functions_desc_4": "当提示词后处理没有选择工具时不支持。",
271271 "Send inline images": "发送图片",
272272 "image_inlining_hint_1": "如果模型支持，就可以在提示词中发送图片。\n发送消息时，点击",
273- "image_inlining_hint_2": "在这里（",
274- "image_inlining_hint_3": "）将图片添加到消息中。",
275273 "Inline Image Quality": "图片画质",
276274 "openai_inline_image_quality_auto": "自动",
277275 "openai_inline_image_quality_low": "低",
278276 "openai_inline_image_quality_high": "高",
279277 "Send inline videos": "发送视频",
280278 "video_inlining_hint_1": "当模型支持时，将视频发送给模型。使用",
281- "video_inlining_hint_2": "在任意消息上添加视频，或",
282- "video_inlining_hint_3": "菜单来添加视频。",
283279 "video_inlining_hint_4": "视频必须在 20MB 以下且时长不超过1分钟。",
284280 "Request inline images": "请求图片返回",
285281 "Allows the model to return image attachments.": "允许模型返回图片附件。",
public/locales/zh-tw.json+2 -6
@@ -236,9 +236,7 @@
236236 "Combines consecutive system messages into one (excluding example dialogues). May improve coherence for some models.": "將連續的系統訊息合併為一個（不包括對話範例）。可能會提高某些模型的一致性。",
237237 "Enable function calling": "啟用函式呼叫",
238238 "Send inline images": "傳送內嵌圖片",
239239 "image_inlining_hint_1": "如果模型支援，則在提示詞中傳送圖片。\n使用任何訊息上的",
240- "image_inlining_hint_2": "動作或",
241- "image_inlining_hint_3": "選單來附加圖片文件到聊天中。",
242240 "Inline Image Quality": "內嵌圖片品質",
243241 "openai_inline_image_quality_auto": "自動",
244242 "openai_inline_image_quality_low": "低",
@@ -2646,9 +2644,7 @@
26462644 "Min Keep": "最小保留",
26472645 "enable_functions_desc_4": "當使用「無工具」提示後處理時不支援！",
26482646 "Send inline videos": "傳送內嵌影片",
26492647 "video_inlining_hint_1": "若模型支援，則在提示中傳送影片。使用",
2650- "video_inlining_hint_2": "操作於任何訊息或使用",
2651- "video_inlining_hint_3": "選單以上傳影片檔案到聊天中。",
26522648 "video_inlining_hint_4": "影片大小必須小於 20 MB 且長度低於 1 分鐘",
26532649 "This setting affects visibility only.": "此設定僅影響可見性。",
26542650 "Allocates a portion of the response length for thinking (min: 1024 tokens, low: 10%, medium: 25%, high: 50%, max: 95%), but minimum 1024 tokens. Auto does not request thinking.": "為推理部分分配回應長度（最大回應長度至少需 1024 符元，低：10%、中：25%、高：50%、最高：95%）。選擇「自動」將不請求推理。",
public/script.js+40 -2
@@ -277,6 +277,7 @@ import { initAccessibility } from './scripts/a11y.js';
277277import { applyStreamFadeIn } from './scripts/util/stream-fadein.js';
278278import { initDomHandlers } from './scripts/dom-handlers.js';
279279import { SimpleMutex } from './scripts/util/SimpleMutex.js';
280+import { AudioPlayer } from './scripts/audio-player.js';
280281
281282// API OBJECT FOR EXTERNAL WIRING
282283globalThis.SillyTavern = {
@@ -1421,7 +1422,7 @@ export async function printMessages() {
14211422
14221423function scrollOnMediaLoad() {
14231424 const started = Date.now();
14241425 const media = chatElement.find('.mes_block img, .mes_block video, .mes_block audio').toArray();
14251426 let mediaLoaded = 0;
14261427
14271428 for (const currentElement of media) {
@@ -1433,7 +1434,7 @@ function scrollOnMediaLoad() {
14331434 currentElement.addEventListener('error', incrementAndCheck);
14341435 }
14351436 }
14361437 if (currentElement instanceof HTMLVideoElementHTMLMediaElement) {
14371438 if (currentElement.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
14381439 incrementAndCheck();
14391440 } else {
@@ -2177,6 +2178,41 @@ export function appendMediaToMessage(mes, messageElement, scrollBehavior = SCROL
21772178 }
21782179
21792180 /**
2181+ * Appends a single audio attachment to the message element.
2182+ * @param {MediaAttachment} attachment Audio attachment object
2183+ * @param {number} index Index of the audio attachment
2184+ * @returns {JQuery<HTMLElement>} The appended audio container element
2185+ */
2186+ function appendAudioAttachment(attachment, index) {
2187+ const template = $('#message_audio_template .mes_audio_container').clone();
2188+ template.attr('data-index', index);
2189+ const audio = template.find('.mes_audio');
2190+ audio.attr('src', attachment.url);
2191+ audio.attr('title', attachment.title || mes.extra.title || '');
2192+
2193+ mediaPromises.push(new Promise((resolve) => {
2194+ function onLoad() {
2195+ resolve();
2196+ }
2197+ function onError() {
2198+ audio.addClass('error');
2199+ resolve();
2200+ }
2201+ if (audio.prop('readyState') >= HTMLMediaElement.HAVE_CURRENT_DATA) {
2202+ onLoad();
2203+ } else {
2204+ audio.off('loadeddata').on('loadeddata', onLoad);
2205+ audio.off('error').on('error', onError);
2206+ }
2207+ }));
2208+
2209+ new AudioPlayer(audio.get(0), template.get(0));
2210+
2211+ mediaBlocks.push(template);
2212+ return template;
2213+ }
2214+
2215+ /**
21802216 * Appends a media attachment to the message element.
21812217 * @param {MediaAttachment} attachment Media attachment object
21822218 * @param {number} index Index of the media attachment
@@ -2191,6 +2227,8 @@ export function appendMediaToMessage(mes, messageElement, scrollBehavior = SCROL
21912227 return appendImageAttachment(attachment, index);
21922228 case MEDIA_TYPE.VIDEO:
21932229 return appendVideoAttachment(attachment, index);
2230+ case MEDIA_TYPE.AUDIO:
2231+ return appendAudioAttachment(attachment, index);
21942232 }
21952233
21962234 console.warn(`Unknown media type: ${attachment.type}, defaulting to image.`, attachment);
public/scripts/audio-player.js+605 -0
@@ -0,0 +1,605 @@
1+import { formatTime } from './utils.js';
2+
3+export class AudioPlayer {
4+ /**
5+ * Creates an audio player instance
6+ * @param {HTMLElement} audioElement - The audio element to control
7+ * @param {HTMLElement} containerElement - The container element with player controls
8+ * @param {Object} options - Configuration options
9+ */
10+ constructor(audioElement, containerElement, options = {}) {
11+ if (!(audioElement instanceof HTMLAudioElement)) {
12+ throw new Error('First argument must be an HTMLAudioElement');
13+ }
14+ if (!(containerElement instanceof HTMLElement)) {
15+ throw new Error('Second argument must be an HTMLElement');
16+ }
17+
18+ this.audio = audioElement;
19+ this.container = containerElement;
20+ this.options = {
21+ title: '',
22+ autoplay: false,
23+ volume: 1.0,
24+ onPlay: null,
25+ onPause: null,
26+ onEnded: null,
27+ onTimeUpdate: null,
28+ onVolumeChange: null,
29+ ...options,
30+ };
31+
32+ this.isDragging = false;
33+ this.isDestroyed = false;
34+
35+ // Store bound event handlers for cleanup
36+ this.boundHandlers = {
37+ // Audio event handlers
38+ audioLoadedMetadata: this.onAudioLoadedMetadata.bind(this),
39+ audioTimeUpdate: this.onAudioTimeUpdate.bind(this),
40+ audioPlay: this.onAudioPlay.bind(this),
41+ audioPause: this.onAudioPause.bind(this),
42+ audioEnded: this.onAudioEnded.bind(this),
43+ audioVolumeChange: this.onAudioVolumeChange.bind(this),
44+ // Control event handlers
45+ playPauseClick: this.onPlayPauseClick.bind(this),
46+ volumeClick: this.onVolumeClick.bind(this),
47+ volumeInput: this.onVolumeInput.bind(this),
48+ progressMouseDown: this.onProgressMouseDown.bind(this),
49+ progressClick: this.onProgressClick.bind(this),
50+ progressMouseMove: this.onProgressMouseMove.bind(this),
51+ documentMouseMove: this.onDocumentMouseMove.bind(this),
52+ documentMouseUp: this.onDocumentMouseUp.bind(this),
53+ };
54+
55+ // MutationObserver for DOM cleanup detection
56+ this.observer = null;
57+
58+ this.init();
59+ }
60+
61+ /**
62+ * Initializes the audio player by setting up elements, events, and initial state
63+ * @returns {void}
64+ */
65+ init() {
66+ this.findElements();
67+ this.bindEvents();
68+ this.setupDOMObserver();
69+
70+ if (this.options.title) {
71+ this.setTitle(this.options.title);
72+ } else if (this.audio.title) {
73+ this.setTitle(this.audio.title);
74+ } else if (this.audio.src) {
75+ const srcParts = this.audio.src.split('/');
76+ this.setTitle(decodeURIComponent(srcParts[srcParts.length - 1]));
77+ }
78+
79+ if (this.options.autoplay) {
80+ this.play();
81+ }
82+
83+ this.setVolume(this.options.volume);
84+
85+ // Initialize time displays
86+ this.updateTimeDisplays();
87+ }
88+
89+ /**
90+ * Finds and caches all required DOM elements within the container
91+ * @returns {void}
92+ */
93+ findElements() {
94+ this.elements = {
95+ title: this.container.querySelector('.audio-player-title'),
96+ playPauseBtn: this.container.querySelector('.audio-player-play-pause'),
97+ currentTime: this.container.querySelector('.audio-player-current-time'),
98+ totalTime: this.container.querySelector('.audio-player-total-time'),
99+ progress: this.container.querySelector('.audio-player-progress'),
100+ progressBar: this.container.querySelector('.audio-player-progress-bar'),
101+ volumeBtn: this.container.querySelector('.audio-player-volume'),
102+ };
103+
104+ // Validate required elements
105+ const requiredElements = ['playPauseBtn', 'currentTime', 'totalTime', 'progress', 'progressBar', 'volumeBtn'];
106+ for (const key of requiredElements) {
107+ if (!this.elements[key]) {
108+ console.warn(`AudioPlayer: Required element .audio-player-${key.replace(/([A-Z])/g, '-$1').toLowerCase()} not found`);
109+ }
110+ }
111+ }
112+
113+ /**
114+ * Sets up a MutationObserver to detect when audio or container elements are removed from DOM
115+ * @returns {void}
116+ */
117+ setupDOMObserver() {
118+ // Watch for removal of audio or container from DOM
119+ this.observer = new MutationObserver((mutations) => {
120+ for (const mutation of mutations) {
121+ for (const node of mutation.removedNodes) {
122+ if (node === this.audio || node === this.container ||
123+ node.contains?.(this.audio) || node.contains?.(this.container)) {
124+ this.destroy();
125+ return;
126+ }
127+ }
128+ }
129+ });
130+
131+ // Observe the parent nodes
132+ const chatParent = this.audio.closest('#chat') ?? document.body;
133+
134+ if (chatParent) {
135+ this.observer.observe(chatParent, { childList: true, subtree: true });
136+ }
137+ }
138+
139+ /**
140+ * Binds all event listeners to audio and control elements
141+ * @returns {void}
142+ */
143+ bindEvents() {
144+ // Audio events
145+ this.audio.addEventListener('loadedmetadata', this.boundHandlers.audioLoadedMetadata);
146+ this.audio.addEventListener('timeupdate', this.boundHandlers.audioTimeUpdate);
147+ this.audio.addEventListener('play', this.boundHandlers.audioPlay);
148+ this.audio.addEventListener('pause', this.boundHandlers.audioPause);
149+ this.audio.addEventListener('ended', this.boundHandlers.audioEnded);
150+ this.audio.addEventListener('volumechange', this.boundHandlers.audioVolumeChange);
151+
152+ // Control events
153+ if (this.elements.playPauseBtn) {
154+ this.elements.playPauseBtn.addEventListener('click', this.boundHandlers.playPauseClick);
155+ }
156+ if (this.elements.volumeBtn) {
157+ this.elements.volumeBtn.addEventListener('click', this.boundHandlers.volumeClick);
158+ }
159+ if (this.elements.progress) {
160+ this.elements.progress.addEventListener('mousedown', this.boundHandlers.progressMouseDown);
161+ this.elements.progress.addEventListener('click', this.boundHandlers.progressClick);
162+ this.elements.progress.addEventListener('mousemove', this.boundHandlers.progressMouseMove);
163+ }
164+ }
165+
166+ /**
167+ * Removes all event listeners from audio and control elements
168+ * @returns {void}
169+ */
170+ unbindEvents() {
171+ // Audio events
172+ this.audio.removeEventListener('loadedmetadata', this.boundHandlers.audioLoadedMetadata);
173+ this.audio.removeEventListener('timeupdate', this.boundHandlers.audioTimeUpdate);
174+ this.audio.removeEventListener('play', this.boundHandlers.audioPlay);
175+ this.audio.removeEventListener('pause', this.boundHandlers.audioPause);
176+ this.audio.removeEventListener('ended', this.boundHandlers.audioEnded);
177+ this.audio.removeEventListener('volumechange', this.boundHandlers.audioVolumeChange);
178+
179+ // Control events
180+ if (this.elements.playPauseBtn) {
181+ this.elements.playPauseBtn.removeEventListener('click', this.boundHandlers.playPauseClick);
182+ }
183+ if (this.elements.volumeBtn) {
184+ this.elements.volumeBtn.removeEventListener('click', this.boundHandlers.volumeClick);
185+ }
186+ if (this.elements.progress) {
187+ this.elements.progress.removeEventListener('mousedown', this.boundHandlers.progressMouseDown);
188+ this.elements.progress.removeEventListener('click', this.boundHandlers.progressClick);
189+ this.elements.progress.removeEventListener('mousemove', this.boundHandlers.progressMouseMove);
190+ }
191+
192+ // Document events
193+ document.removeEventListener('mousemove', this.boundHandlers.documentMouseMove);
194+ document.removeEventListener('mouseup', this.boundHandlers.documentMouseUp);
195+ }
196+
197+ // Audio event handlers
198+ /**
199+ * Handles the audio element's loadedmetadata event
200+ * @returns {void}
201+ */
202+ onAudioLoadedMetadata() {
203+ if (this.isDestroyed) return;
204+ this.updateTimeDisplays();
205+ }
206+
207+ /**
208+ * Handles the audio element's timeupdate event
209+ * @returns {void}
210+ */
211+ onAudioTimeUpdate() {
212+ if (this.isDestroyed || this.isDragging) return;
213+
214+ const percent = (this.audio.currentTime / this.audio.duration) * 100 || 0;
215+ if (this.elements.progressBar) {
216+ /** @type {HTMLElement} */ (this.elements.progressBar).style.width = percent + '%';
217+ }
218+ if (this.elements.currentTime) {
219+ this.elements.currentTime.textContent = formatTime(this.audio.currentTime);
220+ }
221+
222+ if (typeof this.options.onTimeUpdate === 'function') {
223+ this.options.onTimeUpdate.call(this, this.audio.currentTime, this.audio.duration);
224+ }
225+ }
226+
227+ /**
228+ * Handles the audio element's play event
229+ * @returns {void}
230+ */
231+ onAudioPlay() {
232+ if (this.isDestroyed) return;
233+
234+ if (this.elements.playPauseBtn) {
235+ this.elements.playPauseBtn.classList.remove('fa-play');
236+ this.elements.playPauseBtn.classList.add('fa-pause');
237+ this.elements.playPauseBtn.setAttribute('title', 'Pause');
238+ }
239+
240+ if (typeof this.options.onPlay === 'function') {
241+ this.options.onPlay.call(this);
242+ }
243+ }
244+
245+ /**
246+ * Handles the audio element's pause event
247+ * @returns {void}
248+ */
249+ onAudioPause() {
250+ if (this.isDestroyed) return;
251+
252+ if (this.elements.playPauseBtn) {
253+ this.elements.playPauseBtn.classList.remove('fa-pause');
254+ this.elements.playPauseBtn.classList.add('fa-play');
255+ this.elements.playPauseBtn.setAttribute('title', 'Play');
256+ }
257+
258+ if (typeof this.options.onPause === 'function') {
259+ this.options.onPause.call(this);
260+ }
261+ }
262+
263+ /**
264+ * Handles the audio element's ended event
265+ * @returns {void}
266+ */
267+ onAudioEnded() {
268+ if (this.isDestroyed) return;
269+
270+ if (this.elements.playPauseBtn) {
271+ this.elements.playPauseBtn.classList.remove('fa-pause');
272+ this.elements.playPauseBtn.classList.add('fa-play');
273+ this.elements.playPauseBtn.setAttribute('title', 'Play');
274+ }
275+
276+ if (typeof this.options.onEnded === 'function') {
277+ this.options.onEnded.call(this);
278+ }
279+ }
280+
281+ /**
282+ * Handles the audio element's volumechange event
283+ * @returns {void}
284+ */
285+ onAudioVolumeChange() {
286+ if (this.isDestroyed) return;
287+
288+ this.updateVolumeIcon();
289+
290+ if (typeof this.options.onVolumeChange === 'function') {
291+ this.options.onVolumeChange.call(this, this.audio.volume, this.audio.muted);
292+ }
293+ }
294+
295+ // Control event handlers
296+ /**
297+ * Handles click events on the play/pause button
298+ * @param {MouseEvent} e - The click event
299+ * @returns {void}
300+ */
301+ onPlayPauseClick(e) {
302+ e.preventDefault();
303+ this.togglePlay();
304+ }
305+
306+ /**
307+ * Handles click events on the volume button
308+ * @param {MouseEvent} e - The click event
309+ * @returns {void}
310+ */
311+ onVolumeClick(e) {
312+ e.preventDefault();
313+ this.toggleMute();
314+ }
315+
316+ /**
317+ * Handles input events on the volume slider
318+ * @param {InputEvent} e - The input event
319+ * @returns {void}
320+ */
321+ onVolumeInput(e) {
322+ if (!(e.target instanceof HTMLInputElement)) return;
323+ const value = parseFloat(e.target.value);
324+ this.setVolume(value);
325+ }
326+
327+ /**
328+ * Handles mousedown events on the progress bar
329+ * @param {MouseEvent} e - The mousedown event
330+ * @returns {void}
331+ */
332+ onProgressMouseDown(e) {
333+ this.isDragging = true;
334+ this.updateProgress(e);
335+ document.addEventListener('mousemove', this.boundHandlers.documentMouseMove);
336+ document.addEventListener('mouseup', this.boundHandlers.documentMouseUp);
337+ }
338+
339+ /**
340+ * Handles click events on the progress bar
341+ * @param {MouseEvent} e - The click event
342+ * @returns {void}
343+ */
344+ onProgressClick(e) {
345+ if (!this.isDragging) {
346+ this.updateProgress(e);
347+ }
348+ }
349+
350+ /**
351+ * Handles mousemove on the progress bar (no-op if dragging)
352+ * @param {MouseEvent} e - The mousemove event
353+ * @returns {void}
354+ */
355+ onProgressMouseMove(e) {
356+ if (!this.isDragging) {
357+ this.updateProgressTitle(e);
358+ }
359+ }
360+
361+ /**
362+ * Handles document mousemove events during progress bar dragging
363+ * @param {MouseEvent} e - The mousemove event
364+ * @returns {void}
365+ */
366+ onDocumentMouseMove(e) {
367+ if (this.isDragging) {
368+ this.updateProgress(e);
369+ }
370+ }
371+
372+ /**
373+ * Handles document mouseup events to end progress bar dragging
374+ * @returns {void}
375+ */
376+ onDocumentMouseUp() {
377+ if (this.isDragging) {
378+ this.isDragging = false;
379+ document.removeEventListener('mousemove', this.boundHandlers.documentMouseMove);
380+ document.removeEventListener('mouseup', this.boundHandlers.documentMouseUp);
381+ }
382+ }
383+
384+ /**
385+ * Updates the progress bar position and seeks audio based on mouse position
386+ * @param {MouseEvent} e - The mouse event containing position information
387+ * @returns {void}
388+ */
389+ updateProgress(e) {
390+ if (!this.elements.progress) return;
391+
392+ const rect = this.elements.progress.getBoundingClientRect();
393+ const offsetX = e.clientX - rect.left;
394+ const width = rect.width;
395+ const percent = Math.max(0, Math.min(100, (offsetX / width) * 100));
396+
397+ if (this.elements.progressBar) {
398+ /** @type {HTMLElement} */ (this.elements.progressBar).style.width = percent + '%';
399+ }
400+
401+ const seekTime = (percent / 100) * this.audio.duration;
402+ if (isFinite(seekTime)) {
403+ this.audio.currentTime = seekTime;
404+ if (this.elements.currentTime) {
405+ this.elements.currentTime.textContent = formatTime(seekTime);
406+ }
407+ }
408+ }
409+
410+ /**
411+ * Updates the volume icon based on current volume and mute state
412+ * @returns {void}
413+ */
414+ updateVolumeIcon() {
415+ if (!this.elements.volumeBtn) return;
416+
417+ const volume = this.audio.volume;
418+ const isMuted = this.audio.muted;
419+
420+ this.elements.volumeBtn.classList.remove('fa-volume-high', 'fa-volume-low', 'fa-volume-off', 'fa-volume-xmark');
421+
422+ if (isMuted || volume === 0) {
423+ this.elements.volumeBtn.classList.add('fa-volume-xmark');
424+ } else if (volume < 0.5) {
425+ this.elements.volumeBtn.classList.add('fa-volume-low');
426+ } else {
427+ this.elements.volumeBtn.classList.add('fa-volume-high');
428+ }
429+ }
430+
431+ /**
432+ * Updates the current time and total time display elements
433+ * @returns {void}
434+ */
435+ updateTimeDisplays() {
436+ if (this.elements.currentTime) {
437+ this.elements.currentTime.textContent = formatTime(this.audio.currentTime || 0);
438+ }
439+ if (this.elements.totalTime) {
440+ this.elements.totalTime.textContent = formatTime(this.audio.duration || 0);
441+ }
442+ }
443+
444+ /**
445+ * Updates the mouseover title on the progress bar to show time at cursor position
446+ * @param {MouseEvent} e - The mouse event
447+ * @returns {void}
448+ */
449+ updateProgressTitle(e) {
450+ if (!this.elements.progress) return;
451+
452+ const rect = this.elements.progress.getBoundingClientRect();
453+ const offsetX = e.clientX - rect.left;
454+ const width = rect.width;
455+ const percent = Math.max(0, Math.min(100, (offsetX / width) * 100));
456+
457+ this.elements.progress.setAttribute('title', formatTime((percent / 100) * this.audio.duration));
458+ }
459+
460+ // Public methods
461+ /**
462+ * Starts audio playback
463+ * @returns {void}
464+ */
465+ play() {
466+ if (this.isDestroyed) return;
467+ if (this.audio.paused) {
468+ const playPromise = this.audio.play();
469+ if (playPromise !== undefined) {
470+ playPromise.catch(error => {
471+ console.error('Audio play failed:', error);
472+ });
473+ }
474+ }
475+ }
476+
477+ /**
478+ * Pauses audio playback
479+ * @returns {void}
480+ */
481+ pause() {
482+ if (this.isDestroyed) return;
483+ if (!this.audio.paused) {
484+ this.audio.pause();
485+ }
486+ }
487+
488+ /**
489+ * Toggles between play and pause states
490+ * @returns {void}
491+ */
492+ togglePlay() {
493+ if (this.audio.paused) {
494+ this.play();
495+ } else {
496+ this.pause();
497+ }
498+ }
499+
500+ /**
501+ * Seeks to a specific time in the audio
502+ * @param {number} time - The time in seconds to seek to
503+ * @returns {void}
504+ */
505+ seek(time) {
506+ if (this.isDestroyed) return;
507+ if (isFinite(time) && time >= 0 && time <= this.audio.duration) {
508+ this.audio.currentTime = time;
509+ }
510+ }
511+
512+ /**
513+ * Sets the volume level
514+ * @param {number} volume - Volume level between 0.0 and 1.0
515+ * @returns {void}
516+ */
517+ setVolume(volume) {
518+ if (this.isDestroyed) return;
519+ volume = Math.max(0, Math.min(1, volume));
520+ this.audio.volume = volume;
521+
522+ if (volume > 0 && this.audio.muted) {
523+ this.audio.muted = false;
524+ }
525+ }
526+
527+ /**
528+ * Mutes the audio
529+ * @returns {void}
530+ */
531+ mute() {
532+ if (this.isDestroyed) return;
533+ this.audio.muted = true;
534+ }
535+
536+ /**
537+ * Unmutes the audio
538+ * @returns {void}
539+ */
540+ unmute() {
541+ if (this.isDestroyed) return;
542+ this.audio.muted = false;
543+ }
544+
545+ /**
546+ * Toggles the mute state
547+ * @returns {void}
548+ */
549+ toggleMute() {
550+ if (this.isDestroyed) return;
551+ this.audio.muted = !this.audio.muted;
552+ }
553+
554+ /**
555+ * Sets the audio source URL
556+ * @param {string} src - The URL of the audio file
557+ * @returns {void}
558+ */
559+ setSrc(src) {
560+ if (this.isDestroyed) return;
561+ this.audio.src = src;
562+ }
563+
564+ /**
565+ * Sets the title displayed in the player
566+ * @param {string} title - The title text to display
567+ * @returns {void}
568+ */
569+ setTitle(title) {
570+ if (this.isDestroyed) return;
571+ this.options.title = title;
572+ if (this.elements.title) {
573+ this.elements.title.textContent = title;
574+ }
575+ }
576+
577+ /**
578+ * Cleans up the player by removing event listeners and clearing references
579+ * @returns {void}
580+ */
581+ destroy() {
582+ if (this.isDestroyed) return;
583+ this.isDestroyed = true;
584+
585+ // Stop observing DOM changes
586+ if (this.observer) {
587+ this.observer.disconnect();
588+ this.observer = null;
589+ }
590+
591+ // Pause and clear audio
592+ this.pause();
593+ this.audio.src = '';
594+
595+ // Remove all event listeners
596+ this.unbindEvents();
597+
598+ // Clear references to prevent memory leaks
599+ this.audio = null;
600+ this.container = null;
601+ this.elements = null;
602+ this.options = null;
603+ this.boundHandlers = null;
604+ }
605+}
public/scripts/chats.js+74 -4
@@ -216,7 +216,7 @@ export async function populateFileAttachment(message, inputId = 'file_form_input
216216 if (!Array.isArray(message.extra.media)) {
217217 message.extra.media = [];
218218 }
219219 message.extra.media.push({ url: imageUrl, type: mediaType, title: file.name });
220220 message.extra.media_index = message.extra.media.length - 1;
221221 message.extra.inline_image = true;
222222 } else {
@@ -322,7 +322,7 @@ export async function getFileAttachment(url) {
322322 */
323323async function validateFile(file) {
324324 const fileText = await file.text();
325325 const isMedia = file.type.startsWith('image/') || file.type.startsWith('video/') || file.type.startsWith('audio/');
326326 const isBinary = /^[\x00-\x08\x0E-\x1F\x7F-\xFF]*$/.test(fileText);
327327
328328 if (!isMedia && file.size > fileSizeLimit) {
@@ -890,6 +890,11 @@ function expandMessageMedia(messageId, mediaIndex) {
890890 return;
891891 }
892892
893+ if (mediaAttachment.type === MEDIA_TYPE.AUDIO) {
894+ console.warn('Audio media cannot be expanded');
895+ return;
896+ }
897+
893898 /**
894899 * Gets the media element based on its type.
895900 * @returns {HTMLElement} Media element
@@ -973,6 +978,10 @@ async function deleteMessageMedia(messageId, mediaIndex, messageBlock) {
973978 return;
974979 }
975980
981+ const deleteUrls = [];
982+ const deleteFromServerId = 'delete_media_files_checkbox';
983+ let deleteFromServer = true;
984+
976985 const value = await Popup.show.confirm(t`Delete media from message?`, t`This action can't be undone.`, {
977986 okButton: t`Delete one`,
978987 cancelButton: false,
@@ -988,6 +997,17 @@ async function deleteMessageMedia(messageId, mediaIndex, messageBlock) {
988997 result: POPUP_RESULT.CANCELLED,
989998 },
990999 ],
1000+ customInputs: [
1001+ {
1002+ type: 'checkbox',
1003+ label: t`Also delete files from server`,
1004+ id: deleteFromServerId,
1005+ defaultState: true,
1006+ },
1007+ ],
1008+ onClose: (popup) => {
1009+ deleteFromServer = Boolean(popup.inputResults.get(deleteFromServerId) ?? false);
1010+ },
9911011 });
9921012
9931013 if (!value) {
@@ -1007,6 +1027,7 @@ async function deleteMessageMedia(messageId, mediaIndex, messageBlock) {
10071027 return;
10081028 }
10091029
1030+ deleteUrls.push(message.extra.media[mediaIndex].url);
10101031 message.extra.media.splice(mediaIndex, 1);
10111032
10121033 if (message.extra.media_index === mediaIndex) {
@@ -1015,12 +1036,22 @@ async function deleteMessageMedia(messageId, mediaIndex, messageBlock) {
10151036 }
10161037
10171038 if (value === POPUP_RESULT.CUSTOM1) {
1039+ for (const media of message.extra.media) {
1040+ deleteUrls.push(media.url);
1041+ }
10181042 delete message.extra.media;
10191043 delete message.extra.inline_image;
10201044 delete message.extra.title;
10211045 delete message.extra.append_title;
10221046 }
10231047
1048+ if (deleteFromServer) {
1049+ for (const url of deleteUrls) {
1050+ if (!url) continue;
1051+ await deleteMediaFromServer(url, true);
1052+ }
1053+ }
1054+
10241055 await saveChatConditional();
10251056 appendMediaToMessage(message, messageBlock, SCROLL_BEHAVIOR.KEEP);
10261057}
@@ -1055,12 +1086,43 @@ async function switchMessageMediaDisplay(messageId, messageBlock, targetDisplay)
10551086}
10561087
10571088/**
1089+ * Deletes media file from the server.
1090+ * @param {string} url Path to the media file on the server
1091+ * @param {boolean} [silent=false] If true, do not show error messages
1092+ * @returns {Promise<boolean>} True if media file was deleted, false otherwise.
1093+ */
1094+export async function deleteMediaFromServer(url, silent = false) {
1095+ try {
1096+ const result = await fetch('/api/images/delete', {
1097+ method: 'POST',
1098+ headers: getRequestHeaders(),
1099+ body: JSON.stringify({ path: url }),
1100+ });
1101+
1102+ if (!result.ok) {
1103+ if (!silent) {
1104+ const error = await result.text();
1105+ throw new Error(error);
1106+ }
1107+ return false;
1108+ }
1109+
1110+ await eventSource.emit(event_types.MEDIA_ATTACHMENT_DELETED, url);
1111+ return true;
1112+ } catch (error) {
1113+ toastr.error(String(error), t`Could not delete image`);
1114+ console.error('Could not delete image', error);
1115+ return false;
1116+ }
1117+}
1118+
1119+/**
10581120 * Deletes file from the server.
10591121 * @param {string} url Path to the file on the server
10601122 * @param {boolean} [silent=false] If true, do not show error messages
10611123 * @returns {Promise<boolean>} True if file was deleted, false otherwise.
10621124 */
10631125export async function deleteFileFromServer(url, silent = false) {
10641126 try {
10651127 const result = await fetch('/api/files/delete', {
10661128 method: 'POST',
@@ -1068,10 +1130,13 @@ async function deleteFileFromServer(url, silent = false) {
10681130 body: JSON.stringify({ path: url }),
10691131 });
10701132
10711133 if (!result.ok && !silent) {
1134+ if (!silent) {
10721135 const error = await result.text();
10731136 throw new Error(error);
10741137 }
1138+ return false;
1139+ }
10751140
10761141 await eventSource.emit(event_types.FILE_ATTACHMENT_DELETED, url);
10771142 return true;
@@ -2008,6 +2073,11 @@ async function onImageSwiped(messageId, element, direction) {
20082073 return;
20092074 }
20102075
2076+ if (media.length === 1) {
2077+ console.warn('Only one media item in the message, swiping is not applicable');
2078+ return;
2079+ }
2080+
20112081 const currentIndex = getMediaIndex(message);
20122082 const mediaDisplay = getMediaDisplay(message);
20132083
public/scripts/constants.js+4 -0
@@ -87,10 +87,14 @@ export const MEDIA_TYPE = {
8787 if (mimeType.startsWith('video/')) {
8888 return MEDIA_TYPE.VIDEO;
8989 }
90+ if (mimeType.startsWith('audio/')) {
91+ return MEDIA_TYPE.AUDIO;
92+ }
9093 return null;
9194 },
9295 IMAGE: 'image',
9396 VIDEO: 'video',
97+ AUDIO: 'audio',
9498};
9599
96100/**
public/scripts/events.js+1 -0
@@ -91,6 +91,7 @@ export const event_types = {
9191 PRESET_RENAMED_BEFORE: 'preset_renamed_before',
9292 MAIN_API_CHANGED: 'main_api_changed',
9393 WORLDINFO_ENTRIES_LOADED: 'worldinfo_entries_loaded',
94+ MEDIA_ATTACHMENT_DELETED: 'media_attachment_deleted',
9495};
9596
9697export const eventSource = new EventEmitter([event_types.APP_READY]);
public/scripts/extensions/gallery/index.js+3 -14
@@ -18,6 +18,7 @@ import { DragAndDropHandler } from '../../dragdrop.js';
1818import { commonEnumProviders } from '../../slash-commands/SlashCommandCommonEnumsProvider.js';
1919import { t, translate } from '../../i18n.js';
2020import { Popup } from '../../popup.js';
21+import { deleteMediaFromServer } from '../../chats.js';
2122
2223const extensionName = 'gallery';
2324const extensionFolderPath = `scripts/extensions/${extensionName}/`;
@@ -163,21 +164,9 @@ async function getGalleryFolders() {
163164 * @param {string} url - The URL of the image to be deleted.
164165 */
165166async function deleteGalleryItem(url) {
166- try {
167+ const isDeleted = await deleteMediaFromServer(url, false);
167- const response = await fetch('/api/images/delete', {
168+ if (isDeleted){
168- method: 'POST',
169- headers: getRequestHeaders(),
170- body: JSON.stringify({ path: url }),
171- });
172-
173- if (!response.ok) {
174- throw new Error(`HTTP error. Status: ${response.status}`);
175- }
176-
177169 toastr.success(t`Image deleted successfully.`);
178- } catch (error) {
179- console.error('Failed to delete the image:', error);
180- toastr.error(t`Failed to delete the image. Check the console for details.`);
181170 }
182171}
183172
public/scripts/openai.js+102 -19
@@ -50,11 +50,13 @@ import {
5050 createThumbnail,
5151 delay,
5252 download,
53+ getAudioDurationFromDataURL,
5354 getBase64Async,
5455 getFileText,
5556 getImageSizeFromDataURL,
5657 getSortableDelay,
5758 getStringHash,
59+ getVideoDurationFromDataURL,
5860 isDataURL,
5961 isUuid,
6062 isValidUrl,
@@ -330,6 +332,7 @@ export const settingsToUpdate = {
330332 image_inlining: ['#openai_image_inlining', 'image_inlining', true, false],
331333 inline_image_quality: ['#openai_inline_image_quality', 'inline_image_quality', false, false],
332334 video_inlining: ['#openai_video_inlining', 'video_inlining', true, false],
335+ audio_inlining: ['#openai_audio_inlining', 'audio_inlining', true, false],
333336 continue_prefill: ['#continue_prefill', 'continue_prefill', true, false],
334337 continue_postfix: ['#continue_postfix', 'continue_postfix', false, false],
335338 function_calling: ['#openai_function_calling', 'function_calling', true, false],
@@ -425,9 +428,10 @@ const default_settings = {
425428 vertexai_region: 'us-central1',
426429 vertexai_express_project_id: '',
427430 squash_system_messages: false,
428431 image_inlining: falsetrue,
429432 inline_image_quality: 'lowauto',
430433 video_inlining: falsetrue,
434+ audio_inlining: true,
431435 bypass_status_check: false,
432436 continue_prefill: false,
433437 function_calling: false,
@@ -838,6 +842,7 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
838842
839843 const imageInlining = isImageInliningSupported();
840844 const videoInlining = isVideoInliningSupported();
845+ const audioInlining = isAudioInliningSupported();
841846 const canUseTools = ToolManager.isToolCallingSupported();
842847
843848 // Insert chat messages as long as there is budget available
@@ -872,6 +877,9 @@ async function populateChatHistory(messages, prompts, chatCompletion, type = nul
872877 if (videoInlining && media.type === MEDIA_TYPE.VIDEO) {
873878 await chatMessage.addVideo(media.url);
874879 }
880+ if (audioInlining && media.type === MEDIA_TYPE.AUDIO) {
881+ await chatMessage.addAudio(media.url);
882+ }
875883 }
876884
877885 if (Array.isArray(chatPrompt.media) && chatPrompt.media.length) {
@@ -2925,11 +2933,10 @@ class Message {
29252933 }
29262934
29272935 /**
2928- * Adds an image to the message.
2936+ * Ensures the content is an array. If it's a string, converts it to an array with a single text object.
29292937 * @paramreturns {stringany[]} image Image URLContent oras Dataan URL.array
2930- * @returns {Promise<void>}
29312938 */
29322939 async addImageensureContentIsArray(image) {
29332940 const textContent = this.content;
29342941 if (!Array.isArray(this.content)) {
29352942 this.content = [];
@@ -2937,7 +2944,16 @@ class Message {
29372944 this.content.push({ type: 'text', text: textContent });
29382945 }
29392946 }
2947+ return this.content;
2948+ }
29402949
2950+ /**
2951+ * Adds an image to the message.
2952+ * @param {string} image Image URL or Data URL.
2953+ * @returns {Promise<void>}
2954+ */
2955+ async addImage(image) {
2956+ this.content = this.ensureContentIsArray();
29412957 const isDataUrl = isDataURL(image);
29422958 if (!isDataUrl) {
29432959 try {
@@ -2971,14 +2987,7 @@ class Message {
29712987 * @returns {Promise<void>}
29722988 */
29732989 async addVideo(video) {
29742990 const textContentthis.content = this.contentensureContentIsArray();
2975- if (!Array.isArray(this.content)) {
2976- this.content = [];
2977- if (typeof textContent === 'string') {
2978- this.content.push({ type: 'text', text: textContent });
2979- }
2980- }
2981-
29822991 const isDataUrl = isDataURL(video);
29832992 if (!isDataUrl) {
29842993 try {
@@ -2996,17 +3005,51 @@ class Message {
29963005 this.content.push({ type: 'video_url', video_url: { 'url': video } });
29973006
29983007 try {
2999- // Convservative estimate for video token cost without knowing duration
30003008 // Using Gemini calculation (263 tokens per second)
3001- const tokens = 10000; // ~40 second video (60 seconds max)
3009+ const duration = await getVideoDurationFromDataURL(video);
30023010 this.tokens += tokens263 * Math.ceil(duration);
30033011 } catch (error) {
3004- this.tokens += 10000;
3012+ // Convservative estimate for video token cost without knowing duration
3013+ this.tokens += 263 * 40; // ~40 second video (60 seconds max)
30053014 console.error('Failed to get video token cost', error);
30063015 }
30073016 }
30083017
30093018 /**
3019+ * Adds a audio to the message.
3020+ * @param {string} audio Audio URL or Data URL.
3021+ * @returns {Promise<void>}
3022+ */
3023+ async addAudio(audio) {
3024+ this.content = this.ensureContentIsArray();
3025+ const isDataUrl = isDataURL(audio);
3026+ if (!isDataUrl) {
3027+ try {
3028+ const response = await fetch(audio, { method: 'GET', cache: 'force-cache' });
3029+ if (!response.ok) throw new Error('Failed to fetch audio');
3030+ const blob = await response.blob();
3031+ audio = await getBase64Async(blob);
3032+ } catch (error) {
3033+ console.error('Audio adding skipped', error);
3034+ return;
3035+ }
3036+ }
3037+
3038+ this.content.push({ type: 'audio_url', audio_url: { 'url': audio } });
3039+
3040+ try {
3041+ // Using Gemini calculation (32 tokens per second)
3042+ const duration = await getAudioDurationFromDataURL(audio);
3043+ this.tokens += 32 * Math.ceil(duration);
3044+ } catch (error) {
3045+ // Estimate for audio token cost without knowing duration
3046+ const tokens = 32 * 300; // ~5 minute audio
3047+ this.tokens += tokens;
3048+ console.error('Failed to get audio token cost', error);
3049+ }
3050+ }
3051+
3052+ /**
30103053 * Compress an image if it exceeds the size threshold for the current chat completion source.
30113054 * @param {string} image Data URL of the image.
30123055 * @returns {Promise<string>} Compressed image as a Data URL.
@@ -3635,6 +3678,7 @@ function loadOpenAISettings(data, settings) {
36353678 oai_settings.image_inlining = settings.image_inlining ?? default_settings.image_inlining;
36363679 oai_settings.inline_image_quality = settings.inline_image_quality ?? default_settings.inline_image_quality;
36373680 oai_settings.video_inlining = settings.video_inlining ?? default_settings.video_inlining;
3681+ oai_settings.audio_inlining = settings.audio_inlining ?? default_settings.audio_inlining;
36383682 oai_settings.bypass_status_check = settings.bypass_status_check ?? default_settings.bypass_status_check;
36393683 oai_settings.vertexai_express_project_id = settings.vertexai_express_project_id ?? default_settings.vertexai_express_project_id;
36403684 oai_settings.show_thoughts = settings.show_thoughts ?? default_settings.show_thoughts;
@@ -3687,6 +3731,7 @@ function loadOpenAISettings(data, settings) {
36873731 $(`#openai_inline_image_quality option[value="${oai_settings.inline_image_quality}"]`).prop('selected', true);
36883732
36893733 $('#openai_video_inlining').prop('checked', oai_settings.video_inlining);
3734+ $('#openai_audio_inlining').prop('checked', oai_settings.audio_inlining);
36903735
36913736 $('#model_openai_select').val(oai_settings.openai_model);
36923737 $(`#model_openai_select option[value="${oai_settings.openai_model}"`).prop('selected', true);
@@ -4079,6 +4124,7 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
40794124 image_inlining: settings.image_inlining,
40804125 inline_image_quality: settings.inline_image_quality,
40814126 video_inlining: settings.video_inlining,
4127+ audio_inlining: settings.audio_inlining,
40824128 bypass_status_check: settings.bypass_status_check,
40834129 continue_prefill: settings.continue_prefill,
40844130 continue_postfix: settings.continue_postfix,
@@ -5644,6 +5690,36 @@ export function isVideoInliningSupported() {
56445690}
56455691
56465692/**
5693+ * Check if the model supports video inlining
5694+ * @returns {boolean} True if the model supports audio inlining
5695+ */
5696+export function isAudioInliningSupported() {
5697+ if (main_api !== 'openai') {
5698+ return false;
5699+ }
5700+
5701+ if (!oai_settings.audio_inlining) {
5702+ return false;
5703+ }
5704+
5705+ // Only Gemini models support audio for now
5706+ const audioSupportedModels = [
5707+ 'gemini-2.0',
5708+ 'gemini-2.5',
5709+ 'gemini-exp-1206',
5710+ ];
5711+
5712+ switch (oai_settings.chat_completion_source) {
5713+ case chat_completion_sources.MAKERSUITE:
5714+ return audioSupportedModels.some(model => oai_settings.google_model.includes(model));
5715+ case chat_completion_sources.VERTEXAI:
5716+ return audioSupportedModels.some(model => oai_settings.vertexai_model.includes(model));
5717+ default:
5718+ return false;
5719+ }
5720+}
5721+
5722+/**
56475723 * Proxy stuff
56485724 */
56495725export function loadProxyPresets(settings) {
@@ -5905,6 +5981,7 @@ function updateFeatureSupportFlags() {
59055981 openai_function_calling_supported: ToolManager.isToolCallingSupported(),
59065982 openai_image_inlining_supported: isImageInliningSupported(),
59075983 openai_video_inlining_supported: isVideoInliningSupported(),
5984+ openai_audio_inlining_supported: isAudioInliningSupported(),
59085985 };
59095986
59105987 for (const [key, value] of Object.entries(featureFlags)) {
@@ -6229,6 +6306,12 @@ export function initOpenAI() {
62296306 saveSettingsDebounced();
62306307 });
62316308
6309+ $('#openai_audio_inlining').on('input', function () {
6310+ oai_settings.audio_inlining = !!$(this).prop('checked');
6311+ updateFeatureSupportFlags();
6312+ saveSettingsDebounced();
6313+ });
6314+
62326315 $('#continue_prefill').on('input', function () {
62336316 oai_settings.continue_prefill = !!$(this).prop('checked');
62346317 saveSettingsDebounced();
public/scripts/utils.js+51 -0
@@ -929,6 +929,21 @@ export function humanFileSize(bytes, si = false, dp = 1) {
929929}
930930
931931/**
932+ * Formats time in seconds to MM:SS format
933+ * @param {number} seconds - Time in seconds
934+ * @returns {string} Formatted time string
935+ */
936+export function formatTime(seconds) {
937+ if (!isFinite(seconds) || isNaN(seconds)) {
938+ return '0:00';
939+ }
940+
941+ const minutes = Math.floor(seconds / 60);
942+ const secs = Math.floor(seconds % 60);
943+ return `${minutes}:${secs.toString().padStart(2, '0')}`;
944+}
945+
946+/**
932947 * Counts the number of occurrences of a character in a string.
933948 * @param {string} string The string to count occurrences in.
934949 * @param {string} character The character to count occurrences of.
@@ -1164,6 +1179,42 @@ export function getImageSizeFromDataURL(dataUrl) {
11641179}
11651180
11661181/**
1182+ * Gets the duration of a video from a data URL.
1183+ * @param {string} dataUrl Video data URL
1184+ * @returns {Promise<number>} Duration in seconds
1185+ */
1186+export function getVideoDurationFromDataURL(dataUrl) {
1187+ const video = document.createElement('video');
1188+ video.src = dataUrl;
1189+ return new Promise((resolve, reject) => {
1190+ video.onloadedmetadata = function () {
1191+ resolve(video.duration);
1192+ };
1193+ video.onerror = function () {
1194+ reject(new Error('Failed to load video'));
1195+ };
1196+ });
1197+}
1198+
1199+/**
1200+ * Gets the duration of an audio from a data URL.
1201+ * @param {string} dataUrl Audio data URL
1202+ * @returns {Promise<number>} Duration in seconds
1203+ */
1204+export function getAudioDurationFromDataURL(dataUrl) {
1205+ const audio = document.createElement('audio');
1206+ audio.src = dataUrl;
1207+ return new Promise((resolve, reject) => {
1208+ audio.onloadedmetadata = function () {
1209+ resolve(audio.duration);
1210+ };
1211+ audio.onerror = function () {
1212+ reject(new Error('Failed to load audio'));
1213+ };
1214+ });
1215+}
1216+
1217+/**
11671218 * Gets the filename of the character avatar without extension
11681219 * @param {string|number?} [chid=null] - Character ID. If not provided, uses the current character ID
11691220 * @param {object} [options={}] - Options arguments
src/constants.js+7 -0
@@ -477,4 +477,11 @@ export const MEDIA_EXTENSIONS = [
477477 '3gp',
478478 'mkv',
479479 'mpg',
480+ 'mp3',
481+ 'wav',
482+ 'ogg',
483+ 'flac',
484+ 'aac',
485+ 'm4a',
486+ 'aiff',
480487];
src/prompt-converters.js+20 -19
@@ -501,6 +501,20 @@ export function convertGooglePrompt(messages, _model, useSysPrompt, names) {
501501 //create the prompt parts
502502 const parts = [];
503503 message.content.forEach((part) => {
504+ const addDataUrlPart = (/** @type {string} */ url, /** @type {string} */ defaultMimeType) => {
505+ if (url && url.startsWith('data:')) {
506+ const [header, base64Data] = url.split(',');
507+ const mimeType = header.match(/data:([^;]+)/)?.[1] || defaultMimeType;
508+
509+ parts.push({
510+ inlineData: {
511+ mimeType: mimeType,
512+ data: base64Data,
513+ },
514+ });
515+ }
516+ };
517+
504518 if (part.type === 'text') {
505519 parts.push({ text: part.text });
506520 } else if (part.type === 'tool_call_id') {
@@ -523,27 +537,14 @@ export function convertGooglePrompt(messages, _model, useSysPrompt, names) {
523537 toolNameMap[toolCall.id] = toolCall.function.name;
524538 });
525539 } else if (part.type === 'image_url') {
526540 const mimeTypeimageUrl = part.image_url?.url.split(';')[0].split(':')[1];
527- const base64Data = part.image_url.url.split(',')[1];
541+ addDataUrlPart(imageUrl, 'image/png');
528- parts.push({
529- inlineData: {
530- mimeType: mimeType,
531- data: base64Data,
532- },
533- });
534542 } else if (part.type === 'video_url') {
535543 const videoUrl = part.video_url?.url;
536- if (videoUrl && videoUrl.startsWith('data:')) {
544+ addDataUrlPart(videoUrl, 'video/mp4');
537- const [header, data] = videoUrl.split(',');
545+ } else if (part.type === 'audio_url') {
538- const mimeType = header.match(/data:([^;]+)/)?.[1] || 'video/mp4';
546+ const audioUrl = part.audio_url?.url;
539-
547+ addDataUrlPart(audioUrl, 'audio/mpeg');
540- parts.push({
541- inlineData: {
542- mimeType: mimeType,
543- data: data,
544- },
545- });
546- }
547548 }
548549 });
549550