Join recent chat/group queries

587cecb12cdaadf4e09169025e8f0df83c40ee29

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 155172a2b45a660822aa01e7aa69cd8e2f209a04

public/scripts/welcome-screen.js+6 -19 Viewed

 const defaultAssistantAvatar = 'default_Assistant.png';
 const DEFAULT_DISPLAYED = 3;
 const MAX_DISPLAYED = 2015;
 export function getPermanentAssistantAvatar() {
     const assistantAvatar = accountStorage.getItem(assistantAvatarKey);

  * @property {boolean} hidden Chat will be hidden by default
  */
 async function getRecentChats() {
     const charDataresponse = asyncawait fetch() =>'/api/chats/recent', {
 -        const response = await fetch('/api/characters/recent', {
         method: 'POST',
         headers: getRequestHeaders(),
 +        body: JSON.stringify({ max: MAX_DISPLAYED }),
     });
 -        if (!response.ok) {
 -            console.warn('Failed to fetch recent character chats');
 -            return [];
 -        }
 -        return await response.json();
 -    };
 -    const groupData = async () => {
 -        const response = await fetch('/api/groups/recent', {
 -            method: 'POST',
 -            headers: getRequestHeaders(),
 -        });
     if (!response.ok) {
                     console.warn('Failed to fetch recent groupcharacter chats');
         return [];
+    }
 -        return await response.json();
 -    };
     /** @type {RecentChat[]} */
 -    const data = await Promise.all([charData(), groupData()]).then(res => res.flat());
 +    const data = await response.json();
     data.sort((a, b) => sortMoments(timestampToMoment(a.last_mes), timestampToMoment(b.last_mes)))
         .map(chat => ({ chat, character: characters.find(x => x.avatar === chat.avatar), group: groups.find(x => x.id === chat.group) }))

             chat.group = chat.group || '';
         });
     return data.slice(0, MAX_DISPLAYED);
+}
 export async function openPermanentAssistantChat({ tryCreate = true, created = false } = {}) {

src/endpoints/characters.js+1 -105 Viewed

 import path from 'node:path';
 import fs from 'node:fs';
 import { promises as fsPromises } from 'node:fs';
 -import readline from 'node:readline';
 import { Buffer } from 'node:buffer';
 import express from 'express';

 import { invalidateThumbnail } from './thumbnails.js';
 import { importRisuSprites } from './sprites.js';
 import { getUserDirectories } from '../users.js';
 +import { getChatInfo } from './chats.js';
 const defaultAvatarPath = './public/img/ai4.png';
 // With 100 MB limit it would take roughly 3000 characters to reach this limit

     return '';
+}
 -/**
 - * @typedef {Object} ChatInfo
 - * @property {string} [file_name] - The name of the chat file
 - * @property {string} [file_size] - The size of the chat file
 - * @property {number} [chat_items] - The number of chat items in the file
 - * @property {string} [mes] - The last message in the chat
 - * @property {number} [last_mes] - The timestamp of the last message
 - */
+-
 -/**
 - * Reads the information from a chat file.
 - * @param {string} pathToFile
 - * @param {object} additionalData
 - * @returns {Promise<ChatInfo>}
 - */
 -export async function getChatInfo(pathToFile, additionalData = {}, isGroup = false) {
 -    return new Promise(async (res) => {
 -        const fileStream = fs.createReadStream(pathToFile);
 -        const stats = fs.statSync(pathToFile);
 -        const fileSizeInKB = `${(stats.size / 1024).toFixed(2)}kb`;
+-
 -        if (stats.size === 0) {
 -            console.warn(`Found an empty chat file: ${pathToFile}`);
 -            res({});
 -            return;
 -        }
+-
 -        const rl = readline.createInterface({
 -            input: fileStream,
 -            crlfDelay: Infinity,
 -        });
+-
 -        let lastLine;
 -        let itemCounter = 0;
 -        rl.on('line', (line) => {
 -            itemCounter++;
 -            lastLine = line;
 -        });
 -        rl.on('close', () => {
 -            rl.close();
+-
 -            if (lastLine) {
 -                const jsonData = tryParse(lastLine);
 -                if (jsonData && (jsonData.name || jsonData.character_name)) {
 -                    const chatData = {};
+-
 -                    chatData['file_name'] = path.parse(pathToFile).base;
 -                    chatData['file_size'] = fileSizeInKB;
 -                    chatData['chat_items'] = isGroup ? itemCounter : (itemCounter - 1);
 -                    chatData['mes'] = jsonData['mes'] || '[The chat is empty]';
 -                    chatData['last_mes'] = jsonData['send_date'] || stats.mtimeMs;
 -                    Object.assign(chatData, additionalData);
+-
 -                    res(chatData);
 -                } else {
 -                    console.warn('Found an invalid or corrupted chat file:', pathToFile);
 -                    res({});
 -                }
 -            }
 -        });
 -    });
 -}
+-
 export const router = express.Router();
 router.post('/create', async function (request, response) {

+    }
 });
 -router.post('/recent', async function (request, response) {
 -    try {
 -        /** @type {{pngFile: string, filePath: string, mtime: number}[]} */
 -        const allChatFiles = [];
+-
 -        const pngFiles = fs
 -            .readdirSync(request.user.directories.characters, { withFileTypes: true })
 -            .filter(dirent => dirent.isFile() && dirent.name.endsWith('.png'))
 -            .map(dirent => dirent.name);
+-
 -        for (const pngFile of pngFiles) {
 -            const chatsDirectory = pngFile.replace('.png', '');
 -            const pathToChats = path.join(request.user.directories.chats, chatsDirectory);
 -            if (fs.existsSync(pathToChats) && fs.statSync(pathToChats).isDirectory()) {
 -                const chatFiles = fs.readdirSync(pathToChats);
 -                const chatFilesWithDate = chatFiles
 -                    .filter(file => file.endsWith('.jsonl'))
 -                    .map(file => {
 -                        const filePath = path.join(pathToChats, file);
 -                        const stats = fs.statSync(filePath);
 -                        return { pngFile, filePath, mtime: stats.mtimeMs };
 -                    });
 -                allChatFiles.push(...chatFilesWithDate);
 -            }
 -        }
+-
 -        const recentChats = allChatFiles.sort((a, b) => b.mtime - a.mtime).slice(0, 15);
 -        const jsonFilesPromise = recentChats.map((file) => {
 -            return getChatInfo(file.filePath, { avatar: file.pngFile });
 -        });
+-
 -        const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);
 -        const validFiles = chatData.filter(i => i.file_name);
+-
 -        return response.send(validFiles);
 -    } catch (error) {
 -        console.error(error);
 -        return response.sendStatus(500);
 -    }
 -});
+-
 router.post('/chats', validateAvatarUrlMiddleware, async function (request, response) {
     try {
         if (!request.body) return response.sendStatus(400);

src/endpoints/chats.js+139 -0 Viewed

     return chatIntegrity === integritySlug;
+}
 +/**
 + * @typedef {Object} ChatInfo
 + * @property {string} [file_name] - The name of the chat file
 + * @property {string} [file_size] - The size of the chat file
 + * @property {number} [chat_items] - The number of chat items in the file
 + * @property {string} [mes] - The last message in the chat
 + * @property {number} [last_mes] - The timestamp of the last message
 + */
++
 +/**
 + * Reads the information from a chat file.
 + * @param {string} pathToFile
 + * @param {object} additionalData
 + * @returns {Promise<ChatInfo>}
 + */
 +export async function getChatInfo(pathToFile, additionalData = {}, isGroup = false) {
 +    return new Promise(async (res) => {
 +        const fileStream = fs.createReadStream(pathToFile);
 +        const stats = await fs.promises.stat(pathToFile);
 +        const fileSizeInKB = `${(stats.size / 1024).toFixed(2)}kb`;
++
 +        if (stats.size === 0) {
 +            console.warn(`Found an empty chat file: ${pathToFile}`);
 +            res({});
 +            return;
 +        }
++
 +        const rl = readline.createInterface({
 +            input: fileStream,
 +            crlfDelay: Infinity,
 +        });
++
 +        let lastLine;
 +        let itemCounter = 0;
 +        rl.on('line', (line) => {
 +            itemCounter++;
 +            lastLine = line;
 +        });
 +        rl.on('close', () => {
 +            rl.close();
++
 +            if (lastLine) {
 +                const jsonData = tryParse(lastLine);
 +                if (jsonData && (jsonData.name || jsonData.character_name)) {
 +                    const chatData = {};
++
 +                    chatData['file_name'] = path.parse(pathToFile).base;
 +                    chatData['file_size'] = fileSizeInKB;
 +                    chatData['chat_items'] = isGroup ? itemCounter : (itemCounter - 1);
 +                    chatData['mes'] = jsonData['mes'] || '[The chat is empty]';
 +                    chatData['last_mes'] = jsonData['send_date'] || stats.mtimeMs;
 +                    Object.assign(chatData, additionalData);
++
 +                    res(chatData);
 +                } else {
 +                    console.warn('Found an invalid or corrupted chat file:', pathToFile);
 +                    res({});
 +                }
 +            }
 +        });
 +    });
 +}
++
 export const router = express.Router();
 router.post('/save', validateAvatarUrlMiddleware, async function (request, response) {

         return response.status(500).json({ error: 'Search failed' });
+    }
 });
++
 +router.post('/recent', async function (request, response) {
 +    try {
 +        /** @type {{pngFile?: string, groupId?: string, filePath: string, mtime: number}[]} */
 +        const allChatFiles = [];
++
 +        const getCharacterChatFiles = async () => {
 +            const pngDirents = await fs.promises.readdir(request.user.directories.characters, { withFileTypes: true });
 +            const pngFiles = pngDirents.filter(e => e.isFile() && path.extname(e.name) === '.png').map(e => e.name);
++
 +            for (const pngFile of pngFiles) {
 +                const chatsDirectory = pngFile.replace('.png', '');
 +                const pathToChats = path.join(request.user.directories.chats, chatsDirectory);
 +                if (!fs.existsSync(pathToChats)) {
 +                    continue;
 +                }
 +                const pathStats = await fs.promises.stat(pathToChats);
 +                if (pathStats.isDirectory()) {
 +                    const chatFiles = await fs.promises.readdir(pathToChats);
 +                    const jsonlFiles = chatFiles.filter(file => path.extname(file) === '.jsonl');
++
 +                    for (const file of jsonlFiles) {
 +                        const filePath = path.join(pathToChats, file);
 +                        const stats = await fs.promises.stat(filePath);
 +                        allChatFiles.push({ pngFile, filePath, mtime: stats.mtimeMs });
 +                    }
 +                }
 +            }
 +        };
++
 +        const getGroupChatFiles = async () => {
 +            const groupDirents = await fs.promises.readdir(request.user.directories.groups, { withFileTypes: true });
 +            const groups = groupDirents.filter(e => e.isFile() && path.extname(e.name) === '.json').map(e => e.name);
++
 +            for (const group of groups) {
 +                try {
 +                    const groupPath = path.join(request.user.directories.groups, group);
 +                    const groupContents = await fs.promises.readFile(groupPath, 'utf8');
 +                    const groupData = JSON.parse(groupContents);
++
 +                    if (Array.isArray(groupData.chats)) {
 +                        for (const chat of groupData.chats) {
 +                            const filePath = path.join(request.user.directories.groupChats, `${chat}.jsonl`);
 +                            if (!fs.existsSync(filePath)) {
 +                                continue;
 +                            }
 +                            const stats = await fs.promises.stat(filePath);
 +                            allChatFiles.push({ groupId: groupData.id, filePath, mtime: stats.mtimeMs });
 +                        }
 +                    }
 +                } catch (error) {
 +                    // Skip group files that can't be read or parsed
 +                    continue;
 +                }
 +            }
 +        };
++
 +        await Promise.allSettled([getCharacterChatFiles(), getGroupChatFiles()]);
++
 +        const max = parseInt(request.body.max ?? Number.MAX_SAFE_INTEGER);
 +        const recentChats = allChatFiles.sort((a, b) => b.mtime - a.mtime).slice(0, max);
 +        const jsonFilesPromise = recentChats.map((file) => {
 +            return file.groupId
 +                ? getChatInfo(file.filePath, { group: file.groupId }, true)
 +                : getChatInfo(file.filePath, { avatar: file.pngFile }, false);
 +        });
++
 +        const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);
 +        const validFiles = chatData.filter(i => i.file_name);
++
 +        return response.send(validFiles);
 +    } catch (error) {
 +        console.error(error);
 +        return response.sendStatus(500);
 +    }
 +});

src/endpoints/groups.js+0 -39 Viewed

 import { sync as writeFileAtomicSync } from 'write-file-atomic';
 import { humanizedISO8601DateTime } from '../util.js';
 -import { getChatInfo } from './characters.js';
 export const router = express.Router();

     return response.send({ ok: true });
 });
+-
 -router.post('/recent', async (request, response) => {
 -    try {
 -        /** @type {{groupId: string, filePath: string, mtime: number}[]} */
 -        const allChatFiles = [];
+-
 -        const groups = fs.readdirSync(request.user.directories.groups).filter(x => path.extname(x) === '.json');
 -        for (const group of groups) {
 -            const groupPath = path.join(request.user.directories.groups, group);
 -            const groupContents = fs.readFileSync(groupPath, 'utf8');
 -            const groupData = JSON.parse(groupContents);
+-
 -            if (Array.isArray(groupData.chats)) {
 -                for (const chat of groupData.chats) {
 -                    const filePath = path.join(request.user.directories.groupChats, `${chat}.jsonl`);
 -                    if (!fs.existsSync(filePath)) {
 -                        continue;
 -                    }
 -                    const stats = fs.statSync(filePath);
 -                    allChatFiles.push({ groupId: groupData.id, filePath, mtime: stats.mtimeMs });
 -                }
 -            }
 -        }
+-
 -        const recentChats = allChatFiles.sort((a, b) => b.mtime - a.mtime).slice(0, 15);
 -        const jsonFilesPromise = recentChats.map((file) => {
 -            return getChatInfo(file.filePath, { group: file.groupId }, true);
 -        });
+-
 -        const chatData = (await Promise.allSettled(jsonFilesPromise)).filter(x => x.status === 'fulfilled').map(x => x.value);
 -        const validFiles = chatData.filter(i => i.file_name);
+-
 -        return response.send(validFiles);
 -    } catch (error) {
 -        console.error('Error while getting recent group chats', error);
 -        return response.sendStatus(500);
 -    }
 -});