Switch branch
feat/multi-feature-pack maindefault
View all branches
Blame Raw
· · · 469 lines (16.2 KB)
0 contributors 
1import fs from 'node:fs';


2import path from 'node:path';


3import crypto from 'node:crypto';


4


5import express from 'express';


6import writeFileAtomic from 'write-file-atomic';


7


8const readFile = fs.promises.readFile;


9const readdir = fs.promises.readdir;


10


11import { getAllUserHandles, getUserDirectories } from '../users.js';


12


13const STATS_FILE = 'stats.json';


14


15const monthNames = [


16    'January',


17    'February',


18    'March',


19    'April',


20    'May',


21    'June',


22    'July',


23    'August',


24    'September',


25    'October',


26    'November',


27    'December',


28];


29


30/**


31 * @type {Map<string, Object>} The stats object for each user.


32 */


33const STATS = new Map();


34/**


35 * @type {Map<string, number>} The timestamps for each user.


36 */


37const TIMESTAMPS = new Map();


38


39/**


40 * Convert a timestamp to an integer timestamp.


41 * This function can handle several different timestamp formats:


42 * 1. Date.now timestamps (the number of milliseconds since the Unix Epoch)


43 * 2. ST "humanized" timestamps, formatted like `YYYY-MM-DD@HHhMMmSSsMSms`


44 * 3. Date strings in the format `Month DD, YYYY H:MMam/pm`


45 * 4. ISO 8601 formatted strings


46 * 5. Date objects


47 *


48 * The function returns the timestamp as the number of milliseconds since


49 * the Unix Epoch, which can be converted to a JavaScript Date object with new Date().


50 *


51 * @param {string|number|Date} timestamp - The timestamp to convert.


52 * @returns {number} The timestamp in milliseconds since the Unix Epoch, or 0 if the input cannot be parsed.


53 *


54 * @example


55 * // Unix timestamp


56 * parseTimestamp(1609459200);


57 * // ST humanized timestamp


58 * parseTimestamp("2021-01-01 \@00h 00m 00s 000ms");


59 * // Date string


60 * parseTimestamp("January 1, 2021 12:00am");


61 */


62function parseTimestamp(timestamp) {


63    if (!timestamp) {


64        return 0;


65    }


66


67    // Date object


68    if (timestamp instanceof Date) {


69        return timestamp.getTime();


70    }


71


72    // Unix time


73    if (typeof timestamp === 'number' || /^\d+$/.test(timestamp)) {


74        const unixTime = Number(timestamp);


75        const isValid = Number.isFinite(unixTime) && !Number.isNaN(unixTime) && unixTime >= 0;


76        if (!isValid) return 0;


77        return new Date(unixTime).getTime();


78    }


79


80    // ISO 8601 format


81    const isoPattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/;


82    if (isoPattern.test(timestamp)) {


83        return new Date(timestamp).getTime();


84    }


85


86    let dateFormats = [];


87


88    // meridiem-based format


89    const convertFromMeridiemBased = (_, month, day, year, hour, minute, meridiem) => {


90        const monthNum = monthNames.indexOf(month) + 1;


91        const hour24 = meridiem.toLowerCase() === 'pm' ? (parseInt(hour, 10) % 12) + 12 : parseInt(hour, 10) % 12;


92        return `${year}-${monthNum}-${day.padStart(2, '0')}T${hour24.toString().padStart(2, '0')}:${minute.padStart(2, '0')}:00`;


93    };


94    // June 19, 2023 2:20pm


95    dateFormats.push({ callback: convertFromMeridiemBased, pattern: /(\w+)\s(\d{1,2}),\s(\d{4})\s(\d{1,2}):(\d{1,2})(am|pm)/i });


96


97    // ST "humanized" format patterns


98    const convertFromHumanized = (_, year, month, day, hour, min, sec, ms) => {


99        ms = typeof ms !== 'undefined' ? `.${ms.padStart(3, '0')}` : '';


100        return `${year.padStart(4, '0')}-${month.padStart(2, '0')}-${day.padStart(2, '0')}T${hour.padStart(2, '0')}:${min.padStart(2, '0')}:${sec.padStart(2, '0')}${ms}Z`;


101    };


102    // 2024-07-12@01h31m37s123ms


103    dateFormats.push({ callback: convertFromHumanized, pattern: /(\d{4})-(\d{1,2})-(\d{1,2})@(\d{1,2})h(\d{1,2})m(\d{1,2})s(\d{1,3})ms/ });


104    // 2024-7-12@01h31m37s


105    dateFormats.push({ callback: convertFromHumanized, pattern: /(\d{4})-(\d{1,2})-(\d{1,2})@(\d{1,2})h(\d{1,2})m(\d{1,2})s/ });


106    // 2024-6-5 @14h 56m 50s 682ms


107    dateFormats.push({ callback: convertFromHumanized, pattern: /(\d{4})-(\d{1,2})-(\d{1,2}) @(\d{1,2})h (\d{1,2})m (\d{1,2})s (\d{1,3})ms/ });


108


109    for (const x of dateFormats) {


110        const rgxMatch = timestamp.match(x.pattern);


111        if (!rgxMatch) continue;


112        const isoTimestamp = x.callback(...rgxMatch);


113        return new Date(isoTimestamp).getTime();


114    }


115


116    return 0;


117}


118


119/**


120 * Collects and aggregates stats for all characters.


121 *


122 * @param {string} chatsPath - The path to the directory containing the chat files.


123 * @param {string} charactersPath - The path to the directory containing the character files.


124 * @returns {Promise<Object>} The aggregated stats object.


125 */


126async function collectAndCreateStats(chatsPath, charactersPath) {


127    const files = await readdir(charactersPath);


128


129    const pngFiles = files.filter((file) => file.endsWith('.png'));


130


131    let processingPromises = pngFiles.map((file) =>


132        calculateStats(chatsPath, file),


133    );


134    const statsArr = await Promise.all(processingPromises);


135


136    let finalStats = {};


137    for (let stat of statsArr) {


138        finalStats = { ...finalStats, ...stat };


139    }


140    // tag with timestamp on when stats were generated


141    finalStats.timestamp = Date.now();


142    return finalStats;


143}


144


145/**


146 * Recreates the stats object for a user.


147 * @param {string} handle User handle


148 * @param {string} chatsPath Path to the directory containing the chat files.


149 * @param {string} charactersPath Path to the directory containing the character files.


150 */


151export async function recreateStats(handle, chatsPath, charactersPath) {


152    console.info('Collecting and creating stats for user:', handle);


153    const stats = await collectAndCreateStats(chatsPath, charactersPath);


154    STATS.set(handle, stats);


155    await saveStatsToFile();


156}


157


158/**


159 * Loads the stats file into memory. If the file doesn't exist or is invalid,


160 * initializes stats by collecting and creating them for each character.


161 */


162export async function init() {


163    try {


164        const userHandles = await getAllUserHandles();


165        for (const handle of userHandles) {


166            const directories = getUserDirectories(handle);


167            try {


168                const statsFilePath = path.join(directories.root, STATS_FILE);


169                const statsFileContent = await readFile(statsFilePath, 'utf-8');


170                STATS.set(handle, JSON.parse(statsFileContent));


171            } catch (err) {


172                // If the file doesn't exist or is invalid, initialize stats


173                if (err.code === 'ENOENT' || err instanceof SyntaxError) {


174                    await recreateStats(handle, directories.chats, directories.characters);


175                } else {


176                    throw err; // Rethrow the error if it's something we didn't expect


177                }


178            }


179        }


180    } catch (err) {


181        console.error('Failed to initialize stats:', err);


182    }


183    // Save stats every 5 minutes


184    setInterval(saveStatsToFile, 5 * 60 * 1000);


185}


186/**


187 * Saves the current state of charStats to a file, only if the data has changed since the last save.


188 */


189async function saveStatsToFile() {


190    const userHandles = await getAllUserHandles();


191    for (const handle of userHandles) {


192        if (!STATS.has(handle)) {


193            continue;


194        }


195        const charStats = STATS.get(handle);


196        const lastSaveTimestamp = TIMESTAMPS.get(handle) || 0;


197        if (charStats.timestamp > lastSaveTimestamp) {


198            try {


199                const directories = getUserDirectories(handle);


200                const statsFilePath = path.join(directories.root, STATS_FILE);


201                await writeFileAtomic(statsFilePath, JSON.stringify(charStats));


202                TIMESTAMPS.set(handle, Date.now());


203            } catch (error) {


204                console.error('Failed to save stats to file.', error);


205            }


206        }


207    }


208}


209


210/**


211 * Attempts to save charStats to a file and then terminates the process.


212 * If an error occurs during the file write, it logs the error before exiting.


213 */


214export async function onExit() {


215    try {


216        await saveStatsToFile();


217    } catch (err) {


218        console.error('Failed to write stats to file:', err);


219    }


220}


221


222/**


223 * Reads the contents of a file and returns the lines in the file as an array.


224 *


225 * @param {string} filepath - The path of the file to be read.


226 * @returns {Array<string>} - The lines in the file.


227 * @throws Will throw an error if the file cannot be read.


228 */


229function readAndParseFile(filepath) {


230    try {


231        let file = fs.readFileSync(filepath, 'utf8');


232        let lines = file.split('\n');


233        return lines;


234    } catch (error) {


235        console.error(`Error reading file at ${filepath}: ${error}`);


236        return [];


237    }


238}


239


240/**


241 * Calculates the time difference between two dates.


242 *


243 * @param {string} gen_started - The start time in ISO 8601 format.


244 * @param {string} gen_finished - The finish time in ISO 8601 format.


245 * @returns {number} - The difference in time in milliseconds.


246 */


247function calculateGenTime(gen_started, gen_finished) {


248    let startDate = new Date(gen_started);


249    let endDate = new Date(gen_finished);


250    return Number(endDate) - Number(startDate);


251}


252


253/**


254 * Counts the number of words in a string.


255 *


256 * @param {string} str - The string to count words in.


257 * @returns {number} - The number of words in the string.


258 */


259function countWordsInString(str) {


260    const match = str.match(/\b\w+\b/g);


261    return match ? match.length : 0;


262}


263


264/**


265 * calculateStats - Calculate statistics for a given character chat directory.


266 *


267 * @param  {string} chatsPath The directory containing the chat files.


268 * @param  {string} item     The name of the character.


269 * @return {object}          An object containing the calculated statistics.


270 */


271const calculateStats = (chatsPath, item) => {


272    const chatDir = path.join(chatsPath, item.replace('.png', ''));


273    const stats = {


274        total_gen_time: 0,


275        user_word_count: 0,


276        non_user_word_count: 0,


277        user_msg_count: 0,


278        non_user_msg_count: 0,


279        total_swipe_count: 0,


280        chat_size: 0,


281        date_last_chat: 0,


282        date_first_chat: new Date('9999-12-31T23:59:59.999Z').getTime(),


283    };


284    let uniqueGenStartTimes = new Set();


285


286    if (fs.existsSync(chatDir)) {


287        const chats = fs.readdirSync(chatDir);


288        if (Array.isArray(chats) && chats.length) {


289            for (const chat of chats) {


290                const result = calculateTotalGenTimeAndWordCount(


291                    chatDir,


292                    chat,


293                    uniqueGenStartTimes,


294                );


295                stats.total_gen_time += result.totalGenTime || 0;


296                stats.user_word_count += result.userWordCount || 0;


297                stats.non_user_word_count += result.nonUserWordCount || 0;


298                stats.user_msg_count += result.userMsgCount || 0;


299                stats.non_user_msg_count += result.nonUserMsgCount || 0;


300                stats.total_swipe_count += result.totalSwipeCount || 0;


301


302                const chatStat = fs.statSync(path.join(chatDir, chat));


303                stats.chat_size += chatStat.size;


304                stats.date_last_chat = Math.max(


305                    stats.date_last_chat,


306                    Math.floor(chatStat.mtimeMs),


307                );


308                stats.date_first_chat = Math.min(


309                    stats.date_first_chat,


310                    result.firstChatTime,


311                );


312            }


313        }


314    }


315


316    return { [item]: stats };


317};


318


319/**


320 * Sets the current charStats object.


321 * @param {string} handle - The user handle.


322 * @param {Object} stats - The new charStats object.


323 **/


324function setCharStats(handle, stats) {


325    stats.timestamp = Date.now();


326    STATS.set(handle, stats);


327}


328


329/**


330 * Calculates the total generation time and word count for a chat with a character.


331 *


332 * @param {string} chatDir - The directory path where character chat files are stored.


333 * @param {string} chat - The name of the chat file.


334 * @returns {Object} - An object containing the total generation time, user word count, and non-user word count.


335 * @throws Will throw an error if the file cannot be read or parsed.


336 */


337function calculateTotalGenTimeAndWordCount(


338    chatDir,


339    chat,


340    uniqueGenStartTimes,


341) {


342    let filepath = path.join(chatDir, chat);


343    let lines = readAndParseFile(filepath);


344


345    let totalGenTime = 0;


346    let userWordCount = 0;


347    let nonUserWordCount = 0;


348    let nonUserMsgCount = 0;


349    let userMsgCount = 0;


350    let totalSwipeCount = 0;


351    let firstChatTime = new Date('9999-12-31T23:59:59.999Z').getTime();


352


353    for (let line of lines) {


354        if (line.length) {


355            try {


356                let json = JSON.parse(line);


357                if (json.mes) {


358                    let hash = crypto


359                        .createHash('sha256')


360                        .update(json.mes)


361                        .digest('hex');


362                    if (uniqueGenStartTimes.has(hash)) {


363                        continue;


364                    }


365                    if (hash) {


366                        uniqueGenStartTimes.add(hash);


367                    }


368                }


369


370                if (json.gen_started && json.gen_finished) {


371                    let genTime = calculateGenTime(


372                        json.gen_started,


373                        json.gen_finished,


374                    );


375                    totalGenTime += genTime;


376


377                    if (json.swipes && !json.swipe_info) {


378                        // If there are swipes but no swipe_info, estimate the genTime


379                        totalGenTime += genTime * json.swipes.length;


380                    }


381                }


382


383                if (json.mes) {


384                    let wordCount = countWordsInString(json.mes);


385                    json.is_user


386                        ? (userWordCount += wordCount)


387                        : (nonUserWordCount += wordCount);


388                    json.is_user ? userMsgCount++ : nonUserMsgCount++;


389                }


390


391                if (json.swipes && json.swipes.length > 1) {


392                    totalSwipeCount += json.swipes.length - 1; // Subtract 1 to not count the first swipe


393                    for (let i = 1; i < json.swipes.length; i++) {


394                        // Start from the second swipe


395                        let swipeText = json.swipes[i];


396


397                        let wordCount = countWordsInString(swipeText);


398                        json.is_user


399                            ? (userWordCount += wordCount)


400                            : (nonUserWordCount += wordCount);


401                        json.is_user ? userMsgCount++ : nonUserMsgCount++;


402                    }


403                }


404


405                if (json.swipe_info && json.swipe_info.length > 1) {


406                    for (let i = 1; i < json.swipe_info.length; i++) {


407                        // Start from the second swipe


408                        let swipe = json.swipe_info[i];


409                        if (swipe.gen_started && swipe.gen_finished) {


410                            totalGenTime += calculateGenTime(


411                                swipe.gen_started,


412                                swipe.gen_finished,


413                            );


414                        }


415                    }


416                }


417


418                // If this is the first user message, set the first chat time


419                if (json.is_user) {


420                    //get min between firstChatTime and timestampToMoment(json.send_date)


421                    firstChatTime = Math.min(parseTimestamp(json.send_date), firstChatTime);


422                }


423            } catch (error) {


424                console.error(`Error parsing line ${line}: ${error}`);


425            }


426        }


427    }


428    return {


429        totalGenTime,


430        userWordCount,


431        nonUserWordCount,


432        userMsgCount,


433        nonUserMsgCount,


434        totalSwipeCount,


435        firstChatTime,


436    };


437}


438


439export const router = express.Router();


440


441/**


442 * Handle a POST request to get the stats object


443 */


444router.post('/get', function (request, response) {


445    const stats = STATS.get(request.user.profile.handle) || {};


446    response.send(stats);


447});


448


449/**


450 * Triggers the recreation of statistics from chat files.


451 */


452router.post('/recreate', async function (request, response) {


453    try {


454        await recreateStats(request.user.profile.handle, request.user.directories.chats, request.user.directories.characters);


455        return response.sendStatus(200);


456    } catch (error) {


457        console.error(error);


458        return response.sendStatus(500);


459    }


460});


461


462/**


463 * Handle a POST request to update the stats object


464*/


465router.post('/update', function (request, response) {


466    if (!request.body) return response.sendStatus(400);


467    setCharStats(request.user.profile.handle, request.body);


468    return response.sendStatus(200);


469});