Switch branch
feat/multi-feature-pack maindefault
View all branches
Blame Raw
Cohee · e3f41666 · · 381 lines (12.7 KB)
1 contributor 
1import fs from 'node:fs';


2import path from 'node:path';


3


4import express from 'express';


5import _ from 'lodash';


6import { sync as writeFileAtomicSync } from 'write-file-atomic';


7import bytes from 'bytes';


8


9import { SETTINGS_FILE } from '../constants.js';


10import { getConfigValue, generateTimestamp, removeOldBackups } from '../util.js';


11import { getAllUserHandles, getUserDirectories } from '../users.js';


12import { getFileNameValidationFunction } from '../middleware/validateFileName.js';


13


14const ENABLE_EXTENSIONS = !!getConfigValue('extensions.enabled', true, 'boolean');


15const ENABLE_EXTENSIONS_AUTO_UPDATE = !!getConfigValue('extensions.autoUpdate', true, 'boolean');


16const ENABLE_ACCOUNTS = !!getConfigValue('enableUserAccounts', false, 'boolean');


17const ENABLE_REQUEST_COMPRESSION = !!getConfigValue('performance.requestCompression.enabled', false, 'boolean');


18const REQUEST_COMPRESSION_MIN = bytes.parse(getConfigValue('performance.requestCompression.minPayloadSize', '256kb'));


19const REQUEST_COMPRESSION_MAX = bytes.parse(getConfigValue('performance.requestCompression.maxPayloadSize', '8mb'));


20const REQUEST_COMPRESSION_TIMEOUT = Number(getConfigValue('performance.requestCompression.timeout', 3000, 'number'));


21


22// 10 minutes


23const AUTOSAVE_INTERVAL = 10 * 60 * 1000;


24


25/**


26 * Map of functions to trigger settings autosave for a user.


27 * @type {Map<string, function>}


28 */


29const AUTOSAVE_FUNCTIONS = new Map();


30


31/**


32 * Triggers autosave for a user every 10 minutes.


33 * @param {string} handle User handle


34 * @returns {void}


35 */


36function triggerAutoSave(handle) {


37    if (!AUTOSAVE_FUNCTIONS.has(handle)) {


38        const throttledAutoSave = _.throttle(() => backupUserSettings(handle, true), AUTOSAVE_INTERVAL);


39        AUTOSAVE_FUNCTIONS.set(handle, throttledAutoSave);


40    }


41


42    const functionToCall = AUTOSAVE_FUNCTIONS.get(handle);


43    if (functionToCall && typeof functionToCall === 'function') {


44        functionToCall();


45    }


46}


47


48/**


49 * Reads and parses files from a directory.


50 * @param {string} directoryPath Path to the directory


51 * @param {string} fileExtension File extension


52 * @returns {Array} Parsed files


53 */


54function readAndParseFromDirectory(directoryPath, fileExtension = '.json') {


55    const files = fs


56        .readdirSync(directoryPath)


57        .filter(x => path.parse(x).ext == fileExtension)


58        .sort();


59


60    const parsedFiles = [];


61


62    files.forEach(item => {


63        try {


64            const file = fs.readFileSync(path.join(directoryPath, item), 'utf-8');


65            parsedFiles.push(fileExtension == '.json' ? JSON.parse(file) : file);


66        } catch {


67            // skip


68        }


69    });


70


71    return parsedFiles;


72}


73


74/**


75 * Gets a sort function for sorting strings.


76 * @param {*} _


77 * @returns {(a: string, b: string) => number} Sort function


78 */


79function sortByName(_) {


80    return (a, b) => a.localeCompare(b);


81}


82


83/**


84 * Gets backup file prefix for user settings.


85 * @param {string} handle User handle


86 * @returns {string} File prefix


87 */


88export function getSettingsBackupFilePrefix(handle) {


89    return `settings_${handle}_`;


90}


91


92function readPresetsFromDirectory(directoryPath, options = {}) {


93    const {


94        sortFunction,


95        removeFileExtension = false,


96        fileExtension = '.json',


97    } = options;


98


99    const files = fs.readdirSync(directoryPath).sort(sortFunction).filter(x => path.parse(x).ext == fileExtension);


100    const fileContents = [];


101    const fileNames = [];


102


103    files.forEach(item => {


104        try {


105            const file = fs.readFileSync(path.join(directoryPath, item), 'utf8');


106            JSON.parse(file);


107            fileContents.push(file);


108            fileNames.push(removeFileExtension ? item.replace(/\.[^/.]+$/, '') : item);


109        } catch {


110            // skip


111            console.warn(`${item} is not a valid JSON`);


112        }


113    });


114


115    return { fileContents, fileNames };


116}


117


118async function backupSettings() {


119    try {


120        const userHandles = await getAllUserHandles();


121


122        for (const handle of userHandles) {


123            backupUserSettings(handle, true);


124        }


125    } catch (err) {


126        console.error('Could not backup settings file', err);


127    }


128}


129


130/**


131 * Makes a backup of the user's settings file.


132 * @param {string} handle User handle


133 * @param {boolean} preventDuplicates Prevent duplicate backups


134 * @returns {void}


135 */


136function backupUserSettings(handle, preventDuplicates) {


137    const userDirectories = getUserDirectories(handle);


138


139    if (!fs.existsSync(userDirectories.root)) {


140        return;


141    }


142


143    const backupFile = path.join(userDirectories.backups, `${getSettingsBackupFilePrefix(handle)}${generateTimestamp()}.json`);


144    const sourceFile = path.join(userDirectories.root, SETTINGS_FILE);


145


146    if (preventDuplicates && isDuplicateBackup(handle, sourceFile)) {


147        return;


148    }


149


150    if (!fs.existsSync(sourceFile)) {


151        return;


152    }


153


154    fs.copyFileSync(sourceFile, backupFile);


155    removeOldBackups(userDirectories.backups, `settings_${handle}`);


156}


157


158/**


159 * Checks if the backup would be a duplicate.


160 * @param {string} handle User handle


161 * @param {string} sourceFile Source file path


162 * @returns {boolean} True if the backup is a duplicate


163 */


164function isDuplicateBackup(handle, sourceFile) {


165    const latestBackup = getLatestBackup(handle);


166    if (!latestBackup) {


167        return false;


168    }


169    return areFilesEqual(latestBackup, sourceFile);


170}


171


172/**


173 * Returns true if the two files are equal.


174 * @param {string} file1 File path


175 * @param {string} file2 File path


176 */


177function areFilesEqual(file1, file2) {


178    if (!fs.existsSync(file1) || !fs.existsSync(file2)) {


179        return false;


180    }


181


182    const content1 = fs.readFileSync(file1);


183    const content2 = fs.readFileSync(file2);


184    return content1.toString() === content2.toString();


185}


186


187/**


188 * Gets the latest backup file for a user.


189 * @param {string} handle User handle


190 * @returns {string|null} Latest backup file. Null if no backup exists.


191 */


192function getLatestBackup(handle) {


193    const userDirectories = getUserDirectories(handle);


194    const backupFiles = fs.readdirSync(userDirectories.backups)


195        .filter(x => x.startsWith(getSettingsBackupFilePrefix(handle)))


196        .map(x => ({ name: x, ctime: fs.statSync(path.join(userDirectories.backups, x)).ctimeMs }));


197    const latestBackup = backupFiles.sort((a, b) => b.ctime - a.ctime)[0]?.name;


198    if (!latestBackup) {


199        return null;


200    }


201    return path.join(userDirectories.backups, latestBackup);


202}


203


204export const router = express.Router();


205


206router.post('/save', function (request, response) {


207    try {


208        const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE);


209        writeFileAtomicSync(pathToSettings, JSON.stringify(request.body, null, 4), 'utf8');


210        triggerAutoSave(request.user.profile.handle);


211        response.send({ result: 'ok' });


212    } catch (err) {


213        console.error(err);


214        response.send(err);


215    }


216});


217


218// Wintermute's code


219router.post('/get', (request, response) => {


220    let settings;


221    try {


222        const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE);


223        settings = fs.readFileSync(pathToSettings, 'utf8');


224    } catch (e) {


225        return response.sendStatus(500);


226    }


227


228    // NovelAI Settings


229    const { fileContents: novelai_settings, fileNames: novelai_setting_names }


230        = readPresetsFromDirectory(request.user.directories.novelAI_Settings, {


231            sortFunction: sortByName(request.user.directories.novelAI_Settings),


232            removeFileExtension: true,


233        });


234


235    // OpenAI Settings


236    const { fileContents: openai_settings, fileNames: openai_setting_names }


237        = readPresetsFromDirectory(request.user.directories.openAI_Settings, {


238            sortFunction: sortByName(request.user.directories.openAI_Settings), removeFileExtension: true,


239        });


240


241    // TextGenerationWebUI Settings


242    const { fileContents: textgenerationwebui_presets, fileNames: textgenerationwebui_preset_names }


243        = readPresetsFromDirectory(request.user.directories.textGen_Settings, {


244            sortFunction: sortByName(request.user.directories.textGen_Settings), removeFileExtension: true,


245        });


246


247    //Kobold


248    const { fileContents: koboldai_settings, fileNames: koboldai_setting_names }


249        = readPresetsFromDirectory(request.user.directories.koboldAI_Settings, {


250            sortFunction: sortByName(request.user.directories.koboldAI_Settings), removeFileExtension: true,


251        });


252


253    const worldFiles = fs


254        .readdirSync(request.user.directories.worlds)


255        .filter(file => path.extname(file).toLowerCase() === '.json')


256        .sort((a, b) => a.localeCompare(b));


257    const world_names = worldFiles.map(item => path.parse(item).name);


258


259    const themes = readAndParseFromDirectory(request.user.directories.themes);


260    const movingUIPresets = readAndParseFromDirectory(request.user.directories.movingUI);


261    const quickReplyPresets = readAndParseFromDirectory(request.user.directories.quickreplies);


262


263    const instruct = readAndParseFromDirectory(request.user.directories.instruct);


264    const context = readAndParseFromDirectory(request.user.directories.context);


265    const sysprompt = readAndParseFromDirectory(request.user.directories.sysprompt);


266    const reasoning = readAndParseFromDirectory(request.user.directories.reasoning);


267


268    response.send({


269        settings,


270        koboldai_settings,


271        koboldai_setting_names,


272        world_names,


273        novelai_settings,


274        novelai_setting_names,


275        openai_settings,


276        openai_setting_names,


277        textgenerationwebui_presets,


278        textgenerationwebui_preset_names,


279        themes,


280        movingUIPresets,


281        quickReplyPresets,


282        instruct,


283        context,


284        sysprompt,


285        reasoning,


286        enable_extensions: ENABLE_EXTENSIONS,


287        enable_extensions_auto_update: ENABLE_EXTENSIONS_AUTO_UPDATE,


288        enable_accounts: ENABLE_ACCOUNTS,


289        request_compression: {


290            enabled: ENABLE_REQUEST_COMPRESSION,


291            minPayloadSize: REQUEST_COMPRESSION_MIN || 0,


292            maxPayloadSize: REQUEST_COMPRESSION_MAX || 0,


293            timeout: REQUEST_COMPRESSION_TIMEOUT || 0,


294        },


295    });


296});


297


298router.post('/get-snapshots', async (request, response) => {


299    try {


300        const snapshots = fs.readdirSync(request.user.directories.backups);


301        const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);


302        const userSnapshots = snapshots.filter(x => x.startsWith(userFilesPattern));


303


304        const result = userSnapshots.map(x => {


305            const stat = fs.statSync(path.join(request.user.directories.backups, x));


306            return { date: stat.ctimeMs, name: x, size: stat.size };


307        });


308


309        response.json(result);


310    } catch (error) {


311        console.error(error);


312        response.sendStatus(500);


313    }


314});


315


316router.post('/load-snapshot', getFileNameValidationFunction('name'), async (request, response) => {


317    try {


318        const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);


319


320        if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {


321            return response.status(400).send({ error: 'Invalid snapshot name' });


322        }


323


324        const snapshotName = request.body.name;


325        const snapshotPath = path.join(request.user.directories.backups, snapshotName);


326


327        if (!fs.existsSync(snapshotPath)) {


328            return response.sendStatus(404);


329        }


330


331        const content = fs.readFileSync(snapshotPath, 'utf8');


332


333        response.send(content);


334    } catch (error) {


335        console.error(error);


336        response.sendStatus(500);


337    }


338});


339


340router.post('/make-snapshot', async (request, response) => {


341    try {


342        backupUserSettings(request.user.profile.handle, false);


343        response.sendStatus(204);


344    } catch (error) {


345        console.error(error);


346        response.sendStatus(500);


347    }


348});


349


350router.post('/restore-snapshot', getFileNameValidationFunction('name'), async (request, response) => {


351    try {


352        const userFilesPattern = getSettingsBackupFilePrefix(request.user.profile.handle);


353


354        if (!request.body.name || !request.body.name.startsWith(userFilesPattern)) {


355            return response.status(400).send({ error: 'Invalid snapshot name' });


356        }


357


358        const snapshotName = request.body.name;


359        const snapshotPath = path.join(request.user.directories.backups, snapshotName);


360


361        if (!fs.existsSync(snapshotPath)) {


362            return response.sendStatus(404);


363        }


364


365        const pathToSettings = path.join(request.user.directories.root, SETTINGS_FILE);


366        fs.rmSync(pathToSettings, { force: true });


367        fs.copyFileSync(snapshotPath, pathToSettings);


368


369        response.sendStatus(204);


370    } catch (error) {


371        console.error(error);


372        response.sendStatus(500);


373    }


374});


375


376/**


377 * Initializes the settings endpoint


378 */


379export async function init() {


380    await backupSettings();


381}