Switch branch
feat/multi-feature-pack maindefault
View all branches
Blame Raw
Cohee · 51ad27fb · · 1124 lines (39.6 KB)
2 contributors 
1import fs from 'node:fs';


2import path from 'node:path';


3import zlib from 'node:zlib';


4import { Buffer } from 'node:buffer';


5


6import express from 'express';


7import fetch from 'node-fetch';


8import sanitize from 'sanitize-filename';


9import { sync as writeFileAtomicSync } from 'write-file-atomic';


10


11import { getConfigValue, color, setPermissionsSync, isValidUrl } from '../util.js';


12import { write } from '../character-card-parser.js';


13import { serverDirectory } from '../server-directory.js';


14import { Jimp, JimpMime } from '../jimp.js';


15import { DEFAULT_AVATAR_PATH } from '../constants.js';


16


17const contentDirectory = path.join(serverDirectory, 'default/content');


18const scaffoldDirectory = path.join(serverDirectory, 'default/scaffold');


19const contentIndexPath = path.join(contentDirectory, 'index.json');


20const scaffoldIndexPath = path.join(scaffoldDirectory, 'index.json');


21


22const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDomains', []);


23const USER_AGENT = 'SillyTavern';


24


25/**


26 * @typedef {Object} ContentItem


27 * @property {string} filename


28 * @property {string} type


29 * @property {string} [name]


30 * @property {string|null} [folder]


31 */


32


33/**


34 * @typedef {string} ContentType


35 * @enum {string}


36 */


37export const CONTENT_TYPES = {


38    SETTINGS: 'settings',


39    CHARACTER: 'character',


40    SPRITES: 'sprites',


41    BACKGROUND: 'background',


42    WORLD: 'world',


43    AVATAR: 'avatar',


44    THEME: 'theme',


45    WORKFLOW: 'workflow',


46    KOBOLD_PRESET: 'kobold_preset',


47    OPENAI_PRESET: 'openai_preset',


48    NOVEL_PRESET: 'novel_preset',


49    TEXTGEN_PRESET: 'textgen_preset',


50    INSTRUCT: 'instruct',


51    CONTEXT: 'context',


52    MOVING_UI: 'moving_ui',


53    QUICK_REPLIES: 'quick_replies',


54    SYSPROMPT: 'sysprompt',


55    REASONING: 'reasoning',


56    ERROR_PAGE: 'error_page',


57    STYLESHEET: 'stylesheet',


58};


59


60/**


61 * @enum {string}


62 */


63export const CONTENT_SCOPE = {


64    USER: 'user',


65    GLOBAL: 'global',


66};


67


68/**


69 * Gets the scope of a content type.


70 * @param {CONTENT_TYPES} type Content type


71 * @returns {CONTENT_SCOPE} Resolved content scope


72 */


73function getScopeByType(type) {


74    const globalTypes = [


75        CONTENT_TYPES.ERROR_PAGE,


76        CONTENT_TYPES.STYLESHEET,


77    ];


78    return globalTypes.includes(type) ? CONTENT_SCOPE.GLOBAL : CONTENT_SCOPE.USER;


79}


80


81/**


82 * Gets the default presets from the content directory.


83 * @param {import('../users.js').UserDirectoryList} directories User directories


84 * @returns {object[]} Array of default presets


85 */


86export function getDefaultPresets(directories) {


87    try {


88        const contentIndex = getContentIndex(CONTENT_SCOPE.USER);


89        const presets = [];


90


91        for (const contentItem of contentIndex) {


92            if (contentItem.type.endsWith('_preset') || ['instruct', 'context', 'sysprompt', 'reasoning'].includes(contentItem.type)) {


93                contentItem.name = path.parse(contentItem.filename).name;


94                contentItem.folder = getUserTargetByType(contentItem.type, directories);


95                presets.push(contentItem);


96            }


97        }


98


99        return presets;


100    } catch (err) {


101        console.warn('Failed to get default presets', err);


102        return [];


103    }


104}


105


106/**


107 * Gets a default JSON file from the content directory.


108 * @param {string} filename Name of the file to get


109 * @returns {object | null} JSON object or null if the file doesn't exist


110 */


111export function getDefaultPresetFile(filename) {


112    try {


113        const contentPath = path.join(contentDirectory, filename);


114


115        if (!fs.existsSync(contentPath)) {


116            return null;


117        }


118


119        const fileContent = fs.readFileSync(contentPath, 'utf8');


120        return JSON.parse(fileContent);


121    } catch (err) {


122        console.warn(`Failed to get default file ${filename}`, err);


123        return null;


124    }


125}


126


127/**


128 * Seeds content from a content index into a target location.


129 * @param {ContentItem[]} contentIndex Content index


130 * @param {string} contentLogPath Path to the content log file


131 * @param {(type: string) => string | null} resolveTarget Function to resolve the target directory for a content type


132 * @param {string[]} [forceCategories] List of categories to force check (even if content check is skipped)


133 * @returns {boolean} Whether any content was added


134 */


135function seedContent(contentIndex, contentLogPath, resolveTarget, forceCategories) {


136    let anyContentAdded = false;


137    const contentLog = getContentLog(contentLogPath);


138


139    for (const contentItem of contentIndex) {


140        if (contentLog.includes(contentItem.filename) && !forceCategories?.includes(contentItem.type)) {


141            continue;


142        }


143


144        if (!contentItem.folder) {


145            console.warn(`Content file ${contentItem.filename} has no parent folder`);


146            continue;


147        }


148


149        const contentPath = path.join(contentItem.folder, contentItem.filename);


150


151        if (!fs.existsSync(contentPath)) {


152            console.warn(`Content file ${contentItem.filename} is missing`);


153            continue;


154        }


155


156        const contentTarget = resolveTarget(contentItem.type);


157


158        if (!contentTarget) {


159            console.warn(`Content file ${contentItem.filename} has unknown type ${contentItem.type}`);


160            continue;


161        }


162


163        const basePath = path.parse(contentItem.filename).base;


164        const targetPath = path.join(contentTarget, basePath);


165        contentLog.push(contentItem.filename);


166


167        if (fs.existsSync(targetPath)) {


168            console.warn(`Content file ${contentItem.filename} already exists in ${contentTarget}`);


169            continue;


170        }


171


172        fs.mkdirSync(contentTarget, { recursive: true });


173        fs.cpSync(contentPath, targetPath, { recursive: true, force: false });


174        setPermissionsSync(targetPath);


175        console.info(`Content file ${contentItem.filename} copied to ${contentTarget}`);


176        anyContentAdded = true;


177    }


178


179    writeFileAtomicSync(contentLogPath, contentLog.join('\n'));


180    return anyContentAdded;


181}


182


183/**


184 * Seeds content for a user.


185 * @param {ContentItem[]} contentIndex Content index


186 * @param {import('../users.js').UserDirectoryList} directories User directories


187 * @param {string[]} forceCategories List of categories to force check (even if content check is skipped)


188 * @returns {Promise<boolean>} Whether any content was added


189 */


190async function seedContentForUser(contentIndex, directories, forceCategories) {


191    if (!fs.existsSync(directories.root)) {


192        fs.mkdirSync(directories.root, { recursive: true });


193    }


194


195    const contentLogPath = path.join(directories.root, 'content.log');


196    return seedContent(contentIndex, contentLogPath, (type) => getUserTargetByType(type, directories), forceCategories);


197}


198


199/**


200 * Seeds global content that is not user-specific, such as error pages.


201 * @param {ContentItem[]} contentIndex Content index


202 * @returns {Promise<boolean>} Whether any content was added


203 */


204async function seedGlobalContent(contentIndex) {


205    const contentLogPath = path.join(globalThis.DATA_ROOT, 'content.log');


206    return seedContent(contentIndex, contentLogPath, getGlobalTargetByType);


207}


208


209/**


210 * Checks for new content and seeds it for all users.


211 * @param {import('../users.js').UserDirectoryList[]} directoriesList List of user directories


212 * @param {string[]} forceCategories List of categories to force check (even if content check is skipped)


213 * @returns {Promise<void>}


214 */


215export async function checkForNewContent(directoriesList, forceCategories = []) {


216    try {


217        const contentCheckSkip = getConfigValue('skipContentCheck', false, 'boolean');


218        if (contentCheckSkip && forceCategories?.length === 0) {


219            return;


220        }


221


222        const userContentIndex = getContentIndex(CONTENT_SCOPE.USER);


223        const globalContentIndex = getContentIndex(CONTENT_SCOPE.GLOBAL);


224        let anyContentAdded = false;


225


226        const globalSeedResult = await seedGlobalContent(globalContentIndex);


227        if (globalSeedResult) {


228            anyContentAdded = true;


229        }


230


231        for (const directories of directoriesList) {


232            const userSeedResult = await seedContentForUser(userContentIndex, directories, forceCategories);


233


234            if (userSeedResult) {


235                anyContentAdded = true;


236            }


237        }


238


239        if (anyContentAdded && !contentCheckSkip && forceCategories?.length === 0) {


240            console.info();


241            console.info(`${color.blue('If you don\'t want to receive content updates in the future, set')} ${color.yellow('skipContentCheck')} ${color.blue('to true in the config.yaml file.')}`);


242            console.info();


243        }


244    } catch (err) {


245        console.error('Content check failed', err);


246    }


247}


248


249/**


250 * Gets combined content index from the content and scaffold directories.


251 * @param {CONTENT_SCOPE} scope Scope of content to get


252 * @returns {ContentItem[]} Array of content index


253 */


254function getContentIndex(scope = CONTENT_SCOPE.USER) {


255    const result = [];


256


257    if (fs.existsSync(scaffoldIndexPath)) {


258        const scaffoldIndexText = fs.readFileSync(scaffoldIndexPath, 'utf8');


259        const scaffoldIndex = JSON.parse(scaffoldIndexText);


260        if (Array.isArray(scaffoldIndex)) {


261            scaffoldIndex.forEach((item) => {


262                item.folder = scaffoldDirectory;


263                item.scope = getScopeByType(item.type);


264            });


265            result.push(...scaffoldIndex);


266        }


267    }


268


269    if (fs.existsSync(contentIndexPath)) {


270        const contentIndexText = fs.readFileSync(contentIndexPath, 'utf8');


271        const contentIndex = JSON.parse(contentIndexText);


272        if (Array.isArray(contentIndex)) {


273            contentIndex.forEach((item) => {


274                item.folder = contentDirectory;


275                item.scope = getScopeByType(item.type);


276            });


277            result.push(...contentIndex);


278        }


279    }


280


281    return result.filter((item) => item.scope === scope);


282}


283


284/**


285 * Gets content by type and format.


286 * @param {string} type Type of content


287 * @param {'json'|'string'|'raw'} format Format of content


288 * @param {CONTENT_SCOPE} scope Scope of content to get


289 * @returns {string[]|Buffer[]} Array of content


290 */


291export function getContentOfType(type, format, scope = CONTENT_SCOPE.USER) {


292    const contentIndex = getContentIndex(scope);


293    const indexItems = contentIndex.filter((item) => item.type === type && item.folder);


294    const files = [];


295    for (const item of indexItems) {


296        if (!item.folder) {


297            continue;


298        }


299        try {


300            const filePath = path.join(item.folder, item.filename);


301            const fileContent = fs.readFileSync(filePath);


302            switch (format) {


303                case 'json':


304                    files.push(JSON.parse(fileContent.toString()));


305                    break;


306                case 'string':


307                    files.push(fileContent.toString());


308                    break;


309                case 'raw':


310                    files.push(fileContent);


311                    break;


312            }


313        } catch {


314            // Ignore errors


315        }


316    }


317    return files;


318}


319


320/**


321 * Gets the target directory for the specified asset type.


322 * @param {ContentType} type Asset type


323 * @param {import('../users.js').UserDirectoryList} directories User directories


324 * @returns {string | null} Target directory


325 */


326export function getUserTargetByType(type, directories) {


327    switch (type) {


328        case CONTENT_TYPES.SETTINGS:


329            return directories.root;


330        case CONTENT_TYPES.CHARACTER:


331            return directories.characters;


332        case CONTENT_TYPES.SPRITES:


333            return directories.characters;


334        case CONTENT_TYPES.BACKGROUND:


335            return directories.backgrounds;


336        case CONTENT_TYPES.WORLD:


337            return directories.worlds;


338        case CONTENT_TYPES.AVATAR:


339            return directories.avatars;


340        case CONTENT_TYPES.THEME:


341            return directories.themes;


342        case CONTENT_TYPES.WORKFLOW:


343            return directories.comfyWorkflows;


344        case CONTENT_TYPES.KOBOLD_PRESET:


345            return directories.koboldAI_Settings;


346        case CONTENT_TYPES.OPENAI_PRESET:


347            return directories.openAI_Settings;


348        case CONTENT_TYPES.NOVEL_PRESET:


349            return directories.novelAI_Settings;


350        case CONTENT_TYPES.TEXTGEN_PRESET:


351            return directories.textGen_Settings;


352        case CONTENT_TYPES.INSTRUCT:


353            return directories.instruct;


354        case CONTENT_TYPES.CONTEXT:


355            return directories.context;


356        case CONTENT_TYPES.MOVING_UI:


357            return directories.movingUI;


358        case CONTENT_TYPES.QUICK_REPLIES:


359            return directories.quickreplies;


360        case CONTENT_TYPES.SYSPROMPT:


361            return directories.sysprompt;


362        case CONTENT_TYPES.REASONING:


363            return directories.reasoning;


364        default:


365            return null;


366    }


367}


368


369/**


370 * Gets the target directory for global content types.


371 * @param {CONTENT_TYPES} type Content type


372 * @returns {string | null} Target directory


373 */


374export function getGlobalTargetByType(type) {


375    switch (type) {


376        case CONTENT_TYPES.ERROR_PAGE:


377            return path.join(globalThis.DATA_ROOT, '_errors');


378        case CONTENT_TYPES.STYLESHEET:


379            return path.join(globalThis.DATA_ROOT, '_css');


380        default:


381            return null;


382    }


383}


384


385/**


386 * Gets the content log from the content log file.


387 * @param {string} contentLogPath Path to the content log file


388 * @returns {string[]} Array of content log lines


389 */


390function getContentLog(contentLogPath) {


391    if (!fs.existsSync(contentLogPath)) {


392        return [];


393    }


394


395    const contentLogText = fs.readFileSync(contentLogPath, 'utf8');


396    return contentLogText.split('\n');


397}


398


399async function downloadChubLorebook(id) {


400    const [lorebooks, creatorName, projectName] = id.split('/');


401    const result = await fetch(`https://api.chub.ai/api/${lorebooks}/${creatorName}/${projectName}`, {


402        method: 'GET',


403        headers: { 'Accept': 'application/json', 'User-Agent': USER_AGENT },


404    });


405


406    if (!result.ok) {


407        const text = await result.text();


408        console.error('Chub returned error', result.statusText, text);


409        throw new Error('Failed to fetch lorebook metadata');


410    }


411


412    /** @type {any} */


413    const metadata = await result.json();


414    const projectId = metadata.node?.id;


415


416    if (!projectId) {


417        throw new Error('Project ID not found in lorebook metadata');


418    }


419


420    const downloadUrl = `https://api.chub.ai/api/v4/projects/${projectId}/repository/files/raw%252Fsillytavern_raw.json/raw`;


421    const downloadResult = await fetch(downloadUrl, {


422        method: 'GET',


423        headers: { 'Accept': 'application/json', 'User-Agent': USER_AGENT },


424    });


425


426    if (!downloadResult.ok) {


427        const text = await downloadResult.text();


428        console.error('Chub returned error', downloadResult.statusText, text);


429        throw new Error('Failed to download lorebook');


430    }


431


432    const name = projectName;


433    const buffer = Buffer.from(await downloadResult.arrayBuffer());


434    const fileName = `${sanitize(name)}.json`;


435    const fileType = downloadResult.headers.get('content-type');


436


437    return { buffer, fileName, fileType };


438}


439


440async function downloadChubCharacter(id) {


441    const [creatorName, projectName] = id.split('/');


442    const result = await fetch(`https://api.chub.ai/api/characters/${creatorName}/${projectName}?full=true`, {


443        method: 'GET',


444        headers: { 'Accept': 'application/json', 'User-Agent': USER_AGENT },


445    });


446


447    if (!result.ok) {


448        const text = await result.text();


449        console.error('Chub returned error', result.statusText, text);


450        throw new Error('Failed to fetch character metadata');


451    }


452


453    /** @type {any} */


454    const metadata = await result.json();


455    const { definition, topics } = metadata.node;


456


457    /** @type {TavernCardV2} */


458    const characterCard = {


459        data: {


460            name: definition.name,


461            description: definition.personality,


462            personality: definition.tavern_personality,


463            scenario: definition.scenario,


464            first_mes: definition.first_message,


465            mes_example: definition.example_dialogs,


466            creator_notes: definition.description,


467            system_prompt: definition.system_prompt,


468            post_history_instructions: definition.post_history_instructions,


469            alternate_greetings: definition.alternate_greetings,


470            tags: topics,


471            creator: creatorName,


472            character_version: '',


473            character_book: definition.embedded_lorebook,


474            extensions: definition.extensions,


475        },


476        spec: 'chara_card_v2',


477        spec_version: '2.0',


478    };


479


480    const defaultAvatarPath = path.join(serverDirectory, DEFAULT_AVATAR_PATH);


481    const defaultAvatarBuffer = fs.readFileSync(defaultAvatarPath);


482


483    let imageBuffer = defaultAvatarBuffer;


484


485    const imageUrl = metadata.node?.max_res_url;


486


487    if (imageUrl) {


488        const downloadResult = await fetch(imageUrl);


489        if (downloadResult.ok) {


490            imageBuffer = Buffer.from(await downloadResult.arrayBuffer());


491        }


492    }


493


494    const buffer = write(imageBuffer, JSON.stringify(characterCard));


495    const fileName = `${sanitize(characterCard.data.name)}.png`;


496    const fileType = 'image/png';


497


498    return { buffer, fileName, fileType };


499}


500


501/**


502 * Downloads a character card from the Pygsite.


503 * @param {string} id UUID of the character


504 * @returns {Promise<{buffer: Buffer, fileName: string, fileType: string}>}


505 */


506async function downloadPygmalionCharacter(id) {


507    const result = await fetch(`https://server.pygmalion.chat/api/export/character/${id}/v2`);


508


509    if (!result.ok) {


510        const text = await result.text();


511        console.error('Pygsite returned error', result.status, text);


512        throw new Error('Failed to download character');


513    }


514


515    /** @type {any} */


516    const jsonData = await result.json();


517    const characterData = jsonData?.character;


518


519    if (!characterData || typeof characterData !== 'object') {


520        console.error('Pygsite returned invalid character data', jsonData);


521        throw new Error('Failed to download character');


522    }


523


524    try {


525        const avatarUrl = characterData?.data?.avatar;


526


527        if (!avatarUrl) {


528            console.error('Pygsite character does not have an avatar', characterData);


529            throw new Error('Failed to download avatar');


530        }


531


532        const avatarResult = await fetch(avatarUrl);


533        const avatarBuffer = Buffer.from(await avatarResult.arrayBuffer());


534


535        const cardBuffer = write(avatarBuffer, JSON.stringify(characterData));


536


537        return {


538            buffer: cardBuffer,


539            fileName: `${sanitize(id)}.png`,


540            fileType: 'image/png',


541        };


542    } catch (e) {


543        console.error('Failed to download avatar, using JSON instead', e);


544        return {


545            buffer: Buffer.from(JSON.stringify(jsonData)),


546            fileName: `${sanitize(id)}.json`,


547            fileType: 'application/json',


548        };


549    }


550}


551


552/**


553 *


554 * @param {String} str


555 * @returns { { id: string, type: "character" | "lorebook" } | null }


556 */


557function parseChubUrl(str) {


558    const splitStr = str.split('/');


559    const length = splitStr.length;


560


561    if (length < 2) {


562        return null;


563    }


564


565    let domainIndex = -1;


566


567    splitStr.forEach((part, index) => {


568        if (part === 'www.chub.ai' || part === 'chub.ai' || part === 'www.characterhub.org' || part === 'characterhub.org') {


569            domainIndex = index;


570        }


571    });


572


573    const lastTwo = domainIndex !== -1 ? splitStr.slice(domainIndex + 1) : splitStr;


574


575    const firstPart = lastTwo[0].toLowerCase();


576


577    if (firstPart === 'characters' || firstPart === 'lorebooks') {


578        const type = firstPart === 'characters' ? 'character' : 'lorebook';


579        const id = type === 'character' ? lastTwo.slice(1).join('/') : lastTwo.join('/');


580        return {


581            id: id,


582            type: type,


583        };


584    } else if (length === 2) {


585        return {


586            id: lastTwo.join('/'),


587            type: 'character',


588        };


589    }


590


591    return null;


592}


593


594// Warning: Some characters might not exist in JannyAI.me


595async function downloadJannyCharacter(uuid) {


596    // This endpoint is being guarded behind Bot Fight Mode of Cloudflare


597    // So hosted ST on Azure/AWS/GCP/Collab might get blocked by IP


598    // Should work normally on self-host PC/Android


599    const result = await fetch('https://api.jannyai.com/api/v1/download', {


600        method: 'POST',


601        headers: { 'Content-Type': 'application/json' },


602        body: JSON.stringify({


603            'characterId': uuid,


604        }),


605    });


606


607    if (result.ok) {


608        /** @type {any} */


609        const downloadResult = await result.json();


610        if (downloadResult.status === 'ok') {


611            const imageResult = await fetch(downloadResult.downloadUrl);


612            const buffer = Buffer.from(await imageResult.arrayBuffer());


613            const fileName = `${sanitize(uuid)}.png`;


614            const fileType = imageResult.headers.get('content-type');


615


616            return { buffer, fileName, fileType };


617        } else {


618            console.error('Janny failed to download', downloadResult);


619        }


620    } else {


621        console.error('Janny returned error', result.statusText, await result.text());


622    }


623


624    throw new Error('Failed to download character');


625}


626


627//Download Character Cards from AICharactersCards.com (AICC) API.


628async function downloadAICCCharacter(id) {


629    const apiURL = `https://aicharactercards.com/wp-json/pngapi/v1/image/${id}`;


630    try {


631        const response = await fetch(apiURL);


632        if (!response.ok) {


633            throw new Error(`Failed to download character: ${response.statusText}`);


634        }


635


636        const contentType = response.headers.get('content-type') || 'image/png'; // Default to 'image/png' if header is missing


637        const buffer = Buffer.from(await response.arrayBuffer());


638        const fileName = `${sanitize(id)}.png`; // Assuming PNG, but adjust based on actual content or headers


639


640        return {


641            buffer: buffer,


642            fileName: fileName,


643            fileType: contentType,


644        };


645    } catch (error) {


646        console.error('Error downloading character:', error);


647        throw error;


648    }


649}


650


651/**


652 * Parses an aicharactercards URL to extract the path.


653 * @param {string} url URL to parse


654 * @returns {string | null} AICC path


655 */


656function parseAICC(url) {


657    try {


658        if (isValidUrl(url)) {


659            const urlObj = new URL(url);


660            // Split the path and remove empty strings caused by trailing slashes


661            const parts = urlObj.pathname.split('/').filter(Boolean);


662            if (parts.length >= 2) {


663                // Always grab the last two segments (author/character)


664                return `${parts[parts.length - 2]}/${parts[parts.length - 1]}`;


665            }


666        } else {


667            // Fallback for relative paths or raw "author/character" strings


668            const parts = url.split('/').filter(Boolean);


669            if (parts.length >= 2) {


670                return `${parts[parts.length - 2]}/${parts[parts.length - 1]}`;


671            }


672        }


673    } catch (e) {


674        console.error('Error parsing AICC URL:', e);


675    }


676    return null;


677}


678


679/**


680 * Download character card from generic url.


681 * @param {String} url


682 */


683async function downloadGenericPng(url) {


684    try {


685        const result = await fetch(url);


686


687        if (result.ok) {


688            const buffer = Buffer.from(await result.arrayBuffer());


689            let fileName = sanitize(result.url.split('?')[0].split('/').reverse()[0]);


690            const contentType = result.headers.get('content-type') || 'image/png'; //yoink it from AICC function lol


691


692            // The `importCharacter()` function detects the MIME (content-type) of the file


693            // using its file extension. The problem is that not all third-party APIs serve


694            // their cards with a `.png` extension. To support more third-party sites,


695            // dynamically append the `.png` extension to the filename if it doesn't


696            // already have a file extension.


697            if (contentType === 'image/png') {


698                const ext = fileName.match(/\.(\w+)$/); // Same regex used by `importCharacter()`


699                if (!ext) {


700                    fileName += '.png';


701                }


702            }


703


704            return {


705                buffer: buffer,


706                fileName: fileName,


707                fileType: contentType,


708            };


709        }


710    } catch (error) {


711        console.error('Error downloading file: ', error);


712        throw error;


713    }


714    return null;


715}


716


717/**


718 * Parse Risu Realm URL to extract the UUID.


719 * @param {string} url Risu Realm URL


720 * @returns {string | null} UUID of the character


721 */


722function parseRisuUrl(url) {


723    // Example: https://realm.risuai.net/character/7adb0ed8d81855c820b3506980fb40f054ceef010ff0c4bab73730c0ebe92279


724    // or https://realm.risuai.net/character/7adb0ed8-d818-55c8-20b3-506980fb40f0


725    const pattern = /^https?:\/\/realm\.risuai\.net\/character\/([a-f0-9-]+)\/?$/i;


726    const match = url.match(pattern);


727    return match ? match[1] : null;


728}


729


730/**


731 * Download RisuAI character card


732 * @param {string} uuid UUID of the character


733 * @returns {Promise<{buffer: Buffer, fileName: string, fileType: string}>}


734 */


735async function downloadRisuCharacter(uuid) {


736    const result = await fetch(`https://realm.risuai.net/api/v1/download/png-v3/${uuid}?non_commercial=true`);


737


738    if (!result.ok) {


739        const text = await result.text();


740        console.error('RisuAI returned error', result.statusText, text);


741        throw new Error('Failed to download character');


742    }


743


744    const buffer = Buffer.from(await result.arrayBuffer());


745    const fileName = `${sanitize(uuid)}.png`;


746    const fileType = 'image/png';


747


748    return { buffer, fileName, fileType };


749}


750


751/** * Check if the given string is a valid Perchance UUID.


752 * @param {string} uuid UUID string to check


753 * @returns {boolean} True if the UUID is valid, false otherwise


754 */


755function isPerchanceUUID(uuid) {


756    if (!uuid) {


757        return false;


758    }


759


760    //example: Personality_Advisor~6903e991c90fd1dba52c036d917e99c6.gz


761    //charactername~uuid.gz


762


763    const uuidRegex = /^\w+~[a-f0-9]{32}\.gz$/;


764    return uuidRegex.test(uuid);


765}


766


767/**


768 * Parse Perchance URL to extract the character slug.


769 * @param {string} url Perchance character URL


770 * @returns {string} Slug of the character


771 */


772function parsePerchanceSlug(url) {


773    // Example: https://perchance.org/ai-character-chat?data=Personality_Advisor~6903e991c90fd1dba52c036d917e99c6.gz


774    // or: Personality_Advisor~6903e991c90fd1dba52c036d917e99c6.gz


775    return url?.split('~')[1] || '';


776}


777


778/**


779 * Download Perchance character card


780 * @param {string} slug Slug of the character


781 * @returns {Promise<{buffer: Buffer, fileName: string, fileType: string} | null>}


782 */


783async function downloadPerchanceCharacter(slug) {


784    // example of slug


785    // 6903e991c90fd1dba52c036d917e99c6.gz


786    const perchanceBaseURL = 'https://user.uploads.dev/file';


787


788    try {


789        const charURL = `${perchanceBaseURL}/${slug}`;


790        console.log('Downloading Perchance character from URL:', charURL);


791        const result = await fetch(charURL, {


792            headers: { 'Content-Type': 'application/json', 'User-Agent': USER_AGENT },


793        });


794


795        //decompress gzipped content


796        if (result.ok) {


797            const perchanceChar = await extractPerchanceCharacterFromGz(result);


798


799            const avatarUrl = perchanceChar.avatar?.url;


800


801            //check if avatarURL is a base64 of any image type


802            const isAvatarBase64 = avatarUrl && avatarUrl.startsWith('data:image/');


803


804            const charData = {


805                name: perchanceChar.name || 'Unnamed Perchance Character',


806                first_mes: '',


807                tags: [],


808                description: perchanceChar.roleInstruction || '',


809                creator: perchanceChar.metaTitle || '',


810                creator_notes: perchanceChar.metaDescription || '',


811                alternate_greetings: [],


812                character_version: '',


813                mes_example: '',


814                post_history_instructions: '',


815                system_prompt: '',


816                scenario: '',


817                personality: perchanceChar.reminderMessage || '',


818                extensions: {


819                    perchance_data: {


820                        slug: slug,


821                        char_url: charURL,


822                        uuid: perchanceChar.uuid || null,


823                        avatar_url: isAvatarBase64 ? null : (avatarUrl || null),


824                        folder_path: perchanceChar.folderPath || null,


825                        folder_name: perchanceChar.folderName || null,


826                        custom_data: perchanceChar.customData || {},


827                    },


828                },


829            };


830


831            const avatarBuffer = await fetchPerchanceAvatar(avatarUrl, isAvatarBase64);


832


833            // Character card


834            const buffer = write(avatarBuffer, JSON.stringify({


835                'spec': 'chara_card_v2',


836                'spec_version': '2.0',


837                'data': charData,


838            }));


839


840            const fileName = `${charData.name}.png`;


841            const fileType = 'image/png';


842


843            return { buffer, fileName, fileType };


844        }


845    } catch (error) {


846        console.error('Error downloading character:', error);


847        throw error;


848    }


849    return null;


850}


851


852/**


853 * Extracts Perchance character data from a gzipped response.


854 * @param {import('node-fetch').Response} result Fetch response containing gzipped character data


855 * @returns {Promise<Object>} Parsed Perchance character data


856 * @throws {Error} If the character data is invalid or missing required fields


857 */


858async function extractPerchanceCharacterFromGz(result) {


859    const compressedBuffer = await result.arrayBuffer();


860    const decompressedBuffer = zlib.gunzipSync(compressedBuffer);


861


862    // inside the gz file, there is a file of the same name without extensions, but it is a json file


863


864    if (!decompressedBuffer || decompressedBuffer.length === 0) {


865        console.error('Perchance character data is empty or invalid');


866        throw new Error('Failed to download character: Invalid Perchance character data');


867    }


868


869    // Parse the decompressed JSON


870    const perchanceCharData = JSON.parse(decompressedBuffer.toString());


871


872    if (!perchanceCharData?.addCharacter) {


873        console.error('Perchance character data is missing addCharacter field', perchanceCharData);


874        throw new Error('Failed to download character: Invalid Perchance character data');


875    }


876


877    return perchanceCharData.addCharacter;


878}


879


880/** * Fetches the avatar from Perchance URL or uses a default avatar if not available.


881 * @param {string} avatarUrl URL of the avatar


882 * @param {boolean} isAvatarBase64 Flag indicating if the avatar URL is a base64 string


883 * @returns {Promise<Buffer>} Buffer containing the avatar image


884 */


885async function fetchPerchanceAvatar(avatarUrl, isAvatarBase64) {


886    const defaultAvatarPath = path.join(serverDirectory, DEFAULT_AVATAR_PATH);


887    const defaultAvatarBuffer = fs.readFileSync(defaultAvatarPath);


888


889    if (!avatarUrl || (!isAvatarBase64 && !isValidUrl(avatarUrl))) {


890        console.warn('Perchance character does not have an avatar, it is not base64, or it is an invalid url, using default avatar');


891        return defaultAvatarBuffer;


892    }


893


894    if (isAvatarBase64) {


895        // check if avatarUrl is a png


896        const isPng = avatarUrl.startsWith('data:image/png;base64,');


897        const base64 = avatarUrl.split(',')[1];


898        const buffer = Buffer.from(base64, 'base64');


899


900        if (isPng) {


901            return buffer;


902        } else {


903            // use jimp to convert the base64 to PNG if it's not PNG


904            console.debug('Perchance character avatar is not PNG, converting to PNG...');


905            return await Jimp.read(buffer).then(image => image.getBuffer(JimpMime.png));


906        }


907    }


908


909    // Fetch avatar from URL


910    console.log('Fetching Perchance avatar from URL:', avatarUrl);


911    const avatarResponse = await fetch(avatarUrl, { headers: { 'User-Agent': USER_AGENT } });


912


913    if (avatarResponse.ok) {


914        const avatarContentType = avatarResponse.headers.get('content-type');


915        const avatarBuffer = Buffer.from(await avatarResponse.arrayBuffer());


916


917        if (avatarContentType === 'image/png') {


918            return avatarBuffer;


919        } else {


920            console.debug(`Perchance character avatar is not PNG: ${avatarContentType}. Converting to PNG...`);


921


922            // use jimp to convert the image to PNG if it's not PNG


923            return await Jimp.read(avatarBuffer)


924                .then(image => image.getBuffer(JimpMime.png));


925        }


926    }


927


928    console.error('Failed to fetch Perchance avatar:', avatarResponse.statusText);


929    const isPerchanceOrgFileUploader = avatarUrl.includes('https://user-uploads.perchance.org');


930


931    if (isPerchanceOrgFileUploader) {


932        console.warn('Files from https://user-uploads.perchance.org are sometimes blocked by CloudFlare, try reuploading it in https://perchance.org/upload to get the new link from https://user-uploads.dev instead.');


933    }


934


935    console.warn('You can also download the avatar manually and assign it to the character:', avatarUrl);


936    return defaultAvatarBuffer;


937}


938


939/**


940* @param {String} url


941* @returns {String | null } UUID of the character


942*/


943function getUuidFromUrl(url) {


944    // Extract UUID from URL


945    const uuidRegex = /[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/;


946    const matches = url.match(uuidRegex);


947


948    // Check if UUID is found


949    const uuid = matches ? matches[0] : null;


950    return uuid;


951}


952


953/**


954 * Filter to get the domain host of a url instead of a blanket string search.


955 * @param {String} url URL to strip


956 * @returns {String} Domain name


957 */


958export function getHostFromUrl(url) {


959    try {


960        const urlObj = new URL(url);


961        return urlObj.hostname;


962    } catch {


963        return '';


964    }


965}


966


967/**


968 * Checks if host is part of generic download source whitelist.


969 * @param {String} host Host to check


970 * @returns {boolean} If the host is on the whitelist.


971 */


972export function isHostWhitelisted(host) {


973    return WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES.includes(host);


974}


975


976export const router = express.Router();


977


978router.post('/importURL', async (request, response) => {


979    if (!request.body.url) {


980        return response.sendStatus(400);


981    }


982


983    try {


984        const url = request.body.url;


985        const host = getHostFromUrl(url);


986        let result;


987        let type;


988


989        const isChub = host.includes('chub.ai') || host.includes('characterhub.org');


990        const isJannnyContent = host.includes('janitorai');


991        const isPygmalionContent = host.includes('pygmalion.chat');


992        const isAICharacterCardsContent = host.includes('aicharactercards.com');


993        const isRisu = host.includes('realm.risuai.net');


994        const isPerchance = host.includes('perchance.org');


995        const isGeneric = isHostWhitelisted(host);


996


997        if (isPygmalionContent) {


998            const uuid = getUuidFromUrl(url);


999            if (!uuid) {


1000                return response.sendStatus(404);


1001            }


1002


1003            type = 'character';


1004            result = await downloadPygmalionCharacter(uuid);


1005        } else if (isJannnyContent) {


1006            const uuid = getUuidFromUrl(url);


1007            if (!uuid) {


1008                return response.sendStatus(404);


1009            }


1010


1011            type = 'character';


1012            result = await downloadJannyCharacter(uuid);


1013        } else if (isAICharacterCardsContent) {


1014            const AICCParsed = parseAICC(url);


1015            if (!AICCParsed) {


1016                return response.sendStatus(404);


1017            }


1018            type = 'character';


1019            result = await downloadAICCCharacter(AICCParsed);


1020        } else if (isChub) {


1021            const chubParsed = parseChubUrl(url);


1022            type = chubParsed?.type;


1023


1024            if (chubParsed?.type === 'character') {


1025                console.info('Downloading chub character:', chubParsed.id);


1026                result = await downloadChubCharacter(chubParsed.id);


1027            } else if (chubParsed?.type === 'lorebook') {


1028                console.info('Downloading chub lorebook:', chubParsed.id);


1029                result = await downloadChubLorebook(chubParsed.id);


1030            } else {


1031                return response.sendStatus(404);


1032            }


1033        } else if (isRisu) {


1034            const uuid = parseRisuUrl(url);


1035            if (!uuid) {


1036                return response.sendStatus(404);


1037            }


1038


1039            type = 'character';


1040            result = await downloadRisuCharacter(uuid);


1041        } else if (isPerchance) {


1042            const perchanceSlug = parsePerchanceSlug(url);


1043            if (!perchanceSlug) {


1044                return response.sendStatus(404);


1045            }


1046            type = 'character';


1047            result = await downloadPerchanceCharacter(perchanceSlug);


1048        } else if (isGeneric) {


1049            console.info('Downloading from generic url:', url);


1050            type = 'character';


1051            result = await downloadGenericPng(url);


1052        } else {


1053            console.error(`Received an import for "${getHostFromUrl(url)}", but site is not whitelisted. This domain must be added to the config key "whitelistImportDomains" to allow import from this source.`);


1054            return response.sendStatus(404);


1055        }


1056


1057        if (!result) {


1058            return response.sendStatus(404);


1059        }


1060


1061        if (result.fileType) response.set('Content-Type', result.fileType);


1062        response.set('Content-Disposition', `attachment; filename="${encodeURI(result.fileName)}"`);


1063        response.set('X-Custom-Content-Type', type);


1064        return response.send(result.buffer);


1065    } catch (error) {


1066        console.error('Importing custom content failed', error);


1067        return response.sendStatus(500);


1068    }


1069});


1070


1071router.post('/importUUID', async (request, response) => {


1072    if (!request.body.url) {


1073        return response.sendStatus(400);


1074    }


1075


1076    try {


1077        const uuid = request.body.url;


1078        let result;


1079


1080        const isJannny = uuid.includes('_character');


1081        const isPygmalion = (!isJannny && uuid.length == 36);


1082        const isAICC = uuid.startsWith('AICC/');


1083        const isPerchance = isPerchanceUUID(uuid);


1084        const uuidType = uuid.includes('lorebook') ? 'lorebook' : 'character';


1085


1086        if (isPygmalion) {


1087            console.info('Downloading Pygmalion character:', uuid);


1088            result = await downloadPygmalionCharacter(uuid);


1089        } else if (isJannny) {


1090            console.info('Downloading Janitor character:', uuid.split('_')[0]);


1091            result = await downloadJannyCharacter(uuid.split('_')[0]);


1092        } else if (isAICC) {


1093            const [, author, card] = uuid.split('/');


1094            console.info('Downloading AICC character:', `${author}/${card}`);


1095            result = await downloadAICCCharacter(`${author}/${card}`);


1096        } else if (isPerchance) {


1097            console.info('Downloading Perchance character:', uuid);


1098            const parsedUuid = parsePerchanceSlug(uuid);


1099            result = await downloadPerchanceCharacter(parsedUuid);


1100        } else {


1101            if (uuidType === 'character') {


1102                console.info('Downloading chub character:', uuid);


1103                result = await downloadChubCharacter(uuid);


1104            } else if (uuidType === 'lorebook') {


1105                console.info('Downloading chub lorebook:', uuid);


1106                result = await downloadChubLorebook(uuid);


1107            } else {


1108                return response.sendStatus(404);


1109            }


1110        }


1111


1112        if (!result) {


1113            throw new Error('Failed to download content');


1114        }


1115


1116        if (result.fileType) response.set('Content-Type', result.fileType);


1117        response.set('Content-Disposition', `attachment; filename="${result.fileName}"`);


1118        response.set('X-Custom-Content-Type', uuidType);


1119        return response.send(result.buffer);


1120    } catch (error) {


1121        console.error('Importing custom content failed', error);


1122        return response.sendStatus(500);


1123    }


1124});