Move config migration from post-install to src

005a495e964da2fe85636ff731cb86f25bdcc614

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 01c6544e22a3584c78bf9aa72bcee415bb2d910e
6 files changed, +204 -186Ignore whitespace
.github/readme.md+1 -0
@@ -350,6 +350,7 @@ Start.bat --port 8000 --listen false
350| Option | Description | Type |350| Option | Description | Type |
351|-------------------------|----------------------------------------------------------------------|----------|351|-------------------------|----------------------------------------------------------------------|----------|
352| `--version` | Show version number | boolean |352| `--version` | Show version number | boolean |
353| `--configPath` | Override the path to the config.yaml file | string |
353| `--dataRoot` | Root directory for data storage | string |354| `--dataRoot` | Root directory for data storage | string |
354| `--port` | Sets the port under which SillyTavern will run | number |355| `--port` | Sets the port under which SillyTavern will run | number |
355| `--listen` | SillyTavern will listen on all network interfaces | boolean |356| `--listen` | SillyTavern will listen on all network interfaces | boolean |
post-install.js+2 -182
@@ -5,130 +5,15 @@ import fs from 'node:fs';
5import path from 'node:path';5import path from 'node:path';
6import process from 'node:process';6import process from 'node:process';
7import yaml from 'yaml';7import yaml from 'yaml';
8import _ from 'lodash';
9import chalk from 'chalk';8import chalk from 'chalk';
10import { createRequire } from 'node:module';9import { createRequire } from 'node:module';
10import { addMissingConfigValues } from './src/config-init.js';
1111
12/**12/**
13 * Colorizes console output.13 * Colorizes console output.
14 */14 */
15const color = chalk;15const color = chalk;
1616
17const keyMigrationMap = [
18 {
19 oldKey: 'disableThumbnails',
20 newKey: 'thumbnails.enabled',
21 migrate: (value) => !value,
22 },
23 {
24 oldKey: 'thumbnailsQuality',
25 newKey: 'thumbnails.quality',
26 migrate: (value) => value,
27 },
28 {
29 oldKey: 'avatarThumbnailsPng',
30 newKey: 'thumbnails.format',
31 migrate: (value) => (value ? 'png' : 'jpg'),
32 },
33 {
34 oldKey: 'disableChatBackup',
35 newKey: 'backups.chat.enabled',
36 migrate: (value) => !value,
37 },
38 {
39 oldKey: 'numberOfBackups',
40 newKey: 'backups.common.numberOfBackups',
41 migrate: (value) => value,
42 },
43 {
44 oldKey: 'maxTotalChatBackups',
45 newKey: 'backups.chat.maxTotalBackups',
46 migrate: (value) => value,
47 },
48 {
49 oldKey: 'chatBackupThrottleInterval',
50 newKey: 'backups.chat.throttleInterval',
51 migrate: (value) => value,
52 },
53 {
54 oldKey: 'enableExtensions',
55 newKey: 'extensions.enabled',
56 migrate: (value) => value,
57 },
58 {
59 oldKey: 'enableExtensionsAutoUpdate',
60 newKey: 'extensions.autoUpdate',
61 migrate: (value) => value,
62 },
63 {
64 oldKey: 'extras.disableAutoDownload',
65 newKey: 'extensions.models.autoDownload',
66 migrate: (value) => !value,
67 },
68 {
69 oldKey: 'extras.classificationModel',
70 newKey: 'extensions.models.classification',
71 migrate: (value) => value,
72 },
73 {
74 oldKey: 'extras.captioningModel',
75 newKey: 'extensions.models.captioning',
76 migrate: (value) => value,
77 },
78 {
79 oldKey: 'extras.embeddingModel',
80 newKey: 'extensions.models.embedding',
81 migrate: (value) => value,
82 },
83 {
84 oldKey: 'extras.speechToTextModel',
85 newKey: 'extensions.models.speechToText',
86 migrate: (value) => value,
87 },
88 {
89 oldKey: 'extras.textToSpeechModel',
90 newKey: 'extensions.models.textToSpeech',
91 migrate: (value) => value,
92 },
93 {
94 oldKey: 'minLogLevel',
95 newKey: 'logging.minLogLevel',
96 migrate: (value) => value,
97 },
98 {
99 oldKey: 'cardsCacheCapacity',
100 newKey: 'performance.memoryCacheCapacity',
101 migrate: (value) => `${value}mb`,
102 },
103 {
104 oldKey: 'cookieSecret',
105 newKey: 'cookieSecret',
106 migrate: () => void 0,
107 remove: true,
108 },
109];
110
111/**
112 * Gets all keys from an object recursively.
113 * @param {object} obj Object to get all keys from
114 * @param {string} prefix Prefix to prepend to all keys
115 * @returns {string[]} Array of all keys in the object
116 */
117function getAllKeys(obj, prefix = '') {
118 if (typeof obj !== 'object' || Array.isArray(obj) || obj === null) {
119 return [];
120 }
121
122 return _.flatMap(Object.keys(obj), key => {
123 const newPrefix = prefix ? `${prefix}.${key}` : key;
124 if (typeof obj[key] === 'object' && !Array.isArray(obj[key])) {
125 return getAllKeys(obj[key], newPrefix);
126 } else {
127 return [newPrefix];
128 }
129 });
130}
131
132/**17/**
133 * Converts the old config.conf file to the new config.yaml format.18 * Converts the old config.conf file to the new config.yaml format.
134 */19 */
@@ -156,71 +41,6 @@ function convertConfig() {
156}41}
15742
158/**43/**
159 * Compares the current config.yaml with the default config.yaml and adds any missing values.
160 */
161function addMissingConfigValues() {
162 try {
163 const defaultConfig = yaml.parse(fs.readFileSync(path.join(process.cwd(), './default/config.yaml'), 'utf8'));
164 let config = yaml.parse(fs.readFileSync(path.join(process.cwd(), './config.yaml'), 'utf8'));
165
166 // Migrate old keys to new keys
167 const migratedKeys = [];
168 for (const { oldKey, newKey, migrate, remove } of keyMigrationMap) {
169 if (_.has(config, oldKey)) {
170 if (remove) {
171 _.unset(config, oldKey);
172 migratedKeys.push({
173 oldKey,
174 newValue: void 0,
175 });
176 continue;
177 }
178
179 const oldValue = _.get(config, oldKey);
180 const newValue = migrate(oldValue);
181 _.set(config, newKey, newValue);
182 _.unset(config, oldKey);
183
184 migratedKeys.push({
185 oldKey,
186 newKey,
187 oldValue,
188 newValue,
189 });
190 }
191 }
192
193 // Get all keys from the original config
194 const originalKeys = getAllKeys(config);
195
196 // Use lodash's defaultsDeep function to recursively apply default properties
197 config = _.defaultsDeep(config, defaultConfig);
198
199 // Get all keys from the updated config
200 const updatedKeys = getAllKeys(config);
201
202 // Find the keys that were added
203 const addedKeys = _.difference(updatedKeys, originalKeys);
204
205 if (addedKeys.length === 0 && migratedKeys.length === 0) {
206 return;
207 }
208
209 if (addedKeys.length > 0) {
210 console.log('Adding missing config values to config.yaml:', addedKeys);
211 }
212
213 if (migratedKeys.length > 0) {
214 console.log('Migrating config values in config.yaml:', migratedKeys);
215 }
216
217 fs.writeFileSync('./config.yaml', yaml.stringify(config));
218 } catch (error) {
219 console.error(color.red('FATAL: Could not add missing config values to config.yaml'), error);
220 }
221}
222
223/**
224 * Creates the default config files if they don't exist yet.44 * Creates the default config files if they don't exist yet.
225 */45 */
226function createDefaultFiles() {46function createDefaultFiles() {
@@ -288,7 +108,7 @@ try {
288 // 1. Create default config files108 // 1. Create default config files
289 createDefaultFiles();109 createDefaultFiles();
290 // 2. Add missing config values110 // 2. Add missing config values
291 addMissingConfigValues();111 addMissingConfigValues(path.join(process.cwd(), './config.yaml'));
292} catch (error) {112} catch (error) {
293 console.error(error);113 console.error(error);
294}114}
src/command-line.js+3 -2
@@ -1,7 +1,8 @@
1import yargs from 'yargs/yargs';1import yargs from 'yargs/yargs';
2import { hideBin } from 'yargs/helpers';2import { hideBin } from 'yargs/helpers';
3import ipRegex from 'ip-regex';3import ipRegex from 'ip-regex';
4import { canResolve, color, getConfigValue, setConfigFilePath, stringToBool } from './util.js';4import { canResolve, color, getConfigValue, stringToBool } from './util.js';
5import { initConfig } from './config-init.js';
56
6/**7/**
7 * @typedef {object} CommandLineArguments Parsed command line arguments8 * @typedef {object} CommandLineArguments Parsed command line arguments
@@ -185,7 +186,7 @@ export class CommandLineParser {
185 }).parseSync();186 }).parseSync();
186187
187 const configPath = cliArguments.configPath ?? this.default.configPath;188 const configPath = cliArguments.configPath ?? this.default.configPath;
188 setConfigFilePath(configPath);189 initConfig(configPath);
189 /** @type {CommandLineArguments} */190 /** @type {CommandLineArguments} */
190 const result = {191 const result = {
191 configPath: configPath,192 configPath: configPath,
src/config-init.js+197 -0
@@ -0,0 +1,197 @@
1import fs from 'node:fs';
2import path from 'node:path';
3import yaml from 'yaml';
4import color from 'chalk';
5import _ from 'lodash';
6import { serverDirectory } from './server-directory.js';
7import { setConfigFilePath } from './util.js';
8
9const keyMigrationMap = [
10 {
11 oldKey: 'disableThumbnails',
12 newKey: 'thumbnails.enabled',
13 migrate: (value) => !value,
14 },
15 {
16 oldKey: 'thumbnailsQuality',
17 newKey: 'thumbnails.quality',
18 migrate: (value) => value,
19 },
20 {
21 oldKey: 'avatarThumbnailsPng',
22 newKey: 'thumbnails.format',
23 migrate: (value) => (value ? 'png' : 'jpg'),
24 },
25 {
26 oldKey: 'disableChatBackup',
27 newKey: 'backups.chat.enabled',
28 migrate: (value) => !value,
29 },
30 {
31 oldKey: 'numberOfBackups',
32 newKey: 'backups.common.numberOfBackups',
33 migrate: (value) => value,
34 },
35 {
36 oldKey: 'maxTotalChatBackups',
37 newKey: 'backups.chat.maxTotalBackups',
38 migrate: (value) => value,
39 },
40 {
41 oldKey: 'chatBackupThrottleInterval',
42 newKey: 'backups.chat.throttleInterval',
43 migrate: (value) => value,
44 },
45 {
46 oldKey: 'enableExtensions',
47 newKey: 'extensions.enabled',
48 migrate: (value) => value,
49 },
50 {
51 oldKey: 'enableExtensionsAutoUpdate',
52 newKey: 'extensions.autoUpdate',
53 migrate: (value) => value,
54 },
55 {
56 oldKey: 'extras.disableAutoDownload',
57 newKey: 'extensions.models.autoDownload',
58 migrate: (value) => !value,
59 },
60 {
61 oldKey: 'extras.classificationModel',
62 newKey: 'extensions.models.classification',
63 migrate: (value) => value,
64 },
65 {
66 oldKey: 'extras.captioningModel',
67 newKey: 'extensions.models.captioning',
68 migrate: (value) => value,
69 },
70 {
71 oldKey: 'extras.embeddingModel',
72 newKey: 'extensions.models.embedding',
73 migrate: (value) => value,
74 },
75 {
76 oldKey: 'extras.speechToTextModel',
77 newKey: 'extensions.models.speechToText',
78 migrate: (value) => value,
79 },
80 {
81 oldKey: 'extras.textToSpeechModel',
82 newKey: 'extensions.models.textToSpeech',
83 migrate: (value) => value,
84 },
85 {
86 oldKey: 'minLogLevel',
87 newKey: 'logging.minLogLevel',
88 migrate: (value) => value,
89 },
90 {
91 oldKey: 'cardsCacheCapacity',
92 newKey: 'performance.memoryCacheCapacity',
93 migrate: (value) => `${value}mb`,
94 },
95 {
96 oldKey: 'cookieSecret',
97 newKey: 'cookieSecret',
98 migrate: () => void 0,
99 remove: true,
100 },
101];
102
103/**
104 * Gets all keys from an object recursively.
105 * @param {object} obj Object to get all keys from
106 * @param {string} prefix Prefix to prepend to all keys
107 * @returns {string[]} Array of all keys in the object
108 */
109function getAllKeys(obj, prefix = '') {
110 if (typeof obj !== 'object' || Array.isArray(obj) || obj === null) {
111 return [];
112 }
113
114 return _.flatMap(Object.keys(obj), key => {
115 const newPrefix = prefix ? `${prefix}.${key}` : key;
116 if (typeof obj[key] === 'object' && !Array.isArray(obj[key])) {
117 return getAllKeys(obj[key], newPrefix);
118 } else {
119 return [newPrefix];
120 }
121 });
122}
123
124/**
125 * Compares the current config.yaml with the default config.yaml and adds any missing values.
126 * @param {string} configPath Path to config.yaml
127 */
128export function addMissingConfigValues(configPath) {
129 try {
130 const defaultConfig = yaml.parse(fs.readFileSync(path.join(serverDirectory, './default/config.yaml'), 'utf8'));
131 let config = yaml.parse(fs.readFileSync(configPath, 'utf8'));
132
133 // Migrate old keys to new keys
134 const migratedKeys = [];
135 for (const { oldKey, newKey, migrate, remove } of keyMigrationMap) {
136 if (_.has(config, oldKey)) {
137 if (remove) {
138 _.unset(config, oldKey);
139 migratedKeys.push({
140 oldKey,
141 newValue: void 0,
142 });
143 continue;
144 }
145
146 const oldValue = _.get(config, oldKey);
147 const newValue = migrate(oldValue);
148 _.set(config, newKey, newValue);
149 _.unset(config, oldKey);
150
151 migratedKeys.push({
152 oldKey,
153 newKey,
154 oldValue,
155 newValue,
156 });
157 }
158 }
159
160 // Get all keys from the original config
161 const originalKeys = getAllKeys(config);
162
163 // Use lodash's defaultsDeep function to recursively apply default properties
164 config = _.defaultsDeep(config, defaultConfig);
165
166 // Get all keys from the updated config
167 const updatedKeys = getAllKeys(config);
168
169 // Find the keys that were added
170 const addedKeys = _.difference(updatedKeys, originalKeys);
171
172 if (addedKeys.length === 0 && migratedKeys.length === 0) {
173 return;
174 }
175
176 if (addedKeys.length > 0) {
177 console.log('Adding missing config values to config.yaml:', addedKeys);
178 }
179
180 if (migratedKeys.length > 0) {
181 console.log('Migrating config values in config.yaml:', migratedKeys);
182 }
183
184 fs.writeFileSync(configPath, yaml.stringify(config));
185 } catch (error) {
186 console.error(color.red('FATAL: Could not add missing config values to config.yaml'), error);
187 }
188}
189
190/**
191 * Performs early initialization tasks before the server starts.
192 * @param {string} configPath Path to config.yaml
193 */
194export function initConfig(configPath) {
195 setConfigFilePath(configPath);
196 addMissingConfigValues(configPath);
197}
src/transformers.js+0 -1
@@ -12,7 +12,6 @@ configureTransformers();
12function configureTransformers() {12function configureTransformers() {
13 // Limit the number of threads to 1 to avoid issues on Android13 // Limit the number of threads to 1 to avoid issues on Android
14 env.backends.onnx.wasm.numThreads = 1;14 env.backends.onnx.wasm.numThreads = 1;
15 console.log(env.backends.onnx.wasm.wasmPaths);
16 // Use WASM from a local folder to avoid CDN connections15 // Use WASM from a local folder to avoid CDN connections
17 env.backends.onnx.wasm.wasmPaths = path.join(serverDirectory, 'node_modules', 'sillytavern-transformers', 'dist') + path.sep;16 env.backends.onnx.wasm.wasmPaths = path.join(serverDirectory, 'node_modules', 'sillytavern-transformers', 'dist') + path.sep;
18}17}
src/util.js+1 -1
@@ -15,8 +15,8 @@ import yauzl from 'yauzl';
15import mime from 'mime-types';15import mime from 'mime-types';
16import { default as simpleGit } from 'simple-git';16import { default as simpleGit } from 'simple-git';
17import chalk from 'chalk';17import chalk from 'chalk';
18import { LOG_LEVELS } from './constants.js';
19import bytes from 'bytes';18import bytes from 'bytes';
19import { LOG_LEVELS } from './constants.js';
20import { serverDirectory } from './server-directory.js';20import { serverDirectory } from './server-directory.js';
2121
22/**22/**