Move config migration from post-install to src

005a495e964da2fe85636ff731cb86f25bdcc614

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 01c6544e22a3584c78bf9aa72bcee415bb2d910e
6 files changed, +204 -186Ignore whitespace
.github/readme.md+1 -0
@@ -350,6 +350,7 @@ Start.bat --port 8000 --listen false
350350| Option | Description | Type |
351351|-------------------------|----------------------------------------------------------------------|----------|
352352| `--version` | Show version number | boolean |
353+| `--configPath` | Override the path to the config.yaml file | string |
353354| `--dataRoot` | Root directory for data storage | string |
354355| `--port` | Sets the port under which SillyTavern will run | number |
355356| `--listen` | SillyTavern will listen on all network interfaces | boolean |
post-install.js+2 -182
@@ -5,130 +5,15 @@ import fs from 'node:fs';
55import path from 'node:path';
66import process from 'node:process';
77import yaml from 'yaml';
8-import _ from 'lodash';
98import chalk from 'chalk';
109import { createRequire } from 'node:module';
10+import { addMissingConfigValues } from './src/config-init.js';
1111
1212/**
1313 * Colorizes console output.
1414 */
1515const color = chalk;
1616
17-const keyMigrationMap = [
18- {
19- oldKey: 'disableThumbnails',
20- newKey: 'thumbnails.enabled',
21- migrate: (value) => !value,
22- },
23- {
24- oldKey: 'thumbnailsQuality',
25- newKey: 'thumbnails.quality',
26- migrate: (value) => value,
27- },
28- {
29- oldKey: 'avatarThumbnailsPng',
30- newKey: 'thumbnails.format',
31- migrate: (value) => (value ? 'png' : 'jpg'),
32- },
33- {
34- oldKey: 'disableChatBackup',
35- newKey: 'backups.chat.enabled',
36- migrate: (value) => !value,
37- },
38- {
39- oldKey: 'numberOfBackups',
40- newKey: 'backups.common.numberOfBackups',
41- migrate: (value) => value,
42- },
43- {
44- oldKey: 'maxTotalChatBackups',
45- newKey: 'backups.chat.maxTotalBackups',
46- migrate: (value) => value,
47- },
48- {
49- oldKey: 'chatBackupThrottleInterval',
50- newKey: 'backups.chat.throttleInterval',
51- migrate: (value) => value,
52- },
53- {
54- oldKey: 'enableExtensions',
55- newKey: 'extensions.enabled',
56- migrate: (value) => value,
57- },
58- {
59- oldKey: 'enableExtensionsAutoUpdate',
60- newKey: 'extensions.autoUpdate',
61- migrate: (value) => value,
62- },
63- {
64- oldKey: 'extras.disableAutoDownload',
65- newKey: 'extensions.models.autoDownload',
66- migrate: (value) => !value,
67- },
68- {
69- oldKey: 'extras.classificationModel',
70- newKey: 'extensions.models.classification',
71- migrate: (value) => value,
72- },
73- {
74- oldKey: 'extras.captioningModel',
75- newKey: 'extensions.models.captioning',
76- migrate: (value) => value,
77- },
78- {
79- oldKey: 'extras.embeddingModel',
80- newKey: 'extensions.models.embedding',
81- migrate: (value) => value,
82- },
83- {
84- oldKey: 'extras.speechToTextModel',
85- newKey: 'extensions.models.speechToText',
86- migrate: (value) => value,
87- },
88- {
89- oldKey: 'extras.textToSpeechModel',
90- newKey: 'extensions.models.textToSpeech',
91- migrate: (value) => value,
92- },
93- {
94- oldKey: 'minLogLevel',
95- newKey: 'logging.minLogLevel',
96- migrate: (value) => value,
97- },
98- {
99- oldKey: 'cardsCacheCapacity',
100- newKey: 'performance.memoryCacheCapacity',
101- migrate: (value) => `${value}mb`,
102- },
103- {
104- oldKey: 'cookieSecret',
105- newKey: 'cookieSecret',
106- migrate: () => void 0,
107- remove: true,
108- },
109-];
110-
111-/**
112- * Gets all keys from an object recursively.
113- * @param {object} obj Object to get all keys from
114- * @param {string} prefix Prefix to prepend to all keys
115- * @returns {string[]} Array of all keys in the object
116- */
117-function getAllKeys(obj, prefix = '') {
118- if (typeof obj !== 'object' || Array.isArray(obj) || obj === null) {
119- return [];
120- }
121-
122- return _.flatMap(Object.keys(obj), key => {
123- const newPrefix = prefix ? `${prefix}.${key}` : key;
124- if (typeof obj[key] === 'object' && !Array.isArray(obj[key])) {
125- return getAllKeys(obj[key], newPrefix);
126- } else {
127- return [newPrefix];
128- }
129- });
130-}
131-
13217/**
13318 * Converts the old config.conf file to the new config.yaml format.
13419 */
@@ -156,71 +41,6 @@ function convertConfig() {
15641}
15742
15843/**
159- * Compares the current config.yaml with the default config.yaml and adds any missing values.
160- */
161-function addMissingConfigValues() {
162- try {
163- const defaultConfig = yaml.parse(fs.readFileSync(path.join(process.cwd(), './default/config.yaml'), 'utf8'));
164- let config = yaml.parse(fs.readFileSync(path.join(process.cwd(), './config.yaml'), 'utf8'));
165-
166- // Migrate old keys to new keys
167- const migratedKeys = [];
168- for (const { oldKey, newKey, migrate, remove } of keyMigrationMap) {
169- if (_.has(config, oldKey)) {
170- if (remove) {
171- _.unset(config, oldKey);
172- migratedKeys.push({
173- oldKey,
174- newValue: void 0,
175- });
176- continue;
177- }
178-
179- const oldValue = _.get(config, oldKey);
180- const newValue = migrate(oldValue);
181- _.set(config, newKey, newValue);
182- _.unset(config, oldKey);
183-
184- migratedKeys.push({
185- oldKey,
186- newKey,
187- oldValue,
188- newValue,
189- });
190- }
191- }
192-
193- // Get all keys from the original config
194- const originalKeys = getAllKeys(config);
195-
196- // Use lodash's defaultsDeep function to recursively apply default properties
197- config = _.defaultsDeep(config, defaultConfig);
198-
199- // Get all keys from the updated config
200- const updatedKeys = getAllKeys(config);
201-
202- // Find the keys that were added
203- const addedKeys = _.difference(updatedKeys, originalKeys);
204-
205- if (addedKeys.length === 0 && migratedKeys.length === 0) {
206- return;
207- }
208-
209- if (addedKeys.length > 0) {
210- console.log('Adding missing config values to config.yaml:', addedKeys);
211- }
212-
213- if (migratedKeys.length > 0) {
214- console.log('Migrating config values in config.yaml:', migratedKeys);
215- }
216-
217- fs.writeFileSync('./config.yaml', yaml.stringify(config));
218- } catch (error) {
219- console.error(color.red('FATAL: Could not add missing config values to config.yaml'), error);
220- }
221-}
222-
223-/**
22444 * Creates the default config files if they don't exist yet.
22545 */
22646function createDefaultFiles() {
@@ -288,7 +108,7 @@ try {
288108 // 1. Create default config files
289109 createDefaultFiles();
290110 // 2. Add missing config values
291- addMissingConfigValues();
111+ addMissingConfigValues(path.join(process.cwd(), './config.yaml'));
292112} catch (error) {
293113 console.error(error);
294114}
src/command-line.js+3 -2
@@ -1,7 +1,8 @@
11import yargs from 'yargs/yargs';
22import { hideBin } from 'yargs/helpers';
33import ipRegex from 'ip-regex';
44import { canResolve, color, getConfigValue, setConfigFilePath, stringToBool } from './util.js';
5+import { initConfig } from './config-init.js';
56
67/**
78 * @typedef {object} CommandLineArguments Parsed command line arguments
@@ -185,7 +186,7 @@ export class CommandLineParser {
185186 }).parseSync();
186187
187188 const configPath = cliArguments.configPath ?? this.default.configPath;
188189 setConfigFilePathinitConfig(configPath);
189190 /** @type {CommandLineArguments} */
190191 const result = {
191192 configPath: configPath,
src/config-init.js+197 -0
@@ -0,0 +1,197 @@
1+import fs from 'node:fs';
2+import path from 'node:path';
3+import yaml from 'yaml';
4+import color from 'chalk';
5+import _ from 'lodash';
6+import { serverDirectory } from './server-directory.js';
7+import { setConfigFilePath } from './util.js';
8+
9+const keyMigrationMap = [
10+ {
11+ oldKey: 'disableThumbnails',
12+ newKey: 'thumbnails.enabled',
13+ migrate: (value) => !value,
14+ },
15+ {
16+ oldKey: 'thumbnailsQuality',
17+ newKey: 'thumbnails.quality',
18+ migrate: (value) => value,
19+ },
20+ {
21+ oldKey: 'avatarThumbnailsPng',
22+ newKey: 'thumbnails.format',
23+ migrate: (value) => (value ? 'png' : 'jpg'),
24+ },
25+ {
26+ oldKey: 'disableChatBackup',
27+ newKey: 'backups.chat.enabled',
28+ migrate: (value) => !value,
29+ },
30+ {
31+ oldKey: 'numberOfBackups',
32+ newKey: 'backups.common.numberOfBackups',
33+ migrate: (value) => value,
34+ },
35+ {
36+ oldKey: 'maxTotalChatBackups',
37+ newKey: 'backups.chat.maxTotalBackups',
38+ migrate: (value) => value,
39+ },
40+ {
41+ oldKey: 'chatBackupThrottleInterval',
42+ newKey: 'backups.chat.throttleInterval',
43+ migrate: (value) => value,
44+ },
45+ {
46+ oldKey: 'enableExtensions',
47+ newKey: 'extensions.enabled',
48+ migrate: (value) => value,
49+ },
50+ {
51+ oldKey: 'enableExtensionsAutoUpdate',
52+ newKey: 'extensions.autoUpdate',
53+ migrate: (value) => value,
54+ },
55+ {
56+ oldKey: 'extras.disableAutoDownload',
57+ newKey: 'extensions.models.autoDownload',
58+ migrate: (value) => !value,
59+ },
60+ {
61+ oldKey: 'extras.classificationModel',
62+ newKey: 'extensions.models.classification',
63+ migrate: (value) => value,
64+ },
65+ {
66+ oldKey: 'extras.captioningModel',
67+ newKey: 'extensions.models.captioning',
68+ migrate: (value) => value,
69+ },
70+ {
71+ oldKey: 'extras.embeddingModel',
72+ newKey: 'extensions.models.embedding',
73+ migrate: (value) => value,
74+ },
75+ {
76+ oldKey: 'extras.speechToTextModel',
77+ newKey: 'extensions.models.speechToText',
78+ migrate: (value) => value,
79+ },
80+ {
81+ oldKey: 'extras.textToSpeechModel',
82+ newKey: 'extensions.models.textToSpeech',
83+ migrate: (value) => value,
84+ },
85+ {
86+ oldKey: 'minLogLevel',
87+ newKey: 'logging.minLogLevel',
88+ migrate: (value) => value,
89+ },
90+ {
91+ oldKey: 'cardsCacheCapacity',
92+ newKey: 'performance.memoryCacheCapacity',
93+ migrate: (value) => `${value}mb`,
94+ },
95+ {
96+ oldKey: 'cookieSecret',
97+ newKey: 'cookieSecret',
98+ migrate: () => void 0,
99+ remove: true,
100+ },
101+];
102+
103+/**
104+ * Gets all keys from an object recursively.
105+ * @param {object} obj Object to get all keys from
106+ * @param {string} prefix Prefix to prepend to all keys
107+ * @returns {string[]} Array of all keys in the object
108+ */
109+function getAllKeys(obj, prefix = '') {
110+ if (typeof obj !== 'object' || Array.isArray(obj) || obj === null) {
111+ return [];
112+ }
113+
114+ return _.flatMap(Object.keys(obj), key => {
115+ const newPrefix = prefix ? `${prefix}.${key}` : key;
116+ if (typeof obj[key] === 'object' && !Array.isArray(obj[key])) {
117+ return getAllKeys(obj[key], newPrefix);
118+ } else {
119+ return [newPrefix];
120+ }
121+ });
122+}
123+
124+/**
125+ * Compares the current config.yaml with the default config.yaml and adds any missing values.
126+ * @param {string} configPath Path to config.yaml
127+ */
128+export function addMissingConfigValues(configPath) {
129+ try {
130+ const defaultConfig = yaml.parse(fs.readFileSync(path.join(serverDirectory, './default/config.yaml'), 'utf8'));
131+ let config = yaml.parse(fs.readFileSync(configPath, 'utf8'));
132+
133+ // Migrate old keys to new keys
134+ const migratedKeys = [];
135+ for (const { oldKey, newKey, migrate, remove } of keyMigrationMap) {
136+ if (_.has(config, oldKey)) {
137+ if (remove) {
138+ _.unset(config, oldKey);
139+ migratedKeys.push({
140+ oldKey,
141+ newValue: void 0,
142+ });
143+ continue;
144+ }
145+
146+ const oldValue = _.get(config, oldKey);
147+ const newValue = migrate(oldValue);
148+ _.set(config, newKey, newValue);
149+ _.unset(config, oldKey);
150+
151+ migratedKeys.push({
152+ oldKey,
153+ newKey,
154+ oldValue,
155+ newValue,
156+ });
157+ }
158+ }
159+
160+ // Get all keys from the original config
161+ const originalKeys = getAllKeys(config);
162+
163+ // Use lodash's defaultsDeep function to recursively apply default properties
164+ config = _.defaultsDeep(config, defaultConfig);
165+
166+ // Get all keys from the updated config
167+ const updatedKeys = getAllKeys(config);
168+
169+ // Find the keys that were added
170+ const addedKeys = _.difference(updatedKeys, originalKeys);
171+
172+ if (addedKeys.length === 0 && migratedKeys.length === 0) {
173+ return;
174+ }
175+
176+ if (addedKeys.length > 0) {
177+ console.log('Adding missing config values to config.yaml:', addedKeys);
178+ }
179+
180+ if (migratedKeys.length > 0) {
181+ console.log('Migrating config values in config.yaml:', migratedKeys);
182+ }
183+
184+ fs.writeFileSync(configPath, yaml.stringify(config));
185+ } catch (error) {
186+ console.error(color.red('FATAL: Could not add missing config values to config.yaml'), error);
187+ }
188+}
189+
190+/**
191+ * Performs early initialization tasks before the server starts.
192+ * @param {string} configPath Path to config.yaml
193+ */
194+export function initConfig(configPath) {
195+ setConfigFilePath(configPath);
196+ addMissingConfigValues(configPath);
197+}
src/transformers.js+0 -1
@@ -12,7 +12,6 @@ configureTransformers();
1212function configureTransformers() {
1313 // Limit the number of threads to 1 to avoid issues on Android
1414 env.backends.onnx.wasm.numThreads = 1;
15- console.log(env.backends.onnx.wasm.wasmPaths);
1615 // Use WASM from a local folder to avoid CDN connections
1716 env.backends.onnx.wasm.wasmPaths = path.join(serverDirectory, 'node_modules', 'sillytavern-transformers', 'dist') + path.sep;
1817}
src/util.js+1 -1
@@ -15,8 +15,8 @@ import yauzl from 'yauzl';
1515import mime from 'mime-types';
1616import { default as simpleGit } from 'simple-git';
1717import chalk from 'chalk';
18-import { LOG_LEVELS } from './constants.js';
1918import bytes from 'bytes';
19+import { LOG_LEVELS } from './constants.js';
2020import { serverDirectory } from './server-directory.js';
2121
2222/**