Node: Migrate to ES Modules

d52b4fbbde00cf8ca9168ebc249f883e522aba1b

Cohee <18619528+Cohee1207@users.noreply.github.com>

74 files changed, +1299 -1148Showing whitespace changes
.eslintrc.js+3 -0
@@ -16,6 +16,9 @@ module.exports = {
16 env: {16 env: {
17 node: true,17 node: true,
18 },18 },
19 parserOptions: {
20 sourceType: 'module',
21 },
19 },22 },
20 {23 {
21 files: ['src/**/*.mjs'],24 files: ['src/**/*.mjs'],
jsconfig.json+1 -1
@@ -2,7 +2,7 @@
2 "compilerOptions": {2 "compilerOptions": {
3 "module": "ESNext",3 "module": "ESNext",
4 "target": "ESNext",4 "target": "ESNext",
5 "moduleResolution": "node",5 "moduleResolution": "Node",
6 "strictNullChecks": true,6 "strictNullChecks": true,
7 "strictFunctionTypes": true,7 "strictFunctionTypes": true,
8 "checkJs": true,8 "checkJs": true,
package-lock.json+412 -24
@@ -25,8 +25,8 @@
25 "express": "^4.21.0",25 "express": "^4.21.0",
26 "form-data": "^4.0.0",26 "form-data": "^4.0.0",
27 "google-translate-api-browser": "^3.0.1",27 "google-translate-api-browser": "^3.0.1",
28 "he": "^1.2.0",
29 "helmet": "^7.1.0",28 "helmet": "^7.1.0",
29 "html-entities": "^2.5.2",
30 "iconv-lite": "^0.6.3",30 "iconv-lite": "^0.6.3",
31 "ip-matching": "^2.1.2",31 "ip-matching": "^2.1.2",
32 "ipaddr.js": "^2.0.1",32 "ipaddr.js": "^2.0.1",
@@ -34,7 +34,7 @@
34 "lodash": "^4.17.21",34 "lodash": "^4.17.21",
35 "mime-types": "^2.1.35",35 "mime-types": "^2.1.35",
36 "multer": "^1.4.5-lts.1",36 "multer": "^1.4.5-lts.1",
37 "node-fetch": "^2.6.11",37 "node-fetch": "^3.3.2",
38 "node-persist": "^4.0.1",38 "node-persist": "^4.0.1",
39 "open": "^8.4.2",39 "open": "^8.4.2",
40 "png-chunk-text": "^1.0.0",40 "png-chunk-text": "^1.0.0",
@@ -59,9 +59,28 @@
59 "sillytavern": "server.js"59 "sillytavern": "server.js"
60 },60 },
61 "devDependencies": {61 "devDependencies": {
62 "@types/archiver": "^6.0.2",
63 "@types/command-exists": "^1.2.3",
64 "@types/compression": "^1.7.5",
65 "@types/cookie-parser": "^1.4.7",
66 "@types/cookie-session": "^2.0.49",
67 "@types/cors": "^2.8.17",
62 "@types/dompurify": "^3.0.5",68 "@types/dompurify": "^3.0.5",
69 "@types/express": "^5.0.0",
63 "@types/jquery": "^3.5.29",70 "@types/jquery": "^3.5.29",
71 "@types/lodash": "^4.17.10",
72 "@types/mime-types": "^2.1.4",
73 "@types/multer": "^1.4.12",
74 "@types/node-fetch": "^2.6.11",
75 "@types/node-persist": "^3.1.8",
76 "@types/png-chunk-text": "^1.0.3",
77 "@types/png-chunks-encode": "^1.0.2",
78 "@types/png-chunks-extract": "^1.0.2",
79 "@types/response-time": "^2.3.8",
64 "@types/toastr": "^2.1.43",80 "@types/toastr": "^2.1.43",
81 "@types/write-file-atomic": "^4.0.3",
82 "@types/yargs": "^17.0.33",
83 "@types/yauzl": "^2.10.3",
65 "eslint": "^8.57.0"84 "eslint": "^8.57.0"
66 },85 },
67 "engines": {86 "engines": {
@@ -949,6 +968,27 @@
949 "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==",968 "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==",
950 "license": "MIT"969 "license": "MIT"
951 },970 },
971 "node_modules/@types/archiver": {
972 "version": "6.0.2",
973 "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-6.0.2.tgz",
974 "integrity": "sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw==",
975 "dev": true,
976 "license": "MIT",
977 "dependencies": {
978 "@types/readdir-glob": "*"
979 }
980 },
981 "node_modules/@types/body-parser": {
982 "version": "1.19.5",
983 "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz",
984 "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==",
985 "dev": true,
986 "license": "MIT",
987 "dependencies": {
988 "@types/connect": "*",
989 "@types/node": "*"
990 }
991 },
952 "node_modules/@types/cacheable-request": {992 "node_modules/@types/cacheable-request": {
953 "version": "6.0.3",993 "version": "6.0.3",
954 "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz",994 "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz",
@@ -961,6 +1001,64 @@
961 "@types/responselike": "^1.0.0"1001 "@types/responselike": "^1.0.0"
962 }1002 }
963 },1003 },
1004 "node_modules/@types/command-exists": {
1005 "version": "1.2.3",
1006 "resolved": "https://registry.npmjs.org/@types/command-exists/-/command-exists-1.2.3.tgz",
1007 "integrity": "sha512-PpbaE2XWLaWYboXD6k70TcXO/OdOyyRFq5TVpmlUELNxdkkmXU9fkImNosmXU1DtsNrqdUgWd/nJQYXgwmtdXQ==",
1008 "dev": true,
1009 "license": "MIT"
1010 },
1011 "node_modules/@types/compression": {
1012 "version": "1.7.5",
1013 "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.5.tgz",
1014 "integrity": "sha512-AAQvK5pxMpaT+nDvhHrsBhLSYG5yQdtkaJE1WYieSNY2mVFKAgmU4ks65rkZD5oqnGCFLyQpUr1CqI4DmUMyDg==",
1015 "dev": true,
1016 "license": "MIT",
1017 "dependencies": {
1018 "@types/express": "*"
1019 }
1020 },
1021 "node_modules/@types/connect": {
1022 "version": "3.4.38",
1023 "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
1024 "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==",
1025 "dev": true,
1026 "license": "MIT",
1027 "dependencies": {
1028 "@types/node": "*"
1029 }
1030 },
1031 "node_modules/@types/cookie-parser": {
1032 "version": "1.4.7",
1033 "resolved": "https://registry.npmjs.org/@types/cookie-parser/-/cookie-parser-1.4.7.tgz",
1034 "integrity": "sha512-Fvuyi354Z+uayxzIGCwYTayFKocfV7TuDYZClCdIP9ckhvAu/ixDtCB6qx2TT0FKjPLf1f3P/J1rgf6lPs64mw==",
1035 "dev": true,
1036 "license": "MIT",
1037 "dependencies": {
1038 "@types/express": "*"
1039 }
1040 },
1041 "node_modules/@types/cookie-session": {
1042 "version": "2.0.49",
1043 "resolved": "https://registry.npmjs.org/@types/cookie-session/-/cookie-session-2.0.49.tgz",
1044 "integrity": "sha512-4E/bBjlqLhU5l4iGPR+NkVJH593hpNsT4dC3DJDr+ODm6Qpe13kZQVkezRIb+TYDXaBMemS3yLQ+0leba3jlkQ==",
1045 "dev": true,
1046 "license": "MIT",
1047 "dependencies": {
1048 "@types/express": "*",
1049 "@types/keygrip": "*"
1050 }
1051 },
1052 "node_modules/@types/cors": {
1053 "version": "2.8.17",
1054 "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.17.tgz",
1055 "integrity": "sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==",
1056 "dev": true,
1057 "license": "MIT",
1058 "dependencies": {
1059 "@types/node": "*"
1060 }
1061 },
964 "node_modules/@types/dompurify": {1062 "node_modules/@types/dompurify": {
965 "version": "3.0.5",1063 "version": "3.0.5",
966 "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz",1064 "resolved": "https://registry.npmjs.org/@types/dompurify/-/dompurify-3.0.5.tgz",
@@ -971,12 +1069,45 @@
971 "@types/trusted-types": "*"1069 "@types/trusted-types": "*"
972 }1070 }
973 },1071 },
1072 "node_modules/@types/express": {
1073 "version": "5.0.0",
1074 "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz",
1075 "integrity": "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==",
1076 "dev": true,
1077 "license": "MIT",
1078 "dependencies": {
1079 "@types/body-parser": "*",
1080 "@types/express-serve-static-core": "^5.0.0",
1081 "@types/qs": "*",
1082 "@types/serve-static": "*"
1083 }
1084 },
1085 "node_modules/@types/express-serve-static-core": {
1086 "version": "5.0.0",
1087 "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.0.tgz",
1088 "integrity": "sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==",
1089 "dev": true,
1090 "license": "MIT",
1091 "dependencies": {
1092 "@types/node": "*",
1093 "@types/qs": "*",
1094 "@types/range-parser": "*",
1095 "@types/send": "*"
1096 }
1097 },
974 "node_modules/@types/http-cache-semantics": {1098 "node_modules/@types/http-cache-semantics": {
975 "version": "4.0.2",1099 "version": "4.0.2",
976 "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz",1100 "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz",
977 "integrity": "sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==",1101 "integrity": "sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==",
978 "license": "MIT"1102 "license": "MIT"
979 },1103 },
1104 "node_modules/@types/http-errors": {
1105 "version": "2.0.4",
1106 "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz",
1107 "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==",
1108 "dev": true,
1109 "license": "MIT"
1110 },
980 "node_modules/@types/jquery": {1111 "node_modules/@types/jquery": {
981 "version": "3.5.31",1112 "version": "3.5.31",
982 "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.31.tgz",1113 "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.5.31.tgz",
@@ -987,6 +1118,13 @@
987 "@types/sizzle": "*"1118 "@types/sizzle": "*"
988 }1119 }
989 },1120 },
1121 "node_modules/@types/keygrip": {
1122 "version": "1.0.6",
1123 "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.6.tgz",
1124 "integrity": "sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==",
1125 "dev": true,
1126 "license": "MIT"
1127 },
990 "node_modules/@types/keyv": {1128 "node_modules/@types/keyv": {
991 "version": "3.1.4",1129 "version": "3.1.4",
992 "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz",1130 "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz",
@@ -996,12 +1134,43 @@
996 "@types/node": "*"1134 "@types/node": "*"
997 }1135 }
998 },1136 },
1137 "node_modules/@types/lodash": {
1138 "version": "4.17.10",
1139 "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.10.tgz",
1140 "integrity": "sha512-YpS0zzoduEhuOWjAotS6A5AVCva7X4lVlYLF0FYHAY9sdraBfnatttHItlWeZdGhuEkf+OzMNg2ZYAx8t+52uQ==",
1141 "dev": true,
1142 "license": "MIT"
1143 },
999 "node_modules/@types/long": {1144 "node_modules/@types/long": {
1000 "version": "4.0.2",1145 "version": "4.0.2",
1001 "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",1146 "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
1002 "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==",1147 "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==",
1003 "license": "MIT"1148 "license": "MIT"
1004 },1149 },
1150 "node_modules/@types/mime": {
1151 "version": "1.3.5",
1152 "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
1153 "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==",
1154 "dev": true,
1155 "license": "MIT"
1156 },
1157 "node_modules/@types/mime-types": {
1158 "version": "2.1.4",
1159 "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.4.tgz",
1160 "integrity": "sha512-lfU4b34HOri+kAY5UheuFMWPDOI+OPceBSHZKp69gEyTL/mmJ4cnU6Y/rlme3UL3GyOn6Y42hyIEw0/q8sWx5w==",
1161 "dev": true,
1162 "license": "MIT"
1163 },
1164 "node_modules/@types/multer": {
1165 "version": "1.4.12",
1166 "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.12.tgz",
1167 "integrity": "sha512-pQ2hoqvXiJt2FP9WQVLPRO+AmiIm/ZYkavPlIQnx282u4ZrVdztx0pkh3jjpQt0Kz+YI0YhSG264y08UJKoUQg==",
1168 "dev": true,
1169 "license": "MIT",
1170 "dependencies": {
1171 "@types/express": "*"
1172 }
1173 },
1005 "node_modules/@types/node": {1174 "node_modules/@types/node": {
1006 "version": "16.9.1",1175 "version": "16.9.1",
1007 "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz",1176 "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz",
@@ -1009,15 +1178,81 @@
1009 "license": "MIT"1178 "license": "MIT"
1010 },1179 },
1011 "node_modules/@types/node-fetch": {1180 "node_modules/@types/node-fetch": {
1012 "version": "2.6.9",1181 "version": "2.6.11",
1013 "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.9.tgz",1182 "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz",
1014 "integrity": "sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==",1183 "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==",
1015 "license": "MIT",1184 "license": "MIT",
1016 "dependencies": {1185 "dependencies": {
1017 "@types/node": "*",1186 "@types/node": "*",
1018 "form-data": "^4.0.0"1187 "form-data": "^4.0.0"
1019 }1188 }
1020 },1189 },
1190 "node_modules/@types/node-persist": {
1191 "version": "3.1.8",
1192 "resolved": "https://registry.npmjs.org/@types/node-persist/-/node-persist-3.1.8.tgz",
1193 "integrity": "sha512-QLidg6/SadZYPrTKxtxL1A85XBoQlG40bhoMdhu6DH6+eNCMr2j+RGfFZ9I9+IY8W/PDwQonJ+iBWD62jZjMfg==",
1194 "dev": true,
1195 "license": "MIT",
1196 "dependencies": {
1197 "@types/node": "*"
1198 }
1199 },
1200 "node_modules/@types/png-chunk-text": {
1201 "version": "1.0.3",
1202 "resolved": "https://registry.npmjs.org/@types/png-chunk-text/-/png-chunk-text-1.0.3.tgz",
1203 "integrity": "sha512-7keEFz73uNJ9Ar1XMCNnHEXT9pICJnouMQCCYgBEmHMgdkXaQzSTmSvr6tUDSqgdEgmlRAxZd97wprgliyZoCg==",
1204 "dev": true,
1205 "license": "MIT"
1206 },
1207 "node_modules/@types/png-chunks-encode": {
1208 "version": "1.0.2",
1209 "resolved": "https://registry.npmjs.org/@types/png-chunks-encode/-/png-chunks-encode-1.0.2.tgz",
1210 "integrity": "sha512-Dxn0aXEcSg1wVeHjvNlygm/+fKBDzWMCdxJYhjGUTeefFW/jYxWcrg+W7ppLBfH44iJMqeVBHtHBwtYQUeYvgw==",
1211 "dev": true,
1212 "license": "MIT"
1213 },
1214 "node_modules/@types/png-chunks-extract": {
1215 "version": "1.0.2",
1216 "resolved": "https://registry.npmjs.org/@types/png-chunks-extract/-/png-chunks-extract-1.0.2.tgz",
1217 "integrity": "sha512-z6djfFIbrrddtunoMJBOPlyZrnmeuG1kkvHUNi2QfpOb+JMMLuLliHHTmMyRi7k7LiTAut0HbdGCF6ibDtQAHQ==",
1218 "dev": true,
1219 "license": "MIT"
1220 },
1221 "node_modules/@types/qs": {
1222 "version": "6.9.16",
1223 "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz",
1224 "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==",
1225 "dev": true,
1226 "license": "MIT"
1227 },
1228 "node_modules/@types/range-parser": {
1229 "version": "1.2.7",
1230 "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz",
1231 "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==",
1232 "dev": true,
1233 "license": "MIT"
1234 },
1235 "node_modules/@types/readdir-glob": {
1236 "version": "1.1.5",
1237 "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.5.tgz",
1238 "integrity": "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==",
1239 "dev": true,
1240 "license": "MIT",
1241 "dependencies": {
1242 "@types/node": "*"
1243 }
1244 },
1245 "node_modules/@types/response-time": {
1246 "version": "2.3.8",
1247 "resolved": "https://registry.npmjs.org/@types/response-time/-/response-time-2.3.8.tgz",
1248 "integrity": "sha512-7qGaNYvdxc0zRab8oHpYx7AW17qj+G0xuag1eCrw3M2VWPJQ/HyKaaghWygiaOUl0y9x7QGQwppDpqLJ5V9pzw==",
1249 "dev": true,
1250 "license": "MIT",
1251 "dependencies": {
1252 "@types/express": "*",
1253 "@types/node": "*"
1254 }
1255 },
1021 "node_modules/@types/responselike": {1256 "node_modules/@types/responselike": {
1022 "version": "1.0.1",1257 "version": "1.0.1",
1023 "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.1.tgz",1258 "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.1.tgz",
@@ -1027,6 +1262,29 @@
1027 "@types/node": "*"1262 "@types/node": "*"
1028 }1263 }
1029 },1264 },
1265 "node_modules/@types/send": {
1266 "version": "0.17.4",
1267 "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
1268 "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==",
1269 "dev": true,
1270 "license": "MIT",
1271 "dependencies": {
1272 "@types/mime": "^1",
1273 "@types/node": "*"
1274 }
1275 },
1276 "node_modules/@types/serve-static": {
1277 "version": "1.15.7",
1278 "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz",
1279 "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==",
1280 "dev": true,
1281 "license": "MIT",
1282 "dependencies": {
1283 "@types/http-errors": "*",
1284 "@types/node": "*",
1285 "@types/send": "*"
1286 }
1287 },
1030 "node_modules/@types/sizzle": {1288 "node_modules/@types/sizzle": {
1031 "version": "2.3.8",1289 "version": "2.3.8",
1032 "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.8.tgz",1290 "resolved": "https://registry.npmjs.org/@types/sizzle/-/sizzle-2.3.8.tgz",
@@ -1050,6 +1308,43 @@
1050 "dev": true,1308 "dev": true,
1051 "license": "MIT"1309 "license": "MIT"
1052 },1310 },
1311 "node_modules/@types/write-file-atomic": {
1312 "version": "4.0.3",
1313 "resolved": "https://registry.npmjs.org/@types/write-file-atomic/-/write-file-atomic-4.0.3.tgz",
1314 "integrity": "sha512-qdo+vZRchyJIHNeuI1nrpsLw+hnkgqP/8mlaN6Wle/NKhydHmUN9l4p3ZE8yP90AJNJW4uB8HQhedb4f1vNayQ==",
1315 "dev": true,
1316 "license": "MIT",
1317 "dependencies": {
1318 "@types/node": "*"
1319 }
1320 },
1321 "node_modules/@types/yargs": {
1322 "version": "17.0.33",
1323 "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz",
1324 "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==",
1325 "dev": true,
1326 "license": "MIT",
1327 "dependencies": {
1328 "@types/yargs-parser": "*"
1329 }
1330 },
1331 "node_modules/@types/yargs-parser": {
1332 "version": "21.0.3",
1333 "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz",
1334 "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==",
1335 "dev": true,
1336 "license": "MIT"
1337 },
1338 "node_modules/@types/yauzl": {
1339 "version": "2.10.3",
1340 "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz",
1341 "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==",
1342 "dev": true,
1343 "license": "MIT",
1344 "dependencies": {
1345 "@types/node": "*"
1346 }
1347 },
1053 "node_modules/@ungap/structured-clone": {1348 "node_modules/@ungap/structured-clone": {
1054 "version": "1.2.0",1349 "version": "1.2.0",
1055 "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",1350 "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
@@ -3009,6 +3304,29 @@
3009 "pend": "~1.2.0"3304 "pend": "~1.2.0"
3010 }3305 }
3011 },3306 },
3307 "node_modules/fetch-blob": {
3308 "version": "3.2.0",
3309 "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
3310 "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
3311 "funding": [
3312 {
3313 "type": "github",
3314 "url": "https://github.com/sponsors/jimmywarting"
3315 },
3316 {
3317 "type": "paypal",
3318 "url": "https://paypal.me/jimmywarting"
3319 }
3320 ],
3321 "license": "MIT",
3322 "dependencies": {
3323 "node-domexception": "^1.0.0",
3324 "web-streams-polyfill": "^3.0.3"
3325 },
3326 "engines": {
3327 "node": "^12.20 || >= 14.13"
3328 }
3329 },
3012 "node_modules/file-entry-cache": {3330 "node_modules/file-entry-cache": {
3013 "version": "6.0.1",3331 "version": "6.0.1",
3014 "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",3332 "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
@@ -3178,6 +3496,18 @@
3178 "node": ">= 14"3496 "node": ">= 14"
3179 }3497 }
3180 },3498 },
3499 "node_modules/formdata-polyfill": {
3500 "version": "4.0.10",
3501 "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
3502 "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
3503 "license": "MIT",
3504 "dependencies": {
3505 "fetch-blob": "^3.1.2"
3506 },
3507 "engines": {
3508 "node": ">=12.20.0"
3509 }
3510 },
3181 "node_modules/forwarded": {3511 "node_modules/forwarded": {
3182 "version": "0.2.0",3512 "version": "0.2.0",
3183 "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",3513 "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -3489,14 +3819,6 @@
3489 "node": ">= 0.4"3819 "node": ">= 0.4"
3490 }3820 }
3491 },3821 },
3492 "node_modules/he": {
3493 "version": "1.2.0",
3494 "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
3495 "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
3496 "bin": {
3497 "he": "bin/he"
3498 }
3499 },
3500 "node_modules/helmet": {3822 "node_modules/helmet": {
3501 "version": "7.1.0",3823 "version": "7.1.0",
3502 "resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz",3824 "resolved": "https://registry.npmjs.org/helmet/-/helmet-7.1.0.tgz",
@@ -3505,6 +3827,22 @@
3505 "node": ">=16.0.0"3827 "node": ">=16.0.0"
3506 }3828 }
3507 },3829 },
3830 "node_modules/html-entities": {
3831 "version": "2.5.2",
3832 "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz",
3833 "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==",
3834 "funding": [
3835 {
3836 "type": "github",
3837 "url": "https://github.com/sponsors/mdevils"
3838 },
3839 {
3840 "type": "patreon",
3841 "url": "https://patreon.com/mdevils"
3842 }
3843 ],
3844 "license": "MIT"
3845 },
3508 "node_modules/htmlparser2": {3846 "node_modules/htmlparser2": {
3509 "version": "8.0.2",3847 "version": "8.0.2",
3510 "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz",3848 "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz",
@@ -3876,6 +4214,26 @@
3876 "whatwg-fetch": "^3.4.1"4214 "whatwg-fetch": "^3.4.1"
3877 }4215 }
3878 },4216 },
4217 "node_modules/isomorphic-fetch/node_modules/node-fetch": {
4218 "version": "2.7.0",
4219 "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
4220 "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
4221 "license": "MIT",
4222 "dependencies": {
4223 "whatwg-url": "^5.0.0"
4224 },
4225 "engines": {
4226 "node": "4.x || >=6.0.0"
4227 },
4228 "peerDependencies": {
4229 "encoding": "^0.1.0"
4230 },
4231 "peerDependenciesMeta": {
4232 "encoding": {
4233 "optional": true
4234 }
4235 }
4236 },
3879 "node_modules/jackspeak": {4237 "node_modules/jackspeak": {
3880 "version": "2.3.6",4238 "version": "2.3.6",
3881 "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",4239 "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
@@ -4355,23 +4713,30 @@
4355 }4713 }
4356 },4714 },
4357 "node_modules/node-fetch": {4715 "node_modules/node-fetch": {
4358 "version": "2.6.12",4716 "version": "3.3.2",
4359 "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz",4717 "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
4360 "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==",4718 "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
4361 "license": "MIT",4719 "license": "MIT",
4362 "dependencies": {4720 "dependencies": {
4363 "whatwg-url": "^5.0.0"4721 "data-uri-to-buffer": "^4.0.0",
4722 "fetch-blob": "^3.1.4",
4723 "formdata-polyfill": "^4.0.10"
4364 },4724 },
4365 "engines": {4725 "engines": {
4366 "node": "4.x || >=6.0.0"4726 "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
4367 },
4368 "peerDependencies": {
4369 "encoding": "^0.1.0"
4370 },4727 },
4371 "peerDependenciesMeta": {4728 "funding": {
4372 "encoding": {4729 "type": "opencollective",
4373 "optional": true4730 "url": "https://opencollective.com/node-fetch"
4374 }4731 }
4732 },
4733 "node_modules/node-fetch/node_modules/data-uri-to-buffer": {
4734 "version": "4.0.1",
4735 "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
4736 "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
4737 "license": "MIT",
4738 "engines": {
4739 "node": ">= 12"
4375 }4740 }
4376 },4741 },
4377 "node_modules/node-persist": {4742 "node_modules/node-persist": {
@@ -4546,6 +4911,26 @@
4546 "undici-types": "~5.26.4"4911 "undici-types": "~5.26.4"
4547 }4912 }
4548 },4913 },
4914 "node_modules/openai/node_modules/node-fetch": {
4915 "version": "2.7.0",
4916 "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
4917 "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
4918 "license": "MIT",
4919 "dependencies": {
4920 "whatwg-url": "^5.0.0"
4921 },
4922 "engines": {
4923 "node": "4.x || >=6.0.0"
4924 },
4925 "peerDependencies": {
4926 "encoding": "^0.1.0"
4927 },
4928 "peerDependenciesMeta": {
4929 "encoding": {
4930 "optional": true
4931 }
4932 }
4933 },
4549 "node_modules/optionator": {4934 "node_modules/optionator": {
4550 "version": "0.9.3",4935 "version": "0.9.3",
4551 "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",4936 "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
@@ -5808,6 +6193,7 @@
5808 "version": "5.0.0",6193 "version": "5.0.0",
5809 "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.0.0.tgz",6194 "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.0.0.tgz",
5810 "integrity": "sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==",6195 "integrity": "sha512-tk2G5R2KRwBd+ZN0zaEXpmzdKyOYksXwywulIX95MBODjSzMIuQnQ3m8JxgbhnL1LeVo7lqQKsYa1O3Htl7K5g==",
6196 "license": "MIT",
5811 "dependencies": {6197 "dependencies": {
5812 "punycode": "^2.3.1"6198 "punycode": "^2.3.1"
5813 },6199 },
@@ -6012,6 +6398,7 @@
6012 "version": "7.0.0",6398 "version": "7.0.0",
6013 "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",6399 "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
6014 "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",6400 "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
6401 "license": "BSD-2-Clause",
6015 "engines": {6402 "engines": {
6016 "node": ">=12"6403 "node": ">=12"
6017 }6404 }
@@ -6026,6 +6413,7 @@
6026 "version": "14.0.0",6413 "version": "14.0.0",
6027 "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz",6414 "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.0.0.tgz",
6028 "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==",6415 "integrity": "sha512-1lfMEm2IEr7RIV+f4lUNPOqfFL+pO+Xw3fJSqmjX9AbXcXcYOkCe1P6+9VBZB6n94af16NfZf+sSk0JCBZC9aw==",
6416 "license": "MIT",
6029 "dependencies": {6417 "dependencies": {
6030 "tr46": "^5.0.0",6418 "tr46": "^5.0.0",
6031 "webidl-conversions": "^7.0.0"6419 "webidl-conversions": "^7.0.0"
package.json+22 -3
@@ -15,8 +15,8 @@
15 "express": "^4.21.0",15 "express": "^4.21.0",
16 "form-data": "^4.0.0",16 "form-data": "^4.0.0",
17 "google-translate-api-browser": "^3.0.1",17 "google-translate-api-browser": "^3.0.1",
18 "he": "^1.2.0",
19 "helmet": "^7.1.0",18 "helmet": "^7.1.0",
19 "html-entities": "^2.5.2",
20 "iconv-lite": "^0.6.3",20 "iconv-lite": "^0.6.3",
21 "ip-matching": "^2.1.2",21 "ip-matching": "^2.1.2",
22 "ipaddr.js": "^2.0.1",22 "ipaddr.js": "^2.0.1",
@@ -24,7 +24,7 @@
24 "lodash": "^4.17.21",24 "lodash": "^4.17.21",
25 "mime-types": "^2.1.35",25 "mime-types": "^2.1.35",
26 "multer": "^1.4.5-lts.1",26 "multer": "^1.4.5-lts.1",
27 "node-fetch": "^2.6.11",27 "node-fetch": "^3.3.2",
28 "node-persist": "^4.0.1",28 "node-persist": "^4.0.1",
29 "open": "^8.4.2",29 "open": "^8.4.2",
30 "png-chunk-text": "^1.0.0",30 "png-chunk-text": "^1.0.0",
@@ -60,7 +60,7 @@
60 }60 }
61 },61 },
62 "name": "sillytavern",62 "name": "sillytavern",
63 "type": "commonjs",63 "type": "module",
64 "license": "AGPL-3.0",64 "license": "AGPL-3.0",
65 "repository": {65 "repository": {
66 "type": "git",66 "type": "git",
@@ -85,9 +85,28 @@
85 },85 },
86 "main": "server.js",86 "main": "server.js",
87 "devDependencies": {87 "devDependencies": {
88 "@types/archiver": "^6.0.2",
89 "@types/command-exists": "^1.2.3",
90 "@types/compression": "^1.7.5",
91 "@types/cookie-parser": "^1.4.7",
92 "@types/cookie-session": "^2.0.49",
93 "@types/cors": "^2.8.17",
88 "@types/dompurify": "^3.0.5",94 "@types/dompurify": "^3.0.5",
95 "@types/express": "^5.0.0",
89 "@types/jquery": "^3.5.29",96 "@types/jquery": "^3.5.29",
97 "@types/lodash": "^4.17.10",
98 "@types/mime-types": "^2.1.4",
99 "@types/multer": "^1.4.12",
100 "@types/node-fetch": "^2.6.11",
101 "@types/node-persist": "^3.1.8",
102 "@types/png-chunk-text": "^1.0.3",
103 "@types/png-chunks-encode": "^1.0.2",
104 "@types/png-chunks-extract": "^1.0.2",
105 "@types/response-time": "^2.3.8",
90 "@types/toastr": "^2.1.43",106 "@types/toastr": "^2.1.43",
107 "@types/write-file-atomic": "^4.0.3",
108 "@types/yargs": "^17.0.33",
109 "@types/yauzl": "^2.10.3",
91 "eslint": "^8.57.0"110 "eslint": "^8.57.0"
92 }111 }
93}112}
plugins.js+5 -4
@@ -3,11 +3,12 @@
3// 1. node plugins.js update3// 1. node plugins.js update
4// 2. node plugins.js install <plugin-git-url>4// 2. node plugins.js install <plugin-git-url>
5// More operations coming soon.5// More operations coming soon.
6const { default: git } = require('simple-git');6import { default as git } from 'simple-git';
7const fs = require('fs');7import * as fs from 'node:fs';
8const path = require('path');8import * as path from 'node:path';
9const { color } = require('./src/util');9import { color } from './src/util.js';
1010
11const __dirname = import.meta.dirname;
11process.chdir(__dirname);12process.chdir(__dirname);
12const pluginsPath = './plugins';13const pluginsPath = './plugins';
1314
post-install.js+7 -5
@@ -1,11 +1,12 @@
1/**1/**
2 * Scripts to be done before starting the server for the first time.2 * Scripts to be done before starting the server for the first time.
3 */3 */
4const fs = require('fs');4import * as fs from 'node:fs';
5const path = require('path');5import * as path from 'node:path';
6const crypto = require('crypto');6import * as crypto from 'node:crypto';
7const yaml = require('yaml');7import * as yaml from 'yaml';
8const _ = require('lodash');8import _ from 'lodash';
9import { createRequire } from 'node:module';
910
10/**11/**
11 * Colorizes console output.12 * Colorizes console output.
@@ -59,6 +60,7 @@ function convertConfig() {
5960
60 try {61 try {
61 console.log(color.blue('Converting config.conf to config.yaml. Your old config.conf will be renamed to config.conf.bak'));62 console.log(color.blue('Converting config.conf to config.yaml. Your old config.conf will be renamed to config.conf.bak'));
63 const require = createRequire(import.meta.url);
62 const config = require(path.join(process.cwd(), './config.conf'));64 const config = require(path.join(process.cwd(), './config.conf'));
63 fs.copyFileSync('./config.conf', './config.conf.bak');65 fs.copyFileSync('./config.conf', './config.conf.bak');
64 fs.rmSync('./config.conf');66 fs.rmSync('./config.conf');
recover.js+4 -4
@@ -1,7 +1,7 @@
1const yaml = require('yaml');1import * as fs from 'node:fs';
2const fs = require('fs');2import yaml from 'yaml';
3const storage = require('node-persist');3import storage from 'node-persist';
4const users = require('./src/users');4import * as users from './src/users';
55
6const userAccount = process.argv[2];6const userAccount = process.argv[2];
7const userPassword = process.argv[3];7const userPassword = process.argv[3];
server.js+126 -160
@@ -1,32 +1,33 @@
1#!/usr/bin/env node1#!/usr/bin/env node
22
3// native node modules3// native node modules
4const fs = require('fs');4import * as fs from 'node:fs';
5const http = require('http');5import * as http from 'node:http';
6const https = require('https');6import * as https from 'node:https';
7const path = require('path');7import * as path from 'node:path';
8const util = require('util');8import * as util from 'node:util';
9import * as net from 'node:net';
10import * as dns from 'node:dns';
911
10// cli/fs related library imports12// cli/fs related library imports
11const open = require('open');13import * as open from 'open';
12const yargs = require('yargs/yargs');14import yargs from 'yargs/yargs';
13const { hideBin } = require('yargs/helpers');15import { hideBin } from 'yargs/helpers';
1416
15// express/server related library imports17// express/server related library imports
16const cors = require('cors');18import cors from 'cors';
17const doubleCsrf = require('csrf-csrf').doubleCsrf;19import { doubleCsrf } from 'csrf-csrf';
18const express = require('express');20import express from 'express';
19const compression = require('compression');21import compression from 'compression';
20const cookieParser = require('cookie-parser');22import cookieParser from 'cookie-parser';
21const cookieSession = require('cookie-session');23import cookieSession from 'cookie-session';
22const multer = require('multer');24import multer from 'multer';
23const responseTime = require('response-time');25import responseTime from 'response-time';
24const helmet = require('helmet').default;26import helmet from 'helmet';
27import bodyParser from 'body-parser';
2528
26// net related library imports29// net related library imports
27const net = require('net');30import fetch from 'node-fetch';
28const dns = require('dns');
29const fetch = require('node-fetch').default;
3031
31// Unrestrict console logs display limit32// Unrestrict console logs display limit
32util.inspect.defaultOptions.maxArrayLength = null;33util.inspect.defaultOptions.maxArrayLength = null;
@@ -34,20 +35,67 @@ util.inspect.defaultOptions.maxStringLength = null;
34util.inspect.defaultOptions.depth = 4;35util.inspect.defaultOptions.depth = 4;
3536
36// local library imports37// local library imports
37const userModule = require('./src/users');38import * as loader from './src/plugin-loader.js';
38const basicAuthMiddleware = require('./src/middleware/basicAuth');39import * as userModule from './src/users.js';
39const whitelistMiddleware = require('./src/middleware/whitelist');40import basicAuthMiddleware from './src/middleware/basicAuth.js';
40const initRequestProxy = require('./src/request-proxy');41import whitelistMiddleware from './src/middleware/whitelist.js';
41const contentManager = require('./src/endpoints/content-manager');42import multerMonkeyPatch from './src/middleware/multerMonkeyPatch.js';
42const {43import initRequestProxy from './src/request-proxy.js';
44import * as contentManager from './src/endpoints/content-manager.js';
45import {
43 getVersion,46 getVersion,
44 getConfigValue,47 getConfigValue,
45 color,48 color,
46 forwardFetchResponse,49 forwardFetchResponse,
47 removeColorFormatting,50 removeColorFormatting,
48 getSeparator,51 getSeparator,
49} = require('./src/util');52} from './src/util.js';
50const { ensureThumbnailCache } = require('./src/endpoints/thumbnails');53import { UPLOADS_DIRECTORY } from './src/constants.js';
54import { ensureThumbnailCache } from './src/endpoints/thumbnails.js';
55
56// Routers
57import { router as usersPublicRouter } from './src/endpoints/users-public.js';
58import { router as usersPrivateRouter } from './src/endpoints/users-private.js';
59import { router as usersAdminRouter } from './src/endpoints/users-admin.js';
60import { router as movingUIRouter } from './src/endpoints/moving-ui.js';
61import { router as imagesRouter } from './src/endpoints/images.js';
62import { router as quickRepliesRouter } from './src/endpoints/quick-replies.js';
63import { router as avatarsRouter } from './src/endpoints/avatars.js';
64import { router as themesRouter } from './src/endpoints/themes.js';
65import { router as openAiRouter } from './src/endpoints/openai.js';
66import { router as googleRouter } from './src/endpoints/google.js';
67import { router as anthropicRouter } from './src/endpoints/anthropic.js';
68import { router as tokenizersRouter } from './src/endpoints/tokenizers.js';
69import { router as presetsRouter } from './src/endpoints/presets.js';
70import { router as secretsRouter } from './src/endpoints/secrets.js';
71import { router as thumbnailRouter } from './src/endpoints/thumbnails.js';
72import { router as novelAiRouter } from './src/endpoints/novelai.js';
73import { router as extensionsRouter } from './src/endpoints/extensions.js';
74import { router as assetsRouter } from './src/endpoints/assets.js';
75import { router as filesRouter } from './src/endpoints/files.js';
76import { router as charactersRouter } from './src/endpoints/characters.js';
77import { router as chatsRouter } from './src/endpoints/chats.js';
78import { router as groupsRouter } from './src/endpoints/groups.js';
79import { router as worldInfoRouter } from './src/endpoints/worldinfo.js';
80import { router as statsRouter, init as statsInit, onExit as statsOnExit } from './src/endpoints/stats.js';
81import { router as backgroundsRouter } from './src/endpoints/backgrounds.js';
82import { router as spritesRouter } from './src/endpoints/sprites.js';
83import { router as contentManagerRouter } from './src/endpoints/content-manager.js';
84import { router as settingsRouter, init as settingsInit } from './src/endpoints/settings.js';
85import { router as stableDiffusionRouter } from './src/endpoints/stable-diffusion.js';
86import { router as hordeRouter } from './src/endpoints/horde.js';
87import { router as vectorsRouter } from './src/endpoints/vectors.js';
88import { router as translateRouter } from './src/endpoints/translate.js';
89import { router as classifyRouter } from './src/endpoints/classify.js';
90import { router as captionRouter } from './src/endpoints/caption.js';
91import { router as searchRouter } from './src/endpoints/search.js';
92import { router as openRouterRouter } from './src/endpoints/openrouter.js';
93import { router as chatCompletionsRouter } from './src/endpoints/backends/chat-completions.js';
94import { router as koboldRouter } from './src/endpoints/backends/kobold.js';
95import { router as textCompletionsRouter } from './src/endpoints/backends/text-completions.js';
96import { router as scaleAltRouter } from './src/endpoints/backends/scale-alt.js';
97import { router as speechRouter } from './src/endpoints/speech.js';
98import { router as azureRouter } from './src/endpoints/azure.js';
5199
52// Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0.100// Work around a node v20.0.0, v20.1.0, and v20.2.0 bug. The issue was fixed in v20.3.0.
53// https://github.com/nodejs/node/issues/47822#issuecomment-1564708870101// https://github.com/nodejs/node/issues/47822#issuecomment-1564708870
@@ -166,8 +214,8 @@ const cliArguments = yargs(hideBin(process.argv))
166 }).parseSync();214 }).parseSync();
167215
168// change all relative paths216// change all relative paths
169console.log(`Node version: ${process.version}. Running in ${process.env.NODE_ENV} environment.`);217const serverDirectory = import.meta.dirname;
170const serverDirectory = __dirname;218console.log(`Node version: ${process.version}. Running in ${process.env.NODE_ENV} environment. Server directory: ${serverDirectory}`);
171process.chdir(serverDirectory);219process.chdir(serverDirectory);
172220
173const app = express();221const app = express();
@@ -188,7 +236,7 @@ const basicAuthMode = cliArguments.basicAuthMode ?? getConfigValue('basicAuthMod
188const perUserBasicAuth = getConfigValue('perUserBasicAuth', DEFAULT_PER_USER_BASIC_AUTH);236const perUserBasicAuth = getConfigValue('perUserBasicAuth', DEFAULT_PER_USER_BASIC_AUTH);
189const enableAccounts = getConfigValue('enableUserAccounts', DEFAULT_ACCOUNTS);237const enableAccounts = getConfigValue('enableUserAccounts', DEFAULT_ACCOUNTS);
190238
191const uploadsPath = path.join(dataRoot, require('./src/constants').UPLOADS_DIRECTORY);239const uploadsPath = path.join(dataRoot, UPLOADS_DIRECTORY);
192240
193const enableIPv6 = cliArguments.enableIPv6 ?? getConfigValue('protocol.ipv6', DEFAULT_ENABLE_IPV6);241const enableIPv6 = cliArguments.enableIPv6 ?? getConfigValue('protocol.ipv6', DEFAULT_ENABLE_IPV6);
194const enableIPv4 = cliArguments.enableIPv4 ?? getConfigValue('protocol.ipv4', DEFAULT_ENABLE_IPV4);242const enableIPv4 = cliArguments.enableIPv4 ?? getConfigValue('protocol.ipv4', DEFAULT_ENABLE_IPV4);
@@ -232,7 +280,6 @@ if (listen && basicAuthMode) app.use(basicAuthMiddleware);
232app.use(whitelistMiddleware(enableWhitelist, listen));280app.use(whitelistMiddleware(enableWhitelist, listen));
233281
234if (enableCorsProxy) {282if (enableCorsProxy) {
235 const bodyParser = require('body-parser');
236 app.use(bodyParser.json({283 app.use(bodyParser.json({
237 limit: '200mb',284 limit: '200mb',
238 }));285 }));
@@ -379,7 +426,7 @@ app.get('/login', async (request, response) => {
379app.use(express.static(process.cwd() + '/public', {}));426app.use(express.static(process.cwd() + '/public', {}));
380427
381// Public API428// Public API
382app.use('/api/users', require('./src/endpoints/users-public').router);429app.use('/api/users', usersPublicRouter);
383430
384// Everything below this line requires authentication431// Everything below this line requires authentication
385app.use(userModule.requireLoginMiddleware);432app.use(userModule.requireLoginMiddleware);
@@ -387,14 +434,15 @@ app.get('/api/ping', (_, response) => response.sendStatus(204));
387434
388// File uploads435// File uploads
389app.use(multer({ dest: uploadsPath, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar'));436app.use(multer({ dest: uploadsPath, limits: { fieldSize: 10 * 1024 * 1024 } }).single('avatar'));
390app.use(require('./src/middleware/multerMonkeyPatch'));437app.use(multerMonkeyPatch);
391438
392// User data mount439// User data mount
393app.use('/', userModule.router);440app.use('/', userModule.router);
394// Private endpoints441// Private endpoints
395app.use('/api/users', require('./src/endpoints/users-private').router);442
443app.use('/api/users', usersPrivateRouter);
396// Admin endpoints444// Admin endpoints
397app.use('/api/users', require('./src/endpoints/users-admin').router);445app.use('/api/users', usersAdminRouter);
398446
399app.get('/version', async function (_, response) {447app.get('/version', async function (_, response) {
400 const data = await getVersion();448 const data = await getVersion();
@@ -509,126 +557,45 @@ redirect('/api/serpapi/search', '/api/search/serpapi');
509redirect('/api/serpapi/visit', '/api/search/visit');557redirect('/api/serpapi/visit', '/api/search/visit');
510redirect('/api/serpapi/transcript', '/api/search/transcript');558redirect('/api/serpapi/transcript', '/api/search/transcript');
511559
512// Moving UI560app.use('/api/moving-ui', movingUIRouter);
513app.use('/api/moving-ui', require('./src/endpoints/moving-ui').router);561app.use('/api/images', imagesRouter);
514562app.use('/api/quick-replies', quickRepliesRouter);
515// Image management563app.use('/api/avatars', avatarsRouter);
516app.use('/api/images', require('./src/endpoints/images').router);564app.use('/api/themes', themesRouter);
517565app.use('/api/openai', openAiRouter);
518// Quick reply management566app.use('/api/google', googleRouter);
519app.use('/api/quick-replies', require('./src/endpoints/quick-replies').router);567app.use('/api/anthropic', anthropicRouter);
520568app.use('/api/tokenizers', tokenizersRouter);
521// Avatar management569app.use('/api/presets', presetsRouter);
522app.use('/api/avatars', require('./src/endpoints/avatars').router);570app.use('/api/secrets', secretsRouter);
523571app.use('/thumbnail', thumbnailRouter);
524// Theme management572app.use('/api/novelai', novelAiRouter);
525app.use('/api/themes', require('./src/endpoints/themes').router);573app.use('/api/extensions', extensionsRouter);
526574app.use('/api/assets', assetsRouter);
527// OpenAI API575app.use('/api/files', filesRouter);
528app.use('/api/openai', require('./src/endpoints/openai').router);576app.use('/api/characters', charactersRouter);
529577app.use('/api/chats', chatsRouter);
530//Google API578app.use('/api/groups', groupsRouter);
531app.use('/api/google', require('./src/endpoints/google').router);579app.use('/api/worldinfo', worldInfoRouter);
532580app.use('/api/stats', statsRouter);
533//Anthropic API581app.use('/api/backgrounds', backgroundsRouter);
534app.use('/api/anthropic', require('./src/endpoints/anthropic').router);582app.use('/api/sprites', spritesRouter);
535583app.use('/api/content', contentManagerRouter);
536// Tokenizers584app.use('/api/settings', settingsRouter);
537app.use('/api/tokenizers', require('./src/endpoints/tokenizers').router);585app.use('/api/sd', stableDiffusionRouter);
538586app.use('/api/horde', hordeRouter);
539// Preset management587app.use('/api/vector', vectorsRouter);
540app.use('/api/presets', require('./src/endpoints/presets').router);588app.use('/api/translate', translateRouter);
541589app.use('/api/extra/classify', classifyRouter);
542// Secrets managemenet590app.use('/api/extra/caption', captionRouter);
543app.use('/api/secrets', require('./src/endpoints/secrets').router);591app.use('/api/search', searchRouter);
544592app.use('/api/backends/text-completions', textCompletionsRouter);
545// Thumbnail generation. These URLs are saved in chat, so this route cannot be renamed!593app.use('/api/openrouter', openRouterRouter);
546app.use('/thumbnail', require('./src/endpoints/thumbnails').router);594app.use('/api/backends/kobold', koboldRouter);
547595app.use('/api/backends/chat-completions', chatCompletionsRouter);
548// NovelAI generation596app.use('/api/backends/scale-alt', scaleAltRouter);
549app.use('/api/novelai', require('./src/endpoints/novelai').router);597app.use('/api/speech', speechRouter);
550598app.use('/api/azure', azureRouter);
551// Third-party extensions
552app.use('/api/extensions', require('./src/endpoints/extensions').router);
553
554// Asset management
555app.use('/api/assets', require('./src/endpoints/assets').router);
556
557// File management
558app.use('/api/files', require('./src/endpoints/files').router);
559
560// Character management
561app.use('/api/characters', require('./src/endpoints/characters').router);
562
563// Chat management
564app.use('/api/chats', require('./src/endpoints/chats').router);
565
566// Group management
567app.use('/api/groups', require('./src/endpoints/groups').router);
568
569// World info management
570app.use('/api/worldinfo', require('./src/endpoints/worldinfo').router);
571
572// Stats calculation
573const statsEndpoint = require('./src/endpoints/stats');
574app.use('/api/stats', statsEndpoint.router);
575
576// Background management
577app.use('/api/backgrounds', require('./src/endpoints/backgrounds').router);
578
579// Character sprite management
580app.use('/api/sprites', require('./src/endpoints/sprites').router);
581
582// Custom content management
583app.use('/api/content', require('./src/endpoints/content-manager').router);
584
585// Settings load/store
586const settingsEndpoint = require('./src/endpoints/settings');
587app.use('/api/settings', settingsEndpoint.router);
588
589// Stable Diffusion generation
590app.use('/api/sd', require('./src/endpoints/stable-diffusion').router);
591
592// LLM and SD Horde generation
593app.use('/api/horde', require('./src/endpoints/horde').router);
594
595// Vector storage DB
596app.use('/api/vector', require('./src/endpoints/vectors').router);
597
598// Chat translation
599app.use('/api/translate', require('./src/endpoints/translate').router);
600
601// Emotion classification
602app.use('/api/extra/classify', require('./src/endpoints/classify').router);
603
604// Image captioning
605app.use('/api/extra/caption', require('./src/endpoints/caption').router);
606
607// Web search and scraping
608app.use('/api/search', require('./src/endpoints/search').router);
609
610// The different text generation APIs
611
612// Ooba/OpenAI text completions
613app.use('/api/backends/text-completions', require('./src/endpoints/backends/text-completions').router);
614
615// OpenRouter
616app.use('/api/openrouter', require('./src/endpoints/openrouter').router);
617
618// KoboldAI
619app.use('/api/backends/kobold', require('./src/endpoints/backends/kobold').router);
620
621// OpenAI chat completions
622app.use('/api/backends/chat-completions', require('./src/endpoints/backends/chat-completions').router);
623
624// Scale (alt method)
625app.use('/api/backends/scale-alt', require('./src/endpoints/backends/scale-alt').router);
626
627// Speech (text-to-speech and speech-to-text)
628app.use('/api/speech', require('./src/endpoints/speech').router);
629
630// Azure TTS
631app.use('/api/azure', require('./src/endpoints/azure').router);
632599
633const tavernUrlV6 = new URL(600const tavernUrlV6 = new URL(
634 (cliArguments.ssl ? 'https://' : 'http://') +601 (cliArguments.ssl ? 'https://' : 'http://') +
@@ -663,8 +630,8 @@ const preSetupTasks = async function () {
663 await ensureThumbnailCache();630 await ensureThumbnailCache();
664 cleanUploads();631 cleanUploads();
665632
666 await settingsEndpoint.init();633 await settingsInit();
667 await statsEndpoint.init();634 await statsInit();
668635
669 const cleanupPlugins = await loadPlugins();636 const cleanupPlugins = await loadPlugins();
670 const consoleTitle = process.title;637 const consoleTitle = process.title;
@@ -673,7 +640,7 @@ const preSetupTasks = async function () {
673 const exitProcess = async () => {640 const exitProcess = async () => {
674 if (isExiting) return;641 if (isExiting) return;
675 isExiting = true;642 isExiting = true;
676 statsEndpoint.onExit();643 await statsOnExit();
677 if (typeof cleanupPlugins === 'function') {644 if (typeof cleanupPlugins === 'function') {
678 await cleanupPlugins();645 await cleanupPlugins();
679 }646 }
@@ -776,7 +743,6 @@ const postSetupTasks = async function (v6Failed, v4Failed) {
776async function loadPlugins() {743async function loadPlugins() {
777 try {744 try {
778 const pluginDirectory = path.join(serverDirectory, 'plugins');745 const pluginDirectory = path.join(serverDirectory, 'plugins');
779 const loader = require('./src/plugin-loader');
780 const cleanupPlugins = await loader.loadPlugins(app, pluginDirectory);746 const cleanupPlugins = await loader.loadPlugins(app, pluginDirectory);
781 return cleanupPlugins;747 return cleanupPlugins;
782 } catch {748 } catch {
src/additional-headers.js+6 -12
@@ -1,6 +1,6 @@
1const { TEXTGEN_TYPES, OPENROUTER_HEADERS, FEATHERLESS_HEADERS } = require('./constants');1import { TEXTGEN_TYPES, OPENROUTER_HEADERS, FEATHERLESS_HEADERS } from './constants.js';
2const { SECRET_KEYS, readSecret } = require('./endpoints/secrets');2import { SECRET_KEYS, readSecret } from './endpoints/secrets.js';
3const { getConfigValue } = require('./util');3import { getConfigValue } from './util.js';
44
5/**5/**
6 * Gets the headers for the Mancer API.6 * Gets the headers for the Mancer API.
@@ -172,7 +172,7 @@ function getHuggingFaceHeaders(directories) {
172 }) : {};172 }) : {};
173}173}
174174
175function getOverrideHeaders(urlHost) {175export function getOverrideHeaders(urlHost) {
176 const requestOverrides = getConfigValue('requestOverrides', []);176 const requestOverrides = getConfigValue('requestOverrides', []);
177 const overrideHeaders = requestOverrides?.find((e) => e.hosts?.includes(urlHost))?.headers;177 const overrideHeaders = requestOverrides?.find((e) => e.hosts?.includes(urlHost))?.headers;
178 if (overrideHeaders && urlHost) {178 if (overrideHeaders && urlHost) {
@@ -188,7 +188,7 @@ function getOverrideHeaders(urlHost) {
188 * @param {object} args New request arguments188 * @param {object} args New request arguments
189 * @param {string|null} server API server for new request189 * @param {string|null} server API server for new request
190 */190 */
191function setAdditionalHeaders(request, args, server) {191export function setAdditionalHeaders(request, args, server) {
192 setAdditionalHeadersByType(args.headers, request.body.api_type, server, request.user.directories);192 setAdditionalHeadersByType(args.headers, request.body.api_type, server, request.user.directories);
193}193}
194194
@@ -199,7 +199,7 @@ function setAdditionalHeaders(request, args, server) {
199 * @param {string|null} server API server for new request199 * @param {string|null} server API server for new request
200 * @param {import('./users').UserDirectoryList} directories User directories200 * @param {import('./users').UserDirectoryList} directories User directories
201 */201 */
202function setAdditionalHeadersByType(requestHeaders, type, server, directories) {202export function setAdditionalHeadersByType(requestHeaders, type, server, directories) {
203 const headerGetters = {203 const headerGetters = {
204 [TEXTGEN_TYPES.MANCER]: getMancerHeaders,204 [TEXTGEN_TYPES.MANCER]: getMancerHeaders,
205 [TEXTGEN_TYPES.VLLM]: getVllmHeaders,205 [TEXTGEN_TYPES.VLLM]: getVllmHeaders,
@@ -234,9 +234,3 @@ function setAdditionalHeadersByType(requestHeaders, type, server, directories) {
234234
235 Object.assign(requestHeaders, headers);235 Object.assign(requestHeaders, headers);
236}236}
237
238module.exports = {
239 getOverrideHeaders,
240 setAdditionalHeaders,
241 setAdditionalHeadersByType,
242};
src/character-card-parser.js+10 -13
@@ -1,8 +1,8 @@
1const fs = require('fs');1import fs from 'node:fs';
22
3const encode = require('png-chunks-encode');3import encode from 'png-chunks-encode';
4const extract = require('png-chunks-extract');4import extract from 'png-chunks-extract';
5const PNGtext = require('png-chunk-text');5import PNGtext from 'png-chunk-text';
66
7/**7/**
8 * Writes Character metadata to a PNG image buffer.8 * Writes Character metadata to a PNG image buffer.
@@ -11,7 +11,7 @@ const PNGtext = require('png-chunk-text');
11 * @param {string} data Character data to write11 * @param {string} data Character data to write
12 * @returns {Buffer} PNG image buffer with metadata12 * @returns {Buffer} PNG image buffer with metadata
13 */13 */
14const write = (image, data) => {14export const write = (image, data) => {
15 const chunks = extract(image);15 const chunks = extract(image);
16 const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt');16 const tEXtChunks = chunks.filter(chunk => chunk.name === 'tEXt');
1717
@@ -36,7 +36,9 @@ const write = (image, data) => {
3636
37 const base64EncodedData = Buffer.from(JSON.stringify(v3Data), 'utf8').toString('base64');37 const base64EncodedData = Buffer.from(JSON.stringify(v3Data), 'utf8').toString('base64');
38 chunks.splice(-1, 0, PNGtext.encode('ccv3', base64EncodedData));38 chunks.splice(-1, 0, PNGtext.encode('ccv3', base64EncodedData));
39 } catch (error) { }39 } catch (error) {
40 // Ignore errors when adding v3 chunk
41 }
4042
41 const newBuffer = Buffer.from(encode(chunks));43 const newBuffer = Buffer.from(encode(chunks));
42 return newBuffer;44 return newBuffer;
@@ -48,7 +50,7 @@ const write = (image, data) => {
48 * @param {Buffer} image PNG image buffer50 * @param {Buffer} image PNG image buffer
49 * @returns {string} Character data51 * @returns {string} Character data
50 */52 */
51const read = (image) => {53export const read = (image) => {
52 const chunks = extract(image);54 const chunks = extract(image);
5355
54 const textChunks = chunks.filter((chunk) => chunk.name === 'tEXt').map((chunk) => PNGtext.decode(chunk.data));56 const textChunks = chunks.filter((chunk) => chunk.name === 'tEXt').map((chunk) => PNGtext.decode(chunk.data));
@@ -80,7 +82,7 @@ const read = (image) => {
80 * @param {string} format File format82 * @param {string} format File format
81 * @returns {string} Character data83 * @returns {string} Character data
82 */84 */
83const parse = (cardUrl, format) => {85export const parse = (cardUrl, format) => {
84 let fileFormat = format === undefined ? 'png' : format;86 let fileFormat = format === undefined ? 'png' : format;
8587
86 switch (fileFormat) {88 switch (fileFormat) {
@@ -93,8 +95,3 @@ const parse = (cardUrl, format) => {
93 throw new Error('Unsupported format');95 throw new Error('Unsupported format');
94};96};
9597
96module.exports = {
97 parse,
98 write,
99 read,
100};
src/constants.js+21 -46
@@ -1,18 +1,18 @@
1const PUBLIC_DIRECTORIES = {1export const PUBLIC_DIRECTORIES = {
2 images: 'public/img/',2 images: 'public/img/',
3 backups: 'backups/',3 backups: 'backups/',
4 sounds: 'public/sounds',4 sounds: 'public/sounds',
5 extensions: 'public/scripts/extensions',5 extensions: 'public/scripts/extensions',
6};6};
77
8const SETTINGS_FILE = 'settings.json';8export const SETTINGS_FILE = 'settings.json';
99
10/**10/**
11 * @type {import('./users').UserDirectoryList}11 * @type {import('./users').UserDirectoryList}
12 * @readonly12 * @readonly
13 * @enum {string}13 * @enum {string}
14 */14 */
15const USER_DIRECTORY_TEMPLATE = Object.freeze({15export const USER_DIRECTORY_TEMPLATE = Object.freeze({
16 root: '',16 root: '',
17 thumbnails: 'thumbnails',17 thumbnails: 'thumbnails',
18 thumbnailsBg: 'thumbnails/bg',18 thumbnailsBg: 'thumbnails/bg',
@@ -48,7 +48,7 @@ const USER_DIRECTORY_TEMPLATE = Object.freeze({
48 * @type {import('./users').User}48 * @type {import('./users').User}
49 * @readonly49 * @readonly
50 */50 */
51const DEFAULT_USER = Object.freeze({51export const DEFAULT_USER = Object.freeze({
52 handle: 'default-user',52 handle: 'default-user',
53 name: 'User',53 name: 'User',
54 created: Date.now(),54 created: Date.now(),
@@ -58,7 +58,7 @@ const DEFAULT_USER = Object.freeze({
58 salt: '',58 salt: '',
59});59});
6060
61const UNSAFE_EXTENSIONS = [61export const UNSAFE_EXTENSIONS = [
62 '.php',62 '.php',
63 '.exe',63 '.exe',
64 '.com',64 '.com',
@@ -135,7 +135,7 @@ const UNSAFE_EXTENSIONS = [
135 '.ws',135 '.ws',
136];136];
137137
138const GEMINI_SAFETY = [138export const GEMINI_SAFETY = [
139 {139 {
140 category: 'HARM_CATEGORY_HARASSMENT',140 category: 'HARM_CATEGORY_HARASSMENT',
141 threshold: 'BLOCK_NONE',141 threshold: 'BLOCK_NONE',
@@ -158,7 +158,7 @@ const GEMINI_SAFETY = [
158 },158 },
159];159];
160160
161const BISON_SAFETY = [161export const BISON_SAFETY = [
162 {162 {
163 category: 'HARM_CATEGORY_DEROGATORY',163 category: 'HARM_CATEGORY_DEROGATORY',
164 threshold: 'BLOCK_NONE',164 threshold: 'BLOCK_NONE',
@@ -185,7 +185,7 @@ const BISON_SAFETY = [
185 },185 },
186];186];
187187
188const CHAT_COMPLETION_SOURCES = {188export const CHAT_COMPLETION_SOURCES = {
189 OPENAI: 'openai',189 OPENAI: 'openai',
190 WINDOWAI: 'windowai',190 WINDOWAI: 'windowai',
191 CLAUDE: 'claude',191 CLAUDE: 'claude',
@@ -205,10 +205,10 @@ const CHAT_COMPLETION_SOURCES = {
205/**205/**
206 * Path to multer file uploads under the data root.206 * Path to multer file uploads under the data root.
207 */207 */
208const UPLOADS_DIRECTORY = '_uploads';208export const UPLOADS_DIRECTORY = '_uploads';
209209
210// TODO: this is copied from the client code; there should be a way to de-duplicate it eventually210// TODO: this is copied from the client code; there should be a way to de-duplicate it eventually
211const TEXTGEN_TYPES = {211export const TEXTGEN_TYPES = {
212 OOBA: 'ooba',212 OOBA: 'ooba',
213 MANCER: 'mancer',213 MANCER: 'mancer',
214 VLLM: 'vllm',214 VLLM: 'vllm',
@@ -225,7 +225,7 @@ const TEXTGEN_TYPES = {
225 HUGGINGFACE: 'huggingface',225 HUGGINGFACE: 'huggingface',
226};226};
227227
228const INFERMATICAI_KEYS = [228export const INFERMATICAI_KEYS = [
229 'model',229 'model',
230 'prompt',230 'prompt',
231 'max_tokens',231 'max_tokens',
@@ -248,7 +248,7 @@ const INFERMATICAI_KEYS = [
248 'logprobs',248 'logprobs',
249];249];
250250
251const FEATHERLESS_KEYS = [251export const FEATHERLESS_KEYS = [
252 'model',252 'model',
253 'prompt',253 'prompt',
254 'best_of',254 'best_of',
@@ -290,9 +290,8 @@ const FEATHERLESS_KEYS = [
290 'guided_whitespace_pattern',290 'guided_whitespace_pattern',
291];291];
292292
293
294// https://dreamgen.com/docs/api#openai-text293// https://dreamgen.com/docs/api#openai-text
295const DREAMGEN_KEYS = [294export const DREAMGEN_KEYS = [
296 'model',295 'model',
297 'prompt',296 'prompt',
298 'max_tokens',297 'max_tokens',
@@ -309,7 +308,7 @@ const DREAMGEN_KEYS = [
309];308];
310309
311// https://docs.together.ai/reference/completions310// https://docs.together.ai/reference/completions
312const TOGETHERAI_KEYS = [311export const TOGETHERAI_KEYS = [
313 'model',312 'model',
314 'prompt',313 'prompt',
315 'max_tokens',314 'max_tokens',
@@ -325,7 +324,7 @@ const TOGETHERAI_KEYS = [
325];324];
326325
327// https://github.com/jmorganca/ollama/blob/main/docs/api.md#request-with-options326// https://github.com/jmorganca/ollama/blob/main/docs/api.md#request-with-options
328const OLLAMA_KEYS = [327export const OLLAMA_KEYS = [
329 'num_predict',328 'num_predict',
330 'num_ctx',329 'num_ctx',
331 'stop',330 'stop',
@@ -345,20 +344,20 @@ const OLLAMA_KEYS = [
345 'min_p',344 'min_p',
346];345];
347346
348const AVATAR_WIDTH = 512;347export const AVATAR_WIDTH = 512;
349const AVATAR_HEIGHT = 768;348export const AVATAR_HEIGHT = 768;
350349
351const OPENROUTER_HEADERS = {350export const OPENROUTER_HEADERS = {
352 'HTTP-Referer': 'https://sillytavern.app',351 'HTTP-Referer': 'https://sillytavern.app',
353 'X-Title': 'SillyTavern',352 'X-Title': 'SillyTavern',
354};353};
355354
356const FEATHERLESS_HEADERS = {355export const FEATHERLESS_HEADERS = {
357 'HTTP-Referer': 'https://sillytavern.app',356 'HTTP-Referer': 'https://sillytavern.app',
358 'X-Title': 'SillyTavern',357 'X-Title': 'SillyTavern',
359};358};
360359
361const OPENROUTER_KEYS = [360export const OPENROUTER_KEYS = [
362 'max_tokens',361 'max_tokens',
363 'temperature',362 'temperature',
364 'top_k',363 'top_k',
@@ -378,7 +377,7 @@ const OPENROUTER_KEYS = [
378];377];
379378
380// https://github.com/vllm-project/vllm/blob/0f8a91401c89ac0a8018def3756829611b57727f/vllm/entrypoints/openai/protocol.py#L220379// https://github.com/vllm-project/vllm/blob/0f8a91401c89ac0a8018def3756829611b57727f/vllm/entrypoints/openai/protocol.py#L220
381const VLLM_KEYS = [380export const VLLM_KEYS = [
382 'model',381 'model',
383 'prompt',382 'prompt',
384 'best_of',383 'best_of',
@@ -419,27 +418,3 @@ const VLLM_KEYS = [
419 'guided_decoding_backend',418 'guided_decoding_backend',
420 'guided_whitespace_pattern',419 'guided_whitespace_pattern',
421];420];
422
423module.exports = {
424 DEFAULT_USER,
425 SETTINGS_FILE,
426 PUBLIC_DIRECTORIES,
427 USER_DIRECTORY_TEMPLATE,
428 UNSAFE_EXTENSIONS,
429 UPLOADS_DIRECTORY,
430 GEMINI_SAFETY,
431 BISON_SAFETY,
432 TEXTGEN_TYPES,
433 CHAT_COMPLETION_SOURCES,
434 AVATAR_WIDTH,
435 AVATAR_HEIGHT,
436 TOGETHERAI_KEYS,
437 OLLAMA_KEYS,
438 INFERMATICAI_KEYS,
439 DREAMGEN_KEYS,
440 OPENROUTER_HEADERS,
441 OPENROUTER_KEYS,
442 VLLM_KEYS,
443 FEATHERLESS_KEYS,
444 FEATHERLESS_HEADERS,
445};
src/endpoints/anthropic.js+6 -7
@@ -1,9 +1,10 @@
1const { readSecret, SECRET_KEYS } = require('./secrets');1import fetch from 'node-fetch';
2const fetch = require('node-fetch').default;2import express from 'express';
3const express = require('express');
4const { jsonParser } = require('../express-common');
53
6const router = express.Router();4import { readSecret, SECRET_KEYS } from './secrets.js';
5import { jsonParser } from '../express-common.js';
6
7export const router = express.Router();
78
8router.post('/caption-image', jsonParser, async (request, response) => {9router.post('/caption-image', jsonParser, async (request, response) => {
9 try {10 try {
@@ -64,5 +65,3 @@ router.post('/caption-image', jsonParser, async (request, response) => {
64 response.status(500).send('Internal server error');65 response.status(500).send('Internal server error');
65 }66 }
66});67});
67
68module.exports = { router };
src/endpoints/assets.js+14 -14
@@ -1,13 +1,15 @@
1const path = require('path');1import * as path from 'node:path';
2const fs = require('fs');2import * as fs from 'node:fs';
3const mime = require('mime-types');3import { finished } from 'node:stream/promises';
4const express = require('express');4
5const sanitize = require('sanitize-filename');5import mime from 'mime-types';
6const fetch = require('node-fetch').default;6import express from 'express';
7const { finished } = require('stream/promises');7import sanitize from 'sanitize-filename';
8const { UNSAFE_EXTENSIONS } = require('../constants');8import fetch from 'node-fetch';
9const { jsonParser } = require('../express-common');9
10const { clientRelativePath } = require('../util');10import { UNSAFE_EXTENSIONS } from '../constants.js';
11import { jsonParser } from '../express-common.js';
12import { clientRelativePath } from '../util.js';
1113
12const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm', 'character', 'temp'];14const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm', 'character', 'temp'];
1315
@@ -16,7 +18,7 @@ const VALID_CATEGORIES = ['bgm', 'ambient', 'blip', 'live2d', 'vrm', 'character'
16 * @param {string} inputFilename Input filename18 * @param {string} inputFilename Input filename
17 * @returns {{error: boolean, message?: string}} Whether validation failed, and why if so19 * @returns {{error: boolean, message?: string}} Whether validation failed, and why if so
18 */20 */
19function validateAssetFileName(inputFilename) {21export function validateAssetFileName(inputFilename) {
20 if (!/^[a-zA-Z0-9_\-.]+$/.test(inputFilename)) {22 if (!/^[a-zA-Z0-9_\-.]+$/.test(inputFilename)) {
21 return {23 return {
22 error: true,24 error: true,
@@ -93,7 +95,7 @@ function ensureFoldersExist(directories) {
93 }95 }
94}96}
9597
96const router = express.Router();98export const router = express.Router();
9799
98/**100/**
99 * HTTP POST handler function to retrieve name of all files of a given folder path.101 * HTTP POST handler function to retrieve name of all files of a given folder path.
@@ -366,5 +368,3 @@ router.post('/character', jsonParser, async (request, response) => {
366 return response.sendStatus(500);368 return response.sendStatus(500);
367 }369 }
368});370});
369
370module.exports = { router, validateAssetFileName };
src/endpoints/avatars.js+11 -13
@@ -1,16 +1,16 @@
1const express = require('express');1import * as path from 'node:path';
2const path = require('path');2import * as fs from 'node:fs';
3const fs = require('fs');
4const sanitize = require('sanitize-filename');
5const writeFileAtomicSync = require('write-file-atomic').sync;
6const { jsonParser, urlencodedParser } = require('../express-common');
7const { AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants');
8const { getImages, tryParse } = require('../util');
93
10// image processing related library imports4import express from 'express';
11const jimp = require('jimp');5import sanitize from 'sanitize-filename';
6import jimp from 'jimp';
7import { sync as writeFileAtomicSync } from 'write-file-atomic';
128
13const router = express.Router();9import { jsonParser, urlencodedParser } from '../express-common.js';
10import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
11import { getImages, tryParse } from '../util.js';
12
13export const router = express.Router();
1414
15router.post('/get', jsonParser, function (request, response) {15router.post('/get', jsonParser, function (request, response) {
16 var images = getImages(request.user.directories.avatars);16 var images = getImages(request.user.directories.avatars);
@@ -58,5 +58,3 @@ router.post('/upload', urlencodedParser, async (request, response) => {
58 return response.status(400).send('Is not a valid image');58 return response.status(400).send('Is not a valid image');
59 }59 }
60});60});
61
62module.exports = { router };
src/endpoints/azure.js+6 -9
@@ -1,9 +1,10 @@
1const { readSecret, SECRET_KEYS } = require('./secrets');1import fetch from 'node-fetch';
2const fetch = require('node-fetch').default;2import { Router } from 'express';
3const express = require('express');
4const { jsonParser } = require('../express-common');
53
6const router = express.Router();4import { readSecret, SECRET_KEYS } from './secrets.js';
5import { jsonParser } from '../express-common.js';
6
7export const router = Router();
78
8router.post('/list', jsonParser, async (req, res) => {9router.post('/list', jsonParser, async (req, res) => {
9 try {10 try {
@@ -86,7 +87,3 @@ router.post('/generate', jsonParser, async (req, res) => {
86 return res.sendStatus(500);87 return res.sendStatus(500);
87 }88 }
88});89});
89
90module.exports = {
91 router,
92};
src/endpoints/backends/chat-completions.js+38 -51
@@ -1,13 +1,40 @@
1const express = require('express');1import express from 'express';
2const fetch = require('node-fetch').default;2import fetch from 'node-fetch';
33
4const { jsonParser } = require('../../express-common');4import { jsonParser } from '../../express-common.js';
5const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');5import {
6const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');6 CHAT_COMPLETION_SOURCES,
7const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertAI21Messages, mergeMessages } = require('../../prompt-converters');7 GEMINI_SAFETY,
88 BISON_SAFETY,
9const { readSecret, SECRET_KEYS } = require('../secrets');9 OPENROUTER_HEADERS,
10const { getTokenizerModel, getSentencepiceTokenizer, getTiktokenTokenizer, sentencepieceTokenizers, TEXT_COMPLETION_MODELS } = require('../tokenizers');10} from '../../constants.js';
11import {
12 forwardFetchResponse,
13 getConfigValue,
14 tryParse,
15 uuidv4,
16 mergeObjectWithYaml,
17 excludeKeysByYaml,
18 color,
19} from '../../util.js';
20import {
21 convertClaudeMessages,
22 convertGooglePrompt,
23 convertTextCompletionPrompt,
24 convertCohereMessages,
25 convertMistralMessages,
26 convertAI21Messages,
27 mergeMessages,
28} from '../../prompt-converters.js';
29
30import { readSecret, SECRET_KEYS } from '../secrets.js';
31import {
32 getTokenizerModel,
33 getSentencepiceTokenizer,
34 getTiktokenTokenizer,
35 sentencepieceTokenizers,
36 TEXT_COMPLETION_MODELS,
37} from '../tokenizers.js';
1138
12const API_OPENAI = 'https://api.openai.com/v1';39const API_OPENAI = 'https://api.openai.com/v1';
13const API_CLAUDE = 'https://api.anthropic.com/v1';40const API_CLAUDE = 'https://api.anthropic.com/v1';
@@ -42,43 +69,6 @@ function postProcessPrompt(messages, type, charName, userName) {
42}69}
4370
44/**71/**
45 * Ollama strikes back. Special boy #2's steaming routine.
46 * Wrap this abomination into proper SSE stream, again.
47 * @param {Response} jsonStream JSON stream
48 * @param {import('express').Request} request Express request
49 * @param {import('express').Response} response Express response
50 * @returns {Promise<any>} Nothing valuable
51 */
52async function parseCohereStream(jsonStream, request, response) {
53 try {
54 const stream = new CohereStream({ stream: jsonStream.body, eventShape: { type: 'json', messageTerminator: '\n' } });
55
56 for await (const json of stream.iterMessages()) {
57 if (json.message) {
58 const message = json.message || 'Unknown error';
59 const chunk = { error: { message: message } };
60 response.write(`data: ${JSON.stringify(chunk)}\n\n`);
61 } else if (json.event_type === 'text-generation') {
62 const text = json.text || '';
63 const chunk = { choices: [{ text }] };
64 response.write(`data: ${JSON.stringify(chunk)}\n\n`);
65 }
66 }
67
68 console.log('Streaming request finished');
69 response.write('data: [DONE]\n\n');
70 response.end();
71 } catch (error) {
72 console.log('Error forwarding streaming response:', error);
73 if (!response.headersSent) {
74 return response.status(500).send({ error: true });
75 } else {
76 return response.end();
77 }
78 }
79}
80
81/**
82 * Sends a request to Claude API.72 * Sends a request to Claude API.
83 * @param {express.Request} request Express request73 * @param {express.Request} request Express request
84 * @param {express.Response} response Express response74 * @param {express.Response} response Express response
@@ -626,7 +616,7 @@ async function sendCohereRequest(request, response) {
626 }616 }
627}617}
628618
629const router = express.Router();619export const router = express.Router();
630620
631router.post('/status', jsonParser, async function (request, response_getstatus_openai) {621router.post('/status', jsonParser, async function (request, response_getstatus_openai) {
632 if (!request.body) return response_getstatus_openai.sendStatus(400);622 if (!request.body) return response_getstatus_openai.sendStatus(400);
@@ -1069,6 +1059,3 @@ router.post('/generate', jsonParser, function (request, response) {
1069 }1059 }
1070});1060});
10711061
1072module.exports = {
1073 router,
1074};
src/endpoints/backends/kobold.js+8 -10
@@ -1,13 +1,13 @@
1const express = require('express');1import * as fs from 'node:fs';
2const fetch = require('node-fetch').default;2import express from 'express';
3const fs = require('fs');3import fetch from 'node-fetch';
44
5const { jsonParser, urlencodedParser } = require('../../express-common');5import { jsonParser, urlencodedParser } from '../../express-common.js';
6const { forwardFetchResponse, delay } = require('../../util');6import { forwardFetchResponse, delay } from '../../util.js';
7const { getOverrideHeaders, setAdditionalHeaders, setAdditionalHeadersByType } = require('../../additional-headers');7import { getOverrideHeaders, setAdditionalHeaders, setAdditionalHeadersByType } from '../../additional-headers.js';
8const { TEXTGEN_TYPES } = require('../../constants');8import { TEXTGEN_TYPES } from '../../constants.js';
99
10const router = express.Router();10export const router = express.Router();
1111
12router.post('/generate', jsonParser, async function (request, response_generate) {12router.post('/generate', jsonParser, async function (request, response_generate) {
13 if (!request.body) return response_generate.sendStatus(400);13 if (!request.body) return response_generate.sendStatus(400);
@@ -237,5 +237,3 @@ router.post('/transcribe-audio', urlencodedParser, async function (request, resp
237 response.status(500).send('Internal server error');237 response.status(500).send('Internal server error');
238 }238 }
239});239});
240
241module.exports = { router };
src/endpoints/backends/scale-alt.js+5 -8
@@ -1,11 +1,10 @@
1const express = require('express');1import express from 'express';
2const fetch = require('node-fetch').default;2import fetch from 'node-fetch';
33
4const { jsonParser } = require('../../express-common');4import { jsonParser } from '../../express-common.js';
5import { readSecret, SECRET_KEYS } from '../secrets.js';
56
6const { readSecret, SECRET_KEYS } = require('../secrets');7export const router = express.Router();
7
8const router = express.Router();
98
10router.post('/generate', jsonParser, async function (request, response) {9router.post('/generate', jsonParser, async function (request, response) {
11 if (!request.body) return response.sendStatus(400);10 if (!request.body) return response.sendStatus(400);
@@ -97,5 +96,3 @@ router.post('/generate', jsonParser, async function (request, response) {
97 return response.sendStatus(500);96 return response.sendStatus(500);
98 }97 }
99});98});
100
101module.exports = { router };
src/endpoints/backends/text-completions.js+20 -13
@@ -1,14 +1,23 @@
1const express = require('express');1import { Readable } from 'node:stream';
2const fetch = require('node-fetch').default;2import fetch from 'node-fetch';
3const _ = require('lodash');3import express from 'express';
4const Readable = require('stream').Readable;4import _ from 'lodash';
55
6const { jsonParser } = require('../../express-common');6import { jsonParser } from '../../express-common.js';
7const { TEXTGEN_TYPES, TOGETHERAI_KEYS, OLLAMA_KEYS, INFERMATICAI_KEYS, OPENROUTER_KEYS, VLLM_KEYS, DREAMGEN_KEYS, FEATHERLESS_KEYS } = require('../../constants');7import {
8const { forwardFetchResponse, trimV1, getConfigValue } = require('../../util');8 TEXTGEN_TYPES,
9const { setAdditionalHeaders } = require('../../additional-headers');9 TOGETHERAI_KEYS,
1010 OLLAMA_KEYS,
11const router = express.Router();11 INFERMATICAI_KEYS,
12 OPENROUTER_KEYS,
13 VLLM_KEYS,
14 DREAMGEN_KEYS,
15 FEATHERLESS_KEYS,
16} from '../../constants.js';
17import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js';
18import { setAdditionalHeaders } from '../../additional-headers.js';
19
20export const router = express.Router();
1221
13/**22/**
14 * Special boy's steaming routine. Wrap this abomination into proper SSE stream.23 * Special boy's steaming routine. Wrap this abomination into proper SSE stream.
@@ -641,5 +650,3 @@ tabby.post('/download', jsonParser, async function (request, response) {
641router.use('/ollama', ollama);650router.use('/ollama', ollama);
642router.use('/llamacpp', llamacpp);651router.use('/llamacpp', llamacpp);
643router.use('/tabby', tabby);652router.use('/tabby', tabby);
644
645module.exports = { router };
src/endpoints/backgrounds.js+9 -10
@@ -1,13 +1,14 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');
4const sanitize = require('sanitize-filename');
53
6const { jsonParser, urlencodedParser } = require('../express-common');4import express from 'express';
7const { invalidateThumbnail } = require('./thumbnails');5import sanitize from 'sanitize-filename';
8const { getImages } = require('../util');
96
10const router = express.Router();7import { jsonParser, urlencodedParser } from '../express-common.js';
8import { invalidateThumbnail } from './thumbnails.js';
9import { getImages } from '../util.js';
10
11export const router = express.Router();
1112
12router.post('/all', jsonParser, function (request, response) {13router.post('/all', jsonParser, function (request, response) {
13 var images = getImages(request.user.directories.backgrounds);14 var images = getImages(request.user.directories.backgrounds);
@@ -72,5 +73,3 @@ router.post('/upload', urlencodedParser, function (request, response) {
72 response.sendStatus(500);73 response.sendStatus(500);
73 }74 }
74});75});
75
76module.exports = { router };
src/endpoints/caption.js+6 -8
@@ -1,23 +1,23 @@
1const express = require('express');1import express from 'express';
2const { jsonParser } = require('../express-common');2import { jsonParser } from '../express-common.js';
3import { getPipeline, getRawImage } from '../transformers.mjs';
34
4const TASK = 'image-to-text';5const TASK = 'image-to-text';
56
6const router = express.Router();7export const router = express.Router();
78
8router.post('/', jsonParser, async (req, res) => {9router.post('/', jsonParser, async (req, res) => {
9 try {10 try {
10 const { image } = req.body;11 const { image } = req.body;
1112
12 const module = await import('../transformers.mjs');13 const rawImage = await getRawImage(image);
13 const rawImage = await module.default.getRawImage(image);
1414
15 if (!rawImage) {15 if (!rawImage) {
16 console.log('Failed to parse captioned image');16 console.log('Failed to parse captioned image');
17 return res.sendStatus(400);17 return res.sendStatus(400);
18 }18 }
1919
20 const pipe = await module.default.getPipeline(TASK);20 const pipe = await getPipeline(TASK);
21 const result = await pipe(rawImage);21 const result = await pipe(rawImage);
22 const text = result[0].generated_text;22 const text = result[0].generated_text;
23 console.log('Image caption:', text);23 console.log('Image caption:', text);
@@ -28,5 +28,3 @@ router.post('/', jsonParser, async (req, res) => {
28 return res.sendStatus(500);28 return res.sendStatus(500);
29 }29 }
30});30});
31
32module.exports = { router };
src/endpoints/characters.js+22 -24
@@ -1,24 +1,24 @@
1const path = require('path');1import * as path from 'node:path';
2const fs = require('fs');2import * as fs from 'node:fs';
3const fsPromises = require('fs').promises;3import { promises as fsPromises } from 'node:fs';
4const readline = require('readline');4import * as readline from 'node:readline';
5const express = require('express');5
6const sanitize = require('sanitize-filename');6import express from 'express';
7const writeFileAtomicSync = require('write-file-atomic').sync;7import sanitize from 'sanitize-filename';
8const yaml = require('yaml');8import { sync as writeFileAtomicSync } from 'write-file-atomic';
9const _ = require('lodash');9import yaml from 'yaml';
10const mime = require('mime-types');10import _ from 'lodash';
1111import mime from 'mime-types';
12const jimp = require('jimp');12import jimp from 'jimp';
1313
14const { AVATAR_WIDTH, AVATAR_HEIGHT } = require('../constants');14import { AVATAR_WIDTH, AVATAR_HEIGHT } from '../constants.js';
15const { jsonParser, urlencodedParser } = require('../express-common');15import { jsonParser, urlencodedParser } from '../express-common.js';
16const { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer } = require('../util');16import { deepMerge, humanizedISO8601DateTime, tryParse, extractFileFromZipBuffer } from '../util.js';
17const { TavernCardValidator } = require('../validator/TavernCardValidator');17import { TavernCardValidator } from '../validator/TavernCardValidator.js';
18const characterCardParser = require('../character-card-parser.js');18import * as characterCardParser from '../character-card-parser.js';
19const { readWorldInfoFile } = require('./worldinfo');19import { readWorldInfoFile } from './worldinfo.js';
20const { invalidateThumbnail } = require('./thumbnails');20import { invalidateThumbnail } from './thumbnails.js';
21const { importRisuSprites } = require('./sprites');21import { importRisuSprites } from './sprites.js';
22const defaultAvatarPath = './public/img/ai4.png';22const defaultAvatarPath = './public/img/ai4.png';
2323
24// KV-store for parsed character data24// KV-store for parsed character data
@@ -715,7 +715,7 @@ async function importFromPng(uploadPath, { request }, preservedFileName) {
715 return '';715 return '';
716}716}
717717
718const router = express.Router();718export const router = express.Router();
719719
720router.post('/create', urlencodedParser, async function (request, response) {720router.post('/create', urlencodedParser, async function (request, response) {
721 try {721 try {
@@ -1231,5 +1231,3 @@ router.post('/export', jsonParser, async function (request, response) {
1231 response.sendStatus(500);1231 response.sendStatus(500);
1232 }1232 }
1233});1233});
1234
1235module.exports = { router };
src/endpoints/chats.js+11 -12
@@ -1,13 +1,14 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const readline = require('readline');3import * as readline from 'node:readline';
4const express = require('express');
5const sanitize = require('sanitize-filename');
6const writeFileAtomicSync = require('write-file-atomic').sync;
7const _ = require('lodash');
84
9const { jsonParser, urlencodedParser } = require('../express-common');5import express from 'express';
10const { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } = require('../util');6import sanitize from 'sanitize-filename';
7import { sync as writeFileAtomicSync } from 'write-file-atomic';
8import _ from 'lodash';
9
10import { jsonParser, urlencodedParser } from '../express-common.js';
11import { getConfigValue, humanizedISO8601DateTime, tryParse, generateTimestamp, removeOldBackups } from '../util.js';
1112
12/**13/**
13 * Saves a chat to the backups directory.14 * Saves a chat to the backups directory.
@@ -188,7 +189,7 @@ function flattenChubChat(userName, characterName, lines) {
188 return (lines ?? []).map(convert).join('\n');189 return (lines ?? []).map(convert).join('\n');
189}190}
190191
191const router = express.Router();192export const router = express.Router();
192193
193router.post('/save', jsonParser, function (request, response) {194router.post('/save', jsonParser, function (request, response) {
194 try {195 try {
@@ -514,5 +515,3 @@ router.post('/group/save', jsonParser, (request, response) => {
514 getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData);515 getBackupFunction(request.user.profile.handle)(request.user.directories.backups, String(id), jsonlData);
515 return response.send({ ok: true });516 return response.send({ ok: true });
516});517});
517
518module.exports = { router };
src/endpoints/classify.js+7 -9
@@ -1,9 +1,11 @@
1const express = require('express');1import express from 'express';
2const { jsonParser } = require('../express-common');2
3import { getPipeline } from '../transformers.mjs';
4import { jsonParser } from '../express-common.js';
35
4const TASK = 'text-classification';6const TASK = 'text-classification';
57
6const router = express.Router();8export const router = express.Router();
79
8/**10/**
9 * @type {Map<string, object>} Cache for classification results11 * @type {Map<string, object>} Cache for classification results
@@ -12,8 +14,7 @@ const cacheObject = new Map();
1214
13router.post('/labels', jsonParser, async (req, res) => {15router.post('/labels', jsonParser, async (req, res) => {
14 try {16 try {
15 const module = await import('../transformers.mjs');17 const pipe = await getPipeline(TASK);
16 const pipe = await module.default.getPipeline(TASK);
17 const result = Object.keys(pipe.model.config.label2id);18 const result = Object.keys(pipe.model.config.label2id);
18 return res.json({ labels: result });19 return res.json({ labels: result });
19 } catch (error) {20 } catch (error) {
@@ -35,8 +36,7 @@ router.post('/', jsonParser, async (req, res) => {
35 if (cacheObject.has(text)) {36 if (cacheObject.has(text)) {
36 return cacheObject.get(text);37 return cacheObject.get(text);
37 } else {38 } else {
38 const module = await import('../transformers.mjs');39 const pipe = await getPipeline(TASK);
39 const pipe = await module.default.getPipeline(TASK);
40 const result = await pipe(text, { topk: 5 });40 const result = await pipe(text, { topk: 5 });
41 result.sort((a, b) => b.score - a.score);41 result.sort((a, b) => b.score - a.score);
42 cacheObject.set(text, result);42 cacheObject.set(text, result);
@@ -54,5 +54,3 @@ router.post('/', jsonParser, async (req, res) => {
54 return res.sendStatus(500);54 return res.sendStatus(500);
55 }55 }
56});56});
57
58module.exports = { router };
src/endpoints/content-manager.js+17 -24
@@ -1,16 +1,18 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');3
4const fetch = require('node-fetch').default;4import express from 'express';
5const sanitize = require('sanitize-filename');5import fetch from 'node-fetch';
6const { getConfigValue, color } = require('../util');6import sanitize from 'sanitize-filename';
7const { jsonParser } = require('../express-common');7
8const writeFileAtomicSync = require('write-file-atomic').sync;8import { getConfigValue, color } from '../util.js';
9import { jsonParser } from '../express-common.js';
10import { sync as writeFileAtomicSync } from 'write-file-atomic';
9const contentDirectory = path.join(process.cwd(), 'default/content');11const contentDirectory = path.join(process.cwd(), 'default/content');
10const scaffoldDirectory = path.join(process.cwd(), 'default/scaffold');12const scaffoldDirectory = path.join(process.cwd(), 'default/scaffold');
11const contentIndexPath = path.join(contentDirectory, 'index.json');13const contentIndexPath = path.join(contentDirectory, 'index.json');
12const scaffoldIndexPath = path.join(scaffoldDirectory, 'index.json');14const scaffoldIndexPath = path.join(scaffoldDirectory, 'index.json');
13const characterCardParser = require('../character-card-parser.js');15import * as characterCardParser from '../character-card-parser.js';
1416
15const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDomains', []);17const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDomains', []);
1618
@@ -26,7 +28,7 @@ const WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES = getConfigValue('whitelistImportDo
26 * @typedef {string} ContentType28 * @typedef {string} ContentType
27 * @enum {string}29 * @enum {string}
28 */30 */
29const CONTENT_TYPES = {31export const CONTENT_TYPES = {
30 SETTINGS: 'settings',32 SETTINGS: 'settings',
31 CHARACTER: 'character',33 CHARACTER: 'character',
32 SPRITES: 'sprites',34 SPRITES: 'sprites',
@@ -51,7 +53,7 @@ const CONTENT_TYPES = {
51 * @param {import('../users').UserDirectoryList} directories User directories53 * @param {import('../users').UserDirectoryList} directories User directories
52 * @returns {object[]} Array of default presets54 * @returns {object[]} Array of default presets
53 */55 */
54function getDefaultPresets(directories) {56export function getDefaultPresets(directories) {
55 try {57 try {
56 const contentIndex = getContentIndex();58 const contentIndex = getContentIndex();
57 const presets = [];59 const presets = [];
@@ -76,7 +78,7 @@ function getDefaultPresets(directories) {
76 * @param {string} filename Name of the file to get78 * @param {string} filename Name of the file to get
77 * @returns {object | null} JSON object or null if the file doesn't exist79 * @returns {object | null} JSON object or null if the file doesn't exist
78 */80 */
79function getDefaultPresetFile(filename) {81export function getDefaultPresetFile(filename) {
80 try {82 try {
81 const contentPath = path.join(contentDirectory, filename);83 const contentPath = path.join(contentDirectory, filename);
8284
@@ -158,7 +160,7 @@ async function seedContentForUser(contentIndex, directories, forceCategories) {
158 * @param {string[]} forceCategories List of categories to force check (even if content check is skipped)160 * @param {string[]} forceCategories List of categories to force check (even if content check is skipped)
159 * @returns {Promise<void>}161 * @returns {Promise<void>}
160 */162 */
161async function checkForNewContent(directoriesList, forceCategories = []) {163export async function checkForNewContent(directoriesList, forceCategories = []) {
162 try {164 try {
163 const contentCheckSkip = getConfigValue('skipContentCheck', false);165 const contentCheckSkip = getConfigValue('skipContentCheck', false);
164 if (contentCheckSkip && forceCategories?.length === 0) {166 if (contentCheckSkip && forceCategories?.length === 0) {
@@ -224,7 +226,7 @@ function getContentIndex() {
224 * @param {'json'|'string'|'raw'} format Format of content226 * @param {'json'|'string'|'raw'} format Format of content
225 * @returns {string[]|Buffer[]} Array of content227 * @returns {string[]|Buffer[]} Array of content
226 */228 */
227function getContentOfType(type, format) {229export function getContentOfType(type, format) {
228 const contentIndex = getContentIndex();230 const contentIndex = getContentIndex();
229 const indexItems = contentIndex.filter((item) => item.type === type && item.folder);231 const indexItems = contentIndex.filter((item) => item.type === type && item.folder);
230 const files = [];232 const files = [];
@@ -618,7 +620,7 @@ function isHostWhitelisted(host) {
618 return WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES.includes(host);620 return WHITELIST_GENERIC_URL_DOWNLOAD_SOURCES.includes(host);
619}621}
620622
621const router = express.Router();623export const router = express.Router();
622624
623router.post('/importURL', jsonParser, async (request, response) => {625router.post('/importURL', jsonParser, async (request, response) => {
624 if (!request.body.url) {626 if (!request.body.url) {
@@ -753,12 +755,3 @@ router.post('/importUUID', jsonParser, async (request, response) => {
753 return response.sendStatus(500);755 return response.sendStatus(500);
754 }756 }
755});757});
756
757module.exports = {
758 CONTENT_TYPES,
759 checkForNewContent,
760 getDefaultPresets,
761 getDefaultPresetFile,
762 getContentOfType,
763 router,
764};
src/endpoints/extensions.js+10 -10
@@ -1,10 +1,12 @@
1const path = require('path');1import * as path from 'node:path';
2const fs = require('fs');2import * as fs from 'node:fs';
3const express = require('express');3
4const { default: simpleGit } = require('simple-git');4import express from 'express';
5const sanitize = require('sanitize-filename');5import sanitize from 'sanitize-filename';
6const { PUBLIC_DIRECTORIES } = require('../constants');6import { default as simpleGit } from 'simple-git';
7const { jsonParser } = require('../express-common');7
8import { PUBLIC_DIRECTORIES } from '../constants.js';
9import { jsonParser } from '../express-common.js';
810
9/**11/**
10 * This function extracts the extension information from the manifest file.12 * This function extracts the extension information from the manifest file.
@@ -47,7 +49,7 @@ async function checkIfRepoIsUpToDate(extensionPath) {
47 };49 };
48}50}
4951
50const router = express.Router();52export const router = express.Router();
5153
52/**54/**
53 * HTTP POST handler function to clone a git repository from a provided URL, read the extension manifest,55 * HTTP POST handler function to clone a git repository from a provided URL, read the extension manifest,
@@ -240,5 +242,3 @@ router.get('/discover', jsonParser, function (request, response) {
240242
241 return response.send(extensions);243 return response.send(extensions);
242});244});
243
244module.exports = { router };
src/endpoints/files.js+12 -11
@@ -1,12 +1,15 @@
1const path = require('path');1import * as path from 'node:path';
2const fs = require('fs');2import * as fs from 'node:fs';
3const writeFileSyncAtomic = require('write-file-atomic').sync;3
4const express = require('express');4import express from 'express';
5const sanitize = require('sanitize-filename');5import sanitize from 'sanitize-filename';
6const router = express.Router();6import { sync as writeFileSyncAtomic } from 'write-file-atomic';
7const { validateAssetFileName } = require('./assets');7
8const { jsonParser } = require('../express-common');8import { validateAssetFileName } from './assets.js';
9const { clientRelativePath } = require('../util');9import { jsonParser } from '../express-common.js';
10import { clientRelativePath } from '../util.js';
11
12export const router = express.Router();
1013
11router.post('/sanitize-filename', jsonParser, async (request, response) => {14router.post('/sanitize-filename', jsonParser, async (request, response) => {
12 try {15 try {
@@ -97,5 +100,3 @@ router.post('/verify', jsonParser, async (request, response) => {
97 return response.sendStatus(500);100 return response.sendStatus(500);
98 }101 }
99});102});
100
101module.exports = { router };
src/endpoints/google.js+7 -8
@@ -1,12 +1,13 @@
1const { readSecret, SECRET_KEYS } = require('./secrets');1import fetch from 'node-fetch';
2const fetch = require('node-fetch').default;2import express from 'express';
3const express = require('express');3
4const { jsonParser } = require('../express-common');4import { readSecret, SECRET_KEYS } from './secrets.js';
5const { GEMINI_SAFETY } = require('../constants');5import { jsonParser } from '../express-common.js';
6import { GEMINI_SAFETY } from '../constants.js';
67
7const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';8const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
89
9const router = express.Router();10export const router = express.Router();
1011
11router.post('/caption-image', jsonParser, async (request, response) => {12router.post('/caption-image', jsonParser, async (request, response) => {
12 try {13 try {
@@ -67,5 +68,3 @@ router.post('/caption-image', jsonParser, async (request, response) => {
67 response.status(500).send('Internal server error');68 response.status(500).send('Internal server error');
68 }69 }
69});70});
70
71module.exports = { router };
src/endpoints/groups.js+9 -10
@@ -1,13 +1,14 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');
4const sanitize = require('sanitize-filename');
5const writeFileAtomicSync = require('write-file-atomic').sync;
63
7const { jsonParser } = require('../express-common');4import express from 'express';
8const { humanizedISO8601DateTime } = require('../util');5import sanitize from 'sanitize-filename';
6import { sync as writeFileAtomicSync } from 'write-file-atomic';
97
10const router = express.Router();8import { jsonParser } from '../express-common.js';
9import { humanizedISO8601DateTime } from '../util.js';
10
11export const router = express.Router();
1112
12router.post('/all', jsonParser, (request, response) => {13router.post('/all', jsonParser, (request, response) => {
13 const groups = [];14 const groups = [];
@@ -131,5 +132,3 @@ router.post('/delete', jsonParser, async (request, response) => {
131132
132 return response.send({ ok: true });133 return response.send({ ok: true });
133});134});
134
135module.exports = { router };
src/endpoints/horde.js+7 -9
@@ -1,14 +1,14 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const express = require('express');2import express from 'express';
3const { AIHorde, ModelGenerationInputStableSamplers, ModelInterrogationFormTypes, HordeAsyncRequestStates } = require('@zeldafan0225/ai_horde');3import { AIHorde, ModelGenerationInputStableSamplers, ModelInterrogationFormTypes, HordeAsyncRequestStates } from '@zeldafan0225/ai_horde';
4const { getVersion, delay, Cache } = require('../util');4import { getVersion, delay, Cache } from '../util.js';
5const { readSecret, SECRET_KEYS } = require('./secrets');5import { readSecret, SECRET_KEYS } from './secrets.js';
6const { jsonParser } = require('../express-common');6import { jsonParser } from '../express-common.js';
77
8const ANONYMOUS_KEY = '0000000000';8const ANONYMOUS_KEY = '0000000000';
9const HORDE_TEXT_MODEL_METADATA_URL = 'https://raw.githubusercontent.com/db0/AI-Horde-text-model-reference/main/db.json';9const HORDE_TEXT_MODEL_METADATA_URL = 'https://raw.githubusercontent.com/db0/AI-Horde-text-model-reference/main/db.json';
10const cache = new Cache(60 * 1000);10const cache = new Cache(60 * 1000);
11const router = express.Router();11export const router = express.Router();
1212
13/**13/**
14 * Returns the AIHorde client agent.14 * Returns the AIHorde client agent.
@@ -403,5 +403,3 @@ router.post('/generate-image', jsonParser, async (request, response) => {
403 return response.sendStatus(500);403 return response.sendStatus(500);
404 }404 }
405});405});
406
407module.exports = { router };
src/endpoints/images.js+8 -9
@@ -1,10 +1,11 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');
4const sanitize = require('sanitize-filename');
53
6const { jsonParser } = require('../express-common');4import express from 'express';
7const { clientRelativePath, removeFileExtension, getImages } = require('../util');5import sanitize from 'sanitize-filename';
6
7import { jsonParser } from '../express-common.js';
8import { clientRelativePath, removeFileExtension, getImages } from '../util.js';
89
9/**10/**
10 * Ensure the directory for the provided file path exists.11 * Ensure the directory for the provided file path exists.
@@ -21,7 +22,7 @@ function ensureDirectoryExistence(filePath) {
21 fs.mkdirSync(dirname);22 fs.mkdirSync(dirname);
22}23}
2324
24const router = express.Router();25export const router = express.Router();
2526
26/**27/**
27 * Endpoint to handle image uploads.28 * Endpoint to handle image uploads.
@@ -89,5 +90,3 @@ router.post('/list/:folder', (request, response) => {
89 return response.status(500).send({ error: 'Unable to retrieve files' });90 return response.status(500).send({ error: 'Unable to retrieve files' });
90 }91 }
91});92});
92
93module.exports = { router };
src/endpoints/moving-ui.js+6 -8
@@ -1,11 +1,11 @@
1const path = require('path');1import * as path from 'node:path';
2const express = require('express');2import express from 'express';
3const sanitize = require('sanitize-filename');3import sanitize from 'sanitize-filename';
4const writeFileAtomicSync = require('write-file-atomic').sync;4import { sync as writeFileAtomicSync } from 'write-file-atomic';
55
6const { jsonParser } = require('../express-common');6import { jsonParser } from '../express-common.js';
77
8const router = express.Router();8export const router = express.Router();
99
10router.post('/save', jsonParser, (request, response) => {10router.post('/save', jsonParser, (request, response) => {
11 if (!request.body || !request.body.name) {11 if (!request.body || !request.body.name) {
@@ -17,5 +17,3 @@ router.post('/save', jsonParser, (request, response) => {
1717
18 return response.sendStatus(200);18 return response.sendStatus(200);
19});19});
20
21module.exports = { router };
src/endpoints/novelai.js+9 -9
@@ -1,9 +1,11 @@
1const fetch = require('node-fetch').default;1import * as util from 'node:util';
2const express = require('express');2
3const util = require('util');3import fetch from 'node-fetch';
4const { readSecret, SECRET_KEYS } = require('./secrets');4import express from 'express';
5const { readAllChunks, extractFileFromZipBuffer, forwardFetchResponse } = require('../util');5
6const { jsonParser } = require('../express-common');6import { readSecret, SECRET_KEYS } from './secrets.js';
7import { readAllChunks, extractFileFromZipBuffer, forwardFetchResponse } from '../util.js';
8import { jsonParser } from '../express-common.js';
79
8const API_NOVELAI = 'https://api.novelai.net';10const API_NOVELAI = 'https://api.novelai.net';
9const TEXT_NOVELAI = 'https://text.novelai.net';11const TEXT_NOVELAI = 'https://text.novelai.net';
@@ -110,7 +112,7 @@ function getRepPenaltyWhitelist(model) {
110 return null;112 return null;
111}113}
112114
113const router = express.Router();115export const router = express.Router();
114116
115router.post('/status', jsonParser, async function (req, res) {117router.post('/status', jsonParser, async function (req, res) {
116 if (!req.body) return res.sendStatus(400);118 if (!req.body) return res.sendStatus(400);
@@ -432,5 +434,3 @@ router.post('/generate-voice', jsonParser, async (request, response) => {
432 return response.sendStatus(500);434 return response.sendStatus(500);
433 }435 }
434});436});
435
436module.exports = { router };
src/endpoints/openai.js+13 -13
@@ -1,14 +1,16 @@
1const { readSecret, SECRET_KEYS } = require('./secrets');1import * as fs from 'node:fs';
2const fetch = require('node-fetch').default;2
3const express = require('express');3import fetch from 'node-fetch';
4const FormData = require('form-data');4import FormData from 'form-data';
5const fs = require('fs');5import express from 'express';
6const { jsonParser, urlencodedParser } = require('../express-common');6
7const { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1 } = require('../util');7import { jsonParser, urlencodedParser } from '../express-common.js';
8const { setAdditionalHeaders } = require('../additional-headers');8import { getConfigValue, mergeObjectWithYaml, excludeKeysByYaml, trimV1 } from '../util.js';
9const { OPENROUTER_HEADERS } = require('../constants');9import { setAdditionalHeaders } from '../additional-headers.js';
1010import { readSecret, SECRET_KEYS } from './secrets.js';
11const router = express.Router();11import { OPENROUTER_HEADERS } from '../constants.js';
12
13export const router = express.Router();
1214
13router.post('/caption-image', jsonParser, async (request, response) => {15router.post('/caption-image', jsonParser, async (request, response) => {
14 try {16 try {
@@ -341,5 +343,3 @@ custom.post('/generate-voice', jsonParser, async (request, response) => {
341});343});
342344
343router.use('/custom', custom);345router.use('/custom', custom);
344
345module.exports = { router };
src/endpoints/openrouter.js+3 -5
@@ -1,7 +1,7 @@
1const express = require('express');1import express from 'express';
2const { jsonParser } = require('../express-common');2import { jsonParser } from '../express-common.js';
33
4const router = express.Router();4export const router = express.Router();
5const API_OPENROUTER = 'https://openrouter.ai/api/v1';5const API_OPENROUTER = 'https://openrouter.ai/api/v1';
66
7router.post('/models/multimodal', jsonParser, async (_req, res) => {7router.post('/models/multimodal', jsonParser, async (_req, res) => {
@@ -28,5 +28,3 @@ router.post('/models/multimodal', jsonParser, async (_req, res) => {
28 return res.sendStatus(500);28 return res.sendStatus(500);
29 }29 }
30});30});
31
32module.exports = { router };
src/endpoints/presets.js+10 -10
@@ -1,10 +1,12 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');3
4const sanitize = require('sanitize-filename');4import express from 'express';
5const writeFileAtomicSync = require('write-file-atomic').sync;5import sanitize from 'sanitize-filename';
6const { getDefaultPresetFile, getDefaultPresets } = require('./content-manager');6import { sync as writeFileAtomicSync } from 'write-file-atomic';
7const { jsonParser } = require('../express-common');7
8import { getDefaultPresetFile, getDefaultPresets } from './content-manager.js';
9import { jsonParser } from '../express-common.js';
810
9/**11/**
10 * Gets the folder and extension for the preset settings based on the API source ID.12 * Gets the folder and extension for the preset settings based on the API source ID.
@@ -34,7 +36,7 @@ function getPresetSettingsByAPI(apiId, directories) {
34 }36 }
35}37}
3638
37const router = express.Router();39export const router = express.Router();
3840
39router.post('/save', jsonParser, function (request, response) {41router.post('/save', jsonParser, function (request, response) {
40 const name = sanitize(request.body.name);42 const name = sanitize(request.body.name);
@@ -127,5 +129,3 @@ router.post('/delete-openai', jsonParser, function (request, response) {
127129
128 return response.send({ error: true });130 return response.send({ error: true });
129});131});
130
131module.exports = { router };
src/endpoints/quick-replies.js+8 -9
@@ -1,12 +1,13 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');
4const sanitize = require('sanitize-filename');
5const writeFileAtomicSync = require('write-file-atomic').sync;
63
7const { jsonParser } = require('../express-common');4import express from 'express';
5import sanitize from 'sanitize-filename';
6import { sync as writeFileAtomicSync } from 'write-file-atomic';
87
9const router = express.Router();8import { jsonParser } from '../express-common.js';
9
10export const router = express.Router();
1011
11router.post('/save', jsonParser, (request, response) => {12router.post('/save', jsonParser, (request, response) => {
12 if (!request.body || !request.body.name) {13 if (!request.body || !request.body.name) {
@@ -31,5 +32,3 @@ router.post('/delete', jsonParser, (request, response) => {
3132
32 return response.sendStatus(200);33 return response.sendStatus(200);
33});34});
34
35module.exports = { router };
src/endpoints/search.js+8 -9
@@ -1,9 +1,11 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const express = require('express');2import express from 'express';
3const { readSecret, SECRET_KEYS } = require('./secrets');
4const { jsonParser } = require('../express-common');
53
6const router = express.Router();4import { decode } from 'html-entities';
5import { readSecret, SECRET_KEYS } from './secrets.js';
6import { jsonParser } from '../express-common.js';
7
8export const router = express.Router();
79
8// Cosplay as Chrome10// Cosplay as Chrome
9const visitHeaders = {11const visitHeaders = {
@@ -29,7 +31,6 @@ const visitHeaders = {
29 * @returns {Promise<string>} Transcript text31 * @returns {Promise<string>} Transcript text
30 */32 */
31async function extractTranscript(videoPageBody, lang) {33async function extractTranscript(videoPageBody, lang) {
32 const he = require('he');
33 const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;34 const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
34 const splittedHTML = videoPageBody.split('"captions":');35 const splittedHTML = videoPageBody.split('"captions":');
3536
@@ -84,7 +85,7 @@ async function extractTranscript(videoPageBody, lang) {
84 lang: lang ?? captions.captionTracks[0].languageCode,85 lang: lang ?? captions.captionTracks[0].languageCode,
85 }));86 }));
86 // The text is double-encoded87 // The text is double-encoded
87 const transcriptText = transcript.map((line) => he.decode(he.decode(line.text))).join(' ');88 const transcriptText = transcript.map((line) => decode(decode(line.text))).join(' ');
88 return transcriptText;89 return transcriptText;
89}90}
9091
@@ -263,5 +264,3 @@ router.post('/visit', jsonParser, async (request, response) => {
263 return response.sendStatus(500);264 return response.sendStatus(500);
264 }265 }
265});266});
266
267module.exports = { router };
src/endpoints/secrets.js+16 -25
@@ -1,12 +1,13 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');3
4const { getConfigValue } = require('../util');4import express from 'express';
5const writeFileAtomicSync = require('write-file-atomic').sync;5import { sync as writeFileAtomicSync } from 'write-file-atomic';
6const { jsonParser } = require('../express-common');6import { getConfigValue } from '../util.js';
77import { jsonParser } from '../express-common.js';
8const SECRETS_FILE = 'secrets.json';8
9const SECRET_KEYS = {9export const SECRETS_FILE = 'secrets.json';
10export const SECRET_KEYS = {
10 HORDE: 'api_key_horde',11 HORDE: 'api_key_horde',
11 MANCER: 'api_key_mancer',12 MANCER: 'api_key_mancer',
12 VLLM: 'api_key_vllm',13 VLLM: 'api_key_vllm',
@@ -62,7 +63,7 @@ const EXPORTABLE_KEYS = [
62 * @param {string} key Secret key63 * @param {string} key Secret key
63 * @param {string} value Secret value64 * @param {string} value Secret value
64 */65 */
65function writeSecret(directories, key, value) {66export function writeSecret(directories, key, value) {
66 const filePath = path.join(directories.root, SECRETS_FILE);67 const filePath = path.join(directories.root, SECRETS_FILE);
6768
68 if (!fs.existsSync(filePath)) {69 if (!fs.existsSync(filePath)) {
@@ -82,7 +83,7 @@ function writeSecret(directories, key, value) {
82 * @param {string} key Secret key83 * @param {string} key Secret key
83 * @returns84 * @returns
84 */85 */
85function deleteSecret(directories, key) {86export function deleteSecret(directories, key) {
86 const filePath = path.join(directories.root, SECRETS_FILE);87 const filePath = path.join(directories.root, SECRETS_FILE);
8788
88 if (!fs.existsSync(filePath)) {89 if (!fs.existsSync(filePath)) {
@@ -101,7 +102,7 @@ function deleteSecret(directories, key) {
101 * @param {string} key Secret key102 * @param {string} key Secret key
102 * @returns {string} Secret value103 * @returns {string} Secret value
103 */104 */
104function readSecret(directories, key) {105export function readSecret(directories, key) {
105 const filePath = path.join(directories.root, SECRETS_FILE);106 const filePath = path.join(directories.root, SECRETS_FILE);
106107
107 if (!fs.existsSync(filePath)) {108 if (!fs.existsSync(filePath)) {
@@ -118,7 +119,7 @@ function readSecret(directories, key) {
118 * @param {import('../users').UserDirectoryList} directories User directories119 * @param {import('../users').UserDirectoryList} directories User directories
119 * @returns {object} Secret state120 * @returns {object} Secret state
120 */121 */
121function readSecretState(directories) {122export function readSecretState(directories) {
122 const filePath = path.join(directories.root, SECRETS_FILE);123 const filePath = path.join(directories.root, SECRETS_FILE);
123124
124 if (!fs.existsSync(filePath)) {125 if (!fs.existsSync(filePath)) {
@@ -141,7 +142,7 @@ function readSecretState(directories) {
141 * @param {import('../users').UserDirectoryList} directories User directories142 * @param {import('../users').UserDirectoryList} directories User directories
142 * @returns {Record<string, string> | undefined} Secrets143 * @returns {Record<string, string> | undefined} Secrets
143 */144 */
144function getAllSecrets(directories) {145export function getAllSecrets(directories) {
145 const filePath = path.join(directories.root, SECRETS_FILE);146 const filePath = path.join(directories.root, SECRETS_FILE);
146147
147 if (!fs.existsSync(filePath)) {148 if (!fs.existsSync(filePath)) {
@@ -154,7 +155,7 @@ function getAllSecrets(directories) {
154 return secrets;155 return secrets;
155}156}
156157
157const router = express.Router();158export const router = express.Router();
158159
159router.post('/write', jsonParser, (request, response) => {160router.post('/write', jsonParser, (request, response) => {
160 const key = request.body.key;161 const key = request.body.key;
@@ -218,13 +219,3 @@ router.post('/find', jsonParser, (request, response) => {
218 return response.sendStatus(500);219 return response.sendStatus(500);
219 }220 }
220});221});
221
222module.exports = {
223 writeSecret,
224 readSecret,
225 deleteSecret,
226 readSecretState,
227 getAllSecrets,
228 SECRET_KEYS,
229 router,
230};
src/endpoints/settings.js+13 -13
@@ -1,12 +1,14 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');3
4const _ = require('lodash');4import express from 'express';
5const writeFileAtomicSync = require('write-file-atomic').sync;5import _ from 'lodash';
6const { SETTINGS_FILE } = require('../constants');6import { sync as writeFileAtomicSync } from 'write-file-atomic';
7const { getConfigValue, generateTimestamp, removeOldBackups } = require('../util');7
8const { jsonParser } = require('../express-common');8import { SETTINGS_FILE } from '../constants.js';
9const { getAllUserHandles, getUserDirectories } = require('../users');9import { getConfigValue, generateTimestamp, removeOldBackups } from '../util.js';
10import { jsonParser } from '../express-common.js';
11import { getAllUserHandles, getUserDirectories } from '../users.js';
1012
11const ENABLE_EXTENSIONS = getConfigValue('enableExtensions', true);13const ENABLE_EXTENSIONS = getConfigValue('enableExtensions', true);
12const ENABLE_EXTENSIONS_AUTO_UPDATE = getConfigValue('enableExtensionsAutoUpdate', true);14const ENABLE_EXTENSIONS_AUTO_UPDATE = getConfigValue('enableExtensionsAutoUpdate', true);
@@ -190,7 +192,7 @@ function getLatestBackup(handle) {
190 return path.join(userDirectories.backups, latestBackup);192 return path.join(userDirectories.backups, latestBackup);
191}193}
192194
193const router = express.Router();195export const router = express.Router();
194196
195router.post('/save', jsonParser, function (request, response) {197router.post('/save', jsonParser, function (request, response) {
196 try {198 try {
@@ -357,8 +359,6 @@ router.post('/restore-snapshot', jsonParser, async (request, response) => {
357/**359/**
358 * Initializes the settings endpoint360 * Initializes the settings endpoint
359 */361 */
360async function init() {362export async function init() {
361 await backupSettings();363 await backupSettings();
362}364}
363
364module.exports = { router, init };
src/endpoints/speech.js+7 -11
@@ -1,7 +1,9 @@
1const express = require('express');1import express from 'express';
2const { jsonParser } = require('../express-common');2import wavefile from 'wavefile';
3import { jsonParser } from '../express-common.js';
4import { getPipeline } from '../transformers.mjs';
35
4const router = express.Router();6export const router = express.Router();
57
6/**8/**
7 * Gets the audio data from a base64-encoded audio file.9 * Gets the audio data from a base64-encoded audio file.
@@ -9,7 +11,6 @@ const router = express.Router();
9 * @returns {Float64Array} Audio data11 * @returns {Float64Array} Audio data
10 */12 */
11function getWaveFile(audio) {13function getWaveFile(audio) {
12 const wavefile = require('wavefile');
13 const wav = new wavefile.WaveFile();14 const wav = new wavefile.WaveFile();
14 wav.fromDataURI(audio);15 wav.fromDataURI(audio);
15 wav.toBitDepth('32f');16 wav.toBitDepth('32f');
@@ -36,8 +37,7 @@ router.post('/recognize', jsonParser, async (req, res) => {
36 try {37 try {
37 const TASK = 'automatic-speech-recognition';38 const TASK = 'automatic-speech-recognition';
38 const { model, audio, lang } = req.body;39 const { model, audio, lang } = req.body;
39 const module = await import('../transformers.mjs');40 const pipe = await getPipeline(TASK, model);
40 const pipe = await module.default.getPipeline(TASK, model);
41 const wav = getWaveFile(audio);41 const wav = getWaveFile(audio);
42 const start = performance.now();42 const start = performance.now();
43 const result = await pipe(wav, { language: lang || null, task: 'transcribe' });43 const result = await pipe(wav, { language: lang || null, task: 'transcribe' });
@@ -54,11 +54,9 @@ router.post('/recognize', jsonParser, async (req, res) => {
5454
55router.post('/synthesize', jsonParser, async (req, res) => {55router.post('/synthesize', jsonParser, async (req, res) => {
56 try {56 try {
57 const wavefile = require('wavefile');
58 const TASK = 'text-to-speech';57 const TASK = 'text-to-speech';
59 const { text, model, speaker } = req.body;58 const { text, model, speaker } = req.body;
60 const module = await import('../transformers.mjs');59 const pipe = await getPipeline(TASK, model);
61 const pipe = await module.default.getPipeline(TASK, model);
62 const speaker_embeddings = speaker60 const speaker_embeddings = speaker
63 ? new Float32Array(new Uint8Array(Buffer.from(speaker.startsWith('data:') ? speaker.split(',')[1] : speaker, 'base64')).buffer)61 ? new Float32Array(new Uint8Array(Buffer.from(speaker.startsWith('data:') ? speaker.split(',')[1] : speaker, 'base64')).buffer)
64 : null;62 : null;
@@ -78,5 +76,3 @@ router.post('/synthesize', jsonParser, async (req, res) => {
78 return res.sendStatus(500);76 return res.sendStatus(500);
79 }77 }
80});78});
81
82module.exports = { router };
src/endpoints/sprites.js+11 -15
@@ -1,12 +1,13 @@
1import * as fs from 'node:fs';
2import * as path from 'node:path';
13
2const fs = require('fs');4import express from 'express';
3const path = require('path');5import mime from 'mime-types';
4const express = require('express');6import sanitize from 'sanitize-filename';
5const mime = require('mime-types');7import { sync as writeFileAtomicSync } from 'write-file-atomic';
6const sanitize = require('sanitize-filename');8
7const writeFileAtomicSync = require('write-file-atomic').sync;9import { getImageBuffers } from '../util.js';
8const { getImageBuffers } = require('../util');10import { jsonParser, urlencodedParser } from '../express-common.js';
9const { jsonParser, urlencodedParser } = require('../express-common');
1011
11/**12/**
12 * Gets the path to the sprites folder for the provided character name13 * Gets the path to the sprites folder for the provided character name
@@ -45,7 +46,7 @@ function getSpritesPath(directories, name, isSubfolder) {
45 * @param {object} data RisuAI character data46 * @param {object} data RisuAI character data
46 * @returns {void}47 * @returns {void}
47 */48 */
48function importRisuSprites(directories, data) {49export function importRisuSprites(directories, data) {
49 try {50 try {
50 const name = data?.data?.name;51 const name = data?.data?.name;
51 const risuData = data?.data?.extensions?.risuai;52 const risuData = data?.data?.extensions?.risuai;
@@ -106,7 +107,7 @@ function importRisuSprites(directories, data) {
106 }107 }
107}108}
108109
109const router = express.Router();110export const router = express.Router();
110111
111router.get('/get', jsonParser, function (request, response) {112router.get('/get', jsonParser, function (request, response) {
112 const name = String(request.query.name);113 const name = String(request.query.name);
@@ -259,8 +260,3 @@ router.post('/upload', urlencodedParser, async (request, response) => {
259 return response.sendStatus(500);260 return response.sendStatus(500);
260 }261 }
261});262});
262
263module.exports = {
264 router,
265 importRisuSprites,
266};
src/endpoints/stable-diffusion.js+13 -13
@@ -1,13 +1,15 @@
1const express = require('express');1import * as fs from 'node:fs';
2const fetch = require('node-fetch').default;2import * as path from 'node:path';
3const sanitize = require('sanitize-filename');3
4const { getBasicAuthHeader, delay, getHexString } = require('../util.js');4import express from 'express';
5const fs = require('fs');5import fetch from 'node-fetch';
6const path = require('path');6import sanitize from 'sanitize-filename';
7const writeFileAtomicSync = require('write-file-atomic').sync;7import { sync as writeFileAtomicSync } from 'write-file-atomic';
8const { jsonParser } = require('../express-common');8import FormData from 'form-data';
9const { readSecret, SECRET_KEYS } = require('./secrets.js');9
10const FormData = require('form-data');10import { getBasicAuthHeader, delay, getHexString } from '../util.js';
11import { jsonParser } from '../express-common.js';
12import { readSecret, SECRET_KEYS } from './secrets.js';
1113
12/**14/**
13 * Gets the comfy workflows.15 * Gets the comfy workflows.
@@ -21,7 +23,7 @@ function getComfyWorkflows(directories) {
21 .sort(Intl.Collator().compare);23 .sort(Intl.Collator().compare);
22}24}
2325
24const router = express.Router();26export const router = express.Router();
2527
26router.post('/ping', jsonParser, async (request, response) => {28router.post('/ping', jsonParser, async (request, response) => {
27 try {29 try {
@@ -993,5 +995,3 @@ router.use('/pollinations', pollinations);
993router.use('/stability', stability);995router.use('/stability', stability);
994router.use('/blockentropy', blockentropy);996router.use('/blockentropy', blockentropy);
995router.use('/huggingface', huggingface);997router.use('/huggingface', huggingface);
996
997module.exports = { router };
src/endpoints/stats.js+12 -18
@@ -1,14 +1,15 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');3import * as crypto from 'node:crypto';
4const writeFileAtomic = require('write-file-atomic');4
5const crypto = require('crypto');5import express from 'express';
6import writeFileAtomic from 'write-file-atomic';
67
7const readFile = fs.promises.readFile;8const readFile = fs.promises.readFile;
8const readdir = fs.promises.readdir;9const readdir = fs.promises.readdir;
910
10const { jsonParser } = require('../express-common');11import { jsonParser } from '../express-common.js';
11const { getAllUserHandles, getUserDirectories } = require('../users');12import { getAllUserHandles, getUserDirectories } from '../users.js';
1213
13const STATS_FILE = 'stats.json';14const STATS_FILE = 'stats.json';
1415
@@ -146,7 +147,7 @@ async function collectAndCreateStats(chatsPath, charactersPath) {
146 * @param {string} chatsPath Path to the directory containing the chat files.147 * @param {string} chatsPath Path to the directory containing the chat files.
147 * @param {string} charactersPath Path to the directory containing the character files.148 * @param {string} charactersPath Path to the directory containing the character files.
148 */149 */
149async function recreateStats(handle, chatsPath, charactersPath) {150export async function recreateStats(handle, chatsPath, charactersPath) {
150 console.log('Collecting and creating stats for user:', handle);151 console.log('Collecting and creating stats for user:', handle);
151 const stats = await collectAndCreateStats(chatsPath, charactersPath);152 const stats = await collectAndCreateStats(chatsPath, charactersPath);
152 STATS.set(handle, stats);153 STATS.set(handle, stats);
@@ -157,7 +158,7 @@ async function recreateStats(handle, chatsPath, charactersPath) {
157 * Loads the stats file into memory. If the file doesn't exist or is invalid,158 * Loads the stats file into memory. If the file doesn't exist or is invalid,
158 * initializes stats by collecting and creating them for each character.159 * initializes stats by collecting and creating them for each character.
159 */160 */
160async function init() {161export async function init() {
161 try {162 try {
162 const userHandles = await getAllUserHandles();163 const userHandles = await getAllUserHandles();
163 for (const handle of userHandles) {164 for (const handle of userHandles) {
@@ -209,7 +210,7 @@ async function saveStatsToFile() {
209 * Attempts to save charStats to a file and then terminates the process.210 * Attempts to save charStats to a file and then terminates the process.
210 * If an error occurs during the file write, it logs the error before exiting.211 * If an error occurs during the file write, it logs the error before exiting.
211 */212 */
212async function onExit() {213export async function onExit() {
213 try {214 try {
214 await saveStatsToFile();215 await saveStatsToFile();
215 } catch (err) {216 } catch (err) {
@@ -434,7 +435,7 @@ function calculateTotalGenTimeAndWordCount(
434 };435 };
435}436}
436437
437const router = express.Router();438export const router = express.Router();
438439
439/**440/**
440 * Handle a POST request to get the stats object441 * Handle a POST request to get the stats object
@@ -465,10 +466,3 @@ router.post('/update', jsonParser, function (request, response) {
465 setCharStats(request.user.profile.handle, request.body);466 setCharStats(request.user.profile.handle, request.body);
466 return response.sendStatus(200);467 return response.sendStatus(200);
467});468});
468
469module.exports = {
470 router,
471 recreateStats,
472 init,
473 onExit,
474};
src/endpoints/themes.js+9 -9
@@ -1,11 +1,13 @@
1const express = require('express');1import * as path from 'node:path';
2const path = require('path');2import * as fs from 'node:fs';
3const fs = require('fs');
4const sanitize = require('sanitize-filename');
5const writeFileAtomicSync = require('write-file-atomic').sync;
6const { jsonParser } = require('../express-common');
73
8const router = express.Router();4import express from 'express';
5import sanitize from 'sanitize-filename';
6import { sync as writeFileAtomicSync } from 'write-file-atomic';
7
8import { jsonParser } from '../express-common.js';
9
10export const router = express.Router();
911
10router.post('/save', jsonParser, (request, response) => {12router.post('/save', jsonParser, (request, response) => {
11 if (!request.body || !request.body.name) {13 if (!request.body || !request.body.name) {
@@ -36,5 +38,3 @@ router.post('/delete', jsonParser, function (request, response) {
36 return response.sendStatus(500);38 return response.sendStatus(500);
37 }39 }
38});40});
39
40module.exports = { router };
src/endpoints/thumbnails.js+16 -20
@@ -1,14 +1,16 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const fsPromises = require('fs').promises;2import * as fsPromises from 'node:fs/promises';
3const path = require('path');3import * as path from 'node:path';
4const mime = require('mime-types');4
5const express = require('express');5import mime from 'mime-types';
6const sanitize = require('sanitize-filename');6import express from 'express';
7const jimp = require('jimp');7import sanitize from 'sanitize-filename';
8const writeFileAtomicSync = require('write-file-atomic').sync;8import jimp from 'jimp';
9const { getAllUserHandles, getUserDirectories } = require('../users');9import { sync as writeFileAtomicSync } from 'write-file-atomic';
10const { getConfigValue } = require('../util');10
11const { jsonParser } = require('../express-common');11import { getAllUserHandles, getUserDirectories } from '../users.js';
12import { getConfigValue } from '../util.js';
13import { jsonParser } from '../express-common.js';
1214
13const thumbnailsDisabled = getConfigValue('disableThumbnails', false);15const thumbnailsDisabled = getConfigValue('disableThumbnails', false);
14const quality = getConfigValue('thumbnailsQuality', 95);16const quality = getConfigValue('thumbnailsQuality', 95);
@@ -62,7 +64,7 @@ function getOriginalFolder(directories, type) {
62 * @param {'bg' | 'avatar'} type Type of the thumbnail64 * @param {'bg' | 'avatar'} type Type of the thumbnail
63 * @param {string} file Name of the file65 * @param {string} file Name of the file
64 */66 */
65function invalidateThumbnail(directories, type, file) {67export function invalidateThumbnail(directories, type, file) {
66 const folder = getThumbnailFolder(directories, type);68 const folder = getThumbnailFolder(directories, type);
67 if (folder === undefined) throw new Error('Invalid thumbnail type');69 if (folder === undefined) throw new Error('Invalid thumbnail type');
6870
@@ -141,7 +143,7 @@ async function generateThumbnail(directories, type, file) {
141 * Ensures that the thumbnail cache for backgrounds is valid.143 * Ensures that the thumbnail cache for backgrounds is valid.
142 * @returns {Promise<void>} Promise that resolves when the cache is validated144 * @returns {Promise<void>} Promise that resolves when the cache is validated
143 */145 */
144async function ensureThumbnailCache() {146export async function ensureThumbnailCache() {
145 const userHandles = await getAllUserHandles();147 const userHandles = await getAllUserHandles();
146 for (const handle of userHandles) {148 for (const handle of userHandles) {
147 const directories = getUserDirectories(handle);149 const directories = getUserDirectories(handle);
@@ -166,7 +168,7 @@ async function ensureThumbnailCache() {
166 }168 }
167}169}
168170
169const router = express.Router();171export const router = express.Router();
170172
171// Important: This route must be mounted as '/thumbnail'. It is used in the client code and saved to chat files.173// Important: This route must be mounted as '/thumbnail'. It is used in the client code and saved to chat files.
172router.get('/', jsonParser, async function (request, response) {174router.get('/', jsonParser, async function (request, response) {
@@ -227,9 +229,3 @@ router.get('/', jsonParser, async function (request, response) {
227 return response.sendStatus(500);229 return response.sendStatus(500);
228 }230 }
229});231});
230
231module.exports = {
232 invalidateThumbnail,
233 ensureThumbnailCache,
234 router,
235};
src/endpoints/tokenizers.js+22 -29
@@ -1,15 +1,18 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');3
4const { SentencePieceProcessor } = require('@agnai/sentencepiece-js');4import express from 'express';
5const tiktoken = require('tiktoken');5import { sync as writeFileAtomicSync } from 'write-file-atomic';
6const { Tokenizer } = require('@agnai/web-tokenizers');6
7const { convertClaudePrompt } = require('../prompt-converters');7import { Tokenizer } from '@agnai/web-tokenizers';
8const { TEXTGEN_TYPES } = require('../constants');8import { SentencePieceProcessor } from '@agnai/sentencepiece-js';
9const { jsonParser } = require('../express-common');9import tiktoken from 'tiktoken';
10const { setAdditionalHeaders } = require('../additional-headers');10
11const { getConfigValue, isValidUrl } = require('../util');11import { convertClaudePrompt } from '../prompt-converters.js';
12const writeFileAtomicSync = require('write-file-atomic').sync;12import { TEXTGEN_TYPES } from '../constants.js';
13import { jsonParser } from '../express-common.js';
14import { setAdditionalHeaders } from '../additional-headers.js';
15import { getConfigValue, isValidUrl } from '../util.js';
1316
14/**17/**
15 * @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler18 * @typedef { (req: import('express').Request, res: import('express').Response) => Promise<any> } TokenizationHandler
@@ -23,7 +26,7 @@ const tokenizersCache = {};
23/**26/**
24 * @type {string[]}27 * @type {string[]}
25 */28 */
26const TEXT_COMPLETION_MODELS = [29export const TEXT_COMPLETION_MODELS = [
27 'gpt-3.5-turbo-instruct',30 'gpt-3.5-turbo-instruct',
28 'gpt-3.5-turbo-instruct-0914',31 'gpt-3.5-turbo-instruct-0914',
29 'text-davinci-003',32 'text-davinci-003',
@@ -223,7 +226,7 @@ const commandTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyT
223const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');226const qwen2Tokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/qwen2.json', 'src/tokenizers/llama3.json');
224const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');227const nemoTokenizer = new WebTokenizer('https://github.com/SillyTavern/SillyTavern-Tokenizers/raw/main/nemo.json', 'src/tokenizers/llama3.json');
225228
226const sentencepieceTokenizers = [229export const sentencepieceTokenizers = [
227 'llama',230 'llama',
228 'nerdstash',231 'nerdstash',
229 'nerdstash_v2',232 'nerdstash_v2',
@@ -238,7 +241,7 @@ const sentencepieceTokenizers = [
238 * @param {string} model Sentencepiece model name241 * @param {string} model Sentencepiece model name
239 * @returns {SentencePieceTokenizer|null} Sentencepiece tokenizer242 * @returns {SentencePieceTokenizer|null} Sentencepiece tokenizer
240 */243 */
241function getSentencepiceTokenizer(model) {244export function getSentencepiceTokenizer(model) {
242 if (model.includes('llama')) {245 if (model.includes('llama')) {
243 return spp_llama;246 return spp_llama;
244 }247 }
@@ -350,7 +353,7 @@ function getWebTokenizersChunks(tokenizer, ids) {
350 * @param {string} requestModel Models to use for tokenization353 * @param {string} requestModel Models to use for tokenization
351 * @returns {string} Tokenizer model to use354 * @returns {string} Tokenizer model to use
352 */355 */
353function getTokenizerModel(requestModel) {356export function getTokenizerModel(requestModel) {
354 if (requestModel.includes('o1-preview') || requestModel.includes('o1-mini')) {357 if (requestModel.includes('o1-preview') || requestModel.includes('o1-mini')) {
355 return 'gpt-4o';358 return 'gpt-4o';
356 }359 }
@@ -427,7 +430,7 @@ function getTokenizerModel(requestModel) {
427 return 'gpt-3.5-turbo';430 return 'gpt-3.5-turbo';
428}431}
429432
430function getTiktokenTokenizer(model) {433export function getTiktokenTokenizer(model) {
431 if (tokenizersCache[model]) {434 if (tokenizersCache[model]) {
432 return tokenizersCache[model];435 return tokenizersCache[model];
433 }436 }
@@ -444,7 +447,7 @@ function getTiktokenTokenizer(model) {
444 * @param {object[]} messages Array of messages447 * @param {object[]} messages Array of messages
445 * @returns {number} Number of tokens448 * @returns {number} Number of tokens
446 */449 */
447function countWebTokenizerTokens(tokenizer, messages) {450export function countWebTokenizerTokens(tokenizer, messages) {
448 // Should be fine if we use the old conversion method instead of the messages API one i think?451 // Should be fine if we use the old conversion method instead of the messages API one i think?
449 const convertedPrompt = convertClaudePrompt(messages, false, '', false, false, '', false);452 const convertedPrompt = convertClaudePrompt(messages, false, '', false, false, '', false);
450453
@@ -636,7 +639,7 @@ function createWebTokenizerDecodingHandler(tokenizer) {
636 };639 };
637}640}
638641
639const router = express.Router();642export const router = express.Router();
640643
641router.post('/llama/encode', jsonParser, createSentencepieceEncodingHandler(spp_llama));644router.post('/llama/encode', jsonParser, createSentencepieceEncodingHandler(spp_llama));
642router.post('/nerdstash/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd));645router.post('/nerdstash/encode', jsonParser, createSentencepieceEncodingHandler(spp_nerd));
@@ -1002,13 +1005,3 @@ router.post('/remote/textgenerationwebui/encode', jsonParser, async function (re
1002 return response.send({ error: true });1005 return response.send({ error: true });
1003 }1006 }
1004});1007});
1005
1006module.exports = {
1007 TEXT_COMPLETION_MODELS,
1008 getTokenizerModel,
1009 getTiktokenTokenizer,
1010 countWebTokenizerTokens,
1011 getSentencepiceTokenizer,
1012 sentencepieceTokenizers,
1013 router,
1014};
src/endpoints/translate.js+14 -12
@@ -1,10 +1,16 @@
1const fetch = require('node-fetch').default;1import * as https from 'node:https';
2const https = require('https');2import { createRequire } from 'node:module';
3const express = require('express');3import fetch from 'node-fetch';
4const iconv = require('iconv-lite');4import express from 'express';
5const { readSecret, SECRET_KEYS } = require('./secrets');5import iconv from 'iconv-lite';
6const { getConfigValue, uuidv4 } = require('../util');6import bingTranslateApi from 'bing-translate-api';
7const { jsonParser } = require('../express-common');7
8const require = createRequire(import.meta.url);
9const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser');
10
11import { readSecret, SECRET_KEYS } from './secrets.js';
12import { getConfigValue, uuidv4 } from '../util.js';
13import { jsonParser } from '../express-common.js';
814
9const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate';15const DEEPLX_URL_DEFAULT = 'http://127.0.0.1:1188/translate';
10const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate';16const ONERING_URL_DEFAULT = 'http://127.0.0.1:4990/translate';
@@ -23,7 +29,7 @@ function decodeBuffer(buffer) {
23 }29 }
24}30}
2531
26const router = express.Router();32export const router = express.Router();
2733
28router.post('/libre', jsonParser, async (request, response) => {34router.post('/libre', jsonParser, async (request, response) => {
29 const key = readSecret(request.user.directories, SECRET_KEYS.LIBRE);35 const key = readSecret(request.user.directories, SECRET_KEYS.LIBRE);
@@ -82,7 +88,6 @@ router.post('/libre', jsonParser, async (request, response) => {
8288
83router.post('/google', jsonParser, async (request, response) => {89router.post('/google', jsonParser, async (request, response) => {
84 try {90 try {
85 const { generateRequestUrl, normaliseResponse } = require('google-translate-api-browser');
86 const text = request.body.text;91 const text = request.body.text;
87 const lang = request.body.lang;92 const lang = request.body.lang;
8893
@@ -386,7 +391,6 @@ router.post('/deeplx', jsonParser, async (request, response) => {
386});391});
387392
388router.post('/bing', jsonParser, async (request, response) => {393router.post('/bing', jsonParser, async (request, response) => {
389 const bingTranslateApi = require('bing-translate-api');
390 const text = request.body.text;394 const text = request.body.text;
391 let lang = request.body.lang;395 let lang = request.body.lang;
392396
@@ -408,5 +412,3 @@ router.post('/bing', jsonParser, async (request, response) => {
408 return response.sendStatus(500);412 return response.sendStatus(500);
409 });413 });
410});414});
411
412module.exports = { router };
src/endpoints/users-admin.js+11 -14
@@ -1,10 +1,11 @@
1const fsPromises = require('fs').promises;1import { promises as fsPromises } from 'node:fs';
2const storage = require('node-persist');2
3const express = require('express');3import storage from 'node-persist';
4const lodash = require('lodash');4import express from 'express';
5const { jsonParser } = require('../express-common');5import lodash from 'lodash';
6const { checkForNewContent } = require('./content-manager');6import { jsonParser } from '../express-common.js';
7const {7import { checkForNewContent } from './content-manager.js';
8import {
8 KEY_PREFIX,9 KEY_PREFIX,
9 toKey,10 toKey,
10 requireAdminMiddleware,11 requireAdminMiddleware,
@@ -14,10 +15,10 @@ const {
14 getPasswordHash,15 getPasswordHash,
15 getUserDirectories,16 getUserDirectories,
16 ensurePublicDirectoriesExist,17 ensurePublicDirectoriesExist,
17} = require('../users');18} from '../users.js';
18const { DEFAULT_USER } = require('../constants');19import { DEFAULT_USER } from '../constants.js';
1920
20const router = express.Router();21export const router = express.Router();
2122
22router.post('/get', requireAdminMiddleware, jsonParser, async (_request, response) => {23router.post('/get', requireAdminMiddleware, jsonParser, async (_request, response) => {
23 try {24 try {
@@ -249,7 +250,3 @@ router.post('/slugify', requireAdminMiddleware, jsonParser, async (request, resp
249 return response.sendStatus(500);250 return response.sendStatus(500);
250 }251 }
251});252});
252
253module.exports = {
254 router,
255};
src/endpoints/users-private.js+14 -16
@@ -1,18 +1,20 @@
1const path = require('path');1import * as path from 'node:path';
2const fsPromises = require('fs').promises;2import { promises as fsPromises } from 'node:fs';
3const storage = require('node-persist');3import * as crypto from 'node:crypto';
4const express = require('express');4
5const crypto = require('crypto');5import storage from 'node-persist';
6const { jsonParser } = require('../express-common');6import express from 'express';
7const { getUserAvatar, toKey, getPasswordHash, getPasswordSalt, createBackupArchive, ensurePublicDirectoriesExist, toAvatarKey } = require('../users');7
8const { SETTINGS_FILE } = require('../constants');8import { jsonParser } from '../express-common.js';
9const contentManager = require('./content-manager');9import { getUserAvatar, toKey, getPasswordHash, getPasswordSalt, createBackupArchive, ensurePublicDirectoriesExist, toAvatarKey } from '../users.js';
10const { color, Cache } = require('../util');10import { SETTINGS_FILE } from '../constants.js';
11const { checkForNewContent } = require('./content-manager');11import * as contentManager from './content-manager.js';
12import { color, Cache } from '../util.js';
13import { checkForNewContent } from './content-manager.js';
1214
13const RESET_CACHE = new Cache(5 * 60 * 1000);15const RESET_CACHE = new Cache(5 * 60 * 1000);
1416
15const router = express.Router();17export const router = express.Router();
1618
17router.post('/logout', async (request, response) => {19router.post('/logout', async (request, response) => {
18 try {20 try {
@@ -251,7 +253,3 @@ router.post('/reset-step2', jsonParser, async (request, response) => {
251 return response.sendStatus(500);253 return response.sendStatus(500);
252 }254 }
253});255});
254
255module.exports = {
256 router,
257};
src/endpoints/users-public.js+9 -12
@@ -1,15 +1,16 @@
1const crypto = require('crypto');1import * as crypto from 'node:crypto';
2const storage = require('node-persist');2
3const express = require('express');3import storage from 'node-persist';
4const { RateLimiterMemory, RateLimiterRes } = require('rate-limiter-flexible');4import express from 'express';
5const { jsonParser, getIpFromRequest } = require('../express-common');5import { RateLimiterMemory, RateLimiterRes } from 'rate-limiter-flexible';
6const { color, Cache, getConfigValue } = require('../util');6import { jsonParser, getIpFromRequest } from '../express-common.js';
7const { KEY_PREFIX, getUserAvatar, toKey, getPasswordHash, getPasswordSalt } = require('../users');7import { color, Cache, getConfigValue } from '../util.js';
8import { KEY_PREFIX, getUserAvatar, toKey, getPasswordHash, getPasswordSalt } from '../users.js';
89
9const DISCREET_LOGIN = getConfigValue('enableDiscreetLogin', false);10const DISCREET_LOGIN = getConfigValue('enableDiscreetLogin', false);
10const MFA_CACHE = new Cache(5 * 60 * 1000);11const MFA_CACHE = new Cache(5 * 60 * 1000);
1112
12const router = express.Router();13export const router = express.Router();
13const loginLimiter = new RateLimiterMemory({14const loginLimiter = new RateLimiterMemory({
14 points: 5,15 points: 5,
15 duration: 60,16 duration: 60,
@@ -193,7 +194,3 @@ router.post('/recover-step2', jsonParser, async (request, response) => {
193 return response.sendStatus(500);194 return response.sendStatus(500);
194 }195 }
195});196});
196
197module.exports = {
198 router,
199};
src/endpoints/vectors.js+38 -28
@@ -1,10 +1,22 @@
1const vectra = require('vectra');1import * as path from 'node:path';
2const path = require('path');2import * as fs from 'node:fs';
3const fs = require('fs');3
4const express = require('express');4import vectra from 'vectra';
5const sanitize = require('sanitize-filename');5import express from 'express';
6const { jsonParser } = require('../express-common');6import sanitize from 'sanitize-filename';
7const { getConfigValue } = require('../util');7
8import { jsonParser } from '../express-common.js';
9import { getConfigValue } from '../util.js';
10
11import { getNomicAIBatchVector, getNomicAIVector } from '../vectors/nomicai-vectors.js';
12import { getOpenAIVector, getOpenAIBatchVector } from '../vectors/openai-vectors.js';
13import { getTransformersVector, getTransformersBatchVector } from '../vectors/embedding.js';
14import { getExtrasVector, getExtrasBatchVector } from '../vectors/extras-vectors.js';
15import { getMakerSuiteVector, getMakerSuiteBatchVector } from '../vectors/makersuite-vectors.js';
16import { getCohereVector, getCohereBatchVector } from '../vectors/cohere-vectors.js';
17import { getLlamaCppVector, getLlamaCppBatchVector } from '../vectors/llamacpp-vectors.js';
18import { getVllmVector, getVllmBatchVector } from '../vectors/vllm-vectors.js';
19import { getOllamaVector, getOllamaBatchVector } from '../vectors/ollama-vectors.js';
820
9// Don't forget to add new sources to the SOURCES array21// Don't forget to add new sources to the SOURCES array
10const SOURCES = [22const SOURCES = [
@@ -33,25 +45,25 @@ const SOURCES = [
33async function getVector(source, sourceSettings, text, isQuery, directories) {45async function getVector(source, sourceSettings, text, isQuery, directories) {
34 switch (source) {46 switch (source) {
35 case 'nomicai':47 case 'nomicai':
36 return require('../vectors/nomicai-vectors').getNomicAIVector(text, source, directories);48 return getNomicAIVector(text, source, directories);
37 case 'togetherai':49 case 'togetherai':
38 case 'mistral':50 case 'mistral':
39 case 'openai':51 case 'openai':
40 return require('../vectors/openai-vectors').getOpenAIVector(text, source, directories, sourceSettings.model);52 return getOpenAIVector(text, source, directories, sourceSettings.model);
41 case 'transformers':53 case 'transformers':
42 return require('../vectors/embedding').getTransformersVector(text);54 return getTransformersVector(text);
43 case 'extras':55 case 'extras':
44 return require('../vectors/extras-vectors').getExtrasVector(text, sourceSettings.extrasUrl, sourceSettings.extrasKey);56 return getExtrasVector(text, sourceSettings.extrasUrl, sourceSettings.extrasKey);
45 case 'palm':57 case 'palm':
46 return require('../vectors/makersuite-vectors').getMakerSuiteVector(text, directories);58 return getMakerSuiteVector(text, directories);
47 case 'cohere':59 case 'cohere':
48 return require('../vectors/cohere-vectors').getCohereVector(text, isQuery, directories, sourceSettings.model);60 return getCohereVector(text, isQuery, directories, sourceSettings.model);
49 case 'llamacpp':61 case 'llamacpp':
50 return require('../vectors/llamacpp-vectors').getLlamaCppVector(text, sourceSettings.apiUrl, directories);62 return getLlamaCppVector(text, sourceSettings.apiUrl, directories);
51 case 'vllm':63 case 'vllm':
52 return require('../vectors/vllm-vectors').getVllmVector(text, sourceSettings.apiUrl, sourceSettings.model, directories);64 return getVllmVector(text, sourceSettings.apiUrl, sourceSettings.model, directories);
53 case 'ollama':65 case 'ollama':
54 return require('../vectors/ollama-vectors').getOllamaVector(text, sourceSettings.apiUrl, sourceSettings.model, sourceSettings.keep, directories);66 return getOllamaVector(text, sourceSettings.apiUrl, sourceSettings.model, sourceSettings.keep, directories);
55 }67 }
5668
57 throw new Error(`Unknown vector source ${source}`);69 throw new Error(`Unknown vector source ${source}`);
@@ -74,33 +86,33 @@ async function getBatchVector(source, sourceSettings, texts, isQuery, directorie
74 for (let batch of batches) {86 for (let batch of batches) {
75 switch (source) {87 switch (source) {
76 case 'nomicai':88 case 'nomicai':
77 results.push(...await require('../vectors/nomicai-vectors').getNomicAIBatchVector(batch, source, directories));89 results.push(...await getNomicAIBatchVector(batch, source, directories));
78 break;90 break;
79 case 'togetherai':91 case 'togetherai':
80 case 'mistral':92 case 'mistral':
81 case 'openai':93 case 'openai':
82 results.push(...await require('../vectors/openai-vectors').getOpenAIBatchVector(batch, source, directories, sourceSettings.model));94 results.push(...await getOpenAIBatchVector(batch, source, directories, sourceSettings.model));
83 break;95 break;
84 case 'transformers':96 case 'transformers':
85 results.push(...await require('../vectors/embedding').getTransformersBatchVector(batch));97 results.push(...await getTransformersBatchVector(batch));
86 break;98 break;
87 case 'extras':99 case 'extras':
88 results.push(...await require('../vectors/extras-vectors').getExtrasBatchVector(batch, sourceSettings.extrasUrl, sourceSettings.extrasKey));100 results.push(...await getExtrasBatchVector(batch, sourceSettings.extrasUrl, sourceSettings.extrasKey));
89 break;101 break;
90 case 'palm':102 case 'palm':
91 results.push(...await require('../vectors/makersuite-vectors').getMakerSuiteBatchVector(batch, directories));103 results.push(...await getMakerSuiteBatchVector(batch, directories));
92 break;104 break;
93 case 'cohere':105 case 'cohere':
94 results.push(...await require('../vectors/cohere-vectors').getCohereBatchVector(batch, isQuery, directories, sourceSettings.model));106 results.push(...await getCohereBatchVector(batch, isQuery, directories, sourceSettings.model));
95 break;107 break;
96 case 'llamacpp':108 case 'llamacpp':
97 results.push(...await require('../vectors/llamacpp-vectors').getLlamaCppBatchVector(batch, sourceSettings.apiUrl, directories));109 results.push(...await getLlamaCppBatchVector(batch, sourceSettings.apiUrl, directories));
98 break;110 break;
99 case 'vllm':111 case 'vllm':
100 results.push(...await require('../vectors/vllm-vectors').getVllmBatchVector(batch, sourceSettings.apiUrl, sourceSettings.model, directories));112 results.push(...await getVllmBatchVector(batch, sourceSettings.apiUrl, sourceSettings.model, directories));
101 break;113 break;
102 case 'ollama':114 case 'ollama':
103 results.push(...await require('../vectors/ollama-vectors').getOllamaBatchVector(batch, sourceSettings.apiUrl, sourceSettings.model, sourceSettings.keep, directories));115 results.push(...await getOllamaBatchVector(batch, sourceSettings.apiUrl, sourceSettings.model, sourceSettings.keep, directories));
104 break;116 break;
105 default:117 default:
106 throw new Error(`Unknown vector source ${source}`);118 throw new Error(`Unknown vector source ${source}`);
@@ -359,7 +371,7 @@ async function regenerateCorruptedIndexErrorHandler(req, res, error) {
359 return res.sendStatus(500);371 return res.sendStatus(500);
360}372}
361373
362const router = express.Router();374export const router = express.Router();
363375
364router.post('/query', jsonParser, async (req, res) => {376router.post('/query', jsonParser, async (req, res) => {
365 try {377 try {
@@ -495,5 +507,3 @@ router.post('/purge', jsonParser, async (req, res) => {
495 return res.sendStatus(500);507 return res.sendStatus(500);
496 }508 }
497});509});
498
499module.exports = { router };
src/endpoints/worldinfo.js+9 -10
@@ -1,10 +1,11 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const express = require('express');
4const sanitize = require('sanitize-filename');
5const writeFileAtomicSync = require('write-file-atomic').sync;
63
7const { jsonParser, urlencodedParser } = require('../express-common');4import express from 'express';
5import sanitize from 'sanitize-filename';
6import { sync as writeFileAtomicSync } from 'write-file-atomic';
7
8import { jsonParser, urlencodedParser } from '../express-common.js';
89
9/**10/**
10 * Reads a World Info file and returns its contents11 * Reads a World Info file and returns its contents
@@ -13,7 +14,7 @@ const { jsonParser, urlencodedParser } = require('../express-common');
13 * @param {boolean} allowDummy If true, returns an empty object if the file doesn't exist14 * @param {boolean} allowDummy If true, returns an empty object if the file doesn't exist
14 * @returns {object} World Info file contents15 * @returns {object} World Info file contents
15 */16 */
16function readWorldInfoFile(directories, worldInfoName, allowDummy) {17export function readWorldInfoFile(directories, worldInfoName, allowDummy) {
17 const dummyObject = allowDummy ? { entries: {} } : null;18 const dummyObject = allowDummy ? { entries: {} } : null;
1819
19 if (!worldInfoName) {20 if (!worldInfoName) {
@@ -33,7 +34,7 @@ function readWorldInfoFile(directories, worldInfoName, allowDummy) {
33 return worldInfo;34 return worldInfo;
34}35}
3536
36const router = express.Router();37export const router = express.Router();
3738
38router.post('/get', jsonParser, (request, response) => {39router.post('/get', jsonParser, (request, response) => {
39 if (!request.body?.name) {40 if (!request.body?.name) {
@@ -122,5 +123,3 @@ router.post('/edit', jsonParser, (request, response) => {
122123
123 return response.send({ ok: true });124 return response.send({ ok: true });
124});125});
125
126module.exports = { router, readWorldInfoFile };
src/express-common.js+11 -12
@@ -1,28 +1,27 @@
1const express = require('express');1import express from 'express';
2const ipaddr = require('ipaddr.js');2import ipaddr from 'ipaddr.js';
33
4// Instantiate parser middleware here with application-level size limits4// Instantiate parser middleware here with application-level size limits
5const jsonParser = express.json({ limit: '200mb' });5export const jsonParser = express.json({ limit: '200mb' });
6const urlencodedParser = express.urlencoded({ extended: true, limit: '200mb' });6export const urlencodedParser = express.urlencoded({ extended: true, limit: '200mb' });
77
8/**8/**
9 * Gets the IP address of the client from the request object.9 * Gets the IP address of the client from the request object.
10 * @param {import('express'.Request)} req Request object10 * @param {import('express').Request} req Request object
11 * @returns {string} IP address of the client11 * @returns {string} IP address of the client
12 */12 */
13function getIpFromRequest(req) {13export function getIpFromRequest(req) {
14 let clientIp = req.connection.remoteAddress;14 let clientIp = req.socket.remoteAddress;
15 if (!clientIp) {
16 return 'unknown';
17 }
15 let ip = ipaddr.parse(clientIp);18 let ip = ipaddr.parse(clientIp);
16 // Check if the IP address is IPv4-mapped IPv6 address19 // Check if the IP address is IPv4-mapped IPv6 address
17 if (ip.kind() === 'ipv6' && ip instanceof ipaddr.IPv6 && ip.isIPv4MappedAddress()) {20 if (ip.kind() === 'ipv6' && ip instanceof ipaddr.IPv6 && ip.isIPv4MappedAddress()) {
18 const ipv4 = ip.toIPv4Address().toString();21 const ipv4 = ip.toIPv4Address().toString();
19 clientIp = ipv4;22 clientIp = ipv4;
20 } else {23 } else {
21 clientIp = ip;24 clientIp = ip.toString();
22 clientIp = clientIp.toString();
23 }25 }
24 return clientIp;26 return clientIp;
25}27}
26
27
28module.exports = { jsonParser, urlencodedParser, getIpFromRequest };
src/middleware/basicAuth.js+4 -4
@@ -2,9 +2,9 @@
2 * When applied, this middleware will ensure the request contains the required header for basic authentication and only2 * When applied, this middleware will ensure the request contains the required header for basic authentication and only
3 * allow access to the endpoint after successful authentication.3 * allow access to the endpoint after successful authentication.
4 */4 */
5const { getAllUserHandles, toKey, getPasswordHash } = require('../users.js');5import storage from 'node-persist';
6const { getConfig, getConfigValue } = require('../util.js');6import { getAllUserHandles, toKey, getPasswordHash } from '../users.js';
7const storage = require('node-persist');7import { getConfig, getConfigValue } from '../util.js';
88
9const PER_USER_BASIC_AUTH = getConfigValue('perUserBasicAuth', false);9const PER_USER_BASIC_AUTH = getConfigValue('perUserBasicAuth', false);
10const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);10const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);
@@ -49,4 +49,4 @@ const basicAuthMiddleware = async function (request, response, callback) {
49 return unauthorizedResponse(response);49 return unauthorizedResponse(response);
50};50};
5151
52module.exports = basicAuthMiddleware;52export default basicAuthMiddleware;
src/middleware/multerMonkeyPatch.js+1 -3
@@ -14,7 +14,7 @@ function decodeFileName(str) {
14 * @param {import('express').Response} _res Response14 * @param {import('express').Response} _res Response
15 * @param {import('express').NextFunction} next Next middleware15 * @param {import('express').NextFunction} next Next middleware
16 */16 */
17function multerMonkeyPatch(req, _res, next) {17export default function multerMonkeyPatch(req, _res, next) {
18 try {18 try {
19 if (req.file) {19 if (req.file) {
20 req.file.originalname = decodeFileName(req.file.originalname);20 req.file.originalname = decodeFileName(req.file.originalname);
@@ -26,5 +26,3 @@ function multerMonkeyPatch(req, _res, next) {
26 next();26 next();
27 }27 }
28}28}
29
30module.exports = multerMonkeyPatch;
src/middleware/whitelist.js+6 -8
@@ -1,9 +1,9 @@
1const path = require('path');1import * as path from 'node:path';
2const fs = require('fs');2import * as fs from 'node:fs';
3const ipMatching = require('ip-matching');3import ipMatching from 'ip-matching';
44
5const { getIpFromRequest } = require('../express-common');5import { getIpFromRequest } from '../express-common.js';
6const { color, getConfigValue } = require('../util');6import { color, getConfigValue } from '../util.js';
77
8const whitelistPath = path.join(process.cwd(), './whitelist.txt');8const whitelistPath = path.join(process.cwd(), './whitelist.txt');
9const enableForwardedWhitelist = getConfigValue('enableForwardedWhitelist', false);9const enableForwardedWhitelist = getConfigValue('enableForwardedWhitelist', false);
@@ -50,7 +50,7 @@ function getForwardedIp(req) {
50 * @param {boolean} listen If listen mode is enabled via config or command line50 * @param {boolean} listen If listen mode is enabled via config or command line
51 * @returns {import('express').RequestHandler} The middleware function51 * @returns {import('express').RequestHandler} The middleware function
52 */52 */
53function whitelistMiddleware(whitelistMode, listen) {53export default function whitelistMiddleware(whitelistMode, listen) {
54 return function (req, res, next) {54 return function (req, res, next) {
55 const clientIp = getIpFromRequest(req);55 const clientIp = getIpFromRequest(req);
56 const forwardedIp = getForwardedIp(req);56 const forwardedIp = getForwardedIp(req);
@@ -82,5 +82,3 @@ function whitelistMiddleware(whitelistMode, listen) {
82 next();82 next();
83 };83 };
84}84}
85
86module.exports = whitelistMiddleware;
src/plugin-loader.js+7 -10
@@ -1,8 +1,9 @@
1const fs = require('fs');1import * as fs from 'node:fs';
2const path = require('path');2import * as path from 'node:path';
3const url = require('url');3import * as url from 'node:url';
4const express = require('express');4
5const { getConfigValue } = require('./util');5import express from 'express';
6import { getConfigValue } from './util.js';
6const enableServerPlugins = getConfigValue('enableServerPlugins', false);7const enableServerPlugins = getConfigValue('enableServerPlugins', false);
78
8/**9/**
@@ -32,7 +33,7 @@ const isESModule = (file) => path.extname(file) === '.mjs';
32 * @returns {Promise<Function>} Promise that resolves when all plugins are loaded. Resolves to a "cleanup" function to33 * @returns {Promise<Function>} Promise that resolves when all plugins are loaded. Resolves to a "cleanup" function to
33 * be called before the server shuts down.34 * be called before the server shuts down.
34 */35 */
35async function loadPlugins(app, pluginsPath) {36export async function loadPlugins(app, pluginsPath) {
36 const exitHooks = [];37 const exitHooks = [];
37 const emptyFn = () => {};38 const emptyFn = () => {};
3839
@@ -217,7 +218,3 @@ async function initPlugin(app, plugin, exitHooks) {
217218
218 return true;219 return true;
219}220}
220
221module.exports = {
222 loadPlugins,
223};
src/polyfill.js+1 -1
@@ -7,4 +7,4 @@ if (!Array.prototype.findLastIndex) {
7 };7 };
8}8}
99
10module.exports = {};10export default () => {};
src/prompt-converters.js+13 -22
@@ -1,6 +1,8 @@
1require('./polyfill.js');1import * as crypto from 'node:crypto';
2const { getConfigValue } = require('./util.js');2import polyfill from './polyfill.js';
3const crypto = require('crypto');3import { getConfigValue } from './util.js';
4
5polyfill();
46
5const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.');7const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.');
68
@@ -17,7 +19,7 @@ const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get start
17 * @returns {string} Prompt for Claude19 * @returns {string} Prompt for Claude
18 * @copyright Prompt Conversion script taken from RisuAI by kwaroran (GPLv3).20 * @copyright Prompt Conversion script taken from RisuAI by kwaroran (GPLv3).
19 */21 */
20function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill, withSysPromptSupport, useSystemPrompt, addSysHumanMsg, excludePrefixes) {22export function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill, withSysPromptSupport, useSystemPrompt, addSysHumanMsg, excludePrefixes) {
2123
22 //Prepare messages for claude.24 //Prepare messages for claude.
23 //When 'Exclude Human/Assistant prefixes' checked, setting messages role to the 'system'(last message is exception).25 //When 'Exclude Human/Assistant prefixes' checked, setting messages role to the 'system'(last message is exception).
@@ -96,7 +98,7 @@ function convertClaudePrompt(messages, addAssistantPostfix, addAssistantPrefill,
96 * @param {string} charName Character name98 * @param {string} charName Character name
97 * @param {string} userName User name99 * @param {string} userName User name
98 */100 */
99function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools, humanMsgFix, charName = '', userName = '') {101export function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools, humanMsgFix, charName = '', userName = '') {
100 let systemPrompt = [];102 let systemPrompt = [];
101 if (useSysPrompt) {103 if (useSysPrompt) {
102 // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.104 // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
@@ -279,7 +281,7 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools,
279 * @param {string} userName User name281 * @param {string} userName User name
280 * @returns {{chatHistory: object[]}} Prompt for Cohere282 * @returns {{chatHistory: object[]}} Prompt for Cohere
281 */283 */
282function convertCohereMessages(messages, charName = '', userName = '') {284export function convertCohereMessages(messages, charName = '', userName = '') {
283 if (messages.length === 0) {285 if (messages.length === 0) {
284 messages.unshift({286 messages.unshift({
285 role: 'user',287 role: 'user',
@@ -333,7 +335,7 @@ function convertCohereMessages(messages, charName = '', userName = '') {
333 * @param {string} userName User name335 * @param {string} userName User name
334 * @returns {{contents: *[], system_instruction: {parts: {text: string}}}} Prompt for Google MakerSuite models336 * @returns {{contents: *[], system_instruction: {parts: {text: string}}}} Prompt for Google MakerSuite models
335 */337 */
336function convertGooglePrompt(messages, model, useSysPrompt = false, charName = '', userName = '') {338export function convertGooglePrompt(messages, model, useSysPrompt = false, charName = '', userName = '') {
337 // This is a 1x1 transparent PNG339 // This is a 1x1 transparent PNG
338 const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';340 const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
339341
@@ -456,7 +458,7 @@ function convertGooglePrompt(messages, model, useSysPrompt = false, charName = '
456 * @param {string} charName Character name458 * @param {string} charName Character name
457 * @param {string} userName User name459 * @param {string} userName User name
458 */460 */
459function convertAI21Messages(messages, charName = '', userName = '') {461export function convertAI21Messages(messages, charName = '', userName = '') {
460 if (!Array.isArray(messages)) {462 if (!Array.isArray(messages)) {
461 return [];463 return [];
462 }464 }
@@ -528,7 +530,7 @@ function convertAI21Messages(messages, charName = '', userName = '') {
528 * @param {string} charName Character name530 * @param {string} charName Character name
529 * @param {string} userName User name531 * @param {string} userName User name
530 */532 */
531function convertMistralMessages(messages, charName = '', userName = '') {533export function convertMistralMessages(messages, charName = '', userName = '') {
532 if (!Array.isArray(messages)) {534 if (!Array.isArray(messages)) {
533 return [];535 return [];
534 }536 }
@@ -612,7 +614,7 @@ function convertMistralMessages(messages, charName = '', userName = '') {
612 * @param {boolean} strict Enable strict mode: only allow one system message at the start, force user first message614 * @param {boolean} strict Enable strict mode: only allow one system message at the start, force user first message
613 * @returns {any[]} Merged messages615 * @returns {any[]} Merged messages
614 */616 */
615function mergeMessages(messages, charName, userName, strict) {617export function mergeMessages(messages, charName, userName, strict) {
616 let mergedMessages = [];618 let mergedMessages = [];
617619
618 // Remove names from the messages620 // Remove names from the messages
@@ -686,7 +688,7 @@ function mergeMessages(messages, charName, userName, strict) {
686 * @param {object[]} messages Array of messages688 * @param {object[]} messages Array of messages
687 * @returns {string} Prompt for Text Completion API689 * @returns {string} Prompt for Text Completion API
688 */690 */
689function convertTextCompletionPrompt(messages) {691export function convertTextCompletionPrompt(messages) {
690 if (typeof messages === 'string') {692 if (typeof messages === 'string') {
691 return messages;693 return messages;
692 }694 }
@@ -705,14 +707,3 @@ function convertTextCompletionPrompt(messages) {
705 });707 });
706 return messageStrings.join('\n') + '\nassistant:';708 return messageStrings.join('\n') + '\nassistant:';
707}709}
708
709module.exports = {
710 convertClaudePrompt,
711 convertClaudeMessages,
712 convertGooglePrompt,
713 convertTextCompletionPrompt,
714 convertCohereMessages,
715 convertMistralMessages,
716 convertAI21Messages,
717 mergeMessages,
718};
src/request-proxy.js+5 -8
@@ -1,7 +1,8 @@
1const http = require('node:http');1import * as http from 'node:http';
2const https = require('node:https');2import * as https from 'node:https';
33
4const { isValidUrl, color } = require('./util.js');4import { ProxyAgent } from 'proxy-agent';
5import { isValidUrl, color } from './util.js';
56
6const LOG_HEADER = '[Request Proxy]';7const LOG_HEADER = '[Request Proxy]';
78
@@ -13,10 +14,8 @@ const LOG_HEADER = '[Request Proxy]';
13 * @property {string} url Proxy URL.14 * @property {string} url Proxy URL.
14 * @property {string[]} bypass List of URLs to bypass proxy.15 * @property {string[]} bypass List of URLs to bypass proxy.
15 */16 */
16function initRequestProxy({ enabled, url, bypass }) {17export default function initRequestProxy({ enabled, url, bypass }) {
17 try {18 try {
18 const { ProxyAgent } = require('proxy-agent');
19
20 // No proxy is enabled, so return19 // No proxy is enabled, so return
21 if (!enabled) {20 if (!enabled) {
22 return;21 return;
@@ -52,5 +51,3 @@ function initRequestProxy({ enabled, url, bypass }) {
52 console.error(color.red(LOG_HEADER), 'Failed to initialize request proxy:', error);51 console.error(color.red(LOG_HEADER), 'Failed to initialize request proxy:', error);
53 }52 }
54}53}
55
56module.exports = initRequestProxy;
src/transformers.mjs+6 -5
@@ -1,7 +1,8 @@
1import path from 'node:path';
2import fs from 'node:fs';
3
1import { pipeline, env, RawImage, Pipeline } from 'sillytavern-transformers';4import { pipeline, env, RawImage, Pipeline } from 'sillytavern-transformers';
2import { getConfigValue } from './util.js';5import { getConfigValue } from './util.js';
3import path from 'path';
4import fs from 'fs';
56
6configureTransformers();7configureTransformers();
78
@@ -50,7 +51,7 @@ const tasks = {
50 * @param {string} image Base64-encoded image51 * @param {string} image Base64-encoded image
51 * @returns {Promise<RawImage|null>} Object representing the image52 * @returns {Promise<RawImage|null>} Object representing the image
52 */53 */
53async function getRawImage(image) {54export async function getRawImage(image) {
54 try {55 try {
55 const buffer = Buffer.from(image, 'base64');56 const buffer = Buffer.from(image, 'base64');
56 const byteArray = new Uint8Array(buffer);57 const byteArray = new Uint8Array(buffer);
@@ -116,7 +117,7 @@ async function migrateCacheToDataDir() {
116 * @param {string} forceModel The model to use for the pipeline, if any117 * @param {string} forceModel The model to use for the pipeline, if any
117 * @returns {Promise<Pipeline>} Pipeline for the task118 * @returns {Promise<Pipeline>} Pipeline for the task
118 */119 */
119async function getPipeline(task, forceModel = '') {120export async function getPipeline(task, forceModel = '') {
120 await migrateCacheToDataDir();121 await migrateCacheToDataDir();
121122
122 if (tasks[task].pipeline) {123 if (tasks[task].pipeline) {
@@ -138,6 +139,6 @@ async function getPipeline(task, forceModel = '') {
138}139}
139140
140export default {141export default {
141 getPipeline,
142 getRawImage,142 getRawImage,
143 getPipeline,
143};144};
src/users.js+40 -68
@@ -1,22 +1,23 @@
1// Native Node Modules1// Native Node Modules
2const path = require('path');2import * as path from 'node:path';
3const fs = require('fs');3import * as fs from 'node:fs';
4const crypto = require('crypto');4import * as crypto from 'node:crypto';
5const os = require('os');5import * as os from 'node:os';
66
7// Express and other dependencies7// Express and other dependencies
8const storage = require('node-persist');8import storage from 'node-persist';
9const express = require('express');9import express from 'express';
10const mime = require('mime-types');10import mime from 'mime-types';
11const archiver = require('archiver');11import archiver from 'archiver';
12const writeFileAtomicSync = require('write-file-atomic').sync;12import _ from 'lodash';
13const _ = require('lodash');13import { sync as writeFileAtomicSync } from 'write-file-atomic';
1414
15const { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES, SETTINGS_FILE } = require('./constants');15import { USER_DIRECTORY_TEMPLATE, DEFAULT_USER, PUBLIC_DIRECTORIES, SETTINGS_FILE } from './constants.js';
16const { getConfigValue, color, delay, setConfigValue, generateTimestamp } = require('./util');16import { getConfigValue, color, delay, setConfigValue, generateTimestamp } from './util.js';
17const { readSecret, writeSecret } = require('./endpoints/secrets');17import { readSecret, writeSecret } from './endpoints/secrets.js';
1818import { getContentOfType } from './endpoints/content-manager.js';
19const KEY_PREFIX = 'user:';19
20export const KEY_PREFIX = 'user:';
20const AVATAR_PREFIX = 'avatar:';21const AVATAR_PREFIX = 'avatar:';
21const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);22const ENABLE_ACCOUNTS = getConfigValue('enableUserAccounts', false);
22const AUTHELIA_AUTH = getConfigValue('autheliaAuth', false);23const AUTHELIA_AUTH = getConfigValue('autheliaAuth', false);
@@ -94,7 +95,7 @@ const STORAGE_KEYS = {
94 * Ensures that the content directories exist.95 * Ensures that the content directories exist.
95 * @returns {Promise<import('./users').UserDirectoryList[]>} - The list of user directories96 * @returns {Promise<import('./users').UserDirectoryList[]>} - The list of user directories
96 */97 */
97async function ensurePublicDirectoriesExist() {98export async function ensurePublicDirectoriesExist() {
98 for (const dir of Object.values(PUBLIC_DIRECTORIES)) {99 for (const dir of Object.values(PUBLIC_DIRECTORIES)) {
99 if (!fs.existsSync(dir)) {100 if (!fs.existsSync(dir)) {
100 fs.mkdirSync(dir, { recursive: true });101 fs.mkdirSync(dir, { recursive: true });
@@ -117,7 +118,7 @@ async function ensurePublicDirectoriesExist() {
117 * Gets a list of all user directories.118 * Gets a list of all user directories.
118 * @returns {Promise<import('./users').UserDirectoryList[]>} - The list of user directories119 * @returns {Promise<import('./users').UserDirectoryList[]>} - The list of user directories
119 */120 */
120async function getUserDirectoriesList() {121export async function getUserDirectoriesList() {
121 const userHandles = await getAllUserHandles();122 const userHandles = await getAllUserHandles();
122 const directoriesList = userHandles.map(handle => getUserDirectories(handle));123 const directoriesList = userHandles.map(handle => getUserDirectories(handle));
123 return directoriesList;124 return directoriesList;
@@ -126,7 +127,7 @@ async function getUserDirectoriesList() {
126/**127/**
127 * Perform migration from the old user data format to the new one.128 * Perform migration from the old user data format to the new one.
128 */129 */
129async function migrateUserData() {130export async function migrateUserData() {
130 const publicDirectory = path.join(process.cwd(), 'public');131 const publicDirectory = path.join(process.cwd(), 'public');
131132
132 // No need to migrate if the characters directory doesn't exists133 // No need to migrate if the characters directory doesn't exists
@@ -328,14 +329,13 @@ async function migrateUserData() {
328 console.log(color.green('Migration completed!'));329 console.log(color.green('Migration completed!'));
329}330}
330331
331async function migrateSystemPrompts() {332export async function migrateSystemPrompts() {
332 /**333 /**
333 * Gets the default system prompts.334 * Gets the default system prompts.
334 * @returns {Promise<any[]>} - The list of default system prompts335 * @returns {Promise<any[]>} - The list of default system prompts
335 */336 */
336 async function getDefaultSystemPrompts() {337 async function getDefaultSystemPrompts() {
337 try {338 try {
338 const { getContentOfType } = await import('./endpoints/content-manager.js');
339 return getContentOfType('sysprompt', 'json');339 return getContentOfType('sysprompt', 'json');
340 } catch {340 } catch {
341 return [];341 return [];
@@ -391,7 +391,7 @@ async function migrateSystemPrompts() {
391 * @param {string} handle User handle391 * @param {string} handle User handle
392 * @returns {string} The key for the user storage392 * @returns {string} The key for the user storage
393 */393 */
394function toKey(handle) {394export function toKey(handle) {
395 return `${KEY_PREFIX}${handle}`;395 return `${KEY_PREFIX}${handle}`;
396}396}
397397
@@ -400,7 +400,7 @@ function toKey(handle) {
400 * @param {string} handle User handle400 * @param {string} handle User handle
401 * @returns {string} The key for the avatar storage401 * @returns {string} The key for the avatar storage
402 */402 */
403function toAvatarKey(handle) {403export function toAvatarKey(handle) {
404 return `${AVATAR_PREFIX}${handle}`;404 return `${AVATAR_PREFIX}${handle}`;
405}405}
406406
@@ -409,7 +409,7 @@ function toAvatarKey(handle) {
409 * @param {string} dataRoot The root directory for user data409 * @param {string} dataRoot The root directory for user data
410 * @returns {Promise<void>}410 * @returns {Promise<void>}
411 */411 */
412async function initUserStorage(dataRoot) {412export async function initUserStorage(dataRoot) {
413 global.DATA_ROOT = dataRoot;413 global.DATA_ROOT = dataRoot;
414 console.log('Using data root:', color.green(global.DATA_ROOT));414 console.log('Using data root:', color.green(global.DATA_ROOT));
415 console.log();415 console.log();
@@ -430,7 +430,7 @@ async function initUserStorage(dataRoot) {
430 * Get the cookie secret from the config. If it doesn't exist, generate a new one.430 * Get the cookie secret from the config. If it doesn't exist, generate a new one.
431 * @returns {string} The cookie secret431 * @returns {string} The cookie secret
432 */432 */
433function getCookieSecret() {433export function getCookieSecret() {
434 let secret = getConfigValue(STORAGE_KEYS.cookieSecret);434 let secret = getConfigValue(STORAGE_KEYS.cookieSecret);
435435
436 if (!secret) {436 if (!secret) {
@@ -446,7 +446,7 @@ function getCookieSecret() {
446 * Generates a random password salt.446 * Generates a random password salt.
447 * @returns {string} The password salt447 * @returns {string} The password salt
448 */448 */
449function getPasswordSalt() {449export function getPasswordSalt() {
450 return crypto.randomBytes(16).toString('base64');450 return crypto.randomBytes(16).toString('base64');
451}451}
452452
@@ -454,7 +454,7 @@ function getPasswordSalt() {
454 * Get the session name for the current server.454 * Get the session name for the current server.
455 * @returns {string} The session name455 * @returns {string} The session name
456 */456 */
457function getCookieSessionName() {457export function getCookieSessionName() {
458 // Get server hostname and hash it to generate a session suffix458 // Get server hostname and hash it to generate a session suffix
459 const suffix = crypto.createHash('sha256').update(os.hostname()).digest('hex').slice(0, 8);459 const suffix = crypto.createHash('sha256').update(os.hostname()).digest('hex').slice(0, 8);
460 return `session-${suffix}`;460 return `session-${suffix}`;
@@ -466,7 +466,7 @@ function getCookieSessionName() {
466 * @param {string} salt Salt to use for hashing466 * @param {string} salt Salt to use for hashing
467 * @returns {string} Hashed password467 * @returns {string} Hashed password
468 */468 */
469function getPasswordHash(password, salt) {469export function getPasswordHash(password, salt) {
470 return crypto.scryptSync(password.normalize(), salt, 64).toString('base64');470 return crypto.scryptSync(password.normalize(), salt, 64).toString('base64');
471}471}
472472
@@ -475,7 +475,7 @@ function getPasswordHash(password, salt) {
475 * @param {import('express').Request} [request] HTTP request object475 * @param {import('express').Request} [request] HTTP request object
476 * @returns {string} The CSRF secret476 * @returns {string} The CSRF secret
477 */477 */
478function getCsrfSecret(request) {478export function getCsrfSecret(request) {
479 if (!request || !request.user) {479 if (!request || !request.user) {
480 return ANON_CSRF_SECRET;480 return ANON_CSRF_SECRET;
481 }481 }
@@ -494,7 +494,7 @@ function getCsrfSecret(request) {
494 * Gets a list of all user handles.494 * Gets a list of all user handles.
495 * @returns {Promise<string[]>} - The list of user handles495 * @returns {Promise<string[]>} - The list of user handles
496 */496 */
497async function getAllUserHandles() {497export async function getAllUserHandles() {
498 const keys = await storage.keys(x => x.key.startsWith(KEY_PREFIX));498 const keys = await storage.keys(x => x.key.startsWith(KEY_PREFIX));
499 const handles = keys.map(x => x.replace(KEY_PREFIX, ''));499 const handles = keys.map(x => x.replace(KEY_PREFIX, ''));
500 return handles;500 return handles;
@@ -505,7 +505,7 @@ async function getAllUserHandles() {
505 * @param {string} handle User handle505 * @param {string} handle User handle
506 * @returns {UserDirectoryList} User directories506 * @returns {UserDirectoryList} User directories
507 */507 */
508function getUserDirectories(handle) {508export function getUserDirectories(handle) {
509 if (DIRECTORIES_CACHE.has(handle)) {509 if (DIRECTORIES_CACHE.has(handle)) {
510 const cache = DIRECTORIES_CACHE.get(handle);510 const cache = DIRECTORIES_CACHE.get(handle);
511 if (cache) {511 if (cache) {
@@ -526,7 +526,7 @@ function getUserDirectories(handle) {
526 * @param {string} handle User handle526 * @param {string} handle User handle
527 * @returns {Promise<string>} User avatar URL527 * @returns {Promise<string>} User avatar URL
528 */528 */
529async function getUserAvatar(handle) {529export async function getUserAvatar(handle) {
530 try {530 try {
531 // Check if the user has a custom avatar531 // Check if the user has a custom avatar
532 const avatarKey = toAvatarKey(handle);532 const avatarKey = toAvatarKey(handle);
@@ -563,7 +563,7 @@ async function getUserAvatar(handle) {
563 * @param {import('express').Request} request Request object563 * @param {import('express').Request} request Request object
564 * @returns {boolean} Whether the user should be redirected to the login page564 * @returns {boolean} Whether the user should be redirected to the login page
565 */565 */
566function shouldRedirectToLogin(request) {566export function shouldRedirectToLogin(request) {
567 return ENABLE_ACCOUNTS && !request.user;567 return ENABLE_ACCOUNTS && !request.user;
568}568}
569569
@@ -574,7 +574,7 @@ function shouldRedirectToLogin(request) {
574 * @param {boolean} basicAuthMode If Basic auth mode is enabled574 * @param {boolean} basicAuthMode If Basic auth mode is enabled
575 * @returns {Promise<boolean>} Whether auto-login was performed575 * @returns {Promise<boolean>} Whether auto-login was performed
576 */576 */
577async function tryAutoLogin(request, basicAuthMode) {577export async function tryAutoLogin(request, basicAuthMode) {
578 if (!ENABLE_ACCOUNTS || request.user || !request.session) {578 if (!ENABLE_ACCOUNTS || request.user || !request.session) {
579 return false;579 return false;
580 }580 }
@@ -693,7 +693,7 @@ async function basicUserLogin(request) {
693 * @param {import('express').Response} response Response object693 * @param {import('express').Response} response Response object
694 * @param {import('express').NextFunction} next Next function694 * @param {import('express').NextFunction} next Next function
695 */695 */
696async function setUserDataMiddleware(request, response, next) {696export async function setUserDataMiddleware(request, response, next) {
697 // If user accounts are disabled, use the default user697 // If user accounts are disabled, use the default user
698 if (!ENABLE_ACCOUNTS) {698 if (!ENABLE_ACCOUNTS) {
699 const handle = DEFAULT_USER.handle;699 const handle = DEFAULT_USER.handle;
@@ -751,7 +751,7 @@ async function setUserDataMiddleware(request, response, next) {
751 * @param {import('express').Response} response Response object751 * @param {import('express').Response} response Response object
752 * @param {import('express').NextFunction} next Next function752 * @param {import('express').NextFunction} next Next function
753 */753 */
754function requireLoginMiddleware(request, response, next) {754export function requireLoginMiddleware(request, response, next) {
755 if (!request.user) {755 if (!request.user) {
756 return response.sendStatus(403);756 return response.sendStatus(403);
757 }757 }
@@ -787,7 +787,7 @@ function createRouteHandler(directoryFn) {
787 * @param {import('express').NextFunction} next Next function787 * @param {import('express').NextFunction} next Next function
788 * @returns {any}788 * @returns {any}
789 */789 */
790function requireAdminMiddleware(request, response, next) {790export function requireAdminMiddleware(request, response, next) {
791 if (!request.user) {791 if (!request.user) {
792 return response.sendStatus(403);792 return response.sendStatus(403);
793 }793 }
@@ -806,7 +806,7 @@ function requireAdminMiddleware(request, response, next) {
806 * @param {import('express').Response} response Express response object to write to806 * @param {import('express').Response} response Express response object to write to
807 * @returns {Promise<void>} Promise that resolves when the archive is created807 * @returns {Promise<void>} Promise that resolves when the archive is created
808 */808 */
809async function createBackupArchive(handle, response) {809export async function createBackupArchive(handle, response) {
810 const directories = getUserDirectories(handle);810 const directories = getUserDirectories(handle);
811811
812 console.log('Backup requested for', handle);812 console.log('Backup requested for', handle);
@@ -855,7 +855,7 @@ async function getAllUsers() {
855 * Gets all of the enabled users.855 * Gets all of the enabled users.
856 * @returns {Promise<User[]>}856 * @returns {Promise<User[]>}
857 */857 */
858async function getAllEnabledUsers() {858export async function getAllEnabledUsers() {
859 const users = await getAllUsers();859 const users = await getAllUsers();
860 return users.filter(x => x.enabled);860 return users.filter(x => x.enabled);
861}861}
@@ -863,7 +863,7 @@ async function getAllEnabledUsers() {
863/**863/**
864 * Express router for serving files from the user's directories.864 * Express router for serving files from the user's directories.
865 */865 */
866const router = express.Router();866export const router = express.Router();
867router.use('/backgrounds/*', createRouteHandler(req => req.user.directories.backgrounds));867router.use('/backgrounds/*', createRouteHandler(req => req.user.directories.backgrounds));
868router.use('/characters/*', createRouteHandler(req => req.user.directories.characters));868router.use('/characters/*', createRouteHandler(req => req.user.directories.characters));
869router.use('/User%20Avatars/*', createRouteHandler(req => req.user.directories.avatars));869router.use('/User%20Avatars/*', createRouteHandler(req => req.user.directories.avatars));
@@ -871,31 +871,3 @@ router.use('/assets/*', createRouteHandler(req => req.user.directories.assets));
871router.use('/user/images/*', createRouteHandler(req => req.user.directories.userImages));871router.use('/user/images/*', createRouteHandler(req => req.user.directories.userImages));
872router.use('/user/files/*', createRouteHandler(req => req.user.directories.files));872router.use('/user/files/*', createRouteHandler(req => req.user.directories.files));
873router.use('/scripts/extensions/third-party/*', createRouteHandler(req => req.user.directories.extensions));873router.use('/scripts/extensions/third-party/*', createRouteHandler(req => req.user.directories.extensions));
874
875module.exports = {
876 KEY_PREFIX,
877 toKey,
878 toAvatarKey,
879 initUserStorage,
880 ensurePublicDirectoriesExist,
881 getUserDirectoriesList,
882 getAllUserHandles,
883 getUserDirectories,
884 setUserDataMiddleware,
885 requireLoginMiddleware,
886 requireAdminMiddleware,
887 migrateUserData,
888 migrateSystemPrompts,
889 getPasswordSalt,
890 getPasswordHash,
891 getCsrfSecret,
892 getCookieSecret,
893 getCookieSessionName,
894 getUserAvatar,
895 shouldRedirectToLogin,
896 createBackupArchive,
897 tryAutoLogin,
898 getAllUsers,
899 getAllEnabledUsers,
900 router,
901};
src/util.js+45 -73
@@ -1,13 +1,16 @@
1const path = require('path');1import * as path from 'node:path';
2const fs = require('fs');2import * as fs from 'node:fs';
3const commandExistsSync = require('command-exists').sync;3import * as http2 from 'node:http2';
4const writeFileAtomicSync = require('write-file-atomic').sync;4import { Readable } from 'node:stream';
5const _ = require('lodash');5import { createRequire } from 'node:module';
6const yauzl = require('yauzl');6
7const mime = require('mime-types');7import yaml from 'yaml';
8const yaml = require('yaml');8import { sync as commandExistsSync } from 'command-exists';
9const { default: simpleGit } = require('simple-git');9import { sync as writeFileAtomicSync } from 'write-file-atomic';
10const { Readable } = require('stream');10import _ from 'lodash';
11import yauzl from 'yauzl';
12import mime from 'mime-types';
13import { default as simpleGit } from 'simple-git';
1114
12/**15/**
13 * Parsed config object.16 * Parsed config object.
@@ -18,7 +21,7 @@ let CACHED_CONFIG = null;
18 * Returns the config object from the config.yaml file.21 * Returns the config object from the config.yaml file.
19 * @returns {object} Config object22 * @returns {object} Config object
20 */23 */
21function getConfig() {24export function getConfig() {
22 if (CACHED_CONFIG) {25 if (CACHED_CONFIG) {
23 return CACHED_CONFIG;26 return CACHED_CONFIG;
24 }27 }
@@ -46,7 +49,7 @@ function getConfig() {
46 * @param {any} defaultValue - Default value to return if the key is not found49 * @param {any} defaultValue - Default value to return if the key is not found
47 * @returns {any} Value for the given key50 * @returns {any} Value for the given key
48 */51 */
49function getConfigValue(key, defaultValue = null) {52export function getConfigValue(key, defaultValue = null) {
50 const config = getConfig();53 const config = getConfig();
51 return _.get(config, key, defaultValue);54 return _.get(config, key, defaultValue);
52}55}
@@ -56,7 +59,7 @@ function getConfigValue(key, defaultValue = null) {
56 * @param {string} key Key to set59 * @param {string} key Key to set
57 * @param {any} value Value to set60 * @param {any} value Value to set
58 */61 */
59function setConfigValue(key, value) {62export function setConfigValue(key, value) {
60 // Reset cache so that the next getConfig call will read the updated config file63 // Reset cache so that the next getConfig call will read the updated config file
61 CACHED_CONFIG = null;64 CACHED_CONFIG = null;
62 const config = getConfig();65 const config = getConfig();
@@ -69,7 +72,7 @@ function setConfigValue(key, value) {
69 * @param {string} auth username:password72 * @param {string} auth username:password
70 * @returns {string} Basic Auth header value73 * @returns {string} Basic Auth header value
71 */74 */
72function getBasicAuthHeader(auth) {75export function getBasicAuthHeader(auth) {
73 const encoded = Buffer.from(`${auth}`).toString('base64');76 const encoded = Buffer.from(`${auth}`).toString('base64');
74 return `Basic ${encoded}`;77 return `Basic ${encoded}`;
75}78}
@@ -79,7 +82,7 @@ function getBasicAuthHeader(auth) {
79 * Also returns the agent string for the Horde API.82 * Also returns the agent string for the Horde API.
80 * @returns {Promise<{agent: string, pkgVersion: string, gitRevision: string | null, gitBranch: string | null, commitDate: string | null, isLatest: boolean}>} Version info object83 * @returns {Promise<{agent: string, pkgVersion: string, gitRevision: string | null, gitBranch: string | null, commitDate: string | null, isLatest: boolean}>} Version info object
81 */84 */
82async function getVersion() {85export async function getVersion() {
83 let pkgVersion = 'UNKNOWN';86 let pkgVersion = 'UNKNOWN';
84 let gitRevision = null;87 let gitRevision = null;
85 let gitBranch = null;88 let gitBranch = null;
@@ -87,9 +90,10 @@ async function getVersion() {
87 let isLatest = true;90 let isLatest = true;
8891
89 try {92 try {
93 const require = createRequire(import.meta.url);
90 const pkgJson = require(path.join(process.cwd(), './package.json'));94 const pkgJson = require(path.join(process.cwd(), './package.json'));
91 pkgVersion = pkgJson.version;95 pkgVersion = pkgJson.version;
92 if (!process['pkg'] && commandExistsSync('git')) {96 if (commandExistsSync('git')) {
93 const git = simpleGit();97 const git = simpleGit();
94 const cwd = process.cwd();98 const cwd = process.cwd();
95 gitRevision = await git.cwd(cwd).revparse(['--short', 'HEAD']);99 gitRevision = await git.cwd(cwd).revparse(['--short', 'HEAD']);
@@ -117,7 +121,7 @@ async function getVersion() {
117 * @param {number} ms Milliseconds to wait121 * @param {number} ms Milliseconds to wait
118 * @returns {Promise<void>} Promise that resolves after the given amount of milliseconds122 * @returns {Promise<void>} Promise that resolves after the given amount of milliseconds
119 */123 */
120function delay(ms) {124export function delay(ms) {
121 return new Promise(resolve => setTimeout(resolve, ms));125 return new Promise(resolve => setTimeout(resolve, ms));
122}126}
123127
@@ -127,7 +131,7 @@ function delay(ms) {
127 * @returns {string} Random hex string131 * @returns {string} Random hex string
128 * @example getHexString(8) // 'a1b2c3d4'132 * @example getHexString(8) // 'a1b2c3d4'
129 */133 */
130function getHexString(length) {134export function getHexString(length) {
131 const chars = '0123456789abcdef';135 const chars = '0123456789abcdef';
132 let result = '';136 let result = '';
133 for (let i = 0; i < length; i++) {137 for (let i = 0; i < length; i++) {
@@ -142,7 +146,7 @@ function getHexString(length) {
142 * @param {string} fileExtension File extension to look for146 * @param {string} fileExtension File extension to look for
143 * @returns {Promise<Buffer|null>} Buffer containing the extracted file. Null if the file was not found.147 * @returns {Promise<Buffer|null>} Buffer containing the extracted file. Null if the file was not found.
144 */148 */
145async function extractFileFromZipBuffer(archiveBuffer, fileExtension) {149export async function extractFileFromZipBuffer(archiveBuffer, fileExtension) {
146 return await new Promise((resolve, reject) => yauzl.fromBuffer(Buffer.from(archiveBuffer), { lazyEntries: true }, (err, zipfile) => {150 return await new Promise((resolve, reject) => yauzl.fromBuffer(Buffer.from(archiveBuffer), { lazyEntries: true }, (err, zipfile) => {
147 if (err) {151 if (err) {
148 reject(err);152 reject(err);
@@ -181,7 +185,7 @@ async function extractFileFromZipBuffer(archiveBuffer, fileExtension) {
181 * @param {string} zipFilePath Path to the ZIP archive185 * @param {string} zipFilePath Path to the ZIP archive
182 * @returns {Promise<[string, Buffer][]>} Array of image buffers186 * @returns {Promise<[string, Buffer][]>} Array of image buffers
183 */187 */
184async function getImageBuffers(zipFilePath) {188export async function getImageBuffers(zipFilePath) {
185 return new Promise((resolve, reject) => {189 return new Promise((resolve, reject) => {
186 // Check if the zip file exists190 // Check if the zip file exists
187 if (!fs.existsSync(zipFilePath)) {191 if (!fs.existsSync(zipFilePath)) {
@@ -237,7 +241,7 @@ async function getImageBuffers(zipFilePath) {
237 * @param {any} readableStream Readable stream to read from241 * @param {any} readableStream Readable stream to read from
238 * @returns {Promise<Buffer[]>} Array of chunks242 * @returns {Promise<Buffer[]>} Array of chunks
239 */243 */
240async function readAllChunks(readableStream) {244export async function readAllChunks(readableStream) {
241 return new Promise((resolve, reject) => {245 return new Promise((resolve, reject) => {
242 // Consume the readable stream246 // Consume the readable stream
243 const chunks = [];247 const chunks = [];
@@ -261,7 +265,7 @@ function isObject(item) {
261 return (item && typeof item === 'object' && !Array.isArray(item));265 return (item && typeof item === 'object' && !Array.isArray(item));
262}266}
263267
264function deepMerge(target, source) {268export function deepMerge(target, source) {
265 let output = Object.assign({}, target);269 let output = Object.assign({}, target);
266 if (isObject(target) && isObject(source)) {270 if (isObject(target) && isObject(source)) {
267 Object.keys(source).forEach(key => {271 Object.keys(source).forEach(key => {
@@ -278,7 +282,7 @@ function deepMerge(target, source) {
278 return output;282 return output;
279}283}
280284
281const color = {285export const color = {
282 byNum: (mess, fgNum) => {286 byNum: (mess, fgNum) => {
283 mess = mess || '';287 mess = mess || '';
284 fgNum = fgNum === undefined ? 31 : fgNum;288 fgNum = fgNum === undefined ? 31 : fgNum;
@@ -298,7 +302,7 @@ const color = {
298 * Gets a random UUIDv4 string.302 * Gets a random UUIDv4 string.
299 * @returns {string} A UUIDv4 string303 * @returns {string} A UUIDv4 string
300 */304 */
301function uuidv4() {305export function uuidv4() {
302 if ('crypto' in global && 'randomUUID' in global.crypto) {306 if ('crypto' in global && 'randomUUID' in global.crypto) {
303 return global.crypto.randomUUID();307 return global.crypto.randomUUID();
304 }308 }
@@ -309,7 +313,7 @@ function uuidv4() {
309 });313 });
310}314}
311315
312function humanizedISO8601DateTime(date) {316export function humanizedISO8601DateTime(date) {
313 let baseDate = typeof date === 'number' ? new Date(date) : new Date();317 let baseDate = typeof date === 'number' ? new Date(date) : new Date();
314 let humanYear = baseDate.getFullYear();318 let humanYear = baseDate.getFullYear();
315 let humanMonth = (baseDate.getMonth() + 1);319 let humanMonth = (baseDate.getMonth() + 1);
@@ -322,7 +326,7 @@ function humanizedISO8601DateTime(date) {
322 return HumanizedDateTime;326 return HumanizedDateTime;
323}327}
324328
325function tryParse(str) {329export function tryParse(str) {
326 try {330 try {
327 return JSON.parse(str);331 return JSON.parse(str);
328 } catch {332 } catch {
@@ -337,7 +341,7 @@ function tryParse(str) {
337 * @param {string} inputPath The path to be converted.341 * @param {string} inputPath The path to be converted.
338 * @returns The relative URL path from which the client can access the file.342 * @returns The relative URL path from which the client can access the file.
339 */343 */
340function clientRelativePath(root, inputPath) {344export function clientRelativePath(root, inputPath) {
341 if (!inputPath.startsWith(root)) {345 if (!inputPath.startsWith(root)) {
342 throw new Error('Input path does not start with the root directory');346 throw new Error('Input path does not start with the root directory');
343 }347 }
@@ -350,11 +354,11 @@ function clientRelativePath(root, inputPath) {
350 * @param {string} filename The file name to remove the extension from.354 * @param {string} filename The file name to remove the extension from.
351 * @returns The file name, sans extension355 * @returns The file name, sans extension
352 */356 */
353function removeFileExtension(filename) {357export function removeFileExtension(filename) {
354 return filename.replace(/\.[^.]+$/, '');358 return filename.replace(/\.[^.]+$/, '');
355}359}
356360
357function generateTimestamp() {361export function generateTimestamp() {
358 const now = new Date();362 const now = new Date();
359 const year = now.getFullYear();363 const year = now.getFullYear();
360 const month = String(now.getMonth() + 1).padStart(2, '0');364 const month = String(now.getMonth() + 1).padStart(2, '0');
@@ -371,7 +375,7 @@ function generateTimestamp() {
371 * @param {string} directory The root directory to remove backups from.375 * @param {string} directory The root directory to remove backups from.
372 * @param {string} prefix File prefix to filter backups by.376 * @param {string} prefix File prefix to filter backups by.
373 */377 */
374function removeOldBackups(directory, prefix) {378export function removeOldBackups(directory, prefix) {
375 const MAX_BACKUPS = Number(getConfigValue('numberOfBackups', 50));379 const MAX_BACKUPS = Number(getConfigValue('numberOfBackups', 50));
376380
377 let files = fs.readdirSync(directory).filter(f => f.startsWith(prefix));381 let files = fs.readdirSync(directory).filter(f => f.startsWith(prefix));
@@ -389,7 +393,7 @@ function removeOldBackups(directory, prefix) {
389 * @param {'name' | 'date'} sortBy Sort images by name or date393 * @param {'name' | 'date'} sortBy Sort images by name or date
390 * @returns {string[]} List of image file names394 * @returns {string[]} List of image file names
391 */395 */
392function getImages(directoryPath, sortBy = 'name') {396export function getImages(directoryPath, sortBy = 'name') {
393 function getSortFunction() {397 function getSortFunction() {
394 switch (sortBy) {398 switch (sortBy) {
395 case 'name':399 case 'name':
@@ -415,7 +419,7 @@ function getImages(directoryPath, sortBy = 'name') {
415 * @param {import('node-fetch').Response} from The Fetch API response to pipe from.419 * @param {import('node-fetch').Response} from The Fetch API response to pipe from.
416 * @param {import('express').Response} to The Express response to pipe to.420 * @param {import('express').Response} to The Express response to pipe to.
417 */421 */
418function forwardFetchResponse(from, to) {422export function forwardFetchResponse(from, to) {
419 let statusCode = from.status;423 let statusCode = from.status;
420 let statusText = from.statusText;424 let statusText = from.statusText;
421425
@@ -434,6 +438,7 @@ function forwardFetchResponse(from, to) {
434438
435 to.statusCode = statusCode;439 to.statusCode = statusCode;
436 to.statusMessage = statusText;440 to.statusMessage = statusText;
441
437 from.body.pipe(to);442 from.body.pipe(to);
438443
439 to.socket.on('close', function () {444 to.socket.on('close', function () {
@@ -457,10 +462,9 @@ function forwardFetchResponse(from, to) {
457 * @param {object} headers Request headers462 * @param {object} headers Request headers
458 * @returns {Promise<string>} Response body463 * @returns {Promise<string>} Response body
459 */464 */
460function makeHttp2Request(endpoint, method, body, headers) {465export function makeHttp2Request(endpoint, method, body, headers) {
461 return new Promise((resolve, reject) => {466 return new Promise((resolve, reject) => {
462 try {467 try {
463 const http2 = require('http2');
464 const url = new URL(endpoint);468 const url = new URL(endpoint);
465 const client = http2.connect(url.origin);469 const client = http2.connect(url.origin);
466470
@@ -511,7 +515,7 @@ function makeHttp2Request(endpoint, method, body, headers) {
511 * @param {string} yamlString YAML-serialized object515 * @param {string} yamlString YAML-serialized object
512 * @returns516 * @returns
513 */517 */
514function mergeObjectWithYaml(obj, yamlString) {518export function mergeObjectWithYaml(obj, yamlString) {
515 if (!yamlString) {519 if (!yamlString) {
516 return;520 return;
517 }521 }
@@ -540,7 +544,7 @@ function mergeObjectWithYaml(obj, yamlString) {
540 * @param {string} yamlString YAML-serialized array544 * @param {string} yamlString YAML-serialized array
541 * @returns {void} Nothing545 * @returns {void} Nothing
542 */546 */
543function excludeKeysByYaml(obj, yamlString) {547export function excludeKeysByYaml(obj, yamlString) {
544 if (!yamlString) {548 if (!yamlString) {
545 return;549 return;
546 }550 }
@@ -569,14 +573,14 @@ function excludeKeysByYaml(obj, yamlString) {
569 * @param {string} str Input string573 * @param {string} str Input string
570 * @returns {string} Trimmed string574 * @returns {string} Trimmed string
571 */575 */
572function trimV1(str) {576export function trimV1(str) {
573 return String(str ?? '').replace(/\/$/, '').replace(/\/v1$/, '');577 return String(str ?? '').replace(/\/$/, '').replace(/\/v1$/, '');
574}578}
575579
576/**580/**
577 * Simple TTL memory cache.581 * Simple TTL memory cache.
578 */582 */
579class Cache {583export class Cache {
580 /**584 /**
581 * @param {number} ttl Time to live in milliseconds585 * @param {number} ttl Time to live in milliseconds
582 */586 */
@@ -633,7 +637,7 @@ class Cache {
633 * @param {string} text Text with color formatting637 * @param {string} text Text with color formatting
634 * @returns {string} Text without color formatting638 * @returns {string} Text without color formatting
635 */639 */
636function removeColorFormatting(text) {640export function removeColorFormatting(text) {
637 // ANSI escape codes for colors are usually in the format \x1b[<codes>m641 // ANSI escape codes for colors are usually in the format \x1b[<codes>m
638 return text.replace(/\x1b\[\d{1,2}(;\d{1,2})*m/g, '');642 return text.replace(/\x1b\[\d{1,2}(;\d{1,2})*m/g, '');
639}643}
@@ -643,7 +647,7 @@ function removeColorFormatting(text) {
643 * @param {number} n Number of times to repeat the separator647 * @param {number} n Number of times to repeat the separator
644 * @returns {string} Separator string648 * @returns {string} Separator string
645 */649 */
646function getSeparator(n) {650export function getSeparator(n) {
647 return '='.repeat(n);651 return '='.repeat(n);
648}652}
649653
@@ -652,7 +656,7 @@ function getSeparator(n) {
652 * @param {string} url String to check656 * @param {string} url String to check
653 * @returns {boolean} If the URL is valid657 * @returns {boolean} If the URL is valid
654 */658 */
655function isValidUrl(url) {659export function isValidUrl(url) {
656 try {660 try {
657 new URL(url);661 new URL(url);
658 return true;662 return true;
@@ -660,35 +664,3 @@ function isValidUrl(url) {
660 return false;664 return false;
661 }665 }
662}666}
663
664module.exports = {
665 getConfig,
666 getConfigValue,
667 setConfigValue,
668 getVersion,
669 getBasicAuthHeader,
670 extractFileFromZipBuffer,
671 getImageBuffers,
672 readAllChunks,
673 delay,
674 deepMerge,
675 color,
676 uuidv4,
677 humanizedISO8601DateTime,
678 tryParse,
679 clientRelativePath,
680 removeFileExtension,
681 generateTimestamp,
682 removeOldBackups,
683 getImages,
684 forwardFetchResponse,
685 getHexString,
686 mergeObjectWithYaml,
687 excludeKeysByYaml,
688 trimV1,
689 Cache,
690 makeHttp2Request,
691 removeColorFormatting,
692 getSeparator,
693 isValidUrl,
694};
src/validator/TavernCardValidator.js+1 -3
@@ -5,7 +5,7 @@
5 *5 *
6 * @link https://github.com/malfoyslastname/character-card-spec-v26 * @link https://github.com/malfoyslastname/character-card-spec-v2
7 */7 */
8class TavernCardValidator {8export class TavernCardValidator {
9 /**9 /**
10 * @type {string|null}10 * @type {string|null}
11 */11 */
@@ -167,5 +167,3 @@ class TavernCardValidator {
167 return true;167 return true;
168 }168 }
169}169}
170
171module.exports = { TavernCardValidator };
src/vectors/cohere-vectors.js+4 -8
@@ -1,5 +1,5 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const { SECRET_KEYS, readSecret } = require('../endpoints/secrets');2import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js';
33
4/**4/**
5 * Gets the vector for the given text batch from an OpenAI compatible endpoint.5 * Gets the vector for the given text batch from an OpenAI compatible endpoint.
@@ -9,7 +9,7 @@ const { SECRET_KEYS, readSecret } = require('../endpoints/secrets');
9 * @param {string} model - The model to use for the embedding9 * @param {string} model - The model to use for the embedding
10 * @returns {Promise<number[][]>} - The array of vectors for the texts10 * @returns {Promise<number[][]>} - The array of vectors for the texts
11 */11 */
12async function getCohereBatchVector(texts, isQuery, directories, model) {12export async function getCohereBatchVector(texts, isQuery, directories, model) {
13 const key = readSecret(directories, SECRET_KEYS.COHERE);13 const key = readSecret(directories, SECRET_KEYS.COHERE);
1414
15 if (!key) {15 if (!key) {
@@ -55,12 +55,8 @@ async function getCohereBatchVector(texts, isQuery, directories, model) {
55 * @param {string} model - The model to use for the embedding55 * @param {string} model - The model to use for the embedding
56 * @returns {Promise<number[]>} - The vector for the text56 * @returns {Promise<number[]>} - The vector for the text
57 */57 */
58async function getCohereVector(text, isQuery, directories, model) {58export async function getCohereVector(text, isQuery, directories, model) {
59 const vectors = await getCohereBatchVector([text], isQuery, directories, model);59 const vectors = await getCohereBatchVector([text], isQuery, directories, model);
60 return vectors[0];60 return vectors[0];
61}61}
6262
63module.exports = {
64 getCohereBatchVector,
65 getCohereVector,
66};
src/vectors/embedding.js+4 -9
@@ -1,3 +1,4 @@
1import { getPipeline } from '../transformers.mjs';
1const TASK = 'feature-extraction';2const TASK = 'feature-extraction';
23
3/**4/**
@@ -5,9 +6,8 @@ const TASK = 'feature-extraction';
5 * @param {string} text - The text to vectorize6 * @param {string} text - The text to vectorize
6 * @returns {Promise<number[]>} - The vectorized text in form of an array of numbers7 * @returns {Promise<number[]>} - The vectorized text in form of an array of numbers
7 */8 */
8async function getTransformersVector(text) {9export async function getTransformersVector(text) {
9 const module = await import('../transformers.mjs');10 const pipe = await getPipeline(TASK);
10 const pipe = await module.default.getPipeline(TASK);
11 const result = await pipe(text, { pooling: 'mean', normalize: true });11 const result = await pipe(text, { pooling: 'mean', normalize: true });
12 const vector = Array.from(result.data);12 const vector = Array.from(result.data);
13 return vector;13 return vector;
@@ -18,15 +18,10 @@ async function getTransformersVector(text) {
18 * @param {string[]} texts - The texts to vectorize18 * @param {string[]} texts - The texts to vectorize
19 * @returns {Promise<number[][]>} - The vectorized texts in form of an array of arrays of numbers19 * @returns {Promise<number[][]>} - The vectorized texts in form of an array of arrays of numbers
20 */20 */
21async function getTransformersBatchVector(texts) {21export async function getTransformersBatchVector(texts) {
22 const result = [];22 const result = [];
23 for (const text of texts) {23 for (const text of texts) {
24 result.push(await getTransformersVector(text));24 result.push(await getTransformersVector(text));
25 }25 }
26 return result;26 return result;
27}27}
28
29module.exports = {
30 getTransformersVector,
31 getTransformersBatchVector,
32};
src/vectors/extras-vectors.js+3 -8
@@ -1,4 +1,4 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
22
3/**3/**
4 * Gets the vector for the given text from SillyTavern-extras4 * Gets the vector for the given text from SillyTavern-extras
@@ -7,7 +7,7 @@ const fetch = require('node-fetch').default;
7 * @param {string} apiKey - The Extras API key, or empty string if API key not enabled7 * @param {string} apiKey - The Extras API key, or empty string if API key not enabled
8 * @returns {Promise<number[][]>} - The array of vectors for the texts8 * @returns {Promise<number[][]>} - The array of vectors for the texts
9 */9 */
10async function getExtrasBatchVector(texts, apiUrl, apiKey) {10export async function getExtrasBatchVector(texts, apiUrl, apiKey) {
11 return getExtrasVectorImpl(texts, apiUrl, apiKey);11 return getExtrasVectorImpl(texts, apiUrl, apiKey);
12}12}
1313
@@ -18,7 +18,7 @@ async function getExtrasBatchVector(texts, apiUrl, apiKey) {
18 * @param {string} apiKey - The Extras API key, or empty string if API key not enabled18 * @param {string} apiKey - The Extras API key, or empty string if API key not enabled
19 * @returns {Promise<number[]>} - The vector for the text19 * @returns {Promise<number[]>} - The vector for the text
20 */20 */
21async function getExtrasVector(text, apiUrl, apiKey) {21export async function getExtrasVector(text, apiUrl, apiKey) {
22 return getExtrasVectorImpl(text, apiUrl, apiKey);22 return getExtrasVectorImpl(text, apiUrl, apiKey);
23}23}
2424
@@ -71,8 +71,3 @@ async function getExtrasVectorImpl(text, apiUrl, apiKey) {
7171
72 return vector;72 return vector;
73}73}
74
75module.exports = {
76 getExtrasVector,
77 getExtrasBatchVector,
78};
src/vectors/llamacpp-vectors.js+5 -10
@@ -1,6 +1,6 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const { setAdditionalHeadersByType } = require('../additional-headers');2import { setAdditionalHeadersByType } from '../additional-headers.js';
3const { TEXTGEN_TYPES } = require('../constants');3import { TEXTGEN_TYPES } from '../constants.js';
44
5/**5/**
6 * Gets the vector for the given text from LlamaCpp6 * Gets the vector for the given text from LlamaCpp
@@ -9,7 +9,7 @@ const { TEXTGEN_TYPES } = require('../constants');
9 * @param {import('../users').UserDirectoryList} directories - The directories object for the user9 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
10 * @returns {Promise<number[][]>} - The array of vectors for the texts10 * @returns {Promise<number[][]>} - The array of vectors for the texts
11 */11 */
12async function getLlamaCppBatchVector(texts, apiUrl, directories) {12export async function getLlamaCppBatchVector(texts, apiUrl, directories) {
13 const url = new URL(apiUrl);13 const url = new URL(apiUrl);
14 url.pathname = '/v1/embeddings';14 url.pathname = '/v1/embeddings';
1515
@@ -50,12 +50,7 @@ async function getLlamaCppBatchVector(texts, apiUrl, directories) {
50 * @param {import('../users').UserDirectoryList} directories - The directories object for the user50 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
51 * @returns {Promise<number[]>} - The vector for the text51 * @returns {Promise<number[]>} - The vector for the text
52 */52 */
53async function getLlamaCppVector(text, apiUrl, directories) {53export async function getLlamaCppVector(text, apiUrl, directories) {
54 const vectors = await getLlamaCppBatchVector([text], apiUrl, directories);54 const vectors = await getLlamaCppBatchVector([text], apiUrl, directories);
55 return vectors[0];55 return vectors[0];
56}56}
57
58module.exports = {
59 getLlamaCppBatchVector,
60 getLlamaCppVector,
61};
src/vectors/makersuite-vectors.js+4 -9
@@ -1,5 +1,5 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const { SECRET_KEYS, readSecret } = require('../endpoints/secrets');2import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js';
3const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';3const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
44
5/**5/**
@@ -8,7 +8,7 @@ const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
8 * @param {import('../users').UserDirectoryList} directories - The directories object for the user8 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
9 * @returns {Promise<number[][]>} - The array of vectors for the texts9 * @returns {Promise<number[][]>} - The array of vectors for the texts
10 */10 */
11async function getMakerSuiteBatchVector(texts, directories) {11export async function getMakerSuiteBatchVector(texts, directories) {
12 const promises = texts.map(text => getMakerSuiteVector(text, directories));12 const promises = texts.map(text => getMakerSuiteVector(text, directories));
13 return await Promise.all(promises);13 return await Promise.all(promises);
14}14}
@@ -19,7 +19,7 @@ async function getMakerSuiteBatchVector(texts, directories) {
19 * @param {import('../users').UserDirectoryList} directories - The directories object for the user19 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
20 * @returns {Promise<number[]>} - The vector for the text20 * @returns {Promise<number[]>} - The vector for the text
21 */21 */
22async function getMakerSuiteVector(text, directories) {22export async function getMakerSuiteVector(text, directories) {
23 const key = readSecret(directories, SECRET_KEYS.MAKERSUITE);23 const key = readSecret(directories, SECRET_KEYS.MAKERSUITE);
2424
25 if (!key) {25 if (!key) {
@@ -56,8 +56,3 @@ async function getMakerSuiteVector(text, directories) {
56 // noinspection JSValidateTypes56 // noinspection JSValidateTypes
57 return data['embedding']['values'];57 return data['embedding']['values'];
58}58}
59
60module.exports = {
61 getMakerSuiteVector,
62 getMakerSuiteBatchVector,
63};
src/vectors/nomicai-vectors.js+4 -9
@@ -1,5 +1,5 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const { SECRET_KEYS, readSecret } = require('../endpoints/secrets');2import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js';
33
4const SOURCES = {4const SOURCES = {
5 'nomicai': {5 'nomicai': {
@@ -16,7 +16,7 @@ const SOURCES = {
16 * @param {import('../users').UserDirectoryList} directories - The directories object for the user16 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
17 * @returns {Promise<number[][]>} - The array of vectors for the texts17 * @returns {Promise<number[][]>} - The array of vectors for the texts
18 */18 */
19async function getNomicAIBatchVector(texts, source, directories) {19export async function getNomicAIBatchVector(texts, source, directories) {
20 const config = SOURCES[source];20 const config = SOURCES[source];
2121
22 if (!config) {22 if (!config) {
@@ -67,12 +67,7 @@ async function getNomicAIBatchVector(texts, source, directories) {
67 * @param {import('../users').UserDirectoryList} directories - The directories object for the user67 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
68 * @returns {Promise<number[]>} - The vector for the text68 * @returns {Promise<number[]>} - The vector for the text
69 */69 */
70async function getNomicAIVector(text, source, directories) {70export async function getNomicAIVector(text, source, directories) {
71 const vectors = await getNomicAIBatchVector([text], source, directories);71 const vectors = await getNomicAIBatchVector([text], source, directories);
72 return vectors[0];72 return vectors[0];
73}73}
74
75module.exports = {
76 getNomicAIVector,
77 getNomicAIBatchVector,
78};
src/vectors/ollama-vectors.js+5 -10
@@ -1,6 +1,6 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const { setAdditionalHeadersByType } = require('../additional-headers');2import { setAdditionalHeadersByType } from '../additional-headers.js';
3const { TEXTGEN_TYPES } = require('../constants');3import { TEXTGEN_TYPES } from '../constants.js';
44
5/**5/**
6 * Gets the vector for the given text from Ollama6 * Gets the vector for the given text from Ollama
@@ -11,7 +11,7 @@ const { TEXTGEN_TYPES } = require('../constants');
11 * @param {import('../users').UserDirectoryList} directories - The directories object for the user11 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
12 * @returns {Promise<number[][]>} - The array of vectors for the texts12 * @returns {Promise<number[][]>} - The array of vectors for the texts
13 */13 */
14async function getOllamaBatchVector(texts, apiUrl, model, keep, directories) {14export async function getOllamaBatchVector(texts, apiUrl, model, keep, directories) {
15 const result = [];15 const result = [];
16 for (const text of texts) {16 for (const text of texts) {
17 const vector = await getOllamaVector(text, apiUrl, model, keep, directories);17 const vector = await getOllamaVector(text, apiUrl, model, keep, directories);
@@ -29,7 +29,7 @@ async function getOllamaBatchVector(texts, apiUrl, model, keep, directories) {
29 * @param {import('../users').UserDirectoryList} directories - The directories object for the user29 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
30 * @returns {Promise<number[]>} - The vector for the text30 * @returns {Promise<number[]>} - The vector for the text
31 */31 */
32async function getOllamaVector(text, apiUrl, model, keep, directories) {32export async function getOllamaVector(text, apiUrl, model, keep, directories) {
33 const url = new URL(apiUrl);33 const url = new URL(apiUrl);
34 url.pathname = '/api/embeddings';34 url.pathname = '/api/embeddings';
3535
@@ -62,8 +62,3 @@ async function getOllamaVector(text, apiUrl, model, keep, directories) {
6262
63 return data.embedding;63 return data.embedding;
64}64}
65
66module.exports = {
67 getOllamaBatchVector,
68 getOllamaVector,
69};
src/vectors/openai-vectors.js+4 -9
@@ -1,5 +1,5 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const { SECRET_KEYS, readSecret } = require('../endpoints/secrets');2import { SECRET_KEYS, readSecret } from '../endpoints/secrets.js';
33
4const SOURCES = {4const SOURCES = {
5 'togetherai': {5 'togetherai': {
@@ -27,7 +27,7 @@ const SOURCES = {
27 * @param {string} model - The model to use for the embedding27 * @param {string} model - The model to use for the embedding
28 * @returns {Promise<number[][]>} - The array of vectors for the texts28 * @returns {Promise<number[][]>} - The array of vectors for the texts
29 */29 */
30async function getOpenAIBatchVector(texts, source, directories, model = '') {30export async function getOpenAIBatchVector(texts, source, directories, model = '') {
31 const config = SOURCES[source];31 const config = SOURCES[source];
3232
33 if (!config) {33 if (!config) {
@@ -83,12 +83,7 @@ async function getOpenAIBatchVector(texts, source, directories, model = '') {
83 * @param {string} model - The model to use for the embedding83 * @param {string} model - The model to use for the embedding
84 * @returns {Promise<number[]>} - The vector for the text84 * @returns {Promise<number[]>} - The vector for the text
85 */85 */
86async function getOpenAIVector(text, source, directories, model = '') {86export async function getOpenAIVector(text, source, directories, model = '') {
87 const vectors = await getOpenAIBatchVector([text], source, directories, model);87 const vectors = await getOpenAIBatchVector([text], source, directories, model);
88 return vectors[0];88 return vectors[0];
89}89}
90
91module.exports = {
92 getOpenAIVector,
93 getOpenAIBatchVector,
94};
src/vectors/vllm-vectors.js+5 -10
@@ -1,6 +1,6 @@
1const fetch = require('node-fetch').default;1import fetch from 'node-fetch';
2const { setAdditionalHeadersByType } = require('../additional-headers');2import { setAdditionalHeadersByType } from '../additional-headers.js';
3const { TEXTGEN_TYPES } = require('../constants');3import { TEXTGEN_TYPES } from '../constants.js';
44
5/**5/**
6 * Gets the vector for the given text from VLLM6 * Gets the vector for the given text from VLLM
@@ -10,7 +10,7 @@ const { TEXTGEN_TYPES } = require('../constants');
10 * @param {import('../users').UserDirectoryList} directories - The directories object for the user10 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
11 * @returns {Promise<number[][]>} - The array of vectors for the texts11 * @returns {Promise<number[][]>} - The array of vectors for the texts
12 */12 */
13async function getVllmBatchVector(texts, apiUrl, model, directories) {13export async function getVllmBatchVector(texts, apiUrl, model, directories) {
14 const url = new URL(apiUrl);14 const url = new URL(apiUrl);
15 url.pathname = '/v1/embeddings';15 url.pathname = '/v1/embeddings';
1616
@@ -52,12 +52,7 @@ async function getVllmBatchVector(texts, apiUrl, model, directories) {
52 * @param {import('../users').UserDirectoryList} directories - The directories object for the user52 * @param {import('../users').UserDirectoryList} directories - The directories object for the user
53 * @returns {Promise<number[]>} - The vector for the text53 * @returns {Promise<number[]>} - The vector for the text
54 */54 */
55async function getVllmVector(text, apiUrl, model, directories) {55export async function getVllmVector(text, apiUrl, model, directories) {
56 const vectors = await getVllmBatchVector([text], apiUrl, model, directories);56 const vectors = await getVllmBatchVector([text], apiUrl, model, directories);
57 return vectors[0];57 return vectors[0];
58}58}
59
60module.exports = {
61 getVllmBatchVector,
62 getVllmVector,
63};