Add generic text completion API type (100% OAI compatible)

3167019faf9d8626bf5acf653ad28036fd2d78fc

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: e7b53dcb46a9edd1773b76511b4c1e5178d3ccde
11 files changed, +100 -3Ignore whitespace
public/index.html+20 -1
@@ -2187,10 +2187,10 @@
21872187 <div>
21882188 <h4 data-i18n="API Type">API Type</h4>
21892189 <select id="textgen_type">
2190- <option value="ooba" data-i18n="Default (completions compatible)">Default [OpenAI /completions compatible: oobabooga, LM Studio, etc.]</option>
21912190 <option value="aphrodite">Aphrodite</option>
21922191 <option value="dreamgen">DreamGen</option>
21932192 <option value="featherless">Featherless</option>
2193+ <option value="generic" data-i18n="Generic (OpenAI-compatible) [LM Studio, etc.]">Generic (OpenAI-compatible) [LM Studio, etc.]</option>
21942194 <option value="huggingface">HuggingFace (Inference Endpoint)</option>
21952195 <option value="infermaticai">InfermaticAI</option>
21962196 <option value="koboldcpp">KoboldCpp</option>
@@ -2199,6 +2199,7 @@
21992199 <option value="ollama">Ollama</option>
22002200 <option value="openrouter">OpenRouter</option>
22012201 <option value="tabby">TabbyAPI</option>
2202+ <option value="ooba">Text Generation WebUI (oobabooga)</option>
22022203 <option value="togetherai">TogetherAI</option>
22032204 <option value="vllm">vLLM</option>
22042205 </select>
@@ -2321,6 +2322,24 @@
23212322 </select>
23222323 </div>
23232324 </div>
2325+ <div data-tg-type="generic" class="flex-container flexFlowColumn">
2326+ <h4 data-i18n="API key (optional)">API key (optional)</h4>
2327+ <div class="flex-container">
2328+ <input id="api_key_generic" name="api_key_generic" class="text_pole flex1 wide100p" type="text" autocomplete="off">
2329+ <div title="Clear your API key" data-i18n="[title]Clear your API key" class="menu_button fa-solid fa-circle-xmark clear-api-key" data-key="api_key_generic">
2330+ </div>
2331+ </div>
2332+ <div data-for="api_key_generic" class="neutral_warning" data-i18n="For privacy reasons, your API key will be hidden after you reload the page.">
2333+ For privacy reasons, your API key will be hidden after you reload the page.
2334+ </div>
2335+ <div class="flex1">
2336+ <h4 data-i18n="Server url">Server URL</h4>
2337+ <small data-i18n="Example: 127.0.0.1:5000">Example: http://127.0.0.1:5000</small>
2338+ <input id="generic_api_url_text" name="generic_api_url" class="text_pole wide100p" value="" autocomplete="off" data-server-history="generic">
2339+ </div>
2340+ <datalist id="generic_model_fill"></datalist>
2341+ <input id="generic_model_textgenerationwebui" list="generic_model_fill" class="text_pole wide100p" placeholder="Model ID (optional)" data-i18n="[placeholder]Model ID (optional)" type="text">
2342+ </div>
23242343 <div data-tg-type="ooba" class="flex-container flexFlowColumn">
23252344 <div class="flex-container flexFlowColumn">
23262345 <a href="https://github.com/oobabooga/text-generation-webui" target="_blank">
public/script.js+5 -1
@@ -234,7 +234,7 @@ import {
234234import { getBackgrounds, initBackgrounds, loadBackgroundSettings, background_settings } from './scripts/backgrounds.js';
235235import { hideLoader, showLoader } from './scripts/loader.js';
236236import { BulkEditOverlay, CharacterContextMenu } from './scripts/BulkEditOverlay.js';
237237import { loadFeatherlessModels, loadMancerModels, loadOllamaModels, loadTogetherAIModels, loadInfermaticAIModels, loadOpenRouterModels, loadVllmModels, loadAphroditeModels, loadDreamGenModels, initTextGenModels, loadTabbyModels, loadGenericModels } from './scripts/textgen-models.js';
238238import { appendFileContent, hasPendingFileAttachment, populateFileAttachment, decodeStyleTags, encodeStyleTags, isExternalMediaAllowed, getCurrentEntityId, preserveNeutralChat, restoreNeutralChat } from './scripts/chats.js';
239239import { getPresetManager, initPresetManager } from './scripts/preset-manager.js';
240240import { evaluateMacros, getLastMessageId, initMacros } from './scripts/macros.js';
@@ -1221,6 +1221,9 @@ async function getStatusTextgen() {
12211221 } else if (textgen_settings.type === textgen_types.TABBY) {
12221222 loadTabbyModels(data?.data);
12231223 setOnlineStatus(textgen_settings.tabby_model || data?.result);
1224+ } else if (textgen_settings.type === textgen_types.GENERIC) {
1225+ loadGenericModels(data?.data);
1226+ setOnlineStatus(textgen_settings.generic_model || 'Connected');
12241227 } else {
12251228 setOnlineStatus(data?.result);
12261229 }
@@ -10000,6 +10003,7 @@ jQuery(async function () {
1000010003 { id: 'api_key_llamacpp', secret: SECRET_KEYS.LLAMACPP },
1000110004 { id: 'api_key_featherless', secret: SECRET_KEYS.FEATHERLESS },
1000210005 { id: 'api_key_huggingface', secret: SECRET_KEYS.HUGGINGFACE },
10006+ { id: 'api_key_generic', secret: SECRET_KEYS.GENERIC },
1000310007 ];
1000410008
1000510009 for (const key of keys) {
public/scripts/preset-manager.js+1 -0
@@ -585,6 +585,7 @@ class PresetManager {
585585 'openrouter_allow_fallbacks',
586586 'tabby_model',
587587 'derived',
588+ 'generic_model',
588589 ];
589590 const settings = Object.assign({}, getSettingsByApiId(this.apiId));
590591
public/scripts/secrets.js+2 -0
@@ -38,6 +38,7 @@ export const SECRET_KEYS = {
3838 NANOGPT: 'api_key_nanogpt',
3939 TAVILY: 'api_key_tavily',
4040 BFL: 'api_key_bfl',
41+ GENERIC: 'api_key_generic',
4142};
4243
4344const INPUT_MAP = {
@@ -71,6 +72,7 @@ const INPUT_MAP = {
7172 [SECRET_KEYS.HUGGINGFACE]: '#api_key_huggingface',
7273 [SECRET_KEYS.BLOCKENTROPY]: '#api_key_blockentropy',
7374 [SECRET_KEYS.NANOGPT]: '#api_key_nanogpt',
75+ [SECRET_KEYS.GENERIC]: '#api_key_generic',
7476};
7577
7678async function clearSecret() {
public/scripts/slash-commands.js+1 -0
@@ -3687,6 +3687,7 @@ function setBackgroundCallback(_, bg) {
36873687function getModelOptions(quiet) {
36883688 const nullResult = { control: null, options: null };
36893689 const modelSelectMap = [
3690+ { id: 'generic_model_textgenerationwebui', api: 'textgenerationwebui', type: textgen_types.GENERIC },
36903691 { id: 'custom_model_textgenerationwebui', api: 'textgenerationwebui', type: textgen_types.OOBA },
36913692 { id: 'model_togetherai_select', api: 'textgenerationwebui', type: textgen_types.TOGETHERAI },
36923693 { id: 'openrouter_model', api: 'textgenerationwebui', type: textgen_types.OPENROUTER },
public/scripts/textgen-models.js+18 -0
@@ -160,6 +160,24 @@ export async function loadInfermaticAIModels(data) {
160160 }
161161}
162162
163+export function loadGenericModels(data) {
164+ if (!Array.isArray(data)) {
165+ console.error('Invalid Generic models data', data);
166+ return;
167+ }
168+
169+ data.sort((a, b) => a.id.localeCompare(b.id));
170+ const dataList = $('#generic_model_fill');
171+ dataList.empty();
172+
173+ for (const model of data) {
174+ const option = document.createElement('option');
175+ option.value = model.id;
176+ option.text = model.id;
177+ dataList.append(option);
178+ }
179+}
180+
163181export async function loadDreamGenModels(data) {
164182 if (!Array.isArray(data)) {
165183 console.error('Invalid DreamGen models data', data);
public/scripts/textgen-settings.js+10 -0
@@ -33,9 +33,11 @@ export const textgen_types = {
3333 OPENROUTER: 'openrouter',
3434 FEATHERLESS: 'featherless',
3535 HUGGINGFACE: 'huggingface',
36+ GENERIC: 'generic',
3637};
3738
3839const {
40+ GENERIC,
3941 MANCER,
4042 VLLM,
4143 APHRODITE,
@@ -120,6 +122,7 @@ export const SERVER_INPUTS = {
120122 [textgen_types.LLAMACPP]: '#llamacpp_api_url_text',
121123 [textgen_types.OLLAMA]: '#ollama_api_url_text',
122124 [textgen_types.HUGGINGFACE]: '#huggingface_api_url_text',
125+ [textgen_types.GENERIC]: '#generic_api_url_text',
123126};
124127
125128const KOBOLDCPP_ORDER = [6, 0, 1, 3, 4, 2, 5];
@@ -205,6 +208,7 @@ const settings = {
205208 xtc_probability: 0,
206209 nsigma: 0.0,
207210 featherless_model: '',
211+ generic_model: '',
208212};
209213
210214export {
@@ -282,6 +286,7 @@ export const setting_names = [
282286 'xtc_threshold',
283287 'xtc_probability',
284288 'nsigma',
289+ 'generic_model',
285290];
286291
287292const DYNATEMP_BLOCK = document.getElementById('dynatemp_block_ooba');
@@ -1100,6 +1105,11 @@ export function getTextGenModel() {
11001105 return settings.custom_model;
11011106 }
11021107 break;
1108+ case GENERIC:
1109+ if (settings.generic_model) {
1110+ return settings.generic_model;
1111+ }
1112+ break;
11031113 case MANCER:
11041114 return settings.mancer_model;
11051115 case TOGETHERAI:
src/additional-headers.js+14 -0
@@ -172,6 +172,19 @@ function getHuggingFaceHeaders(directories) {
172172 }) : {};
173173}
174174
175+/**
176+ * Gets the headers for the Generic text completion API.
177+ * @param {import('./users.js').UserDirectoryList} directories
178+ * @returns {object} Headers for the request
179+ */
180+function getGenericHeaders(directories) {
181+ const apiKey = readSecret(directories, SECRET_KEYS.GENERIC);
182+
183+ return apiKey ? ({
184+ 'Authorization': `Bearer ${apiKey}`,
185+ }) : {};
186+}
187+
175188export function getOverrideHeaders(urlHost) {
176189 const requestOverrides = getConfigValue('requestOverrides', []);
177190 const overrideHeaders = requestOverrides?.find((e) => e.hosts?.includes(urlHost))?.headers;
@@ -214,6 +227,7 @@ export function setAdditionalHeadersByType(requestHeaders, type, server, directo
214227 [TEXTGEN_TYPES.LLAMACPP]: getLlamaCppHeaders,
215228 [TEXTGEN_TYPES.FEATHERLESS]: getFeatherlessHeaders,
216229 [TEXTGEN_TYPES.HUGGINGFACE]: getHuggingFaceHeaders,
230+ [TEXTGEN_TYPES.GENERIC]: getGenericHeaders,
217231 };
218232
219233 const getHeaders = headerGetters[type];
src/constants.js+19 -0
@@ -225,6 +225,7 @@ export const TEXTGEN_TYPES = {
225225 OPENROUTER: 'openrouter',
226226 FEATHERLESS: 'featherless',
227227 HUGGINGFACE: 'huggingface',
228+ GENERIC: 'generic',
228229};
229230
230231export const INFERMATICAI_KEYS = [
@@ -346,6 +347,24 @@ export const OLLAMA_KEYS = [
346347 'min_p',
347348];
348349
350+// https://platform.openai.com/docs/api-reference/completions
351+export const OPENAI_KEYS = [
352+ 'model',
353+ 'prompt',
354+ 'stream',
355+ 'temperature',
356+ 'top_p',
357+ 'frequency_penalty',
358+ 'presence_penalty',
359+ 'stop',
360+ 'seed',
361+ 'logit_bias',
362+ 'logprobs',
363+ 'max_tokens',
364+ 'n',
365+ 'best_of',
366+];
367+
349368export const AVATAR_WIDTH = 512;
350369export const AVATAR_HEIGHT = 768;
351370
src/endpoints/backends/text-completions.js+9 -1
@@ -13,6 +13,7 @@ import {
1313 VLLM_KEYS,
1414 DREAMGEN_KEYS,
1515 FEATHERLESS_KEYS,
16+ OPENAI_KEYS,
1617} from '../../constants.js';
1718import { forwardFetchResponse, trimV1, getConfigValue } from '../../util.js';
1819import { setAdditionalHeaders } from '../../additional-headers.js';
@@ -113,8 +114,8 @@ router.post('/status', jsonParser, async function (request, response) {
113114 let url = baseUrl;
114115 let result = '';
115116
116-
117117 switch (apiType) {
118+ case TEXTGEN_TYPES.GENERIC:
118119 case TEXTGEN_TYPES.OOBA:
119120 case TEXTGEN_TYPES.VLLM:
120121 case TEXTGEN_TYPES.APHRODITE:
@@ -287,6 +288,7 @@ router.post('/generate', jsonParser, async function (request, response) {
287288 let url = trimV1(baseUrl);
288289
289290 switch (request.body.api_type) {
291+ case TEXTGEN_TYPES.GENERIC:
290292 case TEXTGEN_TYPES.VLLM:
291293 case TEXTGEN_TYPES.FEATHERLESS:
292294 case TEXTGEN_TYPES.APHRODITE:
@@ -347,6 +349,12 @@ router.post('/generate', jsonParser, async function (request, response) {
347349 args.body = JSON.stringify(request.body);
348350 }
349351
352+ if (request.body.api_type === TEXTGEN_TYPES.GENERIC) {
353+ request.body = _.pickBy(request.body, (_, key) => OPENAI_KEYS.includes(key));
354+ if (Array.isArray(request.body.stop)) { request.body.stop = request.body.stop.slice(0, 4); }
355+ args.body = JSON.stringify(request.body);
356+ }
357+
350358 if (request.body.api_type === TEXTGEN_TYPES.OPENROUTER) {
351359 if (Array.isArray(request.body.provider) && request.body.provider.length > 0) {
352360 request.body.provider = {
src/endpoints/secrets.js+1 -0
@@ -50,6 +50,7 @@ export const SECRET_KEYS = {
5050 TAVILY: 'api_key_tavily',
5151 NANOGPT: 'api_key_nanogpt',
5252 BFL: 'api_key_bfl',
53+ GENERIC: 'api_key_generic',
5354};
5455
5556// These are the keys that are safe to expose, even if allowKeysExposure is false