Migrate Cohere to /v2/chat Closes #2916

c2935ba4a70c17e8ba00312f6779ad46ea420db5

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 9e50ff34c99e3d405897e2df0d13fb19a01047dd
24 files changed, +104 -255Ignore whitespace
default/config.yaml+4 -0
@@ -155,5 +155,9 @@ claude:
155155 # (e.g {{random}} macro or lorebooks not as in-chat injections).
156156 # Otherwise, you'll just waste money on cache misses.
157157 enableSystemPromptCache: false
158+# -- COHERE API CONFIGURATION --
159+cohere:
160+ # A placeholder prompt to be used when the message array doesn't end with a user message
161+ userPlaceholder: "Continue"
158162# -- SERVER PLUGIN CONFIGURATION --
159163enableServerPlugins: false
public/index.html+0 -11
@@ -595,17 +595,6 @@
595595 </span>
596596 </div>
597597 </div>
598- <div class="range-block" data-source="cohere">
599- <label for="websearch_toggle" title="Enable Cohere web-search connector" data-i18n="[title]Enable Cohere web-search connector" class="checkbox_label widthFreeExpand">
600- <input id="websearch_toggle" type="checkbox" />
601- <span data-i18n="Web-search">Web-search</span>
602- </label>
603- <div class="toggle-description justifyLeft">
604- <span data-i18n="Allow the model to use the web-search connector.">
605- Allow the model to use the web-search connector.
606- </span>
607- </div>
608- </div>
609598 <div class="range-block" data-source="openai,claude,windowai,openrouter,ai21,scale,makersuite,mistralai,custom,cohere,perplexity,groq,01ai">
610599 <div class="range-block-title" data-i18n="Temperature">
611600 Temperature
public/locales/ar-sa.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "الحد الأقصى لطول الاستجابة (الرموز,الحرف)",
5757 "Multiple swipes per generation": "الضربات الشديدة المتعددة لكل جيل",
5858 "Enable OpenAI completion streaming": "تمكين بث الاكتمال من OpenAI",
59- "Enable Cohere web-search connector": "تمكين موصل بحث الويب Cohere",
60- "Web-search": "البحث في الويب",
61- "Allow the model to use the web-search connector.": "اسمح للنموذج باستخدام موصل بحث الويب.",
6259 "Frequency Penalty": "عقوبة التكرار",
6360 "Presence Penalty": "عقوبة الوجود",
6461 "Count Penalty": "عد ضربة جزاء",
public/locales/de-de.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Maximale Antwortlänge (Tokens)",
5757 "Multiple swipes per generation": "Mehrere Swipes pro Generation",
5858 "Enable OpenAI completion streaming": "OpenAI-Vervollständigungsstreaming aktivieren",
59- "Enable Cohere web-search connector": "Cohere-Websuch-Connector aktivieren",
60- "Web-search": "Web-Suche",
61- "Allow the model to use the web-search connector.": "Erlauben Sie dem Modell, den Websuch-Connector zu verwenden.",
6259 "Frequency Penalty": "Frequenzstrafe",
6360 "Presence Penalty": "Präsenzstrafe",
6461 "Count Penalty": "Strafe zählen",
public/locales/es-es.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Longitud máxima de respuesta (tokens)",
5757 "Multiple swipes per generation": "Múltiples golpes por generación",
5858 "Enable OpenAI completion streaming": "Activar streaming de completado de OpenAI",
59- "Enable Cohere web-search connector": "Habilitar el conector de búsqueda web de Cohere",
60- "Web-search": "Búsqueda Web",
61- "Allow the model to use the web-search connector.": "Permita que el modelo utilice el conector de búsqueda web.",
6259 "Frequency Penalty": "Penalización de frecuencia",
6360 "Presence Penalty": "Penalización de presencia",
6461 "Count Penalty": "Penalización de conteo",
public/locales/fr-fr.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Longueur maximale de la réponse (tokens)",
5757 "Multiple swipes per generation": "Plusieurs balayages par génération",
5858 "Enable OpenAI completion streaming": "Activer le streaming de complétion OpenAI",
59- "Enable Cohere web-search connector": "Activer le connecteur de recherche Web Cohere",
60- "Web-search": "Recherche Internet",
61- "Allow the model to use the web-search connector.": "Autorisez le modèle à utiliser le connecteur de recherche Web.",
6259 "Frequency Penalty": "Pénalité de fréquence",
6360 "Presence Penalty": "Pénalité de présence",
6461 "Count Penalty": "Pénalité de décompte",
public/locales/is-is.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Hámarks lengd svörunar (í táknum)",
5757 "Multiple swipes per generation": "Mörg högg á hverja kynslóð",
5858 "Enable OpenAI completion streaming": "Virkja OpenAI klárastreymi",
59- "Enable Cohere web-search connector": "Virkja Cohere vefleitartengi",
60- "Web-search": "Vefleit",
61- "Allow the model to use the web-search connector.": "Leyfðu líkaninu að nota vefleitartengið.",
6259 "Frequency Penalty": "Tíðnarefning",
6360 "Presence Penalty": "Tilkoma refning",
6461 "Count Penalty": "Telja víti",
public/locales/it-it.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Lunghezza massima della risposta (token)",
5757 "Multiple swipes per generation": "Più passaggi per generazione",
5858 "Enable OpenAI completion streaming": "Abilita lo streaming di completamento OpenAI",
59- "Enable Cohere web-search connector": "Abilita il connettore di ricerca web Cohere",
60- "Web-search": "Ricerca sul web",
61- "Allow the model to use the web-search connector.": "Consenti al modello di utilizzare il connettore di ricerca web.",
6259 "Frequency Penalty": "Penalità di frequenza",
6360 "Presence Penalty": "Penalità di presenza",
6461 "Count Penalty": "Conte Penalità",
public/locales/ja-jp.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "最大応答長（トークン数）",
5757 "Multiple swipes per generation": "世代ごとに複数のスワイプ",
5858 "Enable OpenAI completion streaming": "OpenAIの完了ストリーミングを有効にする",
59- "Enable Cohere web-search connector": "Cohereウェブ検索コネクタを有効にする",
60- "Web-search": "ウェブ検索",
61- "Allow the model to use the web-search connector.": "モデルが Web 検索コネクタを使用できるようにします。",
6259 "Frequency Penalty": "頻度ペナルティ",
6360 "Presence Penalty": "存在ペナルティ",
6461 "Count Penalty": "カウントペナルティ",
public/locales/ko-kr.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "최대 응답 길이 (토큰)",
5757 "Multiple swipes per generation": "세대당 다중 스와이프",
5858 "Enable OpenAI completion streaming": "OpenAI 완성 스트리밍 활성화",
59- "Enable Cohere web-search connector": "Cohere 웹 검색 커넥터 활성화",
60- "Web-search": "웹 서핑",
61- "Allow the model to use the web-search connector.": "모델이 웹 검색 커넥터를 사용하도록 허용합니다.",
6259 "Frequency Penalty": "빈도 패널티",
6360 "Presence Penalty": "존재 패널티",
6461 "Count Penalty": "카운트 페널티",
public/locales/nl-nl.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Maximale lengte van het antwoord (tokens)",
5757 "Multiple swipes per generation": "Meerdere swipes per generatie",
5858 "Enable OpenAI completion streaming": "OpenAI voltooiingsstreaming inschakelen",
59- "Enable Cohere web-search connector": "Schakel de Cohere-webzoekconnector in",
60- "Web-search": "Web-zoeken",
61- "Allow the model to use the web-search connector.": "Sta toe dat het model de webzoekconnector gebruikt.",
6259 "Frequency Penalty": "Frequentieboete",
6360 "Presence Penalty": "Aanwezigheidsboete",
6461 "Count Penalty": "Tel straf",
public/locales/pt-pt.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Comprimento Máximo da Resposta (tokens)",
5757 "Multiple swipes per generation": "Vários furtos por geração",
5858 "Enable OpenAI completion streaming": "Ativar streaming de conclusão do OpenAI",
59- "Enable Cohere web-search connector": "Ativar o conector de pesquisa na Web Cohere",
60- "Web-search": "Pesquisa na internet",
61- "Allow the model to use the web-search connector.": "Permita que o modelo use o conector de pesquisa na Web.",
6259 "Frequency Penalty": "Pena de Frequência",
6360 "Presence Penalty": "Pena de Presença",
6461 "Count Penalty": "Contar penalidade",
public/locales/ru-ru.json+0 -3
@@ -772,7 +772,6 @@
772772 "Type a message, or /? for help": "Введите сообщение, или /? для получения справки",
773773 "Welcome to SillyTavern!": "Добро пожаловать в SillyTavern!",
774774 "Won't be shared with the character card on export.": "Не попадут в карточку персонажа при экспорте.",
775- "Web-search": "Веб-поиск",
776775 "Persona Name:": "Имя персоны:",
777776 "User first message": "Первое сообщение пользователя",
778777 "extension_token_counter": "Токенов:",
@@ -1200,8 +1199,6 @@
12001199 "Streaming_desc": "Выводить текст последовательно по мере его генерации.\rЕсли параметр выключен, ответы будут отображаться сразу целиком, и только после полного завершения генерации.",
12011200 "Max prompt cost:": "Max prompt cost:",
12021201 "TFS": "TFS",
1203- "Enable Cohere web-search connector": "Enable Cohere web-search connector",
1204- "Allow the model to use the web-search connector.": "Allow the model to use the web-search connector.",
12051202 "Count Penalty": "Count Penalty",
12061203 "Min P": "Min P",
12071204 "NSFW": "NSFW",
public/locales/uk-ua.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Довжина відповіді (токени)",
5757 "Multiple swipes per generation": "Кілька свайпів за покоління",
5858 "Enable OpenAI completion streaming": "Увімкнути потокове завершення OpenAI",
59- "Enable Cohere web-search connector": "Увімкнути конектор веб-пошуку Cohere",
60- "Web-search": "Веб-пошук",
61- "Allow the model to use the web-search connector.": "Дозвольте моделі використовувати конектор веб-пошуку.",
6259 "Frequency Penalty": "Штраф за частоту",
6360 "Presence Penalty": "Штраф за наявність",
6461 "Count Penalty": "Рахувати пенальті",
public/locales/vi-vn.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "Độ dài phản hồi tối đa (token)",
5757 "Multiple swipes per generation": "Vuốt nhiều lần trong một lần tạo",
5858 "Enable OpenAI completion streaming": "Bật streaming của OpenAI",
59- "Enable Cohere web-search connector": "Bật web tìm kiếm của Cohere",
60- "Web-search": "Tìm kiếm trên web",
61- "Allow the model to use the web-search connector.": "Cho phép model sử dụng trình kết nối tìm kiếm trên web.",
6259 "Frequency Penalty": "Frequency Penalty",
6360 "Presence Penalty": "Presence Penalty",
6461 "Count Penalty": "Count Penalty",
public/locales/zh-cn.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "最大回复长度（以词符数计）",
5757 "Multiple swipes per generation": "每次生成多个备选回复",
5858 "Enable OpenAI completion streaming": "启用OpenAI文本补全流式传输",
59- "Enable Cohere web-search connector": "启用 Cohere 网络搜索连接器",
60- "Web-search": "联网搜索",
61- "Allow the model to use the web-search connector.": "允许模型使用联网搜索。",
6259 "Frequency Penalty": "频率惩罚",
6360 "Presence Penalty": "存在惩罚",
6461 "Count Penalty": "计数惩罚",
public/locales/zh-tw.json+0 -3
@@ -56,9 +56,6 @@
5656 "Max Response Length (tokens)": "最大回應長度(符記數)",
5757 "Multiple swipes per generation": "每次生成多次滑動",
5858 "Enable OpenAI completion streaming": "啟用 OpenAI 補充串流",
59- "Enable Cohere web-search connector": "啟用 Cohere 網頁搜尋連接器",
60- "Web-search": "網頁搜尋",
61- "Allow the model to use the web-search connector.": "允許模型使用網頁搜尋連接器",
6259 "Frequency Penalty": "頻率懲罰",
6360 "Presence Penalty": "存在懲罰",
6461 "Count Penalty": "計數懲罰",
public/script.js+1 -1
@@ -5436,7 +5436,7 @@ function extractMessageFromData(data) {
54365436 case 'novel':
54375437 return data.output;
54385438 case 'openai':
54395439 return data?.choices?.[0]?.message?.content ?? data?.choices?.[0]?.text ?? data?.text ?? data?.message?.tool_plan ?? data?.message?.content?.[0]?.text ?? '';
54405440 default:
54415441 return '';
54425442 }
public/scripts/openai.js+3 -13
@@ -225,7 +225,6 @@ const default_settings = {
225225 top_a_openai: 0,
226226 repetition_penalty_openai: 1,
227227 stream_openai: false,
228- websearch_cohere: false,
229228 openai_max_context: max_4k,
230229 openai_max_tokens: 300,
231230 wrap_in_quotes: false,
@@ -302,7 +301,6 @@ const oai_settings = {
302301 top_a_openai: 0,
303302 repetition_penalty_openai: 1,
304303 stream_openai: false,
305- websearch_cohere: false,
306304 openai_max_context: max_4k,
307305 openai_max_tokens: 300,
308306 wrap_in_quotes: false,
@@ -1847,7 +1845,6 @@ async function sendOpenAIRequest(type, messages, signal) {
18471845 generate_data['frequency_penalty'] = Math.min(Math.max(Number(oai_settings.freq_pen_openai), 0), 1);
18481846 generate_data['presence_penalty'] = Math.min(Math.max(Number(oai_settings.pres_pen_openai), 0), 1);
18491847 generate_data['stop'] = getCustomStoppingStrings(5);
1850- generate_data['websearch'] = oai_settings.websearch_cohere;
18511848 }
18521849
18531850 if (isPerplexity) {
@@ -1980,8 +1977,10 @@ function getStreamingReply(data) {
19801977 return data?.delta?.text || '';
19811978 } else if (oai_settings.chat_completion_source == chat_completion_sources.MAKERSUITE) {
19821979 return data?.candidates?.[0]?.content?.parts?.[0]?.text || '';
1980+ } else if (oai_settings.chat_completion_source == chat_completion_sources.COHERE) {
1981+ return data?.delta?.message?.content?.text || data?.delta?.message?.tool_plan || '';
19831982 } else {
19841983 return data.choices?.[0]?.delta?.content ?? data.choices?.[0]?.message?.content ?? data.choices?.[0]?.text ?? '';
19851984 }
19861985}
19871986
@@ -2857,7 +2856,6 @@ function loadOpenAISettings(data, settings) {
28572856 oai_settings.min_p_openai = settings.min_p_openai ?? default_settings.min_p_openai;
28582857 oai_settings.repetition_penalty_openai = settings.repetition_penalty_openai ?? default_settings.repetition_penalty_openai;
28592858 oai_settings.stream_openai = settings.stream_openai ?? default_settings.stream_openai;
2860- oai_settings.websearch_cohere = settings.websearch_cohere ?? default_settings.websearch_cohere;
28612859 oai_settings.openai_max_context = settings.openai_max_context ?? default_settings.openai_max_context;
28622860 oai_settings.openai_max_tokens = settings.openai_max_tokens ?? default_settings.openai_max_tokens;
28632861 oai_settings.bias_preset_selected = settings.bias_preset_selected ?? default_settings.bias_preset_selected;
@@ -2931,7 +2929,6 @@ function loadOpenAISettings(data, settings) {
29312929 if (settings.use_makersuite_sysprompt !== undefined) oai_settings.use_makersuite_sysprompt = !!settings.use_makersuite_sysprompt;
29322930 if (settings.use_alt_scale !== undefined) { oai_settings.use_alt_scale = !!settings.use_alt_scale; updateScaleForm(); }
29332931 $('#stream_toggle').prop('checked', oai_settings.stream_openai);
2934- $('#websearch_toggle').prop('checked', oai_settings.websearch_cohere);
29352932 $('#api_url_scale').val(oai_settings.api_url_scale);
29362933 $('#openai_proxy_password').val(oai_settings.proxy_password);
29372934 $('#claude_assistant_prefill').val(oai_settings.assistant_prefill);
@@ -3258,7 +3255,6 @@ async function saveOpenAIPreset(name, settings, triggerUi = true) {
32583255 personality_format: settings.personality_format,
32593256 group_nudge_prompt: settings.group_nudge_prompt,
32603257 stream_openai: settings.stream_openai,
3261- websearch_cohere: settings.websearch_cohere,
32623258 prompts: settings.prompts,
32633259 prompt_order: settings.prompt_order,
32643260 api_url_scale: settings.api_url_scale,
@@ -3682,7 +3678,6 @@ function onSettingsPresetChange() {
36823678 personality_format: ['#personality_format_textarea', 'personality_format', false],
36833679 group_nudge_prompt: ['#group_nudge_prompt_textarea', 'group_nudge_prompt', false],
36843680 stream_openai: ['#stream_toggle', 'stream_openai', true],
3685- websearch_cohere: ['#websearch_toggle', 'websearch_cohere', true],
36863681 prompts: ['', 'prompts', false],
36873682 prompt_order: ['', 'prompt_order', false],
36883683 api_url_scale: ['#api_url_scale', 'api_url_scale', false],
@@ -4846,11 +4841,6 @@ export function initOpenAI() {
48464841 saveSettingsDebounced();
48474842 });
48484843
4849- $('#websearch_toggle').on('change', function () {
4850- oai_settings.websearch_cohere = !!$('#websearch_toggle').prop('checked');
4851- saveSettingsDebounced();
4852- });
4853-
48544844 $('#wrap_in_quotes').on('change', function () {
48554845 oai_settings.wrap_in_quotes = !!$('#wrap_in_quotes').prop('checked');
48564846 saveSettingsDebounced();
public/scripts/sse-stream.js+14 -2
@@ -108,9 +108,21 @@ function getDelay(s) {
108108 * @returns {AsyncGenerator<{data: object, chunk: string}>} The parsed data and the chunk to be sent.
109109 */
110110async function* parseStreamData(json) {
111+ // Cohere
112+ if (typeof json.delta.message === 'object' && ['tool-plan-delta', 'content-delta'].includes(json.type)) {
113+ const text = json?.delta?.message?.content?.text ?? '';
114+ for (let i = 0; i < text.length; i++) {
115+ const str = json.delta.message.content.text[i];
116+ yield {
117+ data: { ...json, delta: { message: { content: { text: str } } } },
118+ chunk: str,
119+ };
120+ }
121+ return;
122+ }
111123 // Claude
112124 if (typeof json.delta === 'object' && typeof json.delta.text === 'string') {
113125 if (typeof json.delta.text === 'string' && json.delta.text.length > 0) {
114126 for (let i = 0; i < json.delta.text.length; i++) {
115127 const str = json.delta.text[i];
116128 yield {
public/scripts/tool-calling.js+31 -1
@@ -381,6 +381,22 @@ export class ToolManager {
381381 }
382382 }
383383 }
384+ const cohereToolEvents = ['message-start', 'tool-call-start', 'tool-call-delta', 'tool-call-end'];
385+ if (cohereToolEvents.includes(parsed?.type) && typeof parsed?.delta?.message === 'object') {
386+ const choiceIndex = 0;
387+ const toolCallIndex = parsed?.index ?? 0;
388+
389+ if (!Array.isArray(toolCalls[choiceIndex])) {
390+ toolCalls[choiceIndex] = [];
391+ }
392+
393+ if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
394+ toolCalls[choiceIndex][toolCallIndex] = {};
395+ }
396+
397+ const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
398+ ToolManager.#applyToolCallDelta(targetToolCall, parsed.delta.message);
399+ }
384400 if (typeof parsed?.content_block === 'object') {
385401 const choiceIndex = 0;
386402 const toolCallIndex = parsed?.index ?? 0;
@@ -483,6 +499,7 @@ export class ToolManager {
483499 chat_completion_sources.CLAUDE,
484500 chat_completion_sources.OPENROUTER,
485501 chat_completion_sources.GROQ,
502+ chat_completion_sources.COHERE,
486503 ];
487504 return supportedSources.includes(oai_settings.chat_completion_source);
488505 }
@@ -509,7 +526,15 @@ export class ToolManager {
509526
510527 // Parsed tool calls from streaming data
511528 if (Array.isArray(data) && data.length > 0 && Array.isArray(data[0])) {
512- return isClaudeToolCall(data[0]) ? data[0].filter(x => x).map(convertClaudeToolCall) : data[0];
529+ if (isClaudeToolCall(data[0])) {
530+ return data[0].filter(x => x).map(convertClaudeToolCall);
531+ }
532+
533+ if (typeof data[0]?.[0]?.tool_calls === 'object') {
534+ return Array.isArray(data[0]?.[0]?.tool_calls) ? data[0][0].tool_calls : [data[0][0].tool_calls];
535+ }
536+
537+ return data[0];
513538 }
514539
515540 // Parsed tool calls from non-streaming data
@@ -530,6 +555,11 @@ export class ToolManager {
530555 return content;
531556 }
532557 }
558+
559+ // Cohere tool calls
560+ if (typeof data?.message?.tool_calls === 'object') {
561+ return Array.isArray(data?.message?.tool_calls) ? data.message.tool_calls : [data.message.tool_calls];
562+ }
533563 }
534564
535565 /**
src/cohere-stream.js+0 -126
@@ -1,126 +0,0 @@
1-const DATA_PREFIX = 'data:';
2-
3-/**
4- * Borrowed from Cohere SDK (MIT License)
5- * https://github.com/cohere-ai/cohere-typescript/blob/main/src/core/streaming-fetcher/Stream.ts
6- * Copyright (c) 2021 Cohere
7- */
8-class CohereStream {
9- /** @type {ReadableStream} */
10- stream;
11- /** @type {string} */
12- prefix;
13- /** @type {string} */
14- messageTerminator;
15- /** @type {string|undefined} */
16- streamTerminator;
17- /** @type {AbortController} */
18- controller = new AbortController();
19-
20- constructor({ stream, eventShape }) {
21- this.stream = stream;
22- if (eventShape.type === 'sse') {
23- this.prefix = DATA_PREFIX;
24- this.messageTerminator = '\n';
25- this.streamTerminator = eventShape.streamTerminator;
26- } else {
27- this.messageTerminator = eventShape.messageTerminator;
28- }
29- }
30-
31- async *iterMessages() {
32- const stream = readableStreamAsyncIterable(this.stream);
33- let buf = '';
34- let prefixSeen = false;
35- let parsedAnyMessages = false;
36- for await (const chunk of stream) {
37- buf += this.decodeChunk(chunk);
38-
39- let terminatorIndex;
40- // Parse the chunk into as many messages as possible
41- while ((terminatorIndex = buf.indexOf(this.messageTerminator)) >= 0) {
42- // Extract the line from the buffer
43- let line = buf.slice(0, terminatorIndex + 1);
44- buf = buf.slice(terminatorIndex + 1);
45-
46- // Skip empty lines
47- if (line.length === 0) {
48- continue;
49- }
50-
51- // Skip the chunk until the prefix is found
52- if (!prefixSeen && this.prefix != null) {
53- const prefixIndex = line.indexOf(this.prefix);
54- if (prefixIndex === -1) {
55- continue;
56- }
57- prefixSeen = true;
58- line = line.slice(prefixIndex + this.prefix.length);
59- }
60-
61- // If the stream terminator is present, return
62- if (this.streamTerminator != null && line.includes(this.streamTerminator)) {
63- return;
64- }
65-
66- // Otherwise, yield message from the prefix to the terminator
67- const message = JSON.parse(line);
68- yield message;
69- prefixSeen = false;
70- parsedAnyMessages = true;
71- }
72- }
73-
74- if (!parsedAnyMessages && buf.length > 0) {
75- try {
76- yield JSON.parse(buf);
77- } catch (e) {
78- console.error('Error parsing message:', e);
79- }
80- }
81- }
82-
83- async *[Symbol.asyncIterator]() {
84- for await (const message of this.iterMessages()) {
85- yield message;
86- }
87- }
88-
89- decodeChunk(chunk) {
90- const decoder = new TextDecoder('utf8');
91- return decoder.decode(chunk);
92- }
93-}
94-
95-function readableStreamAsyncIterable(stream) {
96- if (stream[Symbol.asyncIterator]) {
97- return stream;
98- }
99-
100- const reader = stream.getReader();
101- return {
102- async next() {
103- try {
104- const result = await reader.read();
105- if (result?.done) {
106- reader.releaseLock();
107- } // release lock when stream becomes closed
108- return result;
109- } catch (e) {
110- reader.releaseLock(); // release lock when stream becomes errored
111- throw e;
112- }
113- },
114- async return() {
115- const cancelPromise = reader.cancel();
116- reader.releaseLock();
117- await cancelPromise;
118- return { done: true, value: undefined };
119- },
120- [Symbol.asyncIterator]() {
121- return this;
122- },
123- };
124-}
125-
126-module.exports = CohereStream;
src/endpoints/backends/chat-completions.js+14 -18
@@ -5,7 +5,6 @@ const { jsonParser } = require('../../express-common');
55const { CHAT_COMPLETION_SOURCES, GEMINI_SAFETY, BISON_SAFETY, OPENROUTER_HEADERS } = require('../../constants');
66const { forwardFetchResponse, getConfigValue, tryParse, uuidv4, mergeObjectWithYaml, excludeKeysByYaml, color } = require('../../util');
77const { convertClaudeMessages, convertGooglePrompt, convertTextCompletionPrompt, convertCohereMessages, convertMistralMessages, convertAI21Messages, mergeMessages } = require('../../prompt-converters');
8-const CohereStream = require('../../cohere-stream');
98
109const { readSecret, SECRET_KEYS } = require('../secrets');
1110const { getTokenizerModel, getSentencepiceTokenizer, getTiktokenTokenizer, sentencepieceTokenizers, TEXT_COMPLETION_MODELS } = require('../tokenizers');
@@ -13,7 +12,8 @@ const { getTokenizerModel, getSentencepiceTokenizer, getTiktokenTokenizer, sente
1312const API_OPENAI = 'https://api.openai.com/v1';
1413const API_CLAUDE = 'https://api.anthropic.com/v1';
1514const API_MISTRAL = 'https://api.mistral.ai/v1';
1615const API_COHEREAPI_COHERE_V1 = 'https://api.cohere.ai/v1';
16+const API_COHERE_V2 = 'https://api.cohere.ai/v2';
1717const API_PERPLEXITY = 'https://api.perplexity.ai';
1818const API_GROQ = 'https://api.groq.com/openai/v1';
1919const API_MAKERSUITE = 'https://generativelanguage.googleapis.com';
@@ -553,13 +553,14 @@ async function sendCohereRequest(request, response) {
553553
554554 try {
555555 const convertedHistory = convertCohereMessages(request.body.messages, request.body.char_name, request.body.user_name);
556- const connectors = [];
557556 const tools = [];
558557
559- const canDoWebSearch = !String(request.body.model).includes('c4ai-aya');
558+ if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
560- if (request.body.websearch && canDoWebSearch) {
559+ tools.push(...request.body.tools);
561- connectors.push({
560+ tools.forEach(tool => {
562- id: 'web-search',
561+ if (tool?.function?.parameters?.$schema) {
562+ delete tool.function.parameters.$schema;
563+ }
563564 });
564565 }
565566
@@ -567,9 +568,7 @@ async function sendCohereRequest(request, response) {
567568 const requestBody = {
568569 stream: Boolean(request.body.stream),
569570 model: request.body.model,
570571 messagemessages: convertedHistory.userPromptchatHistory,
571- preamble: convertedHistory.systemPrompt,
572- chat_history: convertedHistory.chatHistory,
573572 temperature: request.body.temperature,
574573 max_tokens: request.body.max_tokens,
575574 k: request.body.top_k,
@@ -578,16 +577,13 @@ async function sendCohereRequest(request, response) {
578577 stop_sequences: request.body.stop,
579578 frequency_penalty: request.body.frequency_penalty,
580579 presence_penalty: request.body.presence_penalty,
581- prompt_truncation: 'AUTO_PRESERVE_ORDER',
582- connectors: connectors,
583580 documents: [],
584581 tools: tools,
585- search_queries_only: false,
586582 };
587583
588584 const canDoSafetyMode = String(request.body.model).endsWith('08-2024');
589585 if (canDoSafetyMode) {
590586 requestBody.safety_mode = 'NONEOFF';
591587 }
592588
593589 console.log('Cohere request:', requestBody);
@@ -603,11 +599,11 @@ async function sendCohereRequest(request, response) {
603599 timeout: 0,
604600 };
605601
606602 const apiUrl = API_COHEREAPI_COHERE_V2 + '/chat';
607603
608604 if (request.body.stream) {
609605 const stream = await global.fetch(apiUrl, config);
610606 parseCohereStreamforwardFetchResponse(stream, request, response);
611607 } else {
612608 const generateResponse = await fetch(apiUrl, config);
613609 if (!generateResponse.ok) {
@@ -658,7 +654,7 @@ router.post('/status', jsonParser, async function (request, response_getstatus_o
658654 headers = {};
659655 mergeObjectWithYaml(headers, request.body.custom_include_headers);
660656 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.COHERE) {
661657 api_url = API_COHEREAPI_COHERE_V1;
662658 api_key_openai = readSecret(request.user.directories, SECRET_KEYS.COHERE);
663659 headers = {};
664660 } else if (request.body.chat_completion_source === CHAT_COMPLETION_SOURCES.ZEROONEAI) {
src/prompt-converters.js+37 -38
@@ -277,56 +277,55 @@ function convertClaudeMessages(messages, prefillString, useSysPrompt, useTools,
277277 * @param {object[]} messages Array of messages
278278 * @param {string} charName Character name
279279 * @param {string} userName User name
280280 * @returns {{systemPrompt: string, chatHistory: object[], userPrompt: string}} Prompt for Cohere
281281 */
282282function convertCohereMessages(messages, charName = '', userName = '') {
283283 constif roleMap(messages.length === 0) {
284- 'system': 'SYSTEM',
284+ messages.unshift({
285285 'user' role: 'USERuser',
286- 'assistant': 'CHATBOT',
286+ content: PROMPT_PLACEHOLDER,
287287 });
288- let systemPrompt = '';
288+ }
289289
290- // Collect all the system messages up until the first instance of a non-system message, and then remove them from the messages array.
290+ messages.forEach((msg, index) => {
291- let i;
291+ // Tool calls require an assistent primer
292- for (i = 0; i < messages.length; i++) {
292+ if (Array.isArray(msg.tool_calls)) {
293293 if (index > 0 && messages[iindex - 1].role !=== 'systemassistant') {
294- break;
294+ msg.content = messages[index - 1].content;
295- }
295+ messages.splice(index - 1, 1);
296- // Append example names if not already done by the frontend (e.g. for group chats).
296+ } else {
297- if (userName && messages[i].name === 'example_user') {
297+ msg.content = `I'm going to call the tool for that: ${msg.tool_calls.map(tc => tc?.function?.name).join(', ')}`;
298- if (!messages[i].content.startsWith(`${userName}: `)) {
299- messages[i].content = `${userName}: ${messages[i].content}`;
300298 }
301299 }
302- if (charName && messages[i].name === 'example_assistant') {
300+ // No names support (who would've thought)
303- if (!messages[i].content.startsWith(`${charName}: `)) {
301+ if (msg.name) {
304- messages[i].content = `${charName}: ${messages[i].content}`;
302+ if (msg.role == 'system' && msg.name == 'example_assistant') {
303+ if (charName && !msg.content.startsWith(`${charName}: `)) {
304+ msg.content = `${charName}: ${msg.content}`;
305+ }
306+ }
307+ if (msg.role == 'system' && msg.name == 'example_user') {
308+ if (userName && !msg.content.startsWith(`${userName}: `)) {
309+ msg.content = `${userName}: ${msg.content}`;
310+ }
311+ }
312+ if (msg.role !== 'system' && !msg.content.startsWith(`${msg.name}: `)) {
313+ msg.content = `${msg.name}: ${msg.content}`;
305314 }
315+ delete msg.name;
306316 }
307- systemPrompt += `${messages[i].content}\n\n`;
317+ });
308- }
309-
310- messages.splice(0, i);
311318
312- if (messages.length === 0) {
319+ // A prompt should end with a user/tool message
313- messages.unshift({
320+ if (!['user', 'tool'].includes(messages[messages.length - 1].role)) {
321+ const userPlaceholder = getConfigValue('cohere.userPlaceholder', PROMPT_PLACEHOLDER || 'Continue');
322+ messages.push({
314323 role: 'user',
315324 content: PROMPT_PLACEHOLDERuserPlaceholder,
316325 });
317326 }
318327
319- const lastNonSystemMessageIndex = messages.findLastIndex(msg => msg.role === 'user' || msg.role === 'assistant');
328+ return { chatHistory: messages };
320- const userPrompt = messages.slice(lastNonSystemMessageIndex).map(msg => msg.content).join('\n\n') || PROMPT_PLACEHOLDER;
321-
322- const chatHistory = messages.slice(0, lastNonSystemMessageIndex).map(msg => {
323- return {
324- role: roleMap[msg.role] || 'USER',
325- message: msg.content,
326- };
327- });
328-
329- return { systemPrompt: systemPrompt.trim(), chatHistory, userPrompt };
330329}
331330
332331/**