Gemini: Add tool calling

c9277cec28792539f7c63cb479682d68d5885ddd

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: e9cf606c70e2cba2aade710acfdfc6b6ca28bb3b
5 files changed, +92 -6Ignore whitespace
public/index.html+1 -1
@@ -1962,7 +1962,7 @@
19621962 </span>
19631963 </div>
19641964 </div>
19651965 <div class="range-block" data-source="openai,cohere,mistralai,custom,claude,openrouter,groq,deepseek,makersuite">
19661966 <label for="openai_function_calling" class="checkbox_label flexWrap widthFreeExpand">
19671967 <input id="openai_function_calling" type="checkbox" />
19681968 <span data-i18n="Enable function calling">Enable function calling</span>
public/scripts/sse-stream.js+5 -0
@@ -137,9 +137,14 @@ async function* parseStreamData(json) {
137137 else if (Array.isArray(json.candidates)) {
138138 for (let i = 0; i < json.candidates.length; i++) {
139139 const isNotPrimary = json.candidates?.[0]?.index > 0;
140+ const hasToolCalls = json?.candidates?.[0]?.content?.parts?.some(p => p?.functionCall);
140141 if (isNotPrimary || json.candidates.length === 0) {
141142 return null;
142143 }
144+ if (hasToolCalls) {
145+ yield { data: json, chunk: '' };
146+ return;
147+ }
143148 if (typeof json.candidates[0].content === 'object' && Array.isArray(json.candidates[i].content.parts)) {
144149 for (let j = 0; j < json.candidates[i].content.parts.length; j++) {
145150 if (typeof json.candidates[i].content.parts[j].text === 'string') {
public/scripts/tool-calling.js+33 -0
@@ -506,6 +506,26 @@ export class ToolManager {
506506 }
507507 }
508508 }
509+ if (Array.isArray(parsed?.candidates)) {
510+ for (let choiceIndex = 0; choiceIndex < parsed.candidates.length; choiceIndex++) {
511+ const candidate = parsed.candidates[choiceIndex];
512+ if (Array.isArray(candidate?.content?.parts)) {
513+ for (let toolCallIndex = 0; toolCallIndex < candidate.content.parts.length; toolCallIndex++) {
514+ const part = candidate.content.parts[toolCallIndex];
515+ if (part.functionCall) {
516+ if (!Array.isArray(toolCalls[choiceIndex])) {
517+ toolCalls[choiceIndex] = [];
518+ }
519+ if (toolCalls[choiceIndex][toolCallIndex] === undefined) {
520+ toolCalls[choiceIndex][toolCallIndex] = {};
521+ }
522+ const targetToolCall = toolCalls[choiceIndex][toolCallIndex];
523+ ToolManager.#applyToolCallDelta(targetToolCall, part.functionCall);
524+ }
525+ }
526+ }
527+ }
528+ }
509529 }
510530
511531 /**
@@ -564,6 +584,7 @@ export class ToolManager {
564584 chat_completion_sources.GROQ,
565585 chat_completion_sources.COHERE,
566586 chat_completion_sources.DEEPSEEK,
587+ chat_completion_sources.MAKERSUITE,
567588 ];
568589 return supportedSources.includes(oai_settings.chat_completion_source);
569590 }
@@ -585,8 +606,11 @@ export class ToolManager {
585606 * @returns {any[]} Tool calls from the response data
586607 */
587608 static #getToolCallsFromData(data) {
609+ const getRandomId = () => Math.random().toString(36).substring(2);
588610 const isClaudeToolCall = c => Array.isArray(c) ? c.filter(x => x).every(isClaudeToolCall) : c?.input && c?.name && c?.id;
611+ const isGoogleToolCall = c => Array.isArray(c) ? c.filter(x => x).every(isGoogleToolCall) : c?.name && c?.args;
589612 const convertClaudeToolCall = c => ({ id: c.id, function: { name: c.name, arguments: c.input } });
613+ const convertGoogleToolCall = (c) => ({ id: getRandomId(), function: { name: c.name, arguments: c.args } });
590614
591615 // Parsed tool calls from streaming data
592616 if (Array.isArray(data) && data.length > 0 && Array.isArray(data[0])) {
@@ -594,6 +618,10 @@ export class ToolManager {
594618 return data[0].filter(x => x).map(convertClaudeToolCall);
595619 }
596620
621+ if (isGoogleToolCall(data[0])) {
622+ return data[0].filter(x => x).map(convertGoogleToolCall);
623+ }
624+
597625 if (typeof data[0]?.[0]?.tool_calls === 'object') {
598626 return Array.isArray(data[0]?.[0]?.tool_calls) ? data[0][0].tool_calls : [data[0][0].tool_calls];
599627 }
@@ -601,6 +629,11 @@ export class ToolManager {
601629 return data[0];
602630 }
603631
632+ // Google AI Studio tool calls
633+ if (Array.isArray(data?.responseContent?.parts)) {
634+ return data.responseContent.parts.filter(p => p.functionCall).map(p => convertGoogleToolCall(p.functionCall));
635+ }
636+
604637 // Parsed tool calls from non-streaming data
605638 if (Array.isArray(data?.choices)) {
606639 // Find a choice with 0-index
src/endpoints/backends/chat-completions.js+15 -1
@@ -385,6 +385,19 @@ async function sendMakerSuiteRequest(request, response) {
385385 tools.push(searchTool);
386386 }
387387
388+ if (Array.isArray(request.body.tools) && request.body.tools.length > 0) {
389+ const functionDeclarations = [];
390+ for (const tool of request.body.tools) {
391+ if (tool.type === 'function') {
392+ if (tool.function.parameters?.$schema) {
393+ delete tool.function.parameters.$schema;
394+ }
395+ functionDeclarations.push(tool.function);
396+ }
397+ }
398+ tools.push({ function_declarations: functionDeclarations });
399+ }
400+
388401 let body = {
389402 contents: prompt.contents,
390403 safetySettings: safetySettings,
@@ -454,10 +467,11 @@ async function sendMakerSuiteRequest(request, response) {
454467 }
455468
456469 const responseContent = candidates[0].content ?? candidates[0].output;
470+ const functionCall = (candidates?.[0]?.content?.parts ?? []).some(part => part.functionCall);
457471 console.warn('Google AI Studio response:', responseContent);
458472
459473 const responseText = typeof responseContent === 'string' ? responseContent : responseContent?.parts?.filter(part => !part.thought)?.map(part => part.text)?.join('\n\n');
460474 if (!responseText && !functionCall) {
461475 let message = 'Google AI Studio Candidate text empty';
462476 console.warn(message, generateResponseJson);
463477 return response.send({ error: { message } });
src/prompt-converters.js+38 -4
@@ -1,5 +1,5 @@
11import crypto from 'node:crypto';
22import { getConfigValue, tryParse } from './util.js';
33
44const PROMPT_PLACEHOLDER = getConfigValue('promptPlaceholder', 'Let\'s get started.');
55
@@ -411,11 +411,12 @@ export function convertGooglePrompt(messages, model, useSysPrompt, names) {
411411 }
412412
413413 const system_instruction = { parts: { text: sys_prompt.trim() } };
414+ const toolNameMap = {};
414415
415416 const contents = [];
416417 messages.forEach((message, index) => {
417418 // fix the roles
418419 if (message.role === 'system' || message.role === 'tool') {
419420 message.role = 'user';
420421 } else if (message.role === 'assistant') {
421422 message.role = 'model';
@@ -423,7 +424,21 @@ export function convertGooglePrompt(messages, model, useSysPrompt, names) {
423424
424425 // Convert the content to an array of parts
425426 if (!Array.isArray(message.content)) {
426- message.content = [{ type: 'text', text: String(message.content ?? '') }];
427+ const content = (() => {
428+ const hasToolCalls = Array.isArray(message.tool_calls) && message.tool_calls.length > 0;
429+ const hasToolCallId = typeof message.tool_call_id === 'string' && message.tool_call_id.length > 0;
430+
431+ if (hasToolCalls) {
432+ return { type: 'tool_calls', tool_calls: message.tool_calls };
433+ }
434+
435+ if (hasToolCallId) {
436+ return { type: 'tool_call_id', tool_call_id: message.tool_call_id, content: String(message.content ?? '') };
437+ }
438+
439+ return { type: 'text', text: String(message.content ?? '') };
440+ })();
441+ message.content = [content];
427442 }
428443
429444 // similar story as claude
@@ -455,6 +470,25 @@ export function convertGooglePrompt(messages, model, useSysPrompt, names) {
455470 message.content.forEach((part) => {
456471 if (part.type === 'text') {
457472 parts.push({ text: part.text });
473+ } else if (part.type === 'tool_call_id') {
474+ const name = toolNameMap[part.tool_call_id] ?? 'unknown';
475+ parts.push({
476+ functionResponse: {
477+ name: name,
478+ response: { name: name, content: part.content },
479+ },
480+ });
481+ } else if (part.type === 'tool_calls') {
482+ part.tool_calls.forEach((toolCall) => {
483+ parts.push({
484+ functionCall: {
485+ name: toolCall.function.name,
486+ args: tryParse(toolCall.function.arguments) ?? toolCall.function.arguments,
487+ },
488+ });
489+
490+ toolNameMap[toolCall.id] = toolCall.function.name;
491+ });
458492 } else if (part.type === 'image_url' && isMultimodal) {
459493 const mimeType = part.image_url.url.split(';')[0].split(':')[1];
460494 const base64Data = part.image_url.url.split(',')[1];
@@ -473,7 +507,7 @@ export function convertGooglePrompt(messages, model, useSysPrompt, names) {
473507 if (part.text) {
474508 contents[contents.length - 1].parts[0].text += '\n\n' + part.text;
475509 }
476510 if (part.inlineData || part.functionCall) {
477511 contents[contents.length - 1].parts.push(part);
478512 }
479513 });