Merge pull request #3475 from SillyTavern/ollama-batch Ollama: Add num_batch config value

c47d997a2d51c564cd5d4f3d86c460617aa4ca44

Cohee <18619528+Cohee1207@users.noreply.github.com>

Signed

default/config.yaml+4 -0 Viewed

   # * 0: Unload the model immediately after the request
   # * N (any positive number): Keep the model loaded for N seconds after the request.
   keepAlive: -1
+  # Controls the "num_batch" (batch size) parameter of the generation request
+  # * -1: Use the default value of the model
+  # * N (positive number): Use the specified value. Must be a power of 2, e.g. 128, 256, 512, etc.
+  batchSize: -1
 # -- ANTHROPIC CLAUDE API CONFIGURATION --
 claude:
   # Enables caching of the system prompt (if supported).

src/constants.js+1 -0 Viewed

 export const OLLAMA_KEYS = [
     'num_predict',
     'num_ctx',
+    'num_batch',
     'stop',
     'temperature',
     'repeat_penalty',

src/endpoints/backends/text-completions.js+4 -0 Viewed

         if (request.body.api_type === TEXTGEN_TYPES.OLLAMA) {
             const keepAlive = getConfigValue('ollama.keepAlive', -1);
+            const numBatch = getConfigValue('ollama.batchSize', -1);
+            if (numBatch > 0) {
+                request.body['num_batch'] = numBatch;
+            }
             args.body = JSON.stringify({
                 model: request.body.model,
                 prompt: request.body.prompt,