Deprecate unscoped vectors

df3d7a048edf0d3fb2e45e47d0581ae01726f2ee

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 981697fda5d82a75c7e26ca9ea2056f31ebba11e

default/config.yaml+0 -3 Viewed

 # Additional model tokenizers can be downloaded on demand.
 # Disabling will fallback to another locally available tokenizer.
 enableDownloadableTokenizers: true
-# Vector storage settings
-vectors:
-  enableModelScopes: false
 # Extension settings
 extras:
   # Disables automatic model download from HuggingFace

public/scripts/extensions/vectors/index.js+0 -26 Viewed

+    }
+}
-async function isModelScopesEnabled() {
-    try {
-        const response = await fetch('/api/vector/scopes-enabled', {
-            method: 'GET',
-            headers: getVectorHeaders(),
-        });
-        if (!response.ok) {
-            return false;
-        }
-        const data = await response.json();
-        return data?.enabled ?? false;
-    } catch (error) {
-        console.error('Vectors: Failed to check model scopes', error);
-        return false;
-    }
-}
 function toggleSettings() {
     $('#vectors_files_settings').toggle(!!settings.enabled_files);
     $('#vectors_chats_settings').toggle(!!settings.enabled_chats);

+    }
     Object.assign(settings, extension_settings.vectors);
-    const scopesEnabled = await isModelScopesEnabled();
     // Migrate from TensorFlow to Transformers
     settings.source = settings.source !== 'local' ? settings.source : 'transformers';

         saveSettingsDebounced();
         toggleSettings();
     });
-    $('#vectors_modelWarning').hide();
     $('#vectors_enabled_files').prop('checked', settings.enabled_files).on('input', () => {
         settings.enabled_files = $('#vectors_enabled_files').prop('checked');
         Object.assign(extension_settings.vectors, settings);

         saveSettingsDebounced();
     });
     $('#vectors_togetherai_model').val(settings.togetherai_model).on('change', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.togetherai_model = String($('#vectors_togetherai_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_openai_model').val(settings.openai_model).on('change', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.openai_model = String($('#vectors_openai_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_cohere_model').val(settings.cohere_model).on('change', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.cohere_model = String($('#vectors_cohere_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_ollama_model').val(settings.ollama_model).on('input', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.ollama_model = String($('#vectors_ollama_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_vllm_model').val(settings.vllm_model).on('input', () => {
-        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.vllm_model = String($('#vectors_vllm_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();

public/scripts/extensions/vectors/settings.html+0 -8 Viewed

                 </i>
             </div>
-            <small id="vectors_modelWarning">
-                <i class="fa-solid fa-exclamation-triangle"></i>
-                <span>
-                    Set <code>vectors.enableModelScopes</code> to true in config.yaml to switch between vectorization models without needing to purge existing vectors.
-                    This option will soon be enabled by default.
-                </span>
-            </small>
             <div class="flex-container alignItemsCenter" id="nomicai_apiKey">
                 <label for="api_key_nomicai" class="flex1">
                     <span data-i18n="NomicAI API Key">NomicAI API Key</span>

src/endpoints/vectors.js+11 -20 Viewed

 const express = require('express');
 const sanitize = require('sanitize-filename');
 const { jsonParser } = require('../express-common');
-const { getConfigValue, color } = require('../util');
+const { getConfigValue } = require('../util');
 // Don't forget to add new sources to the SOURCES array
 const SOURCES = [

                 extrasUrl: String(request.headers['x-extras-url']),
                 extrasKey: String(request.headers['x-extras-key']),
             };
-        case 'local':
+        case 'transformers':
             return {
                 model: getConfigValue('extras.embeddingModel', ''),
             };

                 // TODO: Add support for multiple models
                 model: 'text-embedding-004',
             };
+        case 'mistral':
+            return {
+                model: 'mistral-embed',
+            };
+        case 'nomicai':
+            return {
+                model: 'nomic-embed-text-v1.5',
+            };
         default:
             return {};
+    }

  * @returns {string} The model scope for the source
  */
 function getModelScope(sourceSettings) {
-    const scopesEnabled = getConfigValue('vectors.enableModelScopes', false);
+    return (sourceSettings?.model || '');
-    const warningShown = global.process.env.VECTORS_MODEL_SCOPE_WARNING_SHOWN === 'true';
-    if (!scopesEnabled && !warningShown) {
-        console.log();
-        console.warn(color.red('[DEPRECATION NOTICE]'), 'Model scopes for Vectore Storage are disabled, but will soon be required.');
-        console.log(`To enable model scopes, set the ${color.cyan('vectors.enableModelScopes')} in config.yaml to ${color.green(true)}.`);
-        console.log('This message won\'t be shown again in the current session.');
-        console.log();
-        global.process.env.VECTORS_MODEL_SCOPE_WARNING_SHOWN = 'true';
-    }
-    return scopesEnabled ? (sourceSettings?.model || '') : '';
+}
 /**

 const router = express.Router();
-router.get('/scopes-enabled', (_req, res) => {
-    const scopesEnabled = getConfigValue('vectors.enableModelScopes', false);
-    return res.json({ enabled: scopesEnabled });
-});
 router.post('/query', jsonParser, async (req, res) => {
     try {
         if (!req.body.collectionId || !req.body.searchText) {