Deprecate unscoped vectors

df3d7a048edf0d3fb2e45e47d0581ae01726f2ee

Cohee <18619528+Cohee1207@users.noreply.github.com>

Parents: 981697fda5d82a75c7e26ca9ea2056f31ebba11e

default/config.yaml+0 -3 Viewed

 # Additional model tokenizers can be downloaded on demand.
 # Disabling will fallback to another locally available tokenizer.
 enableDownloadableTokenizers: true
 -# Vector storage settings
 -vectors:
 -  enableModelScopes: false
 # Extension settings
 extras:
   # Disables automatic model download from HuggingFace

public/scripts/extensions/vectors/index.js+0 -26 Viewed

+    }
+}
 -async function isModelScopesEnabled() {
 -    try {
 -        const response = await fetch('/api/vector/scopes-enabled', {
 -            method: 'GET',
 -            headers: getVectorHeaders(),
 -        });
+-
 -        if (!response.ok) {
 -            return false;
 -        }
+-
 -        const data = await response.json();
 -        return data?.enabled ?? false;
 -    } catch (error) {
 -        console.error('Vectors: Failed to check model scopes', error);
 -        return false;
 -    }
 -}
+-
 function toggleSettings() {
     $('#vectors_files_settings').toggle(!!settings.enabled_files);
     $('#vectors_chats_settings').toggle(!!settings.enabled_chats);

+    }
     Object.assign(settings, extension_settings.vectors);
 -    const scopesEnabled = await isModelScopesEnabled();
     // Migrate from TensorFlow to Transformers
     settings.source = settings.source !== 'local' ? settings.source : 'transformers';

         saveSettingsDebounced();
         toggleSettings();
     });
 -    $('#vectors_modelWarning').hide();
     $('#vectors_enabled_files').prop('checked', settings.enabled_files).on('input', () => {
         settings.enabled_files = $('#vectors_enabled_files').prop('checked');
         Object.assign(extension_settings.vectors, settings);

         saveSettingsDebounced();
     });
     $('#vectors_togetherai_model').val(settings.togetherai_model).on('change', () => {
 -        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.togetherai_model = String($('#vectors_togetherai_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_openai_model').val(settings.openai_model).on('change', () => {
 -        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.openai_model = String($('#vectors_openai_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_cohere_model').val(settings.cohere_model).on('change', () => {
 -        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.cohere_model = String($('#vectors_cohere_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_ollama_model').val(settings.ollama_model).on('input', () => {
 -        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.ollama_model = String($('#vectors_ollama_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();
     });
     $('#vectors_vllm_model').val(settings.vllm_model).on('input', () => {
 -        !scopesEnabled && $('#vectors_modelWarning').show();
         settings.vllm_model = String($('#vectors_vllm_model').val());
         Object.assign(extension_settings.vectors, settings);
         saveSettingsDebounced();

public/scripts/extensions/vectors/settings.html+0 -8 Viewed

                 </i>
             </div>
 -            <small id="vectors_modelWarning">
 -                <i class="fa-solid fa-exclamation-triangle"></i>
 -                <span>
 -                    Set <code>vectors.enableModelScopes</code> to true in config.yaml to switch between vectorization models without needing to purge existing vectors.
 -                    This option will soon be enabled by default.
 -                </span>
 -            </small>
+-
             <div class="flex-container alignItemsCenter" id="nomicai_apiKey">
                 <label for="api_key_nomicai" class="flex1">
                     <span data-i18n="NomicAI API Key">NomicAI API Key</span>

src/endpoints/vectors.js+11 -20 Viewed

 const express = require('express');
 const sanitize = require('sanitize-filename');
 const { jsonParser } = require('../express-common');
 const { getConfigValue, color } = require('../util');
 // Don't forget to add new sources to the SOURCES array
 const SOURCES = [

                 extrasUrl: String(request.headers['x-extras-url']),
                 extrasKey: String(request.headers['x-extras-key']),
             };
         case 'localtransformers':
             return {
                 model: getConfigValue('extras.embeddingModel', ''),
             };

                 // TODO: Add support for multiple models
                 model: 'text-embedding-004',
             };
 +        case 'mistral':
 +            return {
 +                model: 'mistral-embed',
 +            };
 +        case 'nomicai':
 +            return {
 +                model: 'nomic-embed-text-v1.5',
 +            };
         default:
             return {};
+    }

  * @returns {string} The model scope for the source
  */
 function getModelScope(sourceSettings) {
 -    const scopesEnabled = getConfigValue('vectors.enableModelScopes', false);
 +    return (sourceSettings?.model || '');
 -    const warningShown = global.process.env.VECTORS_MODEL_SCOPE_WARNING_SHOWN === 'true';
+-
 -    if (!scopesEnabled && !warningShown) {
 -        console.log();
 -        console.warn(color.red('[DEPRECATION NOTICE]'), 'Model scopes for Vectore Storage are disabled, but will soon be required.');
 -        console.log(`To enable model scopes, set the ${color.cyan('vectors.enableModelScopes')} in config.yaml to ${color.green(true)}.`);
 -        console.log('This message won\'t be shown again in the current session.');
 -        console.log();
 -        global.process.env.VECTORS_MODEL_SCOPE_WARNING_SHOWN = 'true';
 -    }
+-
 -    return scopesEnabled ? (sourceSettings?.model || '') : '';
+}
 /**

 const router = express.Router();
 -router.get('/scopes-enabled', (_req, res) => {
 -    const scopesEnabled = getConfigValue('vectors.enableModelScopes', false);
 -    return res.json({ enabled: scopesEnabled });
 -});
+-
 router.post('/query', jsonParser, async (req, res) => {
     try {
         if (!req.body.collectionId || !req.body.searchText) {