Support Qwen tokenizer fro Groq
| @@ -694,6 +694,9 @@ export function getTokenizerModel() { | ||
| 694 | 694 | } |
| 695 | 695 | |
| 696 | 696 | if (oai_settings.chat_completion_source === chat_completion_sources.GROQ) { |
| 697 | + if (oai_settings.groq_model.includes('qwen')) { | |
| 698 | + return qwen2Tokenizer; | |
| 699 | + } | |
| 697 | 700 | if (oai_settings.groq_model.includes('llama-3') || oai_settings.groq_model.includes('llama3')) { |
| 698 | 701 | return llama3Tokenizer; |
| 699 | 702 | } |