diff --git a/apps/server/src/services/llm/config/configuration_helpers.spec.ts b/apps/server/src/services/llm/config/configuration_helpers.spec.ts index 422773202..82d912301 100644 --- a/apps/server/src/services/llm/config/configuration_helpers.spec.ts +++ b/apps/server/src/services/llm/config/configuration_helpers.spec.ts @@ -94,6 +94,83 @@ describe('configuration_helpers', () => { fullIdentifier: '' }); }); + + // Tests for special characters in model names + it('should handle model names with periods', () => { + const result = configHelpers.parseModelIdentifier('gpt-4.1-turbo-preview'); + + expect(result).toStrictEqual({ + modelId: 'gpt-4.1-turbo-preview', + fullIdentifier: 'gpt-4.1-turbo-preview' + }); + }); + + it('should handle model names with provider prefix and periods', () => { + const result = configHelpers.parseModelIdentifier('openai:gpt-4.1-turbo'); + + expect(result).toStrictEqual({ + provider: 'openai', + modelId: 'gpt-4.1-turbo', + fullIdentifier: 'openai:gpt-4.1-turbo' + }); + }); + + it('should handle model names with multiple colons', () => { + const result = configHelpers.parseModelIdentifier('custom:model:v1.2:latest'); + + expect(result).toStrictEqual({ + modelId: 'custom:model:v1.2:latest', + fullIdentifier: 'custom:model:v1.2:latest' + }); + }); + + it('should handle Ollama model names with colons', () => { + const result = configHelpers.parseModelIdentifier('ollama:llama3.1:70b-instruct-q4_K_M'); + + expect(result).toStrictEqual({ + provider: 'ollama', + modelId: 'llama3.1:70b-instruct-q4_K_M', + fullIdentifier: 'ollama:llama3.1:70b-instruct-q4_K_M' + }); + }); + + it('should handle model names with slashes', () => { + const result = configHelpers.parseModelIdentifier('library/mistral:7b-instruct'); + + expect(result).toStrictEqual({ + modelId: 'library/mistral:7b-instruct', + fullIdentifier: 'library/mistral:7b-instruct' + }); + }); + + it('should handle complex model names with special characters', () => { + const complexName = 'org/model-v1.2.3:tag@version#variant'; + const result = configHelpers.parseModelIdentifier(complexName); + + expect(result).toStrictEqual({ + modelId: complexName, + fullIdentifier: complexName + }); + }); + + it('should handle model names with @ symbols', () => { + const result = configHelpers.parseModelIdentifier('claude-3.5-sonnet@20241022'); + + expect(result).toStrictEqual({ + modelId: 'claude-3.5-sonnet@20241022', + fullIdentifier: 'claude-3.5-sonnet@20241022' + }); + }); + + it('should not modify or encode special characters', () => { + const specialChars = 'model!@#$%^&*()_+-=[]{}|;:\'",.<>?/~`'; + const result = configHelpers.parseModelIdentifier(specialChars); + + expect(result).toStrictEqual({ + modelId: specialChars, + fullIdentifier: specialChars + }); + }); }); describe('createModelConfig', () => { @@ -155,6 +232,34 @@ describe('configuration_helpers', () => { expect(result).toBe('llama2'); expect(optionService.getOption).toHaveBeenCalledWith('ollamaDefaultModel'); }); + + // Tests for special characters in model names + it('should handle OpenAI model names with periods', async () => { + const modelName = 'gpt-4.1-turbo-preview'; + vi.mocked(optionService.getOption).mockReturnValue(modelName); + + const result = await configHelpers.getDefaultModelForProvider('openai'); + + expect(result).toBe(modelName); + }); + + it('should handle Anthropic model names with periods and @ symbols', async () => { + const modelName = 'claude-3.5-sonnet@20241022'; + vi.mocked(optionService.getOption).mockReturnValue(modelName); + + const result = await configHelpers.getDefaultModelForProvider('anthropic'); + + expect(result).toBe(modelName); + }); + + it('should handle Ollama model names with colons and slashes', async () => { + const modelName = 'library/llama3.1:70b-instruct-q4_K_M'; + vi.mocked(optionService.getOption).mockReturnValue(modelName); + + const result = await configHelpers.getDefaultModelForProvider('ollama'); + + expect(result).toBe(modelName); + }); }); describe('getProviderSettings', () => { @@ -381,4 +486,122 @@ describe('configuration_helpers', () => { expect(() => configHelpers.clearConfigurationCache()).not.toThrow(); }); }); + + describe('getValidModelConfig', () => { + it('should handle model names with special characters', async () => { + const modelName = 'gpt-4.1-turbo@latest'; + vi.mocked(optionService.getOption) + .mockReturnValueOnce(modelName) // openaiDefaultModel + .mockReturnValueOnce('test-key') // openaiApiKey + .mockReturnValueOnce('') // openaiBaseUrl + .mockReturnValueOnce(''); // openaiDefaultModel + + const result = await configHelpers.getValidModelConfig('openai'); + + expect(result).toStrictEqual({ + model: modelName, + provider: 'openai' + }); + }); + + it('should handle Anthropic model with complex naming', async () => { + const modelName = 'claude-3.5-sonnet-20241022'; + vi.mocked(optionService.getOption) + .mockReturnValueOnce(modelName) // anthropicDefaultModel + .mockReturnValueOnce('anthropic-key') // anthropicApiKey + .mockReturnValueOnce('') // anthropicBaseUrl + .mockReturnValueOnce(''); // anthropicDefaultModel + + const result = await configHelpers.getValidModelConfig('anthropic'); + + expect(result).toStrictEqual({ + model: modelName, + provider: 'anthropic' + }); + }); + + it('should handle Ollama model with colons', async () => { + const modelName = 'custom/llama3.1:70b-q4_K_M@latest'; + vi.mocked(optionService.getOption) + .mockReturnValueOnce(modelName) // ollamaDefaultModel + .mockReturnValueOnce('http://localhost:11434') // ollamaBaseUrl + .mockReturnValueOnce(''); // ollamaDefaultModel + + const result = await configHelpers.getValidModelConfig('ollama'); + + expect(result).toStrictEqual({ + model: modelName, + provider: 'ollama' + }); + }); + }); + + describe('getSelectedModelConfig', () => { + it('should preserve OpenAI model names with special characters', async () => { + const modelName = 'gpt-4.1-turbo-preview@2024'; + vi.mocked(optionService.getOption) + .mockReturnValueOnce('openai') // aiSelectedProvider + .mockReturnValueOnce(modelName) // openaiDefaultModel + .mockReturnValueOnce('test-key') // openaiApiKey + .mockReturnValueOnce('') // openaiBaseUrl + .mockReturnValueOnce(''); // openaiDefaultModel + + const result = await configHelpers.getSelectedModelConfig(); + + expect(result).toStrictEqual({ + model: modelName, + provider: 'openai' + }); + }); + + it('should handle model names with URL-like patterns', async () => { + const modelName = 'https://models.example.com/gpt-4.1'; + vi.mocked(optionService.getOption) + .mockReturnValueOnce('openai') // aiSelectedProvider + .mockReturnValueOnce(modelName) // openaiDefaultModel + .mockReturnValueOnce('test-key') // openaiApiKey + .mockReturnValueOnce('') // openaiBaseUrl + .mockReturnValueOnce(''); // openaiDefaultModel + + const result = await configHelpers.getSelectedModelConfig(); + + expect(result).toStrictEqual({ + model: modelName, + provider: 'openai' + }); + }); + + it('should handle model names that look like file paths', async () => { + const modelName = '/models/custom/gpt-4.1.safetensors'; + vi.mocked(optionService.getOption) + .mockReturnValueOnce('ollama') // aiSelectedProvider + .mockReturnValueOnce(modelName) // ollamaDefaultModel + .mockReturnValueOnce('http://localhost:11434') // ollamaBaseUrl + .mockReturnValueOnce(''); // ollamaDefaultModel + + const result = await configHelpers.getSelectedModelConfig(); + + expect(result).toStrictEqual({ + model: modelName, + provider: 'ollama' + }); + }); + + it('should handle model names with all possible special characters', async () => { + const modelName = 'model!@#$%^&*()_+-=[]{}|;:\'",.<>?/~`'; + vi.mocked(optionService.getOption) + .mockReturnValueOnce('anthropic') // aiSelectedProvider + .mockReturnValueOnce(modelName) // anthropicDefaultModel + .mockReturnValueOnce('test-key') // anthropicApiKey + .mockReturnValueOnce('') // anthropicBaseUrl + .mockReturnValueOnce(''); // anthropicDefaultModel + + const result = await configHelpers.getSelectedModelConfig(); + + expect(result).toStrictEqual({ + model: modelName, + provider: 'anthropic' + }); + }); + }); }); \ No newline at end of file diff --git a/apps/server/src/services/llm/providers/model_selection.spec.ts b/apps/server/src/services/llm/providers/model_selection.spec.ts new file mode 100644 index 000000000..c6fdfae1e --- /dev/null +++ b/apps/server/src/services/llm/providers/model_selection.spec.ts @@ -0,0 +1,389 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { OpenAIService } from './openai_service.js'; +import { AnthropicService } from './anthropic_service.js'; +import { OllamaService } from './ollama_service.js'; +import type { ChatCompletionOptions } from '../ai_interface.js'; +import * as providers from './providers.js'; +import options from '../../options.js'; + +// Mock dependencies +vi.mock('../../options.js', () => ({ + default: { + getOption: vi.fn(), + getOptionBool: vi.fn() + } +})); + +vi.mock('../../log.js', () => ({ + default: { + info: vi.fn(), + error: vi.fn(), + warn: vi.fn() + } +})); + +vi.mock('openai', () => ({ + default: class MockOpenAI { + chat = { + completions: { + create: vi.fn() + } + }; + } +})); + +vi.mock('@anthropic-ai/sdk', () => ({ + default: class MockAnthropic { + messages = { + create: vi.fn() + }; + } +})); + +vi.mock('ollama', () => ({ + Ollama: class MockOllama { + chat = vi.fn(); + show = vi.fn(); + } +})); + +describe('LLM Model Selection with Special Characters', () => { + beforeEach(() => { + vi.clearAllMocks(); + // Set default options + vi.mocked(options.getOption).mockImplementation((key: string) => { + const optionMap: Record = { + 'aiEnabled': 'true', + 'aiTemperature': '0.7', + 'aiSystemPrompt': 'You are a helpful assistant.', + 'openaiApiKey': 'test-api-key', + 'openaiBaseUrl': 'https://api.openai.com/v1', + 'anthropicApiKey': 'test-anthropic-key', + 'anthropicBaseUrl': 'https://api.anthropic.com', + 'ollamaBaseUrl': 'http://localhost:11434' + }; + return optionMap[key] || ''; + }); + vi.mocked(options.getOptionBool).mockReturnValue(true); + }); + + describe('OpenAI Model Names', () => { + it('should correctly handle model names with periods', async () => { + const modelName = 'gpt-4.1-turbo-preview'; + vi.mocked(options.getOption).mockImplementation((key: string) => { + if (key === 'openaiDefaultModel') return modelName; + return ''; + }); + + const service = new OpenAIService(); + const opts: ChatCompletionOptions = { + stream: false + }; + + // Spy on getOpenAIOptions to verify model name is passed correctly + const getOpenAIOptionsSpy = vi.spyOn(providers, 'getOpenAIOptions'); + + try { + await service.generateChatCompletion([{ role: 'user', content: 'test' }], opts); + } catch (error) { + // Expected to fail due to mocked API + } + + expect(getOpenAIOptionsSpy).toHaveBeenCalledWith(opts); + const result = getOpenAIOptionsSpy.mock.results[0].value; + expect(result.model).toBe(modelName); + }); + + it('should handle model names with slashes', async () => { + const modelName = 'openai/gpt-4/turbo-2024'; + vi.mocked(options.getOption).mockImplementation((key: string) => { + if (key === 'openaiDefaultModel') return modelName; + return ''; + }); + + const service = new OpenAIService(); + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const getOpenAIOptionsSpy = vi.spyOn(providers, 'getOpenAIOptions'); + + try { + await service.generateChatCompletion([{ role: 'user', content: 'test' }], opts); + } catch (error) { + // Expected to fail due to mocked API + } + + const result = getOpenAIOptionsSpy.mock.results[0].value; + expect(result.model).toBe(modelName); + }); + + it('should handle model names with colons', async () => { + const modelName = 'custom:gpt-4:finetuned'; + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const getOpenAIOptionsSpy = vi.spyOn(providers, 'getOpenAIOptions'); + + const openaiOptions = providers.getOpenAIOptions(opts); + expect(openaiOptions.model).toBe(modelName); + }); + + it('should handle model names with underscores and hyphens', async () => { + const modelName = 'gpt-4_turbo-preview_v2.1'; + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const openaiOptions = providers.getOpenAIOptions(opts); + expect(openaiOptions.model).toBe(modelName); + }); + + it('should handle model names with special characters in API request', async () => { + const modelName = 'gpt-4.1-turbo@latest'; + vi.mocked(options.getOption).mockImplementation((key: string) => { + if (key === 'openaiDefaultModel') return modelName; + if (key === 'openaiApiKey') return 'test-key'; + if (key === 'openaiBaseUrl') return 'https://api.openai.com/v1'; + return ''; + }); + + const service = new OpenAIService(); + + // Access the private openai client through the service + const client = (service as any).getClient('test-key'); + const createSpy = vi.spyOn(client.chat.completions, 'create'); + + try { + await service.generateChatCompletion( + [{ role: 'user', content: 'test' }], + { stream: false } + ); + } catch (error) { + // Expected due to mock + } + + expect(createSpy).toHaveBeenCalledWith( + expect.objectContaining({ + model: modelName + }) + ); + }); + }); + + describe('Anthropic Model Names', () => { + it('should correctly handle Anthropic model names with periods', async () => { + const modelName = 'claude-3.5-sonnet-20241022'; + vi.mocked(options.getOption).mockImplementation((key: string) => { + if (key === 'anthropicDefaultModel') return modelName; + if (key === 'anthropicApiKey') return 'test-key'; + return ''; + }); + + const opts: ChatCompletionOptions = { + stream: false + }; + + const anthropicOptions = providers.getAnthropicOptions(opts); + expect(anthropicOptions.model).toBe(modelName); + }); + + it('should handle Anthropic model names with colons', async () => { + const modelName = 'anthropic:claude-3:opus'; + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const anthropicOptions = providers.getAnthropicOptions(opts); + expect(anthropicOptions.model).toBe(modelName); + }); + + it('should handle Anthropic model names in API request', async () => { + const modelName = 'claude-3.5-sonnet@beta'; + vi.mocked(options.getOption).mockImplementation((key: string) => { + if (key === 'anthropicDefaultModel') return modelName; + if (key === 'anthropicApiKey') return 'test-key'; + if (key === 'anthropicBaseUrl') return 'https://api.anthropic.com'; + return ''; + }); + + const service = new AnthropicService(); + + // Access the private anthropic client + const client = (service as any).getClient('test-key'); + const createSpy = vi.spyOn(client.messages, 'create'); + + try { + await service.generateChatCompletion( + [{ role: 'user', content: 'test' }], + { stream: false } + ); + } catch (error) { + // Expected due to mock + } + + expect(createSpy).toHaveBeenCalledWith( + expect.objectContaining({ + model: modelName + }) + ); + }); + }); + + describe('Ollama Model Names', () => { + it('should correctly handle Ollama model names with colons', async () => { + const modelName = 'llama3.1:70b-instruct-q4_K_M'; + vi.mocked(options.getOption).mockImplementation((key: string) => { + if (key === 'ollamaDefaultModel') return modelName; + if (key === 'ollamaBaseUrl') return 'http://localhost:11434'; + return ''; + }); + + const opts: ChatCompletionOptions = { + stream: false + }; + + const ollamaOptions = await providers.getOllamaOptions(opts); + expect(ollamaOptions.model).toBe(modelName); + }); + + it('should handle Ollama model names with slashes', async () => { + const modelName = 'library/mistral:7b-instruct-v0.3'; + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const ollamaOptions = await providers.getOllamaOptions(opts); + expect(ollamaOptions.model).toBe(modelName); + }); + + it('should handle Ollama model names with special characters in options', async () => { + const modelName = 'custom/llama3.1:70b-q4_K_M@latest'; + vi.mocked(options.getOption).mockImplementation((key: string) => { + if (key === 'ollamaDefaultModel') return modelName; + if (key === 'ollamaBaseUrl') return 'http://localhost:11434'; + return ''; + }); + + // Test that the model name is preserved in the options + const opts: ChatCompletionOptions = { + stream: false + }; + + const ollamaOptions = await providers.getOllamaOptions(opts); + expect(ollamaOptions.model).toBe(modelName); + + // Also test with model specified in options + const optsWithModel: ChatCompletionOptions = { + model: 'another/model:v2.0@beta', + stream: false + }; + + const ollamaOptionsWithModel = await providers.getOllamaOptions(optsWithModel); + expect(ollamaOptionsWithModel.model).toBe('another/model:v2.0@beta'); + }); + }); + + describe('Model Name Edge Cases', () => { + it('should handle empty model names gracefully', () => { + const opts: ChatCompletionOptions = { + model: '', + stream: false + }; + + expect(() => providers.getOpenAIOptions(opts)).toThrow('No OpenAI model configured'); + }); + + it('should handle model names with unicode characters', async () => { + const modelName = 'gpt-4-日本語-model'; + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const openaiOptions = providers.getOpenAIOptions(opts); + expect(openaiOptions.model).toBe(modelName); + }); + + it('should handle model names with spaces (encoded)', async () => { + const modelName = 'custom model v2.1'; + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const openaiOptions = providers.getOpenAIOptions(opts); + expect(openaiOptions.model).toBe(modelName); + }); + + it('should preserve exact model name without transformation', async () => { + const complexModelName = 'org/model-v1.2.3:tag@version#variant'; + const opts: ChatCompletionOptions = { + model: complexModelName, + stream: false + }; + + // Test for all providers + const openaiOptions = providers.getOpenAIOptions(opts); + expect(openaiOptions.model).toBe(complexModelName); + + const anthropicOptions = providers.getAnthropicOptions(opts); + expect(anthropicOptions.model).toBe(complexModelName); + + const ollamaOptions = await providers.getOllamaOptions(opts); + expect(ollamaOptions.model).toBe(complexModelName); + }); + }); + + describe('Model Configuration Parsing', () => { + it('should not confuse provider prefix with model name containing colons', async () => { + // This model name has a colon but 'custom' is not a known provider + const modelName = 'custom:model:v1.2'; + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const openaiOptions = providers.getOpenAIOptions(opts); + expect(openaiOptions.model).toBe(modelName); + }); + + it('should handle provider prefix correctly', async () => { + // When model has provider prefix, it should still use the full string + const modelName = 'openai:gpt-4.1-turbo'; + const opts: ChatCompletionOptions = { + model: modelName, + stream: false + }; + + const openaiOptions = providers.getOpenAIOptions(opts); + expect(openaiOptions.model).toBe(modelName); + }); + }); + + describe('Integration with REST API', () => { + it('should pass model names correctly through REST chat service', async () => { + const modelName = 'gpt-4.1-turbo-preview@latest'; + + // Mock the configuration helpers + vi.doMock('../config/configuration_helpers.js', () => ({ + getSelectedModelConfig: vi.fn().mockResolvedValue({ + model: modelName, + provider: 'openai' + }), + isAIEnabled: vi.fn().mockResolvedValue(true) + })); + + const { getSelectedModelConfig } = await import('../config/configuration_helpers.js'); + const config = await getSelectedModelConfig(); + + expect(config?.model).toBe(modelName); + }); + }); +}); \ No newline at end of file