Tools/Notes
1
0
Fork 0
mirror of https://github.com/TriliumNext/Notes.git synced 2025-08-10 10:22:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

did I really need to specify the context window size in API requests this whole time?

Browse Source
This commit is contained in:
perf3ct 2025-04-08 23:55:04 +00:00
parent 7373249dee
commit e523d88d23
No known key found for this signature in database
GPG Key ID: 569C4EEC436F5232
1 changed files with 28 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/services/llm/providers/ollama_service.ts
View File

@ -99,6 +99,8 @@ export class OllamaService extends BaseAIService {
messages: messagesToSend,
options: {
temperature,
// Add num_ctx parameter based on model capabilities
num_ctx: await this.getModelContextWindowTokens(model),
// Add response_format for requests that expect JSON
...(expectsJsonResponse ? { response_format: { type: "json_object" } } : {})
},
@ -385,4 +387,30 @@ export class OllamaService extends BaseAIService {
throw error;
}
}
/**
* Gets the context window size in tokens for a given model
* @param modelName The name of the model
* @returns The context window size in tokens
*/
private async getModelContextWindowTokens(modelName: string): Promise<number> {
try {
// Import model capabilities service
const modelCapabilitiesService = (await import('../model_capabilities_service.js')).default;
// Get model capabilities
const modelCapabilities = await modelCapabilitiesService.getModelCapabilities(modelName);
// Get context window tokens with a default fallback
const contextWindowTokens = modelCapabilities.contextWindowTokens || 8192;
log.info(`Using context window size for ${modelName}: ${contextWindowTokens} tokens`);
return contextWindowTokens;
} catch (error: any) {
// Log error but provide a reasonable default
log.error(`Error getting model context window: ${error.message}`);
return 8192; // Default to 8192 tokens if there's an error
}
}
}