mstoeck3/forensic-pathways
2
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Activity

first iteration - buggy

Browse Source
This commit is contained in:
overcuriousity 2025-08-16 18:15:20 +02:00
parent 1d98dd3257
commit 0c7c502b03
12 changed files with 1939 additions and 2437 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

766
src/components/AIQueryInterface.astro
View File

File diff suppressed because it is too large Load Diff

8
src/pages/api/ai/embeddings-status.ts
View File

@ -1,17 +1,19 @@
// src/pages/api/ai/embeddings-status.ts // src/pages/api/ai/embeddings-status.ts - Updated
import type { APIRoute } from 'astro'; import type { APIRoute } from 'astro';
import { embeddingsService } from '../../../utils/embeddings.js';
export const prerender = false; export const prerender = false;
export const GET: APIRoute = async () => { export const GET: APIRoute = async () => {
try { try {
const { embeddingsService } = await import('../../../utils/embeddings.js');
await embeddingsService.waitForInitialization(); await embeddingsService.waitForInitialization();
const stats = embeddingsService.getStats(); const stats = embeddingsService.getStats();
const status = stats.enabled && stats.initialized ? 'ready' : const status = stats.enabled && stats.initialized ? 'ready' :
stats.enabled && !stats.initialized ? 'initializing' : 'disabled'; stats.enabled && !stats.initialized ? 'initializing' : 'disabled';
console.log(`[EMBEDDINGS-STATUS-API] Service status: ${status}, stats:`, stats);
return new Response(JSON.stringify({ return new Response(JSON.stringify({
success: true, success: true,
embeddings: stats, embeddings: stats,
@ -23,6 +25,8 @@ export const GET: APIRoute = async () => {
}); });
} catch (error) { } catch (error) {
console.error('[EMBEDDINGS-STATUS-API] Error checking embeddings status:', error);
return new Response(JSON.stringify({ return new Response(JSON.stringify({
success: false, success: false,
embeddings: { enabled: false, initialized: false, count: 0 }, embeddings: { enabled: false, initialized: false, count: 0 },

79
src/pages/api/ai/enhance-input.ts
View File

@ -1,23 +1,13 @@
// src/pages/api/ai/enhance-input.ts - Enhanced AI service compatibility // src/pages/api/ai/enhance-input.ts - Updated to use refactored services
import type { APIRoute } from 'astro'; import type { APIRoute } from 'astro';
import { withAPIAuth } from '../../../utils/auth.js'; import { withAPIAuth } from '../../../utils/auth.js';
import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js'; import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js';
import { enqueueApiCall } from '../../../utils/rateLimitedQueue.js'; import { enqueueApiCall } from '../../../utils/rateLimitedQueue.js';
import { aiService } from '../../../utils/aiService.js';
import { JSONParser } from '../../../utils/jsonUtils.js';
export const prerender = false; export const prerender = false;
function getEnv(key: string): string {
const value = process.env[key];
if (!value) {
throw new Error(`Missing environment variable: ${key}`);
}
return value;
}
const AI_ENDPOINT = getEnv('AI_ANALYZER_ENDPOINT');
const AI_ANALYZER_API_KEY = getEnv('AI_ANALYZER_API_KEY');
const AI_ANALYZER_MODEL = getEnv('AI_ANALYZER_MODEL');
const rateLimitStore = new Map<string, { count: number; resetTime: number }>(); const rateLimitStore = new Map<string, { count: number; resetTime: number }>();
const RATE_LIMIT_WINDOW = 60 * 1000; const RATE_LIMIT_WINDOW = 60 * 1000;
const RATE_LIMIT_MAX = 5; const RATE_LIMIT_MAX = 5;
@ -49,7 +39,7 @@ function checkRateLimit(userId: string): boolean {
return true; return true;
} }
function cleanupExpiredRateLimits() { function cleanupExpiredRateLimits(): void {
const now = Date.now(); const now = Date.now();
for (const [userId, limit] of rateLimitStore.entries()) { for (const [userId, limit] of rateLimitStore.entries()) {
if (now > limit.resetTime) { if (now > limit.resetTime) {
@ -94,39 +84,6 @@ ${input}
`.trim(); `.trim();
} }
async function callAIService(prompt: string): Promise<Response> {
const endpoint = AI_ENDPOINT;
const apiKey = AI_ANALYZER_API_KEY;
const model = AI_ANALYZER_MODEL;
let headers: Record<string, string> = {
'Content-Type': 'application/json'
};
if (apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`;
console.log('[ENHANCE API] Using API key authentication');
} else {
console.log('[ENHANCE API] No API key - making request without authentication');
}
const requestBody = {
model,
messages: [{ role: 'user', content: prompt }],
max_tokens: 300,
temperature: 0.7,
top_p: 0.9,
frequency_penalty: 0.2,
presence_penalty: 0.1
};
return fetch(`${endpoint}/v1/chat/completions`, {
method: 'POST',
headers,
body: JSON.stringify(requestBody)
});
}
export const POST: APIRoute = async ({ request }) => { export const POST: APIRoute = async ({ request }) => {
try { try {
const authResult = await withAPIAuth(request, 'ai'); const authResult = await withAPIAuth(request, 'ai');
@ -155,28 +112,26 @@ export const POST: APIRoute = async ({ request }) => {
const systemPrompt = createEnhancementPrompt(sanitizedInput); const systemPrompt = createEnhancementPrompt(sanitizedInput);
const taskId = `enhance_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 4)}`; const taskId = `enhance_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 4)}`;
const aiResponse = await enqueueApiCall(() => callAIService(systemPrompt), taskId); console.log(`[ENHANCE-API] Processing enhancement request for user: ${userId}`);
if (!aiResponse.ok) { const aiResponse = await enqueueApiCall(() =>
const errorText = await aiResponse.text(); aiService.callAI(systemPrompt, {
console.error('[ENHANCE API] AI enhancement error:', errorText, 'Status:', aiResponse.status); maxTokens: 300,
return apiServerError.unavailable('Enhancement service unavailable'); temperature: 0.7
} }), taskId);
const aiData = await aiResponse.json(); if (!aiResponse.content) {
const aiContent = aiData.choices?.[0]?.message?.content;
if (!aiContent) {
return apiServerError.unavailable('No enhancement response'); return apiServerError.unavailable('No enhancement response');
} }
let questions; let questions;
try { try {
const cleanedContent = aiContent const cleanedContent = aiResponse.content
.replace(/^```json\s*/i, '') .replace(/^```json\s*/i, '')
.replace(/\s*```\s*$/, '') .replace(/\s*```\s*$/, '')
.trim(); .trim();
questions = JSON.parse(cleanedContent);
questions = JSONParser.safeParseJSON(cleanedContent, []);
if (!Array.isArray(questions)) { if (!Array.isArray(questions)) {
throw new Error('Response is not an array'); throw new Error('Response is not an array');
@ -198,11 +153,11 @@ export const POST: APIRoute = async ({ request }) => {
} }
} catch (error) { } catch (error) {
console.error('Failed to parse enhancement response:', aiContent); console.error('[ENHANCE-API] Failed to parse enhancement response:', aiResponse.content);
questions = []; questions = [];
} }
console.log(`[ENHANCE API] User: ${userId}, Forensics Questions: ${questions.length}, Input length: ${sanitizedInput.length}`); console.log(`[ENHANCE-API] User: ${userId}, Questions generated: ${questions.length}, Input length: ${sanitizedInput.length}`);
return new Response(JSON.stringify({ return new Response(JSON.stringify({
success: true, success: true,
@ -215,7 +170,7 @@ export const POST: APIRoute = async ({ request }) => {
}); });
} catch (error) { } catch (error) {
console.error('Enhancement error:', error); console.error('[ENHANCE-API] Enhancement error:', error);
return apiServerError.internal('Enhancement processing failed'); return apiServerError.internal('Enhancement processing failed');
} }
}; };

64
src/pages/api/ai/query.ts
View File

@ -1,4 +1,4 @@
// src/pages/api/ai/query.ts // src/pages/api/ai/query.ts - Updated to use refactored services
import type { APIRoute } from 'astro'; import type { APIRoute } from 'astro';
import { withAPIAuth } from '../../../utils/auth.js'; import { withAPIAuth } from '../../../utils/auth.js';
import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js'; import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js';
@ -20,15 +20,14 @@ const MAIN_RATE_LIMIT_MAX = parseInt(process.env.AI_RATE_LIMIT_MAX_REQUESTS || '
const MICRO_TASK_TOTAL_LIMIT = parseInt(process.env.AI_MICRO_TASK_TOTAL_LIMIT || '50', 10); const MICRO_TASK_TOTAL_LIMIT = parseInt(process.env.AI_MICRO_TASK_TOTAL_LIMIT || '50', 10);
function sanitizeInput(input: string): string { function sanitizeInput(input: string): string {
let sanitized = input return input
.replace(/```[\s\S]*?```/g, '[CODE_BLOCK_REMOVED]') .replace(/```[\s\S]*?```/g, '[CODE_BLOCK_REMOVED]')
.replace(/\<\/?[^>]+(>|$)/g, '') .replace(/\<\/?[^>]+(>|$)/g, '')
.replace(/\b(system|assistant|user)\s*[:]/gi, '[ROLE_REMOVED]') .replace(/\b(system|assistant|user)\s*[:]/gi, '[ROLE_REMOVED]')
.replace(/\b(ignore|forget|disregard)\s+(previous|all|your)\s+(instructions?|context|rules?)/gi, '[INSTRUCTION_REMOVED]') .replace(/\b(ignore|forget|disregard)\s+(previous|all|your)\s+(instructions?|context|rules?)/gi, '[INSTRUCTION_REMOVED]')
.trim(); .trim()
.slice(0, 2000)
sanitized = sanitized.slice(0, 2000).replace(/\s+/g, ' '); .replace(/\s+/g, ' ');
return sanitized;
} }
function checkRateLimit(userId: string): { allowed: boolean; reason?: string; microTasksRemaining?: number } { function checkRateLimit(userId: string): { allowed: boolean; reason?: string; microTasksRemaining?: number } {
@ -77,7 +76,7 @@ function incrementMicroTaskCount(userId: string, aiCallsMade: number): void {
} }
} }
function cleanupExpiredRateLimits() { function cleanupExpiredRateLimits(): void {
const now = Date.now(); const now = Date.now();
const maxStoreSize = 1000; const maxStoreSize = 1000;
@ -117,51 +116,52 @@ export const POST: APIRoute = async ({ request }) => {
const body = await request.json(); const body = await request.json();
const { query, mode = 'workflow', taskId: clientTaskId } = body; const { query, mode = 'workflow', taskId: clientTaskId } = body;
console.log(`[MICRO-TASK API] Received request - TaskId: ${clientTaskId}, Mode: ${mode}, Query length: ${query?.length || 0}`); console.log(`[AI-API] Received request - TaskId: ${clientTaskId}, Mode: ${mode}, Query length: ${query?.length || 0}`);
console.log(`[MICRO-TASK API] Micro-task calls remaining: ${rateLimitResult.microTasksRemaining}`); console.log(`[AI-API] Micro-task calls remaining: ${rateLimitResult.microTasksRemaining}`);
if (!query || typeof query !== 'string') { if (!query || typeof query !== 'string') {
console.log(`[MICRO-TASK API] Invalid query for task ${clientTaskId}`); console.log(`[AI-API] Invalid query for task ${clientTaskId}`);
return apiError.badRequest('Query required'); return apiError.badRequest('Query required');
} }
if (!['workflow', 'tool'].includes(mode)) { if (!['workflow', 'tool'].includes(mode)) {
console.log(`[MICRO-TASK API] Invalid mode for task ${clientTaskId}: ${mode}`); console.log(`[AI-API] Invalid mode for task ${clientTaskId}: ${mode}`);
return apiError.badRequest('Invalid mode. Must be "workflow" or "tool"'); return apiError.badRequest('Invalid mode. Must be "workflow" or "tool"');
} }
const sanitizedQuery = sanitizeInput(query); const sanitizedQuery = sanitizeInput(query);
if (sanitizedQuery.includes('[FILTERED]')) { if (sanitizedQuery.includes('[FILTERED]')) {
console.log(`[MICRO-TASK API] Filtered input detected for task ${clientTaskId}`); console.log(`[AI-API] Filtered input detected for task ${clientTaskId}`);
return apiError.badRequest('Invalid input detected'); return apiError.badRequest('Invalid input detected');
} }
const taskId = clientTaskId || `ai_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`; const taskId = clientTaskId || `ai_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`;
console.log(`[MICRO-TASK API] About to enqueue micro-task pipeline ${taskId}`); console.log(`[AI-API] Enqueueing pipeline task ${taskId}`);
const result = await enqueueApiCall(() => const result = await enqueueApiCall(() =>
aiPipeline.processQuery(sanitizedQuery, mode) aiPipeline.processQuery(sanitizedQuery, mode)
, taskId); , taskId);
if (!result || !result.recommendation) { if (!result || !result.recommendation) {
return apiServerError.unavailable('No response from micro-task AI pipeline'); return apiServerError.unavailable('No response from AI pipeline');
} }
const stats = result.processingStats; const stats = result.processingStats;
const estimatedAICallsMade = stats.microTasksCompleted + stats.microTasksFailed; const estimatedAICallsMade = stats.microTasksCompleted + stats.microTasksFailed;
incrementMicroTaskCount(userId, estimatedAICallsMade); incrementMicroTaskCount(userId, estimatedAICallsMade);
console.log(`[MICRO-TASK API] Pipeline completed for ${taskId}:`); console.log(`[AI-API] Pipeline completed for ${taskId}:`, {
console.log(` - Mode: ${mode}`); mode,
console.log(` - User: ${userId}`); user: userId,
console.log(` - Query length: ${sanitizedQuery.length}`); queryLength: sanitizedQuery.length,
console.log(` - Processing time: ${stats.processingTimeMs}ms`); processingTime: stats.processingTimeMs,
console.log(` - Micro-tasks completed: ${stats.microTasksCompleted}`); microTasksCompleted: stats.microTasksCompleted,
console.log(` - Micro-tasks failed: ${stats.microTasksFailed}`); microTasksFailed: stats.microTasksFailed,
console.log(` - Estimated AI calls: ${estimatedAICallsMade}`); estimatedAICalls: estimatedAICallsMade,
console.log(` - Embeddings used: ${stats.embeddingsUsed}`); embeddingsUsed: stats.embeddingsUsed,
console.log(` - Final items: ${stats.finalSelectedItems}`); finalItems: stats.finalSelectedItems
});
const currentLimit = rateLimitStore.get(userId); const currentLimit = rateLimitStore.get(userId);
const remainingMicroTasks = currentLimit ? const remainingMicroTasks = currentLimit ?
@ -175,7 +175,7 @@ export const POST: APIRoute = async ({ request }) => {
query: sanitizedQuery, query: sanitizedQuery,
processingStats: { processingStats: {
...result.processingStats, ...result.processingStats,
pipelineType: 'micro-task', pipelineType: 'refactored',
microTasksSuccessRate: stats.microTasksCompleted / (stats.microTasksCompleted + stats.microTasksFailed), microTasksSuccessRate: stats.microTasksCompleted / (stats.microTasksCompleted + stats.microTasksFailed),
averageTaskTime: stats.processingTimeMs / (stats.microTasksCompleted + stats.microTasksFailed), averageTaskTime: stats.processingTimeMs / (stats.microTasksCompleted + stats.microTasksFailed),
estimatedAICallsMade estimatedAICallsMade
@ -191,18 +191,16 @@ export const POST: APIRoute = async ({ request }) => {
}); });
} catch (error) { } catch (error) {
console.error('[MICRO-TASK API] Pipeline error:', error); console.error('[AI-API] Pipeline error:', error);
if (error.message.includes('embeddings')) { if (error.message.includes('embeddings')) {
return apiServerError.unavailable('Embeddings service error - using AI fallback'); return apiServerError.unavailable('Embeddings service error');
} else if (error.message.includes('micro-task')) { } else if (error.message.includes('AI')) {
return apiServerError.unavailable('Micro-task pipeline error - some analysis steps failed'); return apiServerError.unavailable('AI service error');
} else if (error.message.includes('selector')) {
return apiServerError.unavailable('AI selector service error');
} else if (error.message.includes('rate limit')) { } else if (error.message.includes('rate limit')) {
return apiError.rateLimit('AI service rate limits exceeded during micro-task processing'); return apiError.rateLimit('AI service rate limits exceeded');
} else { } else {
return apiServerError.internal('Micro-task AI pipeline error'); return apiServerError.internal('AI pipeline error');
} }
} }
}; };

1672
src/utils/aiPipeline.ts
View File

File diff suppressed because it is too large Load Diff

150
src/utils/aiService.ts Normal file
View File

@ -0,0 +1,150 @@
// src/utils/aiService.ts
import 'dotenv/config';
export interface AIServiceConfig {
endpoint: string;
apiKey: string;
model: string;
}
export interface AICallOptions {
maxTokens?: number;
temperature?: number;
timeout?: number;
}
export interface AIResponse {
content: string;
usage?: {
promptTokens: number;
completionTokens: number;
totalTokens: number;
};
}
class AIService {
private config: AIServiceConfig;
private defaultOptions: AICallOptions;
constructor() {
this.config = {
endpoint: this.getRequiredEnv('AI_ANALYZER_ENDPOINT'),
apiKey: this.getRequiredEnv('AI_ANALYZER_API_KEY'),
model: this.getRequiredEnv('AI_ANALYZER_MODEL')
};
this.defaultOptions = {
maxTokens: 1500,
temperature: 0.3,
timeout: 30000
};
console.log('[AI-SERVICE] Initialized with model:', this.config.model);
}
private getRequiredEnv(key: string): string {
const value = process.env[key];
if (!value) {
throw new Error(`Missing required environment variable: ${key}`);
}
return value;
}
async callAI(prompt: string, options: AICallOptions = {}): Promise<AIResponse> {
const mergedOptions = { ...this.defaultOptions, ...options };
console.log('[AI-SERVICE] Making API call:', {
promptLength: prompt.length,
maxTokens: mergedOptions.maxTokens,
temperature: mergedOptions.temperature
});
const headers: Record<string, string> = {
'Content-Type': 'application/json'
};
if (this.config.apiKey) {
headers['Authorization'] = `Bearer ${this.config.apiKey}`;
}
const requestBody = {
model: this.config.model,
messages: [{ role: 'user', content: prompt }],
max_tokens: mergedOptions.maxTokens,
temperature: mergedOptions.temperature
};
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), mergedOptions.timeout);
const response = await fetch(`${this.config.endpoint}/v1/chat/completions`, {
method: 'POST',
headers,
body: JSON.stringify(requestBody),
signal: controller.signal
});
clearTimeout(timeoutId);
if (!response.ok) {
const errorText = await response.text();
console.error('[AI-SERVICE] API Error:', response.status, errorText);
throw new Error(`AI API error: ${response.status} - ${errorText}`);
}
const data = await response.json();
const content = data.choices?.[0]?.message?.content;
if (!content) {
console.error('[AI-SERVICE] No response content from AI model');
throw new Error('No response from AI model');
}
console.log('[AI-SERVICE] API call successful:', {
responseLength: content.length,
usage: data.usage
});
return {
content: content.trim(),
usage: data.usage
};
} catch (error) {
if (error.name === 'AbortError') {
console.error('[AI-SERVICE] Request timeout');
throw new Error('AI request timeout');
}
console.error('[AI-SERVICE] API call failed:', error.message);
throw error;
}
}
async callMicroTaskAI(prompt: string, maxTokens: number = 500): Promise<AIResponse> {
return this.callAI(prompt, {
maxTokens,
temperature: 0.3,
timeout: 15000
});
}
estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
}
validatePromptLength(prompt: string, maxTokens: number = 35000): void {
const estimatedTokens = this.estimateTokens(prompt);
if (estimatedTokens > maxTokens) {
console.warn('[AI-SERVICE] WARNING: Prompt may exceed model limits:', estimatedTokens);
throw new Error(`Prompt too long: ${estimatedTokens} tokens (max: ${maxTokens})`);
}
}
getConfig(): AIServiceConfig {
return { ...this.config };
}
}
export const aiService = new AIService();

369
src/utils/auditService.ts
View File

@ -1,4 +1,4 @@
// src/utils/auditService.ts // src/utils/auditService.ts - Refactored
import 'dotenv/config'; import 'dotenv/config';
function env(key: string, fallback: string | undefined = undefined): string | undefined { function env(key: string, fallback: string | undefined = undefined): string | undefined {
@ -11,7 +11,6 @@ function env(key: string, fallback: string | undefined = undefined): string | un
return fallback; return fallback;
} }
// CONSOLIDATED AUDIT INTERFACES - Single source of truth
export interface AuditEntry { export interface AuditEntry {
timestamp: number; timestamp: number;
phase: string; phase: string;
@ -30,64 +29,10 @@ interface AuditConfig {
maxEntries: number; maxEntries: number;
} }
interface CompressedAuditEntry {
timestamp: number;
phase: string;
action: string;
inputSummary: string;
outputSummary: string;
confidence: number;
processingTimeMs: number;
metadata: Record<string, any>;
}
export interface ProcessedAuditTrail {
totalTime: number;
avgConfidence: number;
stepCount: number;
highConfidenceSteps: number;
lowConfidenceSteps: number;
phases: Array<{
name: string;
icon: string;
displayName: string;
avgConfidence: number;
totalTime: number;
entries: CompressedAuditEntry[];
}>;
summary: {
analysisQuality: 'excellent' | 'good' | 'fair' | 'poor';
keyInsights: string[];
potentialIssues: string[];
};
}
class AuditService { class AuditService {
private config: AuditConfig; private config: AuditConfig;
private activeAuditTrail: AuditEntry[] = []; private activeAuditTrail: AuditEntry[] = [];
private readonly phaseConfig = {
'initialization': { icon: '🚀', displayName: 'Initialisierung' },
'retrieval': { icon: '🔍', displayName: 'Datensuche' },
'selection': { icon: '🎯', displayName: 'Tool-Auswahl' },
'micro-task': { icon: '⚡', displayName: 'Detail-Analyse' },
'validation': { icon: '✓', displayName: 'Validierung' },
'completion': { icon: '✅', displayName: 'Finalisierung' }
};
private readonly actionTranslations = {
'pipeline-start': 'Analyse gestartet',
'embeddings-search': 'Ähnliche Tools gesucht',
'ai-tool-selection': 'Tools automatisch ausgewählt',
'ai-analysis': 'KI-Analyse durchgeführt',
'phase-tool-selection': 'Phasen-Tools evaluiert',
'tool-evaluation': 'Tool-Bewertung erstellt',
'background-knowledge-selection': 'Hintergrundwissen ausgewählt',
'confidence-scoring': 'Vertrauenswertung berechnet',
'phase-completion': 'Phasenergänzung durchgeführt',
'pipeline-end': 'Analyse abgeschlossen'
};
constructor() { constructor() {
this.config = this.loadConfig(); this.config = this.loadConfig();
console.log('[AUDIT-SERVICE] Initialized:', { console.log('[AUDIT-SERVICE] Initialized:', {
@ -110,7 +55,6 @@ class AuditService {
}; };
} }
// CONSOLIDATED AUDIT ENTRY CREATION - Single method for all audit operations
addEntry( addEntry(
phase: string, phase: string,
action: string, action: string,
@ -134,15 +78,19 @@ class AuditService {
}; };
this.activeAuditTrail.push(entry); this.activeAuditTrail.push(entry);
// Enforce max entries limit
if (this.activeAuditTrail.length > this.config.maxEntries) {
this.activeAuditTrail.shift();
}
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`); console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`);
} }
// GET CURRENT AUDIT TRAIL - For integration with AI pipeline
getCurrentAuditTrail(): AuditEntry[] { getCurrentAuditTrail(): AuditEntry[] {
return [...this.activeAuditTrail]; return [...this.activeAuditTrail];
} }
// CLEAR AUDIT TRAIL - Start fresh for new analysis
clearAuditTrail(): void { clearAuditTrail(): void {
if (this.activeAuditTrail.length > 0) { if (this.activeAuditTrail.length > 0) {
console.log(`[AUDIT-SERVICE] Cleared ${this.activeAuditTrail.length} audit entries`); console.log(`[AUDIT-SERVICE] Cleared ${this.activeAuditTrail.length} audit entries`);
@ -150,7 +98,6 @@ class AuditService {
} }
} }
// FINALIZE AUDIT TRAIL - Complete analysis and return final trail
finalizeAuditTrail(): AuditEntry[] { finalizeAuditTrail(): AuditEntry[] {
const finalTrail = [...this.activeAuditTrail]; const finalTrail = [...this.activeAuditTrail];
console.log(`[AUDIT-SERVICE] Finalized audit trail with ${finalTrail.length} entries`); console.log(`[AUDIT-SERVICE] Finalized audit trail with ${finalTrail.length} entries`);
@ -158,102 +105,6 @@ class AuditService {
return finalTrail; return finalTrail;
} }
processAuditTrail(rawAuditTrail: AuditEntry[]): ProcessedAuditTrail | null {
if (!this.config.enabled) {
console.log('[AUDIT-SERVICE] Processing disabled');
return null;
}
if (!rawAuditTrail || !Array.isArray(rawAuditTrail) || rawAuditTrail.length === 0) {
console.log('[AUDIT-SERVICE] No audit trail data to process');
return null;
}
try {
console.log(`[AUDIT-SERVICE] Processing ${rawAuditTrail.length} audit entries`);
const totalTime = rawAuditTrail.reduce((sum, entry) => sum + (entry.processingTimeMs || 0), 0);
const validConfidenceEntries = rawAuditTrail.filter(entry => typeof entry.confidence === 'number');
const avgConfidence = validConfidenceEntries.length > 0
? Math.round(validConfidenceEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validConfidenceEntries.length)
: 0;
const highConfidenceSteps = rawAuditTrail.filter(entry => (entry.confidence || 0) >= 80).length;
const lowConfidenceSteps = rawAuditTrail.filter(entry => (entry.confidence || 0) < 60).length;
const groupedEntries = rawAuditTrail.reduce((groups, entry) => {
const phase = entry.phase || 'unknown';
if (!groups[phase]) groups[phase] = [];
groups[phase].push(entry);
return groups;
}, {} as Record<string, AuditEntry[]>);
const phases = Object.entries(groupedEntries).map(([phase, entries]) => {
const phaseConfig = this.phaseConfig[phase] || { icon: '📋', displayName: phase };
const validEntries = entries.filter(entry => entry && typeof entry === 'object');
const phaseAvgConfidence = validEntries.length > 0
? Math.round(validEntries.reduce((sum, entry) => sum + (entry.confidence || 0), 0) / validEntries.length)
: 0;
const phaseTotalTime = validEntries.reduce((sum, entry) => sum + (entry.processingTimeMs || 0), 0);
return {
name: phase,
icon: phaseConfig.icon,
displayName: phaseConfig.displayName,
avgConfidence: phaseAvgConfidence,
totalTime: phaseTotalTime,
entries: validEntries
.map(e => this.compressEntry(e))
.filter((e): e is CompressedAuditEntry => e !== null)
};
}).filter(phase => phase.entries.length > 0);
const summary = this.generateSummary(rawAuditTrail, avgConfidence, lowConfidenceSteps);
const result: ProcessedAuditTrail = {
totalTime,
avgConfidence,
stepCount: rawAuditTrail.length,
highConfidenceSteps,
lowConfidenceSteps,
phases,
summary
};
console.log(`[AUDIT-SERVICE] Successfully processed audit trail: ${result.phases.length} phases, ${result.avgConfidence}% avg confidence`);
return result;
} catch (error) {
console.error('[AUDIT-SERVICE] Error processing audit trail:', error);
return null;
}
}
private compressEntry(entry: AuditEntry): CompressedAuditEntry | null {
if (!entry || typeof entry !== 'object') {
console.warn('[AUDIT-SERVICE] Invalid audit entry skipped');
return null;
}
try {
return {
timestamp: entry.timestamp || Date.now(),
phase: entry.phase || 'unknown',
action: entry.action || 'unknown',
inputSummary: this.summarizeData(entry.input),
outputSummary: this.summarizeData(entry.output),
confidence: entry.confidence || 0,
processingTimeMs: entry.processingTimeMs || 0,
metadata: entry.metadata || {}
};
} catch (error) {
console.error('[AUDIT-SERVICE] Error compressing entry:', error);
return null;
}
}
private compressData(data: any): any { private compressData(data: any): any {
if (this.config.detailLevel === 'verbose') { if (this.config.detailLevel === 'verbose') {
return data; return data;
@ -264,30 +115,6 @@ class AuditService {
} }
} }
private summarizeData(data: any): string {
if (data === null || data === undefined) return 'null';
if (typeof data === 'string') {
return data.length > 100 ? data.slice(0, 100) + '...' : data;
}
if (typeof data === 'number' || typeof data === 'boolean') {
return data.toString();
}
if (Array.isArray(data)) {
if (data.length === 0) return '[]';
if (data.length <= 3) return JSON.stringify(data);
return `[${data.slice(0, 3).map(i => typeof i === 'string' ? i : JSON.stringify(i)).join(', ')}, ...+${data.length - 3}]`;
}
if (typeof data === 'object') {
const keys = Object.keys(data);
if (keys.length === 0) return '{}';
if (keys.length <= 3) {
return '{' + keys.map(k => `${k}: ${typeof data[k] === 'string' ? data[k].slice(0, 20) + (data[k].length > 20 ? '...' : '') : JSON.stringify(data[k])}`).join(', ') + '}';
}
return `{${keys.slice(0, 3).join(', ')}, ...+${keys.length - 3} keys}`;
}
return String(data);
}
private summarizeForStorage(data: any): any { private summarizeForStorage(data: any): any {
if (typeof data === 'string' && data.length > 500) { if (typeof data === 'string' && data.length > 500) {
return data.slice(0, 500) + '...[truncated]'; return data.slice(0, 500) + '...[truncated]';
@ -308,71 +135,6 @@ class AuditService {
return data; return data;
} }
private generateSummary(entries: AuditEntry[], avgConfidence: number, lowConfidenceSteps: number): {
analysisQuality: 'excellent' | 'good' | 'fair' | 'poor';
keyInsights: string[];
potentialIssues: string[];
} {
let analysisQuality: 'excellent' | 'good' | 'fair' | 'poor';
if (avgConfidence >= 85 && lowConfidenceSteps === 0) {
analysisQuality = 'excellent';
} else if (avgConfidence >= 70 && lowConfidenceSteps <= 1) {
analysisQuality = 'good';
} else if (avgConfidence >= 60 && lowConfidenceSteps <= 3) {
analysisQuality = 'fair';
} else {
analysisQuality = 'poor';
}
const keyInsights: string[] = [];
const embeddingsUsed = entries.some(e => e.action === 'embeddings-search');
if (embeddingsUsed) {
keyInsights.push('Semantische Suche wurde erfolgreich eingesetzt');
}
const toolSelectionEntries = entries.filter(e => e.action === 'ai-tool-selection');
if (toolSelectionEntries.length > 0) {
const avgSelectionConfidence = toolSelectionEntries.reduce((sum, e) => sum + e.confidence, 0) / toolSelectionEntries.length;
if (avgSelectionConfidence >= 80) {
keyInsights.push('Hohe Konfidenz bei der Tool-Auswahl');
}
}
const potentialIssues: string[] = [];
if (lowConfidenceSteps > 2) {
potentialIssues.push(`${lowConfidenceSteps} Analyseschritte mit niedriger Konfidenz`);
}
const longSteps = entries.filter(e => e.processingTimeMs > 5000);
if (longSteps.length > 0) {
potentialIssues.push(`${longSteps.length} Schritte benötigten mehr als 5 Sekunden`);
}
return {
analysisQuality,
keyInsights,
potentialIssues
};
}
getActionDisplayName(action: string): string {
return this.actionTranslations[action] || action;
}
formatDuration(ms: number): string {
if (ms < 1000) return '< 1s';
if (ms < 60000) return `${Math.ceil(ms / 1000)}s`;
const minutes = Math.floor(ms / 60000);
const seconds = Math.ceil((ms % 60000) / 1000);
return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
}
getConfidenceColor(confidence: number): string {
if (confidence >= 80) return 'var(--color-accent)';
if (confidence >= 60) return 'var(--color-warning)';
return 'var(--color-error)';
}
isEnabled(): boolean { isEnabled(): boolean {
return this.config.enabled; return this.config.enabled;
} }
@ -380,7 +142,122 @@ class AuditService {
getConfig(): AuditConfig { getConfig(): AuditConfig {
return { ...this.config }; return { ...this.config };
} }
// Statistics and analysis methods
getAuditStatistics(auditTrail: AuditEntry[]): {
totalTime: number;
avgConfidence: number;
stepCount: number;
highConfidenceSteps: number;
lowConfidenceSteps: number;
phaseBreakdown: Record<string, { count: number; avgConfidence: number; totalTime: number }>;
} {
if (!auditTrail || auditTrail.length === 0) {
return {
totalTime: 0,
avgConfidence: 0,
stepCount: 0,
highConfidenceSteps: 0,
lowConfidenceSteps: 0,
phaseBreakdown: {}
};
}
const totalTime = auditTrail.reduce((sum, entry) => sum + (entry.processingTimeMs || 0), 0);
const validConfidenceEntries = auditTrail.filter(entry => typeof entry.confidence === 'number');
const avgConfidence = validConfidenceEntries.length > 0
? Math.round(validConfidenceEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validConfidenceEntries.length)
: 0;
const highConfidenceSteps = auditTrail.filter(entry => (entry.confidence || 0) >= 80).length;
const lowConfidenceSteps = auditTrail.filter(entry => (entry.confidence || 0) < 60).length;
// Phase breakdown
const phaseBreakdown: Record<string, { count: number; avgConfidence: number; totalTime: number }> = {};
auditTrail.forEach(entry => {
const phase = entry.phase || 'unknown';
if (!phaseBreakdown[phase]) {
phaseBreakdown[phase] = { count: 0, avgConfidence: 0, totalTime: 0 };
}
phaseBreakdown[phase].count++;
phaseBreakdown[phase].totalTime += entry.processingTimeMs || 0;
});
// Calculate average confidence per phase
Object.keys(phaseBreakdown).forEach(phase => {
const phaseEntries = auditTrail.filter(entry => entry.phase === phase);
const validEntries = phaseEntries.filter(entry => typeof entry.confidence === 'number');
if (validEntries.length > 0) {
phaseBreakdown[phase].avgConfidence = Math.round(
validEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validEntries.length
);
}
});
return {
totalTime,
avgConfidence,
stepCount: auditTrail.length,
highConfidenceSteps,
lowConfidenceSteps,
phaseBreakdown
};
}
validateAuditTrail(auditTrail: AuditEntry[]): {
isValid: boolean;
issues: string[];
warnings: string[];
} {
const issues: string[] = [];
const warnings: string[] = [];
if (!Array.isArray(auditTrail)) {
issues.push('Audit trail is not an array');
return { isValid: false, issues, warnings };
}
if (auditTrail.length === 0) {
warnings.push('Audit trail is empty');
}
auditTrail.forEach((entry, index) => {
if (!entry || typeof entry !== 'object') {
issues.push(`Entry ${index} is not a valid object`);
return;
}
// Required fields validation
const requiredFields = ['timestamp', 'phase', 'action'];
requiredFields.forEach(field => {
if (!(field in entry)) {
issues.push(`Entry ${index} missing required field: ${field}`);
}
});
// Data type validation
if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) {
warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`);
}
if (typeof entry.processingTimeMs !== 'number' || entry.processingTimeMs < 0) {
warnings.push(`Entry ${index} has invalid processing time: ${entry.processingTimeMs}`);
}
if (typeof entry.timestamp !== 'number' || entry.timestamp <= 0) {
issues.push(`Entry ${index} has invalid timestamp: ${entry.timestamp}`);
}
});
return {
isValid: issues.length === 0,
issues,
warnings
};
}
} }
export const auditService = new AuditService(); export const auditService = new AuditService();
export type { CompressedAuditEntry };

98
src/utils/clientUtils.ts
View File

@ -1,9 +1,9 @@
// src/utils/clientUtils.ts // src/utils/clientUtils.ts - Consolidated (removes duplicates from toolHelpers.ts)
// Tool helper functions (moved here to avoid circular imports)
export function createToolSlug(toolName: string): string { export function createToolSlug(toolName: string): string {
if (!toolName || typeof toolName !== 'string') { if (!toolName || typeof toolName !== 'string') {
console.warn('[toolHelpers] Invalid toolName provided to createToolSlug:', toolName); console.warn('[CLIENT-UTILS] Invalid toolName provided to createToolSlug:', toolName);
return ''; return '';
} }
@ -30,6 +30,86 @@ export function isToolHosted(tool: any): boolean {
tool.projectUrl.trim() !== ""; tool.projectUrl.trim() !== "";
} }
// Text and display utilities
export function sanitizeText(text: string): string {
if (typeof text !== 'string') return '';
return text
.replace(/^#{1,6}\s+/gm, '')
.replace(/^\s*[-*+]\s+/gm, '')
.replace(/^\s*\d+\.\s+/gm, '')
.replace(/\*\*(.+?)\*\*/g, '$1')
.replace(/\*(.+?)\*/g, '$1')
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
.replace(/```[\s\S]*?```/g, '[CODE BLOCK]')
.replace(/`([^`]+)`/g, '$1')
.replace(/<[^>]+>/g, '')
.replace(/\n\s*\n\s*\n/g, '\n\n')
.trim();
}
export function escapeHtml(text: string): string {
if (typeof text !== 'string') return String(text);
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
export function truncateText(text: string, maxLength: number): string {
if (!text || text.length <= maxLength) return text;
return text.slice(0, maxLength) + '...';
}
// Data summarization utilities
export function summarizeData(data: any): string {
if (data === null || data === undefined) return 'null';
if (typeof data === 'string') {
return data.length > 100 ? data.slice(0, 100) + '...' : data;
}
if (typeof data === 'number' || typeof data === 'boolean') {
return data.toString();
}
if (Array.isArray(data)) {
if (data.length === 0) return '[]';
if (data.length <= 3) return JSON.stringify(data);
return `[${data.slice(0, 3).map(i => typeof i === 'string' ? i : JSON.stringify(i)).join(', ')}, ...+${data.length - 3}]`;
}
if (typeof data === 'object') {
const keys = Object.keys(data);
if (keys.length === 0) return '{}';
if (keys.length <= 3) {
return '{' + keys.map(k => `${k}: ${typeof data[k] === 'string' ? data[k].slice(0, 20) + (data[k].length > 20 ? '...' : '') : JSON.stringify(data[k])}`).join(', ') + '}';
}
return `{${keys.slice(0, 3).join(', ')}, ...+${keys.length - 3} keys}`;
}
return String(data);
}
// Time formatting utilities
export function formatDuration(ms: number): string {
if (ms < 1000) return '< 1s';
if (ms < 60000) return `${Math.ceil(ms / 1000)}s`;
const minutes = Math.floor(ms / 60000);
const seconds = Math.ceil((ms % 60000) / 1000);
return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
}
// DOM utilities
export function showElement(element: HTMLElement | null): void {
if (element) {
element.style.display = 'block';
element.classList.remove('hidden');
}
}
export function hideElement(element: HTMLElement | null): void {
if (element) {
element.style.display = 'none';
element.classList.add('hidden');
}
}
// Autocomplete functionality (kept from original clientUtils.ts as it's UI-specific)
interface AutocompleteOptions { interface AutocompleteOptions {
minLength?: number; minLength?: number;
maxResults?: number; maxResults?: number;
@ -202,7 +282,7 @@ export class AutocompleteManager {
defaultRender(item: any): string { defaultRender(item: any): string {
const text = typeof item === 'string' ? item : item.name || item.label || item.toString(); const text = typeof item === 'string' ? item : item.name || item.label || item.toString();
return `<div class="autocomplete-item">${this.escapeHtml(text)}</div>`; return `<div class="autocomplete-item">${escapeHtml(text)}</div>`;
} }
renderDropdown(): void { renderDropdown(): void {
@ -284,8 +364,8 @@ export class AutocompleteManager {
align-items: center; align-items: center;
gap: 0.25rem; gap: 0.25rem;
"> ">
${this.escapeHtml(item)} ${escapeHtml(item)}
<button type="button" class="autocomplete-remove" data-item="${this.escapeHtml(item)}" style=" <button type="button" class="autocomplete-remove" data-item="${escapeHtml(item)}" style="
background: none; background: none;
border: none; border: none;
color: white; color: white;
@ -327,12 +407,6 @@ export class AutocompleteManager {
this.selectedIndex = -1; this.selectedIndex = -1;
} }
escapeHtml(text: string): string {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
setDataSource(newDataSource: any[]): void { setDataSource(newDataSource: any[]): void {
this.dataSource = newDataSource; this.dataSource = newDataSource;
} }

239
src/utils/confidenceScoring.ts Normal file
View File

@ -0,0 +1,239 @@
// src/utils/confidenceScoring.ts
import { isToolHosted } from './toolHelpers.js';
import 'dotenv/config';
export interface ConfidenceMetrics {
overall: number;
semanticRelevance: number;
taskSuitability: number;
uncertaintyFactors: string[];
strengthIndicators: string[];
}
export interface ConfidenceConfig {
semanticWeight: number;
suitabilityWeight: number;
minimumThreshold: number;
mediumThreshold: number;
highThreshold: number;
}
export interface AnalysisContext {
userQuery: string;
mode: string;
embeddingsSimilarities: Map<string, number>;
selectedTools?: Array<{
tool: any;
phase: string;
priority: string;
justification?: string;
taskRelevance?: number;
limitations?: string[];
}>;
}
class ConfidenceScoring {
private config: ConfidenceConfig;
constructor() {
this.config = {
semanticWeight: this.getEnvFloat('CONFIDENCE_SEMANTIC_WEIGHT', 0.3),
suitabilityWeight: this.getEnvFloat('CONFIDENCE_SUITABILITY_WEIGHT', 0.7),
minimumThreshold: this.getEnvInt('CONFIDENCE_MINIMUM_THRESHOLD', 40),
mediumThreshold: this.getEnvInt('CONFIDENCE_MEDIUM_THRESHOLD', 60),
highThreshold: this.getEnvInt('CONFIDENCE_HIGH_THRESHOLD', 80)
};
console.log('[CONFIDENCE-SCORING] Initialized with config:', this.config);
}
private getEnvFloat(key: string, defaultValue: number): number {
const value = process.env[key];
return value ? parseFloat(value) : defaultValue;
}
private getEnvInt(key: string, defaultValue: number): number {
const value = process.env[key];
return value ? parseInt(value, 10) : defaultValue;
}
calculateRecommendationConfidence(
tool: any,
context: AnalysisContext,
taskRelevance: number = 70,
limitations: string[] = []
): ConfidenceMetrics {
console.log('[CONFIDENCE-SCORING] Calculating confidence for tool:', tool.name);
const rawSemanticRelevance = context.embeddingsSimilarities.has(tool.name) ?
context.embeddingsSimilarities.get(tool.name)! * 100 : 50;
let enhancedTaskSuitability = taskRelevance;
// Phase alignment bonus for workflow mode
if (context.mode === 'workflow') {
const toolSelection = context.selectedTools?.find((st: any) => st.tool && st.tool.name === tool.name);
if (toolSelection && tool.phases && Array.isArray(tool.phases) && tool.phases.includes(toolSelection.phase)) {
const phaseBonus = Math.min(15, 100 - taskRelevance);
enhancedTaskSuitability = Math.min(100, taskRelevance + phaseBonus);
console.log('[CONFIDENCE-SCORING] Phase alignment bonus applied:', phaseBonus);
}
}
const overall = (
rawSemanticRelevance * this.config.semanticWeight +
enhancedTaskSuitability * this.config.suitabilityWeight
);
const uncertaintyFactors = this.identifyUncertaintyFactors(tool, context, limitations, overall);
const strengthIndicators = this.identifyStrengthIndicators(tool, context, overall);
const result = {
overall: Math.round(overall),
semanticRelevance: Math.round(rawSemanticRelevance),
taskSuitability: Math.round(enhancedTaskSuitability),
uncertaintyFactors,
strengthIndicators
};
console.log('[CONFIDENCE-SCORING] Confidence calculated:', {
tool: tool.name,
overall: result.overall,
semantic: result.semanticRelevance,
task: result.taskSuitability,
uncertaintyCount: uncertaintyFactors.length,
strengthCount: strengthIndicators.length
});
return result;
}
private identifyUncertaintyFactors(
tool: any,
context: AnalysisContext,
limitations: string[],
confidence: number
): string[] {
const factors: string[] = [];
// Add explicit limitations
if (limitations?.length > 0) {
factors.push(...limitations.slice(0, 2));
}
// Semantic similarity concerns
const similarity = context.embeddingsSimilarities.get(tool.name) || 0.5;
if (similarity < 0.7) {
factors.push('Geringe semantische Ähnlichkeit zur Anfrage');
}
// Skill level vs query complexity mismatches
if (tool.skillLevel === 'expert' && /schnell|rapid|triage|urgent|sofort/i.test(context.userQuery)) {
factors.push('Experten-Tool für zeitkritisches Szenario');
}
if (tool.skillLevel === 'novice' && /komplex|erweitert|tiefgehend|advanced|forensisch/i.test(context.userQuery)) {
factors.push('Einsteiger-Tool für komplexe Analyse');
}
// Technical accessibility concerns
if (tool.type === 'software' && !isToolHosted(tool) && tool.accessType === 'download') {
factors.push('Installation und Setup erforderlich');
}
// Licensing concerns
if (tool.license === 'Proprietary') {
factors.push('Kommerzielle Software - Lizenzkosten zu beachten');
}
// Overall confidence concerns
if (confidence < 60) {
factors.push('Moderate Gesamtbewertung - alternative Ansätze empfohlen');
}
return factors.slice(0, 4);
}
private identifyStrengthIndicators(tool: any, context: AnalysisContext, confidence: number): string[] {
const indicators: string[] = [];
// High semantic relevance
const similarity = context.embeddingsSimilarities.get(tool.name) || 0.5;
if (similarity >= 0.7) {
indicators.push('Sehr gute semantische Übereinstimmung mit Ihrer Anfrage');
}
// Documentation availability
if (tool.knowledgebase === true) {
indicators.push('Umfassende Dokumentation und Wissensbasis verfügbar');
}
// Immediate availability
if (isToolHosted(tool)) {
indicators.push('Sofort verfügbar über gehostete Lösung');
}
// Balanced skill requirements
if (tool.skillLevel === 'intermediate' || tool.skillLevel === 'advanced') {
indicators.push('Ausgewogenes Verhältnis zwischen Funktionalität und Benutzerfreundlichkeit');
}
// Method-query alignment
if (tool.type === 'method' && /methodik|vorgehen|prozess|ansatz/i.test(context.userQuery)) {
indicators.push('Methodischer Ansatz passt zu Ihrer prozeduralen Anfrage');
}
return indicators.slice(0, 4);
}
calculateSelectionConfidence(result: any, candidateCount: number): number {
if (!result?.selectedTools) {
console.log('[CONFIDENCE-SCORING] No selected tools for confidence calculation');
return 30;
}
const selectionRatio = result.selectedTools.length / candidateCount;
const hasReasoning = result.reasoning && result.reasoning.length > 50;
let confidence = 60;
// Selection ratio scoring
if (selectionRatio > 0.05 && selectionRatio < 0.3) confidence += 20;
else if (selectionRatio <= 0.05) confidence -= 10;
else confidence -= 15;
// Quality indicators
if (hasReasoning) confidence += 15;
if (result.selectedConcepts?.length > 0) confidence += 5;
const finalConfidence = Math.min(95, Math.max(25, confidence));
console.log('[CONFIDENCE-SCORING] Selection confidence calculated:', {
candidateCount,
selectedCount: result.selectedTools.length,
selectionRatio: selectionRatio.toFixed(3),
hasReasoning,
confidence: finalConfidence
});
return finalConfidence;
}
getConfidenceLevel(confidence: number): 'weak' | 'moderate' | 'strong' {
if (confidence >= this.config.highThreshold) return 'strong';
if (confidence >= this.config.mediumThreshold) return 'moderate';
return 'weak';
}
getConfidenceColor(confidence: number): string {
if (confidence >= this.config.highThreshold) return 'var(--color-accent)';
if (confidence >= this.config.mediumThreshold) return 'var(--color-warning)';
return 'var(--color-error)';
}
getConfig(): ConfidenceConfig {
return { ...this.config };
}
}
export const confidenceScoring = new ConfidenceScoring();

327
src/utils/embeddings.ts
View File

@ -1,11 +1,11 @@
// src/utils/embeddings.ts // src/utils/embeddings.ts - Refactored
import { promises as fs } from 'fs'; import { promises as fs } from 'fs';
import path from 'path'; import path from 'path';
import { getCompressedToolsDataForAI } from './dataService.js'; import { getCompressedToolsDataForAI } from './dataService.js';
import 'dotenv/config'; import 'dotenv/config';
import crypto from 'crypto'; import crypto from 'crypto';
interface EmbeddingData { export interface EmbeddingData {
id: string; id: string;
type: 'tool' | 'concept'; type: 'tool' | 'concept';
name: string; name: string;
@ -20,14 +20,23 @@ interface EmbeddingData {
}; };
} }
export interface SimilarityResult extends EmbeddingData {
similarity: number;
}
interface EmbeddingsDatabase { interface EmbeddingsDatabase {
version: string; version: string;
lastUpdated: number; lastUpdated: number;
embeddings: EmbeddingData[]; embeddings: EmbeddingData[];
} }
interface SimilarityResult extends EmbeddingData { interface EmbeddingsConfig {
similarity: number; enabled: boolean;
endpoint?: string;
apiKey?: string;
model?: string;
batchSize: number;
batchDelay: number;
} }
class EmbeddingsService { class EmbeddingsService {
@ -35,48 +44,33 @@ class EmbeddingsService {
private isInitialized = false; private isInitialized = false;
private initializationPromise: Promise<void> | null = null; private initializationPromise: Promise<void> | null = null;
private readonly embeddingsPath = path.join(process.cwd(), 'data', 'embeddings.json'); private readonly embeddingsPath = path.join(process.cwd(), 'data', 'embeddings.json');
private readonly batchSize: number; private config: EmbeddingsConfig;
private readonly batchDelay: number;
private enabled: boolean = false;
constructor() { constructor() {
this.batchSize = parseInt(process.env.AI_EMBEDDINGS_BATCH_SIZE || '20', 10); this.config = this.loadConfig();
this.batchDelay = parseInt(process.env.AI_EMBEDDINGS_BATCH_DELAY_MS || '1000', 10); console.log('[EMBEDDINGS-SERVICE] Initialized:', {
enabled: this.config.enabled,
this.enabled = true; hasEndpoint: !!this.config.endpoint,
hasModel: !!this.config.model
});
} }
private async checkEnabledStatus(): Promise<void> { private loadConfig(): EmbeddingsConfig {
try { const enabled = process.env.AI_EMBEDDINGS_ENABLED === 'true';
const envEnabled = process.env.AI_EMBEDDINGS_ENABLED; const endpoint = process.env.AI_EMBEDDINGS_ENDPOINT;
const apiKey = process.env.AI_EMBEDDINGS_API_KEY;
const model = process.env.AI_EMBEDDINGS_MODEL;
const batchSize = parseInt(process.env.AI_EMBEDDINGS_BATCH_SIZE || '20', 10);
const batchDelay = parseInt(process.env.AI_EMBEDDINGS_BATCH_DELAY_MS || '1000', 10);
if (envEnabled === 'true') { return {
const endpoint = process.env.AI_EMBEDDINGS_ENDPOINT; enabled,
const model = process.env.AI_EMBEDDINGS_MODEL; endpoint,
apiKey,
if (!endpoint || !model) { model,
console.warn('[EMBEDDINGS] Embeddings enabled but API configuration missing - disabling'); batchSize,
this.enabled = false; batchDelay
return; };
}
console.log('[EMBEDDINGS] All requirements met - enabling embeddings');
this.enabled = true;
return;
}
try {
await fs.stat(this.embeddingsPath);
console.log('[EMBEDDINGS] Existing embeddings file found - enabling');
this.enabled = true;
} catch {
console.log('[EMBEDDINGS] Embeddings not explicitly enabled - disabling');
this.enabled = false;
}
} catch (error) {
console.error('[EMBEDDINGS] Error checking enabled status:', error);
this.enabled = false;
}
} }
async initialize(): Promise<void> { async initialize(): Promise<void> {
@ -93,46 +87,43 @@ class EmbeddingsService {
} }
private async performInitialization(): Promise<void> { private async performInitialization(): Promise<void> {
await this.checkEnabledStatus();
if (!this.enabled) {
console.log('[EMBEDDINGS] Embeddings disabled, skipping initialization');
return;
}
const initStart = Date.now(); const initStart = Date.now();
try { try {
console.log('[EMBEDDINGS] Initializing embeddings system…'); console.log('[EMBEDDINGS-SERVICE] Starting initialization');
if (!this.config.enabled) {
console.log('[EMBEDDINGS-SERVICE] Service disabled via configuration');
return;
}
await fs.mkdir(path.dirname(this.embeddingsPath), { recursive: true }); await fs.mkdir(path.dirname(this.embeddingsPath), { recursive: true });
const toolsData = await getCompressedToolsDataForAI(); const toolsData = await getCompressedToolsDataForAI();
const currentDataHash = await this.hashToolsFile(); const currentDataHash = await this.hashToolsFile();
const existing = await this.loadEmbeddings(); const existing = await this.loadEmbeddings();
console.log('[EMBEDDINGS] Current hash:', currentDataHash);
console.log('[EMBEDDINGS] Existing file version:', existing?.version);
console.log('[EMBEDDINGS] Existing embeddings length:', existing?.embeddings?.length);
const cacheIsUsable = const cacheIsUsable = existing &&
existing &&
existing.version === currentDataHash && existing.version === currentDataHash &&
Array.isArray(existing.embeddings) && Array.isArray(existing.embeddings) &&
existing.embeddings.length > 0; existing.embeddings.length > 0;
if (cacheIsUsable) { if (cacheIsUsable) {
console.log('[EMBEDDINGS] Using cached embeddings'); console.log('[EMBEDDINGS-SERVICE] Using cached embeddings');
this.embeddings = existing.embeddings; this.embeddings = existing.embeddings;
} else { } else {
console.log('[EMBEDDINGS] Generating new embeddings…'); console.log('[EMBEDDINGS-SERVICE] Generating new embeddings');
await this.generateEmbeddings(toolsData, currentDataHash); await this.generateEmbeddings(toolsData, currentDataHash);
} }
this.isInitialized = true; this.isInitialized = true;
console.log(`[EMBEDDINGS] Initialized with ${this.embeddings.length} embeddings in ${Date.now() - initStart} ms`); console.log(`[EMBEDDINGS-SERVICE] Initialized successfully with ${this.embeddings.length} embeddings in ${Date.now() - initStart}ms`);
} catch (err) {
console.error('[EMBEDDINGS] Failed to initialize:', err); } catch (error) {
console.error('[EMBEDDINGS-SERVICE] Initialization failed:', error);
this.isInitialized = false; this.isInitialized = false;
throw err; throw error;
} finally { } finally {
this.initializationPromise = null; this.initializationPromise = null;
} }
@ -140,7 +131,7 @@ class EmbeddingsService {
private async hashToolsFile(): Promise<string> { private async hashToolsFile(): Promise<string> {
const file = path.join(process.cwd(), 'src', 'data', 'tools.yaml'); const file = path.join(process.cwd(), 'src', 'data', 'tools.yaml');
const raw = await fs.readFile(file, 'utf8'); const raw = await fs.readFile(file, 'utf8');
return crypto.createHash('sha256').update(raw).digest('hex'); return crypto.createHash('sha256').update(raw).digest('hex');
} }
@ -149,7 +140,7 @@ class EmbeddingsService {
const data = await fs.readFile(this.embeddingsPath, 'utf8'); const data = await fs.readFile(this.embeddingsPath, 'utf8');
return JSON.parse(data); return JSON.parse(data);
} catch (error) { } catch (error) {
console.log('[EMBEDDINGS] No existing embeddings found'); console.log('[EMBEDDINGS-SERVICE] No existing embeddings file found');
return null; return null;
} }
} }
@ -162,7 +153,7 @@ class EmbeddingsService {
}; };
await fs.writeFile(this.embeddingsPath, JSON.stringify(database, null, 2)); await fs.writeFile(this.embeddingsPath, JSON.stringify(database, null, 2));
console.log(`[EMBEDDINGS] Saved ${this.embeddings.length} embeddings to disk`); console.log(`[EMBEDDINGS-SERVICE] Saved ${this.embeddings.length} embeddings to disk`);
} }
private createContentString(item: any): string { private createContentString(item: any): string {
@ -178,30 +169,23 @@ class EmbeddingsService {
} }
private async generateEmbeddingsBatch(contents: string[]): Promise<number[][]> { private async generateEmbeddingsBatch(contents: string[]): Promise<number[][]> {
const endpoint = process.env.AI_EMBEDDINGS_ENDPOINT; if (!this.config.endpoint || !this.config.model) {
const apiKey = process.env.AI_EMBEDDINGS_API_KEY; throw new Error('Missing embeddings API configuration');
const model = process.env.AI_EMBEDDINGS_MODEL;
if (!endpoint || !model) {
const missing: string[] = [];
if (!endpoint) missing.push('AI_EMBEDDINGS_ENDPOINT');
if (!model) missing.push('AI_EMBEDDINGS_MODEL');
throw new Error(`Missing embeddings API configuration: ${missing.join(', ')}`);
} }
const headers: Record<string, string> = { const headers: Record<string, string> = {
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}; };
if (apiKey) { if (this.config.apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`; headers['Authorization'] = `Bearer ${this.config.apiKey}`;
} }
const response = await fetch(endpoint, { const response = await fetch(this.config.endpoint, {
method: 'POST', method: 'POST',
headers, headers,
body: JSON.stringify({ body: JSON.stringify({
model, model: this.config.model,
input: contents input: contents
}) })
}); });
@ -233,11 +217,16 @@ class EmbeddingsService {
const contents = allItems.map(item => this.createContentString(item)); const contents = allItems.map(item => this.createContentString(item));
this.embeddings = []; this.embeddings = [];
for (let i = 0; i < contents.length; i += this.batchSize) { console.log(`[EMBEDDINGS-SERVICE] Generating embeddings for ${contents.length} items`);
const batch = contents.slice(i, i + this.batchSize);
const batchItems = allItems.slice(i, i + this.batchSize);
console.log(`[EMBEDDINGS] Processing batch ${Math.ceil((i + 1) / this.batchSize)} of ${Math.ceil(contents.length / this.batchSize)}`); for (let i = 0; i < contents.length; i += this.config.batchSize) {
const batch = contents.slice(i, i + this.config.batchSize);
const batchItems = allItems.slice(i, i + this.config.batchSize);
const batchNumber = Math.ceil((i + 1) / this.config.batchSize);
const totalBatches = Math.ceil(contents.length / this.config.batchSize);
console.log(`[EMBEDDINGS-SERVICE] Processing batch ${batchNumber}/${totalBatches}`);
try { try {
const embeddings = await this.generateEmbeddingsBatch(batch); const embeddings = await this.generateEmbeddingsBatch(batch);
@ -260,12 +249,12 @@ class EmbeddingsService {
}); });
}); });
if (i + this.batchSize < contents.length) { if (i + this.config.batchSize < contents.length) {
await new Promise(resolve => setTimeout(resolve, this.batchDelay)); await new Promise(resolve => setTimeout(resolve, this.config.batchDelay));
} }
} catch (error) { } catch (error) {
console.error(`[EMBEDDINGS] Failed to process batch ${Math.ceil((i + 1) / this.batchSize)}:`, error); console.error(`[EMBEDDINGS-SERVICE] Batch ${batchNumber} failed:`, error);
throw error; throw error;
} }
} }
@ -273,18 +262,21 @@ class EmbeddingsService {
await this.saveEmbeddings(version); await this.saveEmbeddings(version);
} }
public async embedText(text: string): Promise<number[]> { async embedText(text: string): Promise<number[]> {
if (!this.enabled || !this.isInitialized) { if (!this.isEnabled() || !this.isInitialized) {
throw new Error('Embeddings service not available'); throw new Error('Embeddings service not available');
} }
const [embedding] = await this.generateEmbeddingsBatch([text.toLowerCase()]); const [embedding] = await this.generateEmbeddingsBatch([text.toLowerCase()]);
return embedding; return embedding;
} }
async waitForInitialization(): Promise<void> { async waitForInitialization(): Promise<void> {
await this.checkEnabledStatus(); if (!this.config.enabled) {
return Promise.resolve();
}
if (!this.enabled || this.isInitialized) { if (this.isInitialized) {
return Promise.resolve(); return Promise.resolve();
} }
@ -296,13 +288,6 @@ class EmbeddingsService {
return this.initialize(); return this.initialize();
} }
async forceRecheckEnvironment(): Promise<void> {
this.enabled = false;
this.isInitialized = false;
await this.checkEnabledStatus();
console.log('[EMBEDDINGS] Environment status re-checked, enabled:', this.enabled);
}
private cosineSimilarity(a: number[], b: number[]): number { private cosineSimilarity(a: number[], b: number[]): number {
let dotProduct = 0; let dotProduct = 0;
let normA = 0; let normA = 0;
@ -318,145 +303,67 @@ class EmbeddingsService {
} }
async findSimilar(query: string, maxResults: number = 30, threshold: number = 0.3): Promise<SimilarityResult[]> { async findSimilar(query: string, maxResults: number = 30, threshold: number = 0.3): Promise<SimilarityResult[]> {
if (!this.enabled) { if (!this.config.enabled) {
console.log('[EMBEDDINGS] Service disabled for similarity search'); console.log('[EMBEDDINGS-SERVICE] Service disabled, returning empty results');
return [];
}
if (!this.isInitialized || this.embeddings.length === 0) {
console.log('[EMBEDDINGS-SERVICE] Not initialized or no embeddings available');
return []; return [];
} }
try { try {
if (this.isInitialized && this.embeddings.length > 0) { console.log(`[EMBEDDINGS-SERVICE] Finding similar items for query: "${query}"`);
console.log(`[EMBEDDINGS] Using embeddings data for similarity search: ${query}`);
const queryEmbeddings = await this.generateEmbeddingsBatch([query.toLowerCase()]); const queryEmbeddings = await this.generateEmbeddingsBatch([query.toLowerCase()]);
const queryEmbedding = queryEmbeddings[0]; const queryEmbedding = queryEmbeddings[0];
console.log(`[EMBEDDINGS] Computing similarities for ${this.embeddings.length} items`); const similarities: SimilarityResult[] = this.embeddings.map(item => ({
...item,
similarity: this.cosineSimilarity(queryEmbedding, item.embedding)
}));
const similarities: SimilarityResult[] = this.embeddings.map(item => ({ const topScore = Math.max(...similarities.map(s => s.similarity));
...item, const dynamicThreshold = Math.max(threshold, topScore * 0.85);
similarity: this.cosineSimilarity(queryEmbedding, item.embedding)
}));
const topScore = Math.max(...similarities.map(s => s.similarity)); const results = similarities
const dynamicCutOff = Math.max(threshold, topScore * 0.85); .filter(item => item.similarity >= dynamicThreshold)
.sort((a, b) => b.similarity - a.similarity)
.slice(0, maxResults);
const results = similarities console.log(`[EMBEDDINGS-SERVICE] Found ${results.length} similar items (threshold: ${dynamicThreshold.toFixed(3)})`);
.filter(item => item.similarity >= dynamicCutOff)
.sort((a, b) => b.similarity - a.similarity)
.slice(0, maxResults);
if (results.length > 0) {
const orderingValid = results.every((item, index) => { console.log('[EMBEDDINGS-SERVICE] Top 5 matches:');
if (index === 0) return true; results.slice(0, 5).forEach((item, idx) => {
return item.similarity <= results[index - 1].similarity; console.log(` ${idx + 1}. ${item.name} (${item.type}) = ${item.similarity.toFixed(4)}`);
}); });
if (!orderingValid) {
console.error('[EMBEDDINGS] CRITICAL: Similarity ordering is broken!');
results.forEach((item, idx) => {
console.error(` ${idx}: ${item.name} = ${item.similarity.toFixed(4)}`);
});
}
console.log(`[EMBEDDINGS] Found ${results.length} similar items (threshold: ${threshold})`);
if (results.length > 0) {
console.log('[EMBEDDINGS] Top 10 similarity matches:');
results.slice(0, 10).forEach((item, idx) => {
console.log(` ${idx + 1}. ${item.name} (${item.type}) = ${item.similarity.toFixed(4)}`);
});
const topSimilarity = results[0].similarity;
const hasHigherSimilarity = results.some(item => item.similarity > topSimilarity);
if (hasHigherSimilarity) {
console.error('[EMBEDDINGS] CRITICAL: Top result is not actually the highest similarity!');
}
}
return results;
} else {
console.log(`[EMBEDDINGS] No embeddings data, using fallback text matching: ${query}`);
const { getToolsData } = await import('./dataService.js');
const toolsData = await getToolsData();
const queryLower = query.toLowerCase();
const queryWords = queryLower.split(/\s+/).filter(w => w.length > 2);
const similarities: SimilarityResult[] = toolsData.tools
.map((tool: any) => {
let similarity = 0;
if (tool.name.toLowerCase().includes(queryLower)) {
similarity += 0.8;
}
if (tool.description && tool.description.toLowerCase().includes(queryLower)) {
similarity += 0.6;
}
if (tool.tags && Array.isArray(tool.tags)) {
const matchingTags = tool.tags.filter((tag: string) =>
tag.toLowerCase().includes(queryLower) || queryLower.includes(tag.toLowerCase())
);
if (tool.tags.length > 0) {
similarity += (matchingTags.length / tool.tags.length) * 0.4;
}
}
const toolText = `${tool.name} ${tool.description || ''} ${(tool.tags || []).join(' ')}`.toLowerCase();
const matchingWords = queryWords.filter(word => toolText.includes(word));
if (queryWords.length > 0) {
similarity += (matchingWords.length / queryWords.length) * 0.3;
}
return {
id: `tool_${tool.name.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase()}`,
type: 'tool' as const,
name: tool.name,
content: toolText,
embedding: [],
metadata: {
domains: tool.domains || [],
phases: tool.phases || [],
tags: tool.tags || [],
skillLevel: tool.skillLevel,
type: tool.type
},
similarity: Math.min(similarity, 1.0)
};
})
.filter(item => item.similarity >= threshold)
.sort((a, b) => b.similarity - a.similarity)
.slice(0, maxResults);
console.log(`[EMBEDDINGS] Fallback found ${similarities.length} similar items`);
return similarities;
} }
return results;
} catch (error) { } catch (error) {
console.error('[EMBEDDINGS] Failed to find similar items:', error); console.error('[EMBEDDINGS-SERVICE] Similarity search failed:', error);
return []; return [];
} }
} }
isEnabled(): boolean { isEnabled(): boolean {
if (!this.enabled && !this.isInitialized) { return this.config.enabled;
this.checkEnabledStatus().catch(console.error);
}
return this.enabled;
} }
getStats(): { enabled: boolean; initialized: boolean; count: number } { getStats(): { enabled: boolean; initialized: boolean; count: number } {
return { return {
enabled: this.enabled, enabled: this.config.enabled,
initialized: this.isInitialized, initialized: this.isInitialized,
count: this.embeddings.length count: this.embeddings.length
}; };
} }
getConfig(): EmbeddingsConfig {
return { ...this.config };
}
} }
const embeddingsService = new EmbeddingsService(); export const embeddingsService = new EmbeddingsService();
export { embeddingsService, type EmbeddingData, type SimilarityResult };

132
src/utils/jsonUtils.ts Normal file
View File

@ -0,0 +1,132 @@
// src/utils/jsonUtils.ts
export class JSONParser {
static safeParseJSON(jsonString: string, fallback: any = null): any {
try {
let cleaned = jsonString.trim();
// Remove code block markers
const jsonBlockPatterns = [
/```json\s*([\s\S]*?)\s*```/i,
/```\s*([\s\S]*?)\s*```/i,
/\{[\s\S]*\}/,
];
for (const pattern of jsonBlockPatterns) {
const match = cleaned.match(pattern);
if (match) {
cleaned = match[1] || match[0];
break;
}
}
// Handle truncated JSON
if (!cleaned.endsWith('}') && !cleaned.endsWith(']')) {
console.warn('[JSON-PARSER] JSON appears truncated, attempting recovery');
cleaned = this.repairTruncatedJSON(cleaned);
}
const parsed = JSON.parse(cleaned);
// Ensure proper structure for tool selection responses
if (parsed && typeof parsed === 'object') {
if (!parsed.selectedTools) parsed.selectedTools = [];
if (!parsed.selectedConcepts) parsed.selectedConcepts = [];
if (!Array.isArray(parsed.selectedTools)) parsed.selectedTools = [];
if (!Array.isArray(parsed.selectedConcepts)) parsed.selectedConcepts = [];
}
return parsed;
} catch (error) {
console.warn('[JSON-PARSER] JSON parsing failed:', error.message);
return fallback;
}
}
private static repairTruncatedJSON(cleaned: string): string {
let braceCount = 0;
let bracketCount = 0;
let inString = false;
let escaped = false;
let lastCompleteStructure = '';
for (let i = 0; i < cleaned.length; i++) {
const char = cleaned[i];
if (escaped) {
escaped = false;
continue;
}
if (char === '\\') {
escaped = true;
continue;
}
if (char === '"' && !escaped) {
inString = !inString;
continue;
}
if (!inString) {
if (char === '{') braceCount++;
if (char === '}') braceCount--;
if (char === '[') bracketCount++;
if (char === ']') bracketCount--;
if (braceCount === 0 && bracketCount === 0 && (char === '}' || char === ']')) {
lastCompleteStructure = cleaned.substring(0, i + 1);
}
}
}
if (lastCompleteStructure) {
return lastCompleteStructure;
} else {
if (braceCount > 0) cleaned += '}';
if (bracketCount > 0) cleaned += ']';
return cleaned;
}
}
static extractToolsFromMalformedJSON(jsonString: string): { selectedTools: string[]; selectedConcepts: string[] } {
const selectedTools: string[] = [];
const selectedConcepts: string[] = [];
const toolsMatch = jsonString.match(/"selectedTools"\s*:\s*\[([\s\S]*?)\]/i);
if (toolsMatch) {
const toolMatches = toolsMatch[1].match(/"([^"]+)"/g);
if (toolMatches) {
selectedTools.push(...toolMatches.map(match => match.replace(/"/g, '')));
}
}
const conceptsMatch = jsonString.match(/"selectedConcepts"\s*:\s*\[([\s\S]*?)\]/i);
if (conceptsMatch) {
const conceptMatches = conceptsMatch[1].match(/"([^"]+)"/g);
if (conceptMatches) {
selectedConcepts.push(...conceptMatches.map(match => match.replace(/"/g, '')));
}
}
// Fallback: extract any quoted strings that look like tool names
if (selectedTools.length === 0 && selectedConcepts.length === 0) {
const allMatches = jsonString.match(/"([^"]+)"/g);
if (allMatches) {
const possibleNames = allMatches
.map(match => match.replace(/"/g, ''))
.filter(name =>
name.length > 2 &&
!['selectedTools', 'selectedConcepts', 'reasoning'].includes(name) &&
!name.includes(':') &&
!name.match(/^\d+$/)
)
.slice(0, 15);
selectedTools.push(...possibleNames);
}
}
return { selectedTools, selectedConcepts };
}
}

458
src/utils/toolSelector.ts Normal file
View File

@ -0,0 +1,458 @@
// src/utils/toolSelector.ts
import { aiService } from './aiService.js';
import { embeddingsService, type SimilarityResult } from './embeddings.js';
import { confidenceScoring } from './confidenceScoring.js';
import { getPrompt } from '../config/prompts.js';
import 'dotenv/config';
export interface ToolSelectionConfig {
maxSelectedItems: number;
embeddingCandidates: number;
similarityThreshold: number;
embeddingSelectionLimit: number;
embeddingConceptsLimit: number;
noEmbeddingsToolLimit: number;
noEmbeddingsConceptLimit: number;
embeddingsMinTools: number;
embeddingsMaxReductionRatio: number;
methodSelectionRatio: number;
softwareSelectionRatio: number;
}
export interface SelectionContext {
userQuery: string;
mode: string;
embeddingsSimilarities: Map<string, number>;
seenToolNames: Set<string>;
selectedTools?: Array<{
tool: any;
phase: string;
priority: string;
justification?: string;
taskRelevance?: number;
limitations?: string[];
}>;
}
export interface ToolSelectionResult {
selectedTools: any[];
selectedConcepts: any[];
selectionMethod: string;
confidence: number;
}
class ToolSelector {
private config: ToolSelectionConfig;
constructor() {
this.config = {
maxSelectedItems: this.getEnvInt('AI_MAX_SELECTED_ITEMS', 25),
embeddingCandidates: this.getEnvInt('AI_EMBEDDING_CANDIDATES', 50),
similarityThreshold: this.getEnvFloat('AI_SIMILARITY_THRESHOLD', 0.3),
embeddingSelectionLimit: this.getEnvInt('AI_EMBEDDING_SELECTION_LIMIT', 30),
embeddingConceptsLimit: this.getEnvInt('AI_EMBEDDING_CONCEPTS_LIMIT', 15),
noEmbeddingsToolLimit: this.getEnvInt('AI_NO_EMBEDDINGS_TOOL_LIMIT', 25),
noEmbeddingsConceptLimit: this.getEnvInt('AI_NO_EMBEDDINGS_CONCEPT_LIMIT', 10),
embeddingsMinTools: this.getEnvInt('AI_EMBEDDINGS_MIN_TOOLS', 8),
embeddingsMaxReductionRatio: this.getEnvFloat('AI_EMBEDDINGS_MAX_REDUCTION_RATIO', 0.75),
methodSelectionRatio: this.getEnvFloat('AI_METHOD_SELECTION_RATIO', 0.4),
softwareSelectionRatio: this.getEnvFloat('AI_SOFTWARE_SELECTION_RATIO', 0.5)
};
console.log('[TOOL-SELECTOR] Initialized with config:', this.config);
}
private getEnvInt(key: string, defaultValue: number): number {
const value = process.env[key];
return value ? parseInt(value, 10) : defaultValue;
}
private getEnvFloat(key: string, defaultValue: number): number {
const value = process.env[key];
return value ? parseFloat(value) : defaultValue;
}
async getIntelligentCandidates(
userQuery: string,
toolsData: any,
mode: string,
context: SelectionContext
): Promise<{
tools: any[];
concepts: any[];
domains: any[];
phases: any[];
'domain-agnostic-software': any[];
selectionMethod: string;
}> {
console.log('[TOOL-SELECTOR] Getting intelligent candidates for query');
let candidateTools: any[] = [];
let candidateConcepts: any[] = [];
let selectionMethod = 'unknown';
context.embeddingsSimilarities.clear();
try {
await embeddingsService.waitForInitialization();
} catch (error) {
console.error('[TOOL-SELECTOR] Embeddings initialization failed:', error);
}
if (embeddingsService.isEnabled()) {
console.log('[TOOL-SELECTOR] Using embeddings for candidate selection');
const similarItems = await embeddingsService.findSimilar(
userQuery,
this.config.embeddingCandidates,
this.config.similarityThreshold
) as SimilarityResult[];
console.log('[TOOL-SELECTOR] Embeddings found', similarItems.length, 'similar items');
// Store similarities for confidence calculation
similarItems.forEach(item => {
context.embeddingsSimilarities.set(item.name, item.similarity);
});
const toolsMap = new Map(toolsData.tools.map((tool: any) => [tool.name, tool]));
const conceptsMap = new Map(toolsData.concepts.map((concept: any) => [concept.name, concept]));
const similarTools = similarItems
.filter((item: any) => item.type === 'tool')
.map((item: any) => toolsMap.get(item.name))
.filter((tool: any): tool is NonNullable<any> => tool !== undefined && tool !== null);
const similarConcepts = similarItems
.filter((item: any) => item.type === 'concept')
.map((item: any) => conceptsMap.get(item.name))
.filter((concept: any): concept is NonNullable<any> => concept !== undefined && concept !== null);
const totalAvailableTools = toolsData.tools.length;
const reductionRatio = similarTools.length / totalAvailableTools;
if (similarTools.length >= this.config.embeddingsMinTools && reductionRatio <= this.config.embeddingsMaxReductionRatio) {
candidateTools = similarTools;
candidateConcepts = similarConcepts;
selectionMethod = 'embeddings_candidates';
console.log('[TOOL-SELECTOR] Using embeddings filtering:', totalAvailableTools, '→', similarTools.length, 'tools');
} else {
console.log('[TOOL-SELECTOR] Embeddings filtering insufficient, using full dataset');
candidateTools = toolsData.tools;
candidateConcepts = toolsData.concepts;
selectionMethod = 'full_dataset';
}
} else {
console.log('[TOOL-SELECTOR] Embeddings disabled, using full dataset');
candidateTools = toolsData.tools;
candidateConcepts = toolsData.concepts;
selectionMethod = 'full_dataset';
}
const selection = await this.performAISelection(
userQuery,
candidateTools,
candidateConcepts,
mode,
selectionMethod,
context
);
return {
tools: selection.selectedTools,
concepts: selection.selectedConcepts,
domains: toolsData.domains,
phases: toolsData.phases,
'domain-agnostic-software': toolsData['domain-agnostic-software'],
selectionMethod
};
}
private async performAISelection(
userQuery: string,
candidateTools: any[],
candidateConcepts: any[],
mode: string,
selectionMethod: string,
context: SelectionContext
): Promise<ToolSelectionResult> {
console.log('[TOOL-SELECTOR] Performing AI selection');
const candidateMethods = candidateTools.filter((tool: any) => tool && tool.type === 'method');
const candidateSoftware = candidateTools.filter((tool: any) => tool && tool.type === 'software');
console.log('[TOOL-SELECTOR] Candidates:', candidateMethods.length, 'methods,', candidateSoftware.length, 'software,', candidateConcepts.length, 'concepts');
const methodsWithFullData = candidateMethods.map(this.createToolData);
const softwareWithFullData = candidateSoftware.map(this.createToolData);
const conceptsWithFullData = candidateConcepts.map(this.createConceptData);
let toolsToSend: any[];
let conceptsToSend: any[];
if (selectionMethod === 'embeddings_candidates') {
const totalLimit = this.config.embeddingSelectionLimit;
const methodLimit = Math.ceil(totalLimit * this.config.methodSelectionRatio);
const softwareLimit = Math.floor(totalLimit * this.config.softwareSelectionRatio);
toolsToSend = [
...methodsWithFullData.slice(0, methodLimit),
...softwareWithFullData.slice(0, softwareLimit)
];
const remainingCapacity = totalLimit - toolsToSend.length;
if (remainingCapacity > 0) {
if (methodsWithFullData.length > methodLimit) {
toolsToSend.push(...methodsWithFullData.slice(methodLimit, methodLimit + remainingCapacity));
} else if (softwareWithFullData.length > softwareLimit) {
toolsToSend.push(...softwareWithFullData.slice(softwareLimit, softwareLimit + remainingCapacity));
}
}
conceptsToSend = conceptsWithFullData.slice(0, this.config.embeddingConceptsLimit);
} else {
const maxTools = this.config.noEmbeddingsToolLimit;
const maxConcepts = this.config.noEmbeddingsConceptLimit;
const methodLimit = Math.ceil(maxTools * 0.4);
const softwareLimit = Math.floor(maxTools * 0.5);
toolsToSend = [
...methodsWithFullData.slice(0, methodLimit),
...softwareWithFullData.slice(0, softwareLimit)
];
const remainingCapacity = maxTools - toolsToSend.length;
if (remainingCapacity > 0) {
if (methodsWithFullData.length > methodLimit) {
toolsToSend.push(...methodsWithFullData.slice(methodLimit, methodLimit + remainingCapacity));
} else if (softwareWithFullData.length > softwareLimit) {
toolsToSend.push(...softwareWithFullData.slice(softwareLimit, softwareLimit + remainingCapacity));
}
}
conceptsToSend = conceptsWithFullData.slice(0, maxConcepts);
}
const basePrompt = getPrompt('toolSelection', mode, userQuery, selectionMethod, this.config.maxSelectedItems);
const prompt = getPrompt('toolSelectionWithData', basePrompt, toolsToSend, conceptsToSend);
// Validate prompt length
aiService.validatePromptLength(prompt);
console.log('[TOOL-SELECTOR] Sending to AI:',
toolsToSend.filter((t: any) => t.type === 'method').length, 'methods,',
toolsToSend.filter((t: any) => t.type === 'software').length, 'software,',
conceptsToSend.length, 'concepts'
);
try {
const response = await aiService.callAI(prompt, { maxTokens: 2500 });
const result = this.safeParseJSON(response.content, null);
if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) {
console.error('[TOOL-SELECTOR] AI selection returned invalid structure');
throw new Error('AI selection failed to return valid tool and concept selection');
}
const totalSelected = result.selectedTools.length + result.selectedConcepts.length;
if (totalSelected === 0) {
throw new Error('AI selection returned empty selection');
}
const toolsMap = new Map(candidateTools.map((tool: any) => [tool.name, tool]));
const conceptsMap = new Map(candidateConcepts.map((concept: any) => [concept.name, concept]));
const selectedTools = result.selectedTools
.map((name: string) => toolsMap.get(name))
.filter((tool: any): tool is NonNullable<any> => tool !== undefined && tool !== null);
const selectedConcepts = result.selectedConcepts
.map((name: string) => conceptsMap.get(name))
.filter((concept: any): concept is NonNullable<any> => concept !== undefined && concept !== null);
const selectedMethods = selectedTools.filter((t: any) => t && t.type === 'method');
const selectedSoftware = selectedTools.filter((t: any) => t && t.type === 'software');
console.log('[TOOL-SELECTOR] AI selected:', selectedMethods.length, 'methods,', selectedSoftware.length, 'software,', selectedConcepts.length, 'concepts');
const confidence = confidenceScoring.calculateSelectionConfidence(result, candidateTools.length + candidateConcepts.length);
return {
selectedTools,
selectedConcepts,
selectionMethod,
confidence
};
} catch (error) {
console.error('[TOOL-SELECTOR] AI selection failed:', error);
throw error;
}
}
async selectToolsForPhase(
userQuery: string,
phase: any,
availableTools: any[],
context: SelectionContext
): Promise<Array<{
toolName: string;
taskRelevance: number;
justification: string;
limitations: string[];
}>> {
console.log('[TOOL-SELECTOR] Selecting tools for phase:', phase.id);
if (availableTools.length === 0) {
console.log('[TOOL-SELECTOR] No tools available for phase:', phase.id);
return [];
}
const prompt = getPrompt('phaseToolSelection', userQuery, phase, availableTools);
try {
const response = await aiService.callMicroTaskAI(prompt, 1000);
const selections = this.safeParseJSON(response.content, []);
if (Array.isArray(selections)) {
const validSelections = selections.filter((sel: any) => {
const matchingTool = availableTools.find((tool: any) => tool && tool.name === sel.toolName);
if (!matchingTool) {
console.warn('[TOOL-SELECTOR] Invalid tool selection for phase:', phase.id, sel.toolName);
}
return !!matchingTool;
});
console.log('[TOOL-SELECTOR] Valid selections for phase:', phase.id, validSelections.length);
return validSelections;
}
return [];
} catch (error) {
console.error('[TOOL-SELECTOR] Phase tool selection failed:', error);
return [];
}
}
private createToolData = (tool: any) => ({
name: tool.name,
type: tool.type,
description: tool.description,
domains: tool.domains,
phases: tool.phases,
platforms: tool.platforms || [],
tags: tool.tags || [],
skillLevel: tool.skillLevel,
license: tool.license,
accessType: tool.accessType,
projectUrl: tool.projectUrl,
knowledgebase: tool.knowledgebase,
related_concepts: tool.related_concepts || [],
related_software: tool.related_software || []
});
private createConceptData = (concept: any) => ({
name: concept.name,
type: 'concept',
description: concept.description,
domains: concept.domains,
phases: concept.phases,
tags: concept.tags || [],
skillLevel: concept.skillLevel,
related_concepts: concept.related_concepts || [],
related_software: concept.related_software || []
});
private safeParseJSON(jsonString: string, fallback: any = null): any {
try {
let cleaned = jsonString.trim();
// Remove code block markers
const jsonBlockPatterns = [
/```json\s*([\s\S]*?)\s*```/i,
/```\s*([\s\S]*?)\s*```/i,
/\{[\s\S]*\}/,
];
for (const pattern of jsonBlockPatterns) {
const match = cleaned.match(pattern);
if (match) {
cleaned = match[1] || match[0];
break;
}
}
// Handle truncated JSON
if (!cleaned.endsWith('}') && !cleaned.endsWith(']')) {
console.warn('[TOOL-SELECTOR] JSON appears truncated, attempting recovery');
let braceCount = 0;
let bracketCount = 0;
let inString = false;
let escaped = false;
let lastCompleteStructure = '';
for (let i = 0; i < cleaned.length; i++) {
const char = cleaned[i];
if (escaped) {
escaped = false;
continue;
}
if (char === '\\') {
escaped = true;
continue;
}
if (char === '"' && !escaped) {
inString = !inString;
continue;
}
if (!inString) {
if (char === '{') braceCount++;
if (char === '}') braceCount--;
if (char === '[') bracketCount++;
if (char === ']') bracketCount--;
if (braceCount === 0 && bracketCount === 0 && (char === '}' || char === ']')) {
lastCompleteStructure = cleaned.substring(0, i + 1);
}
}
}
if (lastCompleteStructure) {
cleaned = lastCompleteStructure;
} else {
if (braceCount > 0) cleaned += '}';
if (bracketCount > 0) cleaned += ']';
}
}
const parsed = JSON.parse(cleaned);
// Ensure proper structure
if (parsed && typeof parsed === 'object') {
if (!parsed.selectedTools) parsed.selectedTools = [];
if (!parsed.selectedConcepts) parsed.selectedConcepts = [];
if (!Array.isArray(parsed.selectedTools)) parsed.selectedTools = [];
if (!Array.isArray(parsed.selectedConcepts)) parsed.selectedConcepts = [];
}
return parsed;
} catch (error) {
console.warn('[TOOL-SELECTOR] JSON parsing failed:', error.message);
return fallback;
}
}
getConfig(): ToolSelectionConfig {
return { ...this.config };
}
}
export const toolSelector = new ToolSelector();