first iteration - buggy
This commit is contained in:
overcuriousity
parent
1d98dd3257
commit
0c7c502b03
File diff suppressed because it is too large
Load Diff
@ -1,17 +1,19 @@
|
||||
// src/pages/api/ai/embeddings-status.ts
|
||||
// src/pages/api/ai/embeddings-status.ts - Updated
|
||||
import type { APIRoute } from 'astro';
|
||||
import { embeddingsService } from '../../../utils/embeddings.js';
|
||||
|
||||
export const prerender = false;
|
||||
|
||||
export const GET: APIRoute = async () => {
|
||||
try {
|
||||
const { embeddingsService } = await import('../../../utils/embeddings.js');
|
||||
await embeddingsService.waitForInitialization();
|
||||
|
||||
const stats = embeddingsService.getStats();
|
||||
const status = stats.enabled && stats.initialized ? 'ready' :
|
||||
stats.enabled && !stats.initialized ? 'initializing' : 'disabled';
|
||||
|
||||
console.log(`[EMBEDDINGS-STATUS-API] Service status: ${status}, stats:`, stats);
|
||||
|
||||
return new Response(JSON.stringify({
|
||||
success: true,
|
||||
embeddings: stats,
|
||||
@ -23,6 +25,8 @@ export const GET: APIRoute = async () => {
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('[EMBEDDINGS-STATUS-API] Error checking embeddings status:', error);
|
||||
|
||||
return new Response(JSON.stringify({
|
||||
success: false,
|
||||
embeddings: { enabled: false, initialized: false, count: 0 },
|
||||
|
||||
@ -1,23 +1,13 @@
|
||||
// src/pages/api/ai/enhance-input.ts - Enhanced AI service compatibility
|
||||
// src/pages/api/ai/enhance-input.ts - Updated to use refactored services
|
||||
import type { APIRoute } from 'astro';
|
||||
import { withAPIAuth } from '../../../utils/auth.js';
|
||||
import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js';
|
||||
import { enqueueApiCall } from '../../../utils/rateLimitedQueue.js';
|
||||
import { aiService } from '../../../utils/aiService.js';
|
||||
import { JSONParser } from '../../../utils/jsonUtils.js';
|
||||
|
||||
export const prerender = false;
|
||||
|
||||
function getEnv(key: string): string {
|
||||
const value = process.env[key];
|
||||
if (!value) {
|
||||
throw new Error(`Missing environment variable: ${key}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
const AI_ENDPOINT = getEnv('AI_ANALYZER_ENDPOINT');
|
||||
const AI_ANALYZER_API_KEY = getEnv('AI_ANALYZER_API_KEY');
|
||||
const AI_ANALYZER_MODEL = getEnv('AI_ANALYZER_MODEL');
|
||||
|
||||
const rateLimitStore = new Map<string, { count: number; resetTime: number }>();
|
||||
const RATE_LIMIT_WINDOW = 60 * 1000;
|
||||
const RATE_LIMIT_MAX = 5;
|
||||
@ -49,7 +39,7 @@ function checkRateLimit(userId: string): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
function cleanupExpiredRateLimits() {
|
||||
function cleanupExpiredRateLimits(): void {
|
||||
const now = Date.now();
|
||||
for (const [userId, limit] of rateLimitStore.entries()) {
|
||||
if (now > limit.resetTime) {
|
||||
@ -94,39 +84,6 @@ ${input}
|
||||
`.trim();
|
||||
}
|
||||
|
||||
async function callAIService(prompt: string): Promise<Response> {
|
||||
const endpoint = AI_ENDPOINT;
|
||||
const apiKey = AI_ANALYZER_API_KEY;
|
||||
const model = AI_ANALYZER_MODEL;
|
||||
|
||||
let headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json'
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
console.log('[ENHANCE API] Using API key authentication');
|
||||
} else {
|
||||
console.log('[ENHANCE API] No API key - making request without authentication');
|
||||
}
|
||||
|
||||
const requestBody = {
|
||||
model,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
max_tokens: 300,
|
||||
temperature: 0.7,
|
||||
top_p: 0.9,
|
||||
frequency_penalty: 0.2,
|
||||
presence_penalty: 0.1
|
||||
};
|
||||
|
||||
return fetch(`${endpoint}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(requestBody)
|
||||
});
|
||||
}
|
||||
|
||||
export const POST: APIRoute = async ({ request }) => {
|
||||
try {
|
||||
const authResult = await withAPIAuth(request, 'ai');
|
||||
@ -155,28 +112,26 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
const systemPrompt = createEnhancementPrompt(sanitizedInput);
|
||||
const taskId = `enhance_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 4)}`;
|
||||
|
||||
const aiResponse = await enqueueApiCall(() => callAIService(systemPrompt), taskId);
|
||||
console.log(`[ENHANCE-API] Processing enhancement request for user: ${userId}`);
|
||||
|
||||
if (!aiResponse.ok) {
|
||||
const errorText = await aiResponse.text();
|
||||
console.error('[ENHANCE API] AI enhancement error:', errorText, 'Status:', aiResponse.status);
|
||||
return apiServerError.unavailable('Enhancement service unavailable');
|
||||
}
|
||||
const aiResponse = await enqueueApiCall(() =>
|
||||
aiService.callAI(systemPrompt, {
|
||||
maxTokens: 300,
|
||||
temperature: 0.7
|
||||
}), taskId);
|
||||
|
||||
const aiData = await aiResponse.json();
|
||||
const aiContent = aiData.choices?.[0]?.message?.content;
|
||||
|
||||
if (!aiContent) {
|
||||
if (!aiResponse.content) {
|
||||
return apiServerError.unavailable('No enhancement response');
|
||||
}
|
||||
|
||||
let questions;
|
||||
try {
|
||||
const cleanedContent = aiContent
|
||||
const cleanedContent = aiResponse.content
|
||||
.replace(/^```json\s*/i, '')
|
||||
.replace(/\s*```\s*$/, '')
|
||||
.trim();
|
||||
questions = JSON.parse(cleanedContent);
|
||||
|
||||
questions = JSONParser.safeParseJSON(cleanedContent, []);
|
||||
|
||||
if (!Array.isArray(questions)) {
|
||||
throw new Error('Response is not an array');
|
||||
@ -198,11 +153,11 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to parse enhancement response:', aiContent);
|
||||
console.error('[ENHANCE-API] Failed to parse enhancement response:', aiResponse.content);
|
||||
questions = [];
|
||||
}
|
||||
|
||||
console.log(`[ENHANCE API] User: ${userId}, Forensics Questions: ${questions.length}, Input length: ${sanitizedInput.length}`);
|
||||
console.log(`[ENHANCE-API] User: ${userId}, Questions generated: ${questions.length}, Input length: ${sanitizedInput.length}`);
|
||||
|
||||
return new Response(JSON.stringify({
|
||||
success: true,
|
||||
@ -215,7 +170,7 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Enhancement error:', error);
|
||||
console.error('[ENHANCE-API] Enhancement error:', error);
|
||||
return apiServerError.internal('Enhancement processing failed');
|
||||
}
|
||||
};
|
||||
@ -1,4 +1,4 @@
|
||||
// src/pages/api/ai/query.ts
|
||||
// src/pages/api/ai/query.ts - Updated to use refactored services
|
||||
import type { APIRoute } from 'astro';
|
||||
import { withAPIAuth } from '../../../utils/auth.js';
|
||||
import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js';
|
||||
@ -20,15 +20,14 @@ const MAIN_RATE_LIMIT_MAX = parseInt(process.env.AI_RATE_LIMIT_MAX_REQUESTS || '
|
||||
const MICRO_TASK_TOTAL_LIMIT = parseInt(process.env.AI_MICRO_TASK_TOTAL_LIMIT || '50', 10);
|
||||
|
||||
function sanitizeInput(input: string): string {
|
||||
let sanitized = input
|
||||
return input
|
||||
.replace(/```[\s\S]*?```/g, '[CODE_BLOCK_REMOVED]')
|
||||
.replace(/\<\/?[^>]+(>|$)/g, '')
|
||||
.replace(/\b(system|assistant|user)\s*[:]/gi, '[ROLE_REMOVED]')
|
||||
.replace(/\b(ignore|forget|disregard)\s+(previous|all|your)\s+(instructions?|context|rules?)/gi, '[INSTRUCTION_REMOVED]')
|
||||
.trim();
|
||||
|
||||
sanitized = sanitized.slice(0, 2000).replace(/\s+/g, ' ');
|
||||
return sanitized;
|
||||
.trim()
|
||||
.slice(0, 2000)
|
||||
.replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
function checkRateLimit(userId: string): { allowed: boolean; reason?: string; microTasksRemaining?: number } {
|
||||
@ -77,7 +76,7 @@ function incrementMicroTaskCount(userId: string, aiCallsMade: number): void {
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupExpiredRateLimits() {
|
||||
function cleanupExpiredRateLimits(): void {
|
||||
const now = Date.now();
|
||||
const maxStoreSize = 1000;
|
||||
|
||||
@ -117,51 +116,52 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
const body = await request.json();
|
||||
const { query, mode = 'workflow', taskId: clientTaskId } = body;
|
||||
|
||||
console.log(`[MICRO-TASK API] Received request - TaskId: ${clientTaskId}, Mode: ${mode}, Query length: ${query?.length || 0}`);
|
||||
console.log(`[MICRO-TASK API] Micro-task calls remaining: ${rateLimitResult.microTasksRemaining}`);
|
||||
console.log(`[AI-API] Received request - TaskId: ${clientTaskId}, Mode: ${mode}, Query length: ${query?.length || 0}`);
|
||||
console.log(`[AI-API] Micro-task calls remaining: ${rateLimitResult.microTasksRemaining}`);
|
||||
|
||||
if (!query || typeof query !== 'string') {
|
||||
console.log(`[MICRO-TASK API] Invalid query for task ${clientTaskId}`);
|
||||
console.log(`[AI-API] Invalid query for task ${clientTaskId}`);
|
||||
return apiError.badRequest('Query required');
|
||||
}
|
||||
|
||||
if (!['workflow', 'tool'].includes(mode)) {
|
||||
console.log(`[MICRO-TASK API] Invalid mode for task ${clientTaskId}: ${mode}`);
|
||||
console.log(`[AI-API] Invalid mode for task ${clientTaskId}: ${mode}`);
|
||||
return apiError.badRequest('Invalid mode. Must be "workflow" or "tool"');
|
||||
}
|
||||
|
||||
const sanitizedQuery = sanitizeInput(query);
|
||||
if (sanitizedQuery.includes('[FILTERED]')) {
|
||||
console.log(`[MICRO-TASK API] Filtered input detected for task ${clientTaskId}`);
|
||||
console.log(`[AI-API] Filtered input detected for task ${clientTaskId}`);
|
||||
return apiError.badRequest('Invalid input detected');
|
||||
}
|
||||
|
||||
const taskId = clientTaskId || `ai_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`;
|
||||
|
||||
console.log(`[MICRO-TASK API] About to enqueue micro-task pipeline ${taskId}`);
|
||||
console.log(`[AI-API] Enqueueing pipeline task ${taskId}`);
|
||||
|
||||
const result = await enqueueApiCall(() =>
|
||||
aiPipeline.processQuery(sanitizedQuery, mode)
|
||||
, taskId);
|
||||
|
||||
if (!result || !result.recommendation) {
|
||||
return apiServerError.unavailable('No response from micro-task AI pipeline');
|
||||
return apiServerError.unavailable('No response from AI pipeline');
|
||||
}
|
||||
|
||||
const stats = result.processingStats;
|
||||
const estimatedAICallsMade = stats.microTasksCompleted + stats.microTasksFailed;
|
||||
incrementMicroTaskCount(userId, estimatedAICallsMade);
|
||||
|
||||
console.log(`[MICRO-TASK API] Pipeline completed for ${taskId}:`);
|
||||
console.log(` - Mode: ${mode}`);
|
||||
console.log(` - User: ${userId}`);
|
||||
console.log(` - Query length: ${sanitizedQuery.length}`);
|
||||
console.log(` - Processing time: ${stats.processingTimeMs}ms`);
|
||||
console.log(` - Micro-tasks completed: ${stats.microTasksCompleted}`);
|
||||
console.log(` - Micro-tasks failed: ${stats.microTasksFailed}`);
|
||||
console.log(` - Estimated AI calls: ${estimatedAICallsMade}`);
|
||||
console.log(` - Embeddings used: ${stats.embeddingsUsed}`);
|
||||
console.log(` - Final items: ${stats.finalSelectedItems}`);
|
||||
console.log(`[AI-API] Pipeline completed for ${taskId}:`, {
|
||||
mode,
|
||||
user: userId,
|
||||
queryLength: sanitizedQuery.length,
|
||||
processingTime: stats.processingTimeMs,
|
||||
microTasksCompleted: stats.microTasksCompleted,
|
||||
microTasksFailed: stats.microTasksFailed,
|
||||
estimatedAICalls: estimatedAICallsMade,
|
||||
embeddingsUsed: stats.embeddingsUsed,
|
||||
finalItems: stats.finalSelectedItems
|
||||
});
|
||||
|
||||
const currentLimit = rateLimitStore.get(userId);
|
||||
const remainingMicroTasks = currentLimit ?
|
||||
@ -175,7 +175,7 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
query: sanitizedQuery,
|
||||
processingStats: {
|
||||
...result.processingStats,
|
||||
pipelineType: 'micro-task',
|
||||
pipelineType: 'refactored',
|
||||
microTasksSuccessRate: stats.microTasksCompleted / (stats.microTasksCompleted + stats.microTasksFailed),
|
||||
averageTaskTime: stats.processingTimeMs / (stats.microTasksCompleted + stats.microTasksFailed),
|
||||
estimatedAICallsMade
|
||||
@ -191,18 +191,16 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('[MICRO-TASK API] Pipeline error:', error);
|
||||
console.error('[AI-API] Pipeline error:', error);
|
||||
|
||||
if (error.message.includes('embeddings')) {
|
||||
return apiServerError.unavailable('Embeddings service error - using AI fallback');
|
||||
} else if (error.message.includes('micro-task')) {
|
||||
return apiServerError.unavailable('Micro-task pipeline error - some analysis steps failed');
|
||||
} else if (error.message.includes('selector')) {
|
||||
return apiServerError.unavailable('AI selector service error');
|
||||
return apiServerError.unavailable('Embeddings service error');
|
||||
} else if (error.message.includes('AI')) {
|
||||
return apiServerError.unavailable('AI service error');
|
||||
} else if (error.message.includes('rate limit')) {
|
||||
return apiError.rateLimit('AI service rate limits exceeded during micro-task processing');
|
||||
return apiError.rateLimit('AI service rate limits exceeded');
|
||||
} else {
|
||||
return apiServerError.internal('Micro-task AI pipeline error');
|
||||
return apiServerError.internal('AI pipeline error');
|
||||
}
|
||||
}
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
150
src/utils/aiService.ts
Normal file
150
src/utils/aiService.ts
Normal file
@ -0,0 +1,150 @@
|
||||
// src/utils/aiService.ts
|
||||
import 'dotenv/config';
|
||||
|
||||
export interface AIServiceConfig {
|
||||
endpoint: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface AICallOptions {
|
||||
maxTokens?: number;
|
||||
temperature?: number;
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
export interface AIResponse {
|
||||
content: string;
|
||||
usage?: {
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
};
|
||||
}
|
||||
|
||||
class AIService {
|
||||
private config: AIServiceConfig;
|
||||
private defaultOptions: AICallOptions;
|
||||
|
||||
constructor() {
|
||||
this.config = {
|
||||
endpoint: this.getRequiredEnv('AI_ANALYZER_ENDPOINT'),
|
||||
apiKey: this.getRequiredEnv('AI_ANALYZER_API_KEY'),
|
||||
model: this.getRequiredEnv('AI_ANALYZER_MODEL')
|
||||
};
|
||||
|
||||
this.defaultOptions = {
|
||||
maxTokens: 1500,
|
||||
temperature: 0.3,
|
||||
timeout: 30000
|
||||
};
|
||||
|
||||
console.log('[AI-SERVICE] Initialized with model:', this.config.model);
|
||||
}
|
||||
|
||||
private getRequiredEnv(key: string): string {
|
||||
const value = process.env[key];
|
||||
if (!value) {
|
||||
throw new Error(`Missing required environment variable: ${key}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async callAI(prompt: string, options: AICallOptions = {}): Promise<AIResponse> {
|
||||
const mergedOptions = { ...this.defaultOptions, ...options };
|
||||
|
||||
console.log('[AI-SERVICE] Making API call:', {
|
||||
promptLength: prompt.length,
|
||||
maxTokens: mergedOptions.maxTokens,
|
||||
temperature: mergedOptions.temperature
|
||||
});
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json'
|
||||
};
|
||||
|
||||
if (this.config.apiKey) {
|
||||
headers['Authorization'] = `Bearer ${this.config.apiKey}`;
|
||||
}
|
||||
|
||||
const requestBody = {
|
||||
model: this.config.model,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
max_tokens: mergedOptions.maxTokens,
|
||||
temperature: mergedOptions.temperature
|
||||
};
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), mergedOptions.timeout);
|
||||
|
||||
const response = await fetch(`${this.config.endpoint}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
signal: controller.signal
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
console.error('[AI-SERVICE] API Error:', response.status, errorText);
|
||||
throw new Error(`AI API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
|
||||
if (!content) {
|
||||
console.error('[AI-SERVICE] No response content from AI model');
|
||||
throw new Error('No response from AI model');
|
||||
}
|
||||
|
||||
console.log('[AI-SERVICE] API call successful:', {
|
||||
responseLength: content.length,
|
||||
usage: data.usage
|
||||
});
|
||||
|
||||
return {
|
||||
content: content.trim(),
|
||||
usage: data.usage
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
if (error.name === 'AbortError') {
|
||||
console.error('[AI-SERVICE] Request timeout');
|
||||
throw new Error('AI request timeout');
|
||||
}
|
||||
|
||||
console.error('[AI-SERVICE] API call failed:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async callMicroTaskAI(prompt: string, maxTokens: number = 500): Promise<AIResponse> {
|
||||
return this.callAI(prompt, {
|
||||
maxTokens,
|
||||
temperature: 0.3,
|
||||
timeout: 15000
|
||||
});
|
||||
}
|
||||
|
||||
estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
validatePromptLength(prompt: string, maxTokens: number = 35000): void {
|
||||
const estimatedTokens = this.estimateTokens(prompt);
|
||||
if (estimatedTokens > maxTokens) {
|
||||
console.warn('[AI-SERVICE] WARNING: Prompt may exceed model limits:', estimatedTokens);
|
||||
throw new Error(`Prompt too long: ${estimatedTokens} tokens (max: ${maxTokens})`);
|
||||
}
|
||||
}
|
||||
|
||||
getConfig(): AIServiceConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
}
|
||||
|
||||
export const aiService = new AIService();
|
||||
@ -1,4 +1,4 @@
|
||||
// src/utils/auditService.ts
|
||||
// src/utils/auditService.ts - Refactored
|
||||
import 'dotenv/config';
|
||||
|
||||
function env(key: string, fallback: string | undefined = undefined): string | undefined {
|
||||
@ -11,7 +11,6 @@ function env(key: string, fallback: string | undefined = undefined): string | un
|
||||
return fallback;
|
||||
}
|
||||
|
||||
// CONSOLIDATED AUDIT INTERFACES - Single source of truth
|
||||
export interface AuditEntry {
|
||||
timestamp: number;
|
||||
phase: string;
|
||||
@ -30,64 +29,10 @@ interface AuditConfig {
|
||||
maxEntries: number;
|
||||
}
|
||||
|
||||
interface CompressedAuditEntry {
|
||||
timestamp: number;
|
||||
phase: string;
|
||||
action: string;
|
||||
inputSummary: string;
|
||||
outputSummary: string;
|
||||
confidence: number;
|
||||
processingTimeMs: number;
|
||||
metadata: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface ProcessedAuditTrail {
|
||||
totalTime: number;
|
||||
avgConfidence: number;
|
||||
stepCount: number;
|
||||
highConfidenceSteps: number;
|
||||
lowConfidenceSteps: number;
|
||||
phases: Array<{
|
||||
name: string;
|
||||
icon: string;
|
||||
displayName: string;
|
||||
avgConfidence: number;
|
||||
totalTime: number;
|
||||
entries: CompressedAuditEntry[];
|
||||
}>;
|
||||
summary: {
|
||||
analysisQuality: 'excellent' | 'good' | 'fair' | 'poor';
|
||||
keyInsights: string[];
|
||||
potentialIssues: string[];
|
||||
};
|
||||
}
|
||||
|
||||
class AuditService {
|
||||
private config: AuditConfig;
|
||||
private activeAuditTrail: AuditEntry[] = [];
|
||||
|
||||
private readonly phaseConfig = {
|
||||
'initialization': { icon: '🚀', displayName: 'Initialisierung' },
|
||||
'retrieval': { icon: '🔍', displayName: 'Datensuche' },
|
||||
'selection': { icon: '🎯', displayName: 'Tool-Auswahl' },
|
||||
'micro-task': { icon: '⚡', displayName: 'Detail-Analyse' },
|
||||
'validation': { icon: '✓', displayName: 'Validierung' },
|
||||
'completion': { icon: '✅', displayName: 'Finalisierung' }
|
||||
};
|
||||
|
||||
private readonly actionTranslations = {
|
||||
'pipeline-start': 'Analyse gestartet',
|
||||
'embeddings-search': 'Ähnliche Tools gesucht',
|
||||
'ai-tool-selection': 'Tools automatisch ausgewählt',
|
||||
'ai-analysis': 'KI-Analyse durchgeführt',
|
||||
'phase-tool-selection': 'Phasen-Tools evaluiert',
|
||||
'tool-evaluation': 'Tool-Bewertung erstellt',
|
||||
'background-knowledge-selection': 'Hintergrundwissen ausgewählt',
|
||||
'confidence-scoring': 'Vertrauenswertung berechnet',
|
||||
'phase-completion': 'Phasenergänzung durchgeführt',
|
||||
'pipeline-end': 'Analyse abgeschlossen'
|
||||
};
|
||||
|
||||
constructor() {
|
||||
this.config = this.loadConfig();
|
||||
console.log('[AUDIT-SERVICE] Initialized:', {
|
||||
@ -110,7 +55,6 @@ class AuditService {
|
||||
};
|
||||
}
|
||||
|
||||
// CONSOLIDATED AUDIT ENTRY CREATION - Single method for all audit operations
|
||||
addEntry(
|
||||
phase: string,
|
||||
action: string,
|
||||
@ -134,15 +78,19 @@ class AuditService {
|
||||
};
|
||||
|
||||
this.activeAuditTrail.push(entry);
|
||||
|
||||
// Enforce max entries limit
|
||||
if (this.activeAuditTrail.length > this.config.maxEntries) {
|
||||
this.activeAuditTrail.shift();
|
||||
}
|
||||
|
||||
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`);
|
||||
}
|
||||
|
||||
// GET CURRENT AUDIT TRAIL - For integration with AI pipeline
|
||||
getCurrentAuditTrail(): AuditEntry[] {
|
||||
return [...this.activeAuditTrail];
|
||||
}
|
||||
|
||||
// CLEAR AUDIT TRAIL - Start fresh for new analysis
|
||||
clearAuditTrail(): void {
|
||||
if (this.activeAuditTrail.length > 0) {
|
||||
console.log(`[AUDIT-SERVICE] Cleared ${this.activeAuditTrail.length} audit entries`);
|
||||
@ -150,7 +98,6 @@ class AuditService {
|
||||
}
|
||||
}
|
||||
|
||||
// FINALIZE AUDIT TRAIL - Complete analysis and return final trail
|
||||
finalizeAuditTrail(): AuditEntry[] {
|
||||
const finalTrail = [...this.activeAuditTrail];
|
||||
console.log(`[AUDIT-SERVICE] Finalized audit trail with ${finalTrail.length} entries`);
|
||||
@ -158,102 +105,6 @@ class AuditService {
|
||||
return finalTrail;
|
||||
}
|
||||
|
||||
processAuditTrail(rawAuditTrail: AuditEntry[]): ProcessedAuditTrail | null {
|
||||
if (!this.config.enabled) {
|
||||
console.log('[AUDIT-SERVICE] Processing disabled');
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!rawAuditTrail || !Array.isArray(rawAuditTrail) || rawAuditTrail.length === 0) {
|
||||
console.log('[AUDIT-SERVICE] No audit trail data to process');
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
console.log(`[AUDIT-SERVICE] Processing ${rawAuditTrail.length} audit entries`);
|
||||
|
||||
const totalTime = rawAuditTrail.reduce((sum, entry) => sum + (entry.processingTimeMs || 0), 0);
|
||||
const validConfidenceEntries = rawAuditTrail.filter(entry => typeof entry.confidence === 'number');
|
||||
const avgConfidence = validConfidenceEntries.length > 0
|
||||
? Math.round(validConfidenceEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validConfidenceEntries.length)
|
||||
: 0;
|
||||
|
||||
const highConfidenceSteps = rawAuditTrail.filter(entry => (entry.confidence || 0) >= 80).length;
|
||||
const lowConfidenceSteps = rawAuditTrail.filter(entry => (entry.confidence || 0) < 60).length;
|
||||
|
||||
const groupedEntries = rawAuditTrail.reduce((groups, entry) => {
|
||||
const phase = entry.phase || 'unknown';
|
||||
if (!groups[phase]) groups[phase] = [];
|
||||
groups[phase].push(entry);
|
||||
return groups;
|
||||
}, {} as Record<string, AuditEntry[]>);
|
||||
|
||||
const phases = Object.entries(groupedEntries).map(([phase, entries]) => {
|
||||
const phaseConfig = this.phaseConfig[phase] || { icon: '📋', displayName: phase };
|
||||
const validEntries = entries.filter(entry => entry && typeof entry === 'object');
|
||||
|
||||
const phaseAvgConfidence = validEntries.length > 0
|
||||
? Math.round(validEntries.reduce((sum, entry) => sum + (entry.confidence || 0), 0) / validEntries.length)
|
||||
: 0;
|
||||
|
||||
const phaseTotalTime = validEntries.reduce((sum, entry) => sum + (entry.processingTimeMs || 0), 0);
|
||||
|
||||
return {
|
||||
name: phase,
|
||||
icon: phaseConfig.icon,
|
||||
displayName: phaseConfig.displayName,
|
||||
avgConfidence: phaseAvgConfidence,
|
||||
totalTime: phaseTotalTime,
|
||||
entries: validEntries
|
||||
.map(e => this.compressEntry(e))
|
||||
.filter((e): e is CompressedAuditEntry => e !== null)
|
||||
};
|
||||
}).filter(phase => phase.entries.length > 0);
|
||||
|
||||
const summary = this.generateSummary(rawAuditTrail, avgConfidence, lowConfidenceSteps);
|
||||
|
||||
const result: ProcessedAuditTrail = {
|
||||
totalTime,
|
||||
avgConfidence,
|
||||
stepCount: rawAuditTrail.length,
|
||||
highConfidenceSteps,
|
||||
lowConfidenceSteps,
|
||||
phases,
|
||||
summary
|
||||
};
|
||||
|
||||
console.log(`[AUDIT-SERVICE] Successfully processed audit trail: ${result.phases.length} phases, ${result.avgConfidence}% avg confidence`);
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
console.error('[AUDIT-SERVICE] Error processing audit trail:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private compressEntry(entry: AuditEntry): CompressedAuditEntry | null {
|
||||
if (!entry || typeof entry !== 'object') {
|
||||
console.warn('[AUDIT-SERVICE] Invalid audit entry skipped');
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return {
|
||||
timestamp: entry.timestamp || Date.now(),
|
||||
phase: entry.phase || 'unknown',
|
||||
action: entry.action || 'unknown',
|
||||
inputSummary: this.summarizeData(entry.input),
|
||||
outputSummary: this.summarizeData(entry.output),
|
||||
confidence: entry.confidence || 0,
|
||||
processingTimeMs: entry.processingTimeMs || 0,
|
||||
metadata: entry.metadata || {}
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('[AUDIT-SERVICE] Error compressing entry:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private compressData(data: any): any {
|
||||
if (this.config.detailLevel === 'verbose') {
|
||||
return data;
|
||||
@ -264,30 +115,6 @@ class AuditService {
|
||||
}
|
||||
}
|
||||
|
||||
private summarizeData(data: any): string {
|
||||
if (data === null || data === undefined) return 'null';
|
||||
if (typeof data === 'string') {
|
||||
return data.length > 100 ? data.slice(0, 100) + '...' : data;
|
||||
}
|
||||
if (typeof data === 'number' || typeof data === 'boolean') {
|
||||
return data.toString();
|
||||
}
|
||||
if (Array.isArray(data)) {
|
||||
if (data.length === 0) return '[]';
|
||||
if (data.length <= 3) return JSON.stringify(data);
|
||||
return `[${data.slice(0, 3).map(i => typeof i === 'string' ? i : JSON.stringify(i)).join(', ')}, ...+${data.length - 3}]`;
|
||||
}
|
||||
if (typeof data === 'object') {
|
||||
const keys = Object.keys(data);
|
||||
if (keys.length === 0) return '{}';
|
||||
if (keys.length <= 3) {
|
||||
return '{' + keys.map(k => `${k}: ${typeof data[k] === 'string' ? data[k].slice(0, 20) + (data[k].length > 20 ? '...' : '') : JSON.stringify(data[k])}`).join(', ') + '}';
|
||||
}
|
||||
return `{${keys.slice(0, 3).join(', ')}, ...+${keys.length - 3} keys}`;
|
||||
}
|
||||
return String(data);
|
||||
}
|
||||
|
||||
private summarizeForStorage(data: any): any {
|
||||
if (typeof data === 'string' && data.length > 500) {
|
||||
return data.slice(0, 500) + '...[truncated]';
|
||||
@ -308,71 +135,6 @@ class AuditService {
|
||||
return data;
|
||||
}
|
||||
|
||||
private generateSummary(entries: AuditEntry[], avgConfidence: number, lowConfidenceSteps: number): {
|
||||
analysisQuality: 'excellent' | 'good' | 'fair' | 'poor';
|
||||
keyInsights: string[];
|
||||
potentialIssues: string[];
|
||||
} {
|
||||
let analysisQuality: 'excellent' | 'good' | 'fair' | 'poor';
|
||||
if (avgConfidence >= 85 && lowConfidenceSteps === 0) {
|
||||
analysisQuality = 'excellent';
|
||||
} else if (avgConfidence >= 70 && lowConfidenceSteps <= 1) {
|
||||
analysisQuality = 'good';
|
||||
} else if (avgConfidence >= 60 && lowConfidenceSteps <= 3) {
|
||||
analysisQuality = 'fair';
|
||||
} else {
|
||||
analysisQuality = 'poor';
|
||||
}
|
||||
|
||||
const keyInsights: string[] = [];
|
||||
const embeddingsUsed = entries.some(e => e.action === 'embeddings-search');
|
||||
if (embeddingsUsed) {
|
||||
keyInsights.push('Semantische Suche wurde erfolgreich eingesetzt');
|
||||
}
|
||||
|
||||
const toolSelectionEntries = entries.filter(e => e.action === 'ai-tool-selection');
|
||||
if (toolSelectionEntries.length > 0) {
|
||||
const avgSelectionConfidence = toolSelectionEntries.reduce((sum, e) => sum + e.confidence, 0) / toolSelectionEntries.length;
|
||||
if (avgSelectionConfidence >= 80) {
|
||||
keyInsights.push('Hohe Konfidenz bei der Tool-Auswahl');
|
||||
}
|
||||
}
|
||||
|
||||
const potentialIssues: string[] = [];
|
||||
if (lowConfidenceSteps > 2) {
|
||||
potentialIssues.push(`${lowConfidenceSteps} Analyseschritte mit niedriger Konfidenz`);
|
||||
}
|
||||
|
||||
const longSteps = entries.filter(e => e.processingTimeMs > 5000);
|
||||
if (longSteps.length > 0) {
|
||||
potentialIssues.push(`${longSteps.length} Schritte benötigten mehr als 5 Sekunden`);
|
||||
}
|
||||
|
||||
return {
|
||||
analysisQuality,
|
||||
keyInsights,
|
||||
potentialIssues
|
||||
};
|
||||
}
|
||||
|
||||
getActionDisplayName(action: string): string {
|
||||
return this.actionTranslations[action] || action;
|
||||
}
|
||||
|
||||
formatDuration(ms: number): string {
|
||||
if (ms < 1000) return '< 1s';
|
||||
if (ms < 60000) return `${Math.ceil(ms / 1000)}s`;
|
||||
const minutes = Math.floor(ms / 60000);
|
||||
const seconds = Math.ceil((ms % 60000) / 1000);
|
||||
return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
|
||||
}
|
||||
|
||||
getConfidenceColor(confidence: number): string {
|
||||
if (confidence >= 80) return 'var(--color-accent)';
|
||||
if (confidence >= 60) return 'var(--color-warning)';
|
||||
return 'var(--color-error)';
|
||||
}
|
||||
|
||||
isEnabled(): boolean {
|
||||
return this.config.enabled;
|
||||
}
|
||||
@ -380,7 +142,122 @@ class AuditService {
|
||||
getConfig(): AuditConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
|
||||
// Statistics and analysis methods
|
||||
getAuditStatistics(auditTrail: AuditEntry[]): {
|
||||
totalTime: number;
|
||||
avgConfidence: number;
|
||||
stepCount: number;
|
||||
highConfidenceSteps: number;
|
||||
lowConfidenceSteps: number;
|
||||
phaseBreakdown: Record<string, { count: number; avgConfidence: number; totalTime: number }>;
|
||||
} {
|
||||
if (!auditTrail || auditTrail.length === 0) {
|
||||
return {
|
||||
totalTime: 0,
|
||||
avgConfidence: 0,
|
||||
stepCount: 0,
|
||||
highConfidenceSteps: 0,
|
||||
lowConfidenceSteps: 0,
|
||||
phaseBreakdown: {}
|
||||
};
|
||||
}
|
||||
|
||||
const totalTime = auditTrail.reduce((sum, entry) => sum + (entry.processingTimeMs || 0), 0);
|
||||
const validConfidenceEntries = auditTrail.filter(entry => typeof entry.confidence === 'number');
|
||||
const avgConfidence = validConfidenceEntries.length > 0
|
||||
? Math.round(validConfidenceEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validConfidenceEntries.length)
|
||||
: 0;
|
||||
|
||||
const highConfidenceSteps = auditTrail.filter(entry => (entry.confidence || 0) >= 80).length;
|
||||
const lowConfidenceSteps = auditTrail.filter(entry => (entry.confidence || 0) < 60).length;
|
||||
|
||||
// Phase breakdown
|
||||
const phaseBreakdown: Record<string, { count: number; avgConfidence: number; totalTime: number }> = {};
|
||||
|
||||
auditTrail.forEach(entry => {
|
||||
const phase = entry.phase || 'unknown';
|
||||
if (!phaseBreakdown[phase]) {
|
||||
phaseBreakdown[phase] = { count: 0, avgConfidence: 0, totalTime: 0 };
|
||||
}
|
||||
|
||||
phaseBreakdown[phase].count++;
|
||||
phaseBreakdown[phase].totalTime += entry.processingTimeMs || 0;
|
||||
});
|
||||
|
||||
// Calculate average confidence per phase
|
||||
Object.keys(phaseBreakdown).forEach(phase => {
|
||||
const phaseEntries = auditTrail.filter(entry => entry.phase === phase);
|
||||
const validEntries = phaseEntries.filter(entry => typeof entry.confidence === 'number');
|
||||
|
||||
if (validEntries.length > 0) {
|
||||
phaseBreakdown[phase].avgConfidence = Math.round(
|
||||
validEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validEntries.length
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
totalTime,
|
||||
avgConfidence,
|
||||
stepCount: auditTrail.length,
|
||||
highConfidenceSteps,
|
||||
lowConfidenceSteps,
|
||||
phaseBreakdown
|
||||
};
|
||||
}
|
||||
|
||||
validateAuditTrail(auditTrail: AuditEntry[]): {
|
||||
isValid: boolean;
|
||||
issues: string[];
|
||||
warnings: string[];
|
||||
} {
|
||||
const issues: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
if (!Array.isArray(auditTrail)) {
|
||||
issues.push('Audit trail is not an array');
|
||||
return { isValid: false, issues, warnings };
|
||||
}
|
||||
|
||||
if (auditTrail.length === 0) {
|
||||
warnings.push('Audit trail is empty');
|
||||
}
|
||||
|
||||
auditTrail.forEach((entry, index) => {
|
||||
if (!entry || typeof entry !== 'object') {
|
||||
issues.push(`Entry ${index} is not a valid object`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Required fields validation
|
||||
const requiredFields = ['timestamp', 'phase', 'action'];
|
||||
requiredFields.forEach(field => {
|
||||
if (!(field in entry)) {
|
||||
issues.push(`Entry ${index} missing required field: ${field}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Data type validation
|
||||
if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) {
|
||||
warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`);
|
||||
}
|
||||
|
||||
if (typeof entry.processingTimeMs !== 'number' || entry.processingTimeMs < 0) {
|
||||
warnings.push(`Entry ${index} has invalid processing time: ${entry.processingTimeMs}`);
|
||||
}
|
||||
|
||||
if (typeof entry.timestamp !== 'number' || entry.timestamp <= 0) {
|
||||
issues.push(`Entry ${index} has invalid timestamp: ${entry.timestamp}`);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
isValid: issues.length === 0,
|
||||
issues,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const auditService = new AuditService();
|
||||
export type { CompressedAuditEntry };
|
||||
@ -1,9 +1,9 @@
|
||||
// src/utils/clientUtils.ts
|
||||
|
||||
// src/utils/clientUtils.ts - Consolidated (removes duplicates from toolHelpers.ts)
|
||||
|
||||
// Tool helper functions (moved here to avoid circular imports)
|
||||
export function createToolSlug(toolName: string): string {
|
||||
if (!toolName || typeof toolName !== 'string') {
|
||||
console.warn('[toolHelpers] Invalid toolName provided to createToolSlug:', toolName);
|
||||
console.warn('[CLIENT-UTILS] Invalid toolName provided to createToolSlug:', toolName);
|
||||
return '';
|
||||
}
|
||||
|
||||
@ -30,6 +30,86 @@ export function isToolHosted(tool: any): boolean {
|
||||
tool.projectUrl.trim() !== "";
|
||||
}
|
||||
|
||||
// Text and display utilities
|
||||
export function sanitizeText(text: string): string {
|
||||
if (typeof text !== 'string') return '';
|
||||
|
||||
return text
|
||||
.replace(/^#{1,6}\s+/gm, '')
|
||||
.replace(/^\s*[-*+]\s+/gm, '')
|
||||
.replace(/^\s*\d+\.\s+/gm, '')
|
||||
.replace(/\*\*(.+?)\*\*/g, '$1')
|
||||
.replace(/\*(.+?)\*/g, '$1')
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
|
||||
.replace(/```[\s\S]*?```/g, '[CODE BLOCK]')
|
||||
.replace(/`([^`]+)`/g, '$1')
|
||||
.replace(/<[^>]+>/g, '')
|
||||
.replace(/\n\s*\n\s*\n/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function escapeHtml(text: string): string {
|
||||
if (typeof text !== 'string') return String(text);
|
||||
const div = document.createElement('div');
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
export function truncateText(text: string, maxLength: number): string {
|
||||
if (!text || text.length <= maxLength) return text;
|
||||
return text.slice(0, maxLength) + '...';
|
||||
}
|
||||
|
||||
// Data summarization utilities
|
||||
export function summarizeData(data: any): string {
|
||||
if (data === null || data === undefined) return 'null';
|
||||
if (typeof data === 'string') {
|
||||
return data.length > 100 ? data.slice(0, 100) + '...' : data;
|
||||
}
|
||||
if (typeof data === 'number' || typeof data === 'boolean') {
|
||||
return data.toString();
|
||||
}
|
||||
if (Array.isArray(data)) {
|
||||
if (data.length === 0) return '[]';
|
||||
if (data.length <= 3) return JSON.stringify(data);
|
||||
return `[${data.slice(0, 3).map(i => typeof i === 'string' ? i : JSON.stringify(i)).join(', ')}, ...+${data.length - 3}]`;
|
||||
}
|
||||
if (typeof data === 'object') {
|
||||
const keys = Object.keys(data);
|
||||
if (keys.length === 0) return '{}';
|
||||
if (keys.length <= 3) {
|
||||
return '{' + keys.map(k => `${k}: ${typeof data[k] === 'string' ? data[k].slice(0, 20) + (data[k].length > 20 ? '...' : '') : JSON.stringify(data[k])}`).join(', ') + '}';
|
||||
}
|
||||
return `{${keys.slice(0, 3).join(', ')}, ...+${keys.length - 3} keys}`;
|
||||
}
|
||||
return String(data);
|
||||
}
|
||||
|
||||
// Time formatting utilities
|
||||
export function formatDuration(ms: number): string {
|
||||
if (ms < 1000) return '< 1s';
|
||||
if (ms < 60000) return `${Math.ceil(ms / 1000)}s`;
|
||||
const minutes = Math.floor(ms / 60000);
|
||||
const seconds = Math.ceil((ms % 60000) / 1000);
|
||||
return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`;
|
||||
}
|
||||
|
||||
// DOM utilities
|
||||
export function showElement(element: HTMLElement | null): void {
|
||||
if (element) {
|
||||
element.style.display = 'block';
|
||||
element.classList.remove('hidden');
|
||||
}
|
||||
}
|
||||
|
||||
export function hideElement(element: HTMLElement | null): void {
|
||||
if (element) {
|
||||
element.style.display = 'none';
|
||||
element.classList.add('hidden');
|
||||
}
|
||||
}
|
||||
|
||||
// Autocomplete functionality (kept from original clientUtils.ts as it's UI-specific)
|
||||
interface AutocompleteOptions {
|
||||
minLength?: number;
|
||||
maxResults?: number;
|
||||
@ -202,7 +282,7 @@ export class AutocompleteManager {
|
||||
|
||||
defaultRender(item: any): string {
|
||||
const text = typeof item === 'string' ? item : item.name || item.label || item.toString();
|
||||
return `<div class="autocomplete-item">${this.escapeHtml(text)}</div>`;
|
||||
return `<div class="autocomplete-item">${escapeHtml(text)}</div>`;
|
||||
}
|
||||
|
||||
renderDropdown(): void {
|
||||
@ -284,8 +364,8 @@ export class AutocompleteManager {
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
">
|
||||
${this.escapeHtml(item)}
|
||||
<button type="button" class="autocomplete-remove" data-item="${this.escapeHtml(item)}" style="
|
||||
${escapeHtml(item)}
|
||||
<button type="button" class="autocomplete-remove" data-item="${escapeHtml(item)}" style="
|
||||
background: none;
|
||||
border: none;
|
||||
color: white;
|
||||
@ -327,12 +407,6 @@ export class AutocompleteManager {
|
||||
this.selectedIndex = -1;
|
||||
}
|
||||
|
||||
escapeHtml(text: string): string {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
setDataSource(newDataSource: any[]): void {
|
||||
this.dataSource = newDataSource;
|
||||
}
|
||||
|
||||
239
src/utils/confidenceScoring.ts
Normal file
239
src/utils/confidenceScoring.ts
Normal file
@ -0,0 +1,239 @@
|
||||
// src/utils/confidenceScoring.ts
|
||||
import { isToolHosted } from './toolHelpers.js';
|
||||
import 'dotenv/config';
|
||||
|
||||
export interface ConfidenceMetrics {
|
||||
overall: number;
|
||||
semanticRelevance: number;
|
||||
taskSuitability: number;
|
||||
uncertaintyFactors: string[];
|
||||
strengthIndicators: string[];
|
||||
}
|
||||
|
||||
export interface ConfidenceConfig {
|
||||
semanticWeight: number;
|
||||
suitabilityWeight: number;
|
||||
minimumThreshold: number;
|
||||
mediumThreshold: number;
|
||||
highThreshold: number;
|
||||
}
|
||||
|
||||
export interface AnalysisContext {
|
||||
userQuery: string;
|
||||
mode: string;
|
||||
embeddingsSimilarities: Map<string, number>;
|
||||
selectedTools?: Array<{
|
||||
tool: any;
|
||||
phase: string;
|
||||
priority: string;
|
||||
justification?: string;
|
||||
taskRelevance?: number;
|
||||
limitations?: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
class ConfidenceScoring {
|
||||
private config: ConfidenceConfig;
|
||||
|
||||
constructor() {
|
||||
this.config = {
|
||||
semanticWeight: this.getEnvFloat('CONFIDENCE_SEMANTIC_WEIGHT', 0.3),
|
||||
suitabilityWeight: this.getEnvFloat('CONFIDENCE_SUITABILITY_WEIGHT', 0.7),
|
||||
minimumThreshold: this.getEnvInt('CONFIDENCE_MINIMUM_THRESHOLD', 40),
|
||||
mediumThreshold: this.getEnvInt('CONFIDENCE_MEDIUM_THRESHOLD', 60),
|
||||
highThreshold: this.getEnvInt('CONFIDENCE_HIGH_THRESHOLD', 80)
|
||||
};
|
||||
|
||||
console.log('[CONFIDENCE-SCORING] Initialized with config:', this.config);
|
||||
}
|
||||
|
||||
private getEnvFloat(key: string, defaultValue: number): number {
|
||||
const value = process.env[key];
|
||||
return value ? parseFloat(value) : defaultValue;
|
||||
}
|
||||
|
||||
private getEnvInt(key: string, defaultValue: number): number {
|
||||
const value = process.env[key];
|
||||
return value ? parseInt(value, 10) : defaultValue;
|
||||
}
|
||||
|
||||
calculateRecommendationConfidence(
|
||||
tool: any,
|
||||
context: AnalysisContext,
|
||||
taskRelevance: number = 70,
|
||||
limitations: string[] = []
|
||||
): ConfidenceMetrics {
|
||||
console.log('[CONFIDENCE-SCORING] Calculating confidence for tool:', tool.name);
|
||||
|
||||
const rawSemanticRelevance = context.embeddingsSimilarities.has(tool.name) ?
|
||||
context.embeddingsSimilarities.get(tool.name)! * 100 : 50;
|
||||
|
||||
let enhancedTaskSuitability = taskRelevance;
|
||||
|
||||
// Phase alignment bonus for workflow mode
|
||||
if (context.mode === 'workflow') {
|
||||
const toolSelection = context.selectedTools?.find((st: any) => st.tool && st.tool.name === tool.name);
|
||||
if (toolSelection && tool.phases && Array.isArray(tool.phases) && tool.phases.includes(toolSelection.phase)) {
|
||||
const phaseBonus = Math.min(15, 100 - taskRelevance);
|
||||
enhancedTaskSuitability = Math.min(100, taskRelevance + phaseBonus);
|
||||
console.log('[CONFIDENCE-SCORING] Phase alignment bonus applied:', phaseBonus);
|
||||
}
|
||||
}
|
||||
|
||||
const overall = (
|
||||
rawSemanticRelevance * this.config.semanticWeight +
|
||||
enhancedTaskSuitability * this.config.suitabilityWeight
|
||||
);
|
||||
|
||||
const uncertaintyFactors = this.identifyUncertaintyFactors(tool, context, limitations, overall);
|
||||
const strengthIndicators = this.identifyStrengthIndicators(tool, context, overall);
|
||||
|
||||
const result = {
|
||||
overall: Math.round(overall),
|
||||
semanticRelevance: Math.round(rawSemanticRelevance),
|
||||
taskSuitability: Math.round(enhancedTaskSuitability),
|
||||
uncertaintyFactors,
|
||||
strengthIndicators
|
||||
};
|
||||
|
||||
console.log('[CONFIDENCE-SCORING] Confidence calculated:', {
|
||||
tool: tool.name,
|
||||
overall: result.overall,
|
||||
semantic: result.semanticRelevance,
|
||||
task: result.taskSuitability,
|
||||
uncertaintyCount: uncertaintyFactors.length,
|
||||
strengthCount: strengthIndicators.length
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private identifyUncertaintyFactors(
|
||||
tool: any,
|
||||
context: AnalysisContext,
|
||||
limitations: string[],
|
||||
confidence: number
|
||||
): string[] {
|
||||
const factors: string[] = [];
|
||||
|
||||
// Add explicit limitations
|
||||
if (limitations?.length > 0) {
|
||||
factors.push(...limitations.slice(0, 2));
|
||||
}
|
||||
|
||||
// Semantic similarity concerns
|
||||
const similarity = context.embeddingsSimilarities.get(tool.name) || 0.5;
|
||||
if (similarity < 0.7) {
|
||||
factors.push('Geringe semantische Ähnlichkeit zur Anfrage');
|
||||
}
|
||||
|
||||
// Skill level vs query complexity mismatches
|
||||
if (tool.skillLevel === 'expert' && /schnell|rapid|triage|urgent|sofort/i.test(context.userQuery)) {
|
||||
factors.push('Experten-Tool für zeitkritisches Szenario');
|
||||
}
|
||||
|
||||
if (tool.skillLevel === 'novice' && /komplex|erweitert|tiefgehend|advanced|forensisch/i.test(context.userQuery)) {
|
||||
factors.push('Einsteiger-Tool für komplexe Analyse');
|
||||
}
|
||||
|
||||
// Technical accessibility concerns
|
||||
if (tool.type === 'software' && !isToolHosted(tool) && tool.accessType === 'download') {
|
||||
factors.push('Installation und Setup erforderlich');
|
||||
}
|
||||
|
||||
// Licensing concerns
|
||||
if (tool.license === 'Proprietary') {
|
||||
factors.push('Kommerzielle Software - Lizenzkosten zu beachten');
|
||||
}
|
||||
|
||||
// Overall confidence concerns
|
||||
if (confidence < 60) {
|
||||
factors.push('Moderate Gesamtbewertung - alternative Ansätze empfohlen');
|
||||
}
|
||||
|
||||
return factors.slice(0, 4);
|
||||
}
|
||||
|
||||
private identifyStrengthIndicators(tool: any, context: AnalysisContext, confidence: number): string[] {
|
||||
const indicators: string[] = [];
|
||||
|
||||
// High semantic relevance
|
||||
const similarity = context.embeddingsSimilarities.get(tool.name) || 0.5;
|
||||
if (similarity >= 0.7) {
|
||||
indicators.push('Sehr gute semantische Übereinstimmung mit Ihrer Anfrage');
|
||||
}
|
||||
|
||||
// Documentation availability
|
||||
if (tool.knowledgebase === true) {
|
||||
indicators.push('Umfassende Dokumentation und Wissensbasis verfügbar');
|
||||
}
|
||||
|
||||
// Immediate availability
|
||||
if (isToolHosted(tool)) {
|
||||
indicators.push('Sofort verfügbar über gehostete Lösung');
|
||||
}
|
||||
|
||||
// Balanced skill requirements
|
||||
if (tool.skillLevel === 'intermediate' || tool.skillLevel === 'advanced') {
|
||||
indicators.push('Ausgewogenes Verhältnis zwischen Funktionalität und Benutzerfreundlichkeit');
|
||||
}
|
||||
|
||||
// Method-query alignment
|
||||
if (tool.type === 'method' && /methodik|vorgehen|prozess|ansatz/i.test(context.userQuery)) {
|
||||
indicators.push('Methodischer Ansatz passt zu Ihrer prozeduralen Anfrage');
|
||||
}
|
||||
|
||||
return indicators.slice(0, 4);
|
||||
}
|
||||
|
||||
calculateSelectionConfidence(result: any, candidateCount: number): number {
|
||||
if (!result?.selectedTools) {
|
||||
console.log('[CONFIDENCE-SCORING] No selected tools for confidence calculation');
|
||||
return 30;
|
||||
}
|
||||
|
||||
const selectionRatio = result.selectedTools.length / candidateCount;
|
||||
const hasReasoning = result.reasoning && result.reasoning.length > 50;
|
||||
|
||||
let confidence = 60;
|
||||
|
||||
// Selection ratio scoring
|
||||
if (selectionRatio > 0.05 && selectionRatio < 0.3) confidence += 20;
|
||||
else if (selectionRatio <= 0.05) confidence -= 10;
|
||||
else confidence -= 15;
|
||||
|
||||
// Quality indicators
|
||||
if (hasReasoning) confidence += 15;
|
||||
if (result.selectedConcepts?.length > 0) confidence += 5;
|
||||
|
||||
const finalConfidence = Math.min(95, Math.max(25, confidence));
|
||||
|
||||
console.log('[CONFIDENCE-SCORING] Selection confidence calculated:', {
|
||||
candidateCount,
|
||||
selectedCount: result.selectedTools.length,
|
||||
selectionRatio: selectionRatio.toFixed(3),
|
||||
hasReasoning,
|
||||
confidence: finalConfidence
|
||||
});
|
||||
|
||||
return finalConfidence;
|
||||
}
|
||||
|
||||
getConfidenceLevel(confidence: number): 'weak' | 'moderate' | 'strong' {
|
||||
if (confidence >= this.config.highThreshold) return 'strong';
|
||||
if (confidence >= this.config.mediumThreshold) return 'moderate';
|
||||
return 'weak';
|
||||
}
|
||||
|
||||
getConfidenceColor(confidence: number): string {
|
||||
if (confidence >= this.config.highThreshold) return 'var(--color-accent)';
|
||||
if (confidence >= this.config.mediumThreshold) return 'var(--color-warning)';
|
||||
return 'var(--color-error)';
|
||||
}
|
||||
|
||||
getConfig(): ConfidenceConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
}
|
||||
|
||||
export const confidenceScoring = new ConfidenceScoring();
|
||||
@ -1,11 +1,11 @@
|
||||
// src/utils/embeddings.ts
|
||||
// src/utils/embeddings.ts - Refactored
|
||||
import { promises as fs } from 'fs';
|
||||
import path from 'path';
|
||||
import { getCompressedToolsDataForAI } from './dataService.js';
|
||||
import 'dotenv/config';
|
||||
import crypto from 'crypto';
|
||||
|
||||
interface EmbeddingData {
|
||||
export interface EmbeddingData {
|
||||
id: string;
|
||||
type: 'tool' | 'concept';
|
||||
name: string;
|
||||
@ -20,14 +20,23 @@ interface EmbeddingData {
|
||||
};
|
||||
}
|
||||
|
||||
export interface SimilarityResult extends EmbeddingData {
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
interface EmbeddingsDatabase {
|
||||
version: string;
|
||||
lastUpdated: number;
|
||||
embeddings: EmbeddingData[];
|
||||
}
|
||||
|
||||
interface SimilarityResult extends EmbeddingData {
|
||||
similarity: number;
|
||||
interface EmbeddingsConfig {
|
||||
enabled: boolean;
|
||||
endpoint?: string;
|
||||
apiKey?: string;
|
||||
model?: string;
|
||||
batchSize: number;
|
||||
batchDelay: number;
|
||||
}
|
||||
|
||||
class EmbeddingsService {
|
||||
@ -35,48 +44,33 @@ class EmbeddingsService {
|
||||
private isInitialized = false;
|
||||
private initializationPromise: Promise<void> | null = null;
|
||||
private readonly embeddingsPath = path.join(process.cwd(), 'data', 'embeddings.json');
|
||||
private readonly batchSize: number;
|
||||
private readonly batchDelay: number;
|
||||
private enabled: boolean = false;
|
||||
private config: EmbeddingsConfig;
|
||||
|
||||
constructor() {
|
||||
this.batchSize = parseInt(process.env.AI_EMBEDDINGS_BATCH_SIZE || '20', 10);
|
||||
this.batchDelay = parseInt(process.env.AI_EMBEDDINGS_BATCH_DELAY_MS || '1000', 10);
|
||||
|
||||
this.enabled = true;
|
||||
this.config = this.loadConfig();
|
||||
console.log('[EMBEDDINGS-SERVICE] Initialized:', {
|
||||
enabled: this.config.enabled,
|
||||
hasEndpoint: !!this.config.endpoint,
|
||||
hasModel: !!this.config.model
|
||||
});
|
||||
}
|
||||
|
||||
private async checkEnabledStatus(): Promise<void> {
|
||||
try {
|
||||
const envEnabled = process.env.AI_EMBEDDINGS_ENABLED;
|
||||
private loadConfig(): EmbeddingsConfig {
|
||||
const enabled = process.env.AI_EMBEDDINGS_ENABLED === 'true';
|
||||
const endpoint = process.env.AI_EMBEDDINGS_ENDPOINT;
|
||||
const apiKey = process.env.AI_EMBEDDINGS_API_KEY;
|
||||
const model = process.env.AI_EMBEDDINGS_MODEL;
|
||||
const batchSize = parseInt(process.env.AI_EMBEDDINGS_BATCH_SIZE || '20', 10);
|
||||
const batchDelay = parseInt(process.env.AI_EMBEDDINGS_BATCH_DELAY_MS || '1000', 10);
|
||||
|
||||
if (envEnabled === 'true') {
|
||||
const endpoint = process.env.AI_EMBEDDINGS_ENDPOINT;
|
||||
const model = process.env.AI_EMBEDDINGS_MODEL;
|
||||
|
||||
if (!endpoint || !model) {
|
||||
console.warn('[EMBEDDINGS] Embeddings enabled but API configuration missing - disabling');
|
||||
this.enabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[EMBEDDINGS] All requirements met - enabling embeddings');
|
||||
this.enabled = true;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.stat(this.embeddingsPath);
|
||||
console.log('[EMBEDDINGS] Existing embeddings file found - enabling');
|
||||
this.enabled = true;
|
||||
} catch {
|
||||
console.log('[EMBEDDINGS] Embeddings not explicitly enabled - disabling');
|
||||
this.enabled = false;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[EMBEDDINGS] Error checking enabled status:', error);
|
||||
this.enabled = false;
|
||||
}
|
||||
return {
|
||||
enabled,
|
||||
endpoint,
|
||||
apiKey,
|
||||
model,
|
||||
batchSize,
|
||||
batchDelay
|
||||
};
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
@ -93,46 +87,43 @@ class EmbeddingsService {
|
||||
}
|
||||
|
||||
private async performInitialization(): Promise<void> {
|
||||
await this.checkEnabledStatus();
|
||||
if (!this.enabled) {
|
||||
console.log('[EMBEDDINGS] Embeddings disabled, skipping initialization');
|
||||
return;
|
||||
}
|
||||
|
||||
const initStart = Date.now();
|
||||
|
||||
try {
|
||||
console.log('[EMBEDDINGS] Initializing embeddings system…');
|
||||
console.log('[EMBEDDINGS-SERVICE] Starting initialization');
|
||||
|
||||
if (!this.config.enabled) {
|
||||
console.log('[EMBEDDINGS-SERVICE] Service disabled via configuration');
|
||||
return;
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(this.embeddingsPath), { recursive: true });
|
||||
|
||||
const toolsData = await getCompressedToolsDataForAI();
|
||||
const currentDataHash = await this.hashToolsFile();
|
||||
const toolsData = await getCompressedToolsDataForAI();
|
||||
const currentDataHash = await this.hashToolsFile();
|
||||
|
||||
const existing = await this.loadEmbeddings();
|
||||
console.log('[EMBEDDINGS] Current hash:', currentDataHash);
|
||||
console.log('[EMBEDDINGS] Existing file version:', existing?.version);
|
||||
console.log('[EMBEDDINGS] Existing embeddings length:', existing?.embeddings?.length);
|
||||
const existing = await this.loadEmbeddings();
|
||||
|
||||
const cacheIsUsable =
|
||||
existing &&
|
||||
const cacheIsUsable = existing &&
|
||||
existing.version === currentDataHash &&
|
||||
Array.isArray(existing.embeddings) &&
|
||||
existing.embeddings.length > 0;
|
||||
|
||||
if (cacheIsUsable) {
|
||||
console.log('[EMBEDDINGS] Using cached embeddings');
|
||||
this.embeddings = existing.embeddings;
|
||||
console.log('[EMBEDDINGS-SERVICE] Using cached embeddings');
|
||||
this.embeddings = existing.embeddings;
|
||||
} else {
|
||||
console.log('[EMBEDDINGS] Generating new embeddings…');
|
||||
console.log('[EMBEDDINGS-SERVICE] Generating new embeddings');
|
||||
await this.generateEmbeddings(toolsData, currentDataHash);
|
||||
}
|
||||
|
||||
this.isInitialized = true;
|
||||
console.log(`[EMBEDDINGS] Initialized with ${this.embeddings.length} embeddings in ${Date.now() - initStart} ms`);
|
||||
} catch (err) {
|
||||
console.error('[EMBEDDINGS] Failed to initialize:', err);
|
||||
console.log(`[EMBEDDINGS-SERVICE] Initialized successfully with ${this.embeddings.length} embeddings in ${Date.now() - initStart}ms`);
|
||||
|
||||
} catch (error) {
|
||||
console.error('[EMBEDDINGS-SERVICE] Initialization failed:', error);
|
||||
this.isInitialized = false;
|
||||
throw err;
|
||||
throw error;
|
||||
} finally {
|
||||
this.initializationPromise = null;
|
||||
}
|
||||
@ -140,7 +131,7 @@ class EmbeddingsService {
|
||||
|
||||
private async hashToolsFile(): Promise<string> {
|
||||
const file = path.join(process.cwd(), 'src', 'data', 'tools.yaml');
|
||||
const raw = await fs.readFile(file, 'utf8');
|
||||
const raw = await fs.readFile(file, 'utf8');
|
||||
return crypto.createHash('sha256').update(raw).digest('hex');
|
||||
}
|
||||
|
||||
@ -149,7 +140,7 @@ class EmbeddingsService {
|
||||
const data = await fs.readFile(this.embeddingsPath, 'utf8');
|
||||
return JSON.parse(data);
|
||||
} catch (error) {
|
||||
console.log('[EMBEDDINGS] No existing embeddings found');
|
||||
console.log('[EMBEDDINGS-SERVICE] No existing embeddings file found');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -162,7 +153,7 @@ class EmbeddingsService {
|
||||
};
|
||||
|
||||
await fs.writeFile(this.embeddingsPath, JSON.stringify(database, null, 2));
|
||||
console.log(`[EMBEDDINGS] Saved ${this.embeddings.length} embeddings to disk`);
|
||||
console.log(`[EMBEDDINGS-SERVICE] Saved ${this.embeddings.length} embeddings to disk`);
|
||||
}
|
||||
|
||||
private createContentString(item: any): string {
|
||||
@ -178,30 +169,23 @@ class EmbeddingsService {
|
||||
}
|
||||
|
||||
private async generateEmbeddingsBatch(contents: string[]): Promise<number[][]> {
|
||||
const endpoint = process.env.AI_EMBEDDINGS_ENDPOINT;
|
||||
const apiKey = process.env.AI_EMBEDDINGS_API_KEY;
|
||||
const model = process.env.AI_EMBEDDINGS_MODEL;
|
||||
|
||||
if (!endpoint || !model) {
|
||||
const missing: string[] = [];
|
||||
if (!endpoint) missing.push('AI_EMBEDDINGS_ENDPOINT');
|
||||
if (!model) missing.push('AI_EMBEDDINGS_MODEL');
|
||||
throw new Error(`Missing embeddings API configuration: ${missing.join(', ')}`);
|
||||
if (!this.config.endpoint || !this.config.model) {
|
||||
throw new Error('Missing embeddings API configuration');
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json'
|
||||
};
|
||||
|
||||
if (apiKey) {
|
||||
headers['Authorization'] = `Bearer ${apiKey}`;
|
||||
if (this.config.apiKey) {
|
||||
headers['Authorization'] = `Bearer ${this.config.apiKey}`;
|
||||
}
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
const response = await fetch(this.config.endpoint, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
model: this.config.model,
|
||||
input: contents
|
||||
})
|
||||
});
|
||||
@ -233,11 +217,16 @@ class EmbeddingsService {
|
||||
const contents = allItems.map(item => this.createContentString(item));
|
||||
this.embeddings = [];
|
||||
|
||||
for (let i = 0; i < contents.length; i += this.batchSize) {
|
||||
const batch = contents.slice(i, i + this.batchSize);
|
||||
const batchItems = allItems.slice(i, i + this.batchSize);
|
||||
console.log(`[EMBEDDINGS-SERVICE] Generating embeddings for ${contents.length} items`);
|
||||
|
||||
console.log(`[EMBEDDINGS] Processing batch ${Math.ceil((i + 1) / this.batchSize)} of ${Math.ceil(contents.length / this.batchSize)}`);
|
||||
for (let i = 0; i < contents.length; i += this.config.batchSize) {
|
||||
const batch = contents.slice(i, i + this.config.batchSize);
|
||||
const batchItems = allItems.slice(i, i + this.config.batchSize);
|
||||
|
||||
const batchNumber = Math.ceil((i + 1) / this.config.batchSize);
|
||||
const totalBatches = Math.ceil(contents.length / this.config.batchSize);
|
||||
|
||||
console.log(`[EMBEDDINGS-SERVICE] Processing batch ${batchNumber}/${totalBatches}`);
|
||||
|
||||
try {
|
||||
const embeddings = await this.generateEmbeddingsBatch(batch);
|
||||
@ -260,12 +249,12 @@ class EmbeddingsService {
|
||||
});
|
||||
});
|
||||
|
||||
if (i + this.batchSize < contents.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, this.batchDelay));
|
||||
if (i + this.config.batchSize < contents.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, this.config.batchDelay));
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`[EMBEDDINGS] Failed to process batch ${Math.ceil((i + 1) / this.batchSize)}:`, error);
|
||||
console.error(`[EMBEDDINGS-SERVICE] Batch ${batchNumber} failed:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -273,18 +262,21 @@ class EmbeddingsService {
|
||||
await this.saveEmbeddings(version);
|
||||
}
|
||||
|
||||
public async embedText(text: string): Promise<number[]> {
|
||||
if (!this.enabled || !this.isInitialized) {
|
||||
async embedText(text: string): Promise<number[]> {
|
||||
if (!this.isEnabled() || !this.isInitialized) {
|
||||
throw new Error('Embeddings service not available');
|
||||
}
|
||||
|
||||
const [embedding] = await this.generateEmbeddingsBatch([text.toLowerCase()]);
|
||||
return embedding;
|
||||
}
|
||||
|
||||
async waitForInitialization(): Promise<void> {
|
||||
await this.checkEnabledStatus();
|
||||
if (!this.config.enabled) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
if (!this.enabled || this.isInitialized) {
|
||||
if (this.isInitialized) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
@ -296,13 +288,6 @@ class EmbeddingsService {
|
||||
return this.initialize();
|
||||
}
|
||||
|
||||
async forceRecheckEnvironment(): Promise<void> {
|
||||
this.enabled = false;
|
||||
this.isInitialized = false;
|
||||
await this.checkEnabledStatus();
|
||||
console.log('[EMBEDDINGS] Environment status re-checked, enabled:', this.enabled);
|
||||
}
|
||||
|
||||
private cosineSimilarity(a: number[], b: number[]): number {
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
@ -318,145 +303,67 @@ class EmbeddingsService {
|
||||
}
|
||||
|
||||
async findSimilar(query: string, maxResults: number = 30, threshold: number = 0.3): Promise<SimilarityResult[]> {
|
||||
if (!this.enabled) {
|
||||
console.log('[EMBEDDINGS] Service disabled for similarity search');
|
||||
if (!this.config.enabled) {
|
||||
console.log('[EMBEDDINGS-SERVICE] Service disabled, returning empty results');
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!this.isInitialized || this.embeddings.length === 0) {
|
||||
console.log('[EMBEDDINGS-SERVICE] Not initialized or no embeddings available');
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.isInitialized && this.embeddings.length > 0) {
|
||||
console.log(`[EMBEDDINGS] Using embeddings data for similarity search: ${query}`);
|
||||
console.log(`[EMBEDDINGS-SERVICE] Finding similar items for query: "${query}"`);
|
||||
|
||||
const queryEmbeddings = await this.generateEmbeddingsBatch([query.toLowerCase()]);
|
||||
const queryEmbedding = queryEmbeddings[0];
|
||||
const queryEmbeddings = await this.generateEmbeddingsBatch([query.toLowerCase()]);
|
||||
const queryEmbedding = queryEmbeddings[0];
|
||||
|
||||
console.log(`[EMBEDDINGS] Computing similarities for ${this.embeddings.length} items`);
|
||||
const similarities: SimilarityResult[] = this.embeddings.map(item => ({
|
||||
...item,
|
||||
similarity: this.cosineSimilarity(queryEmbedding, item.embedding)
|
||||
}));
|
||||
|
||||
const similarities: SimilarityResult[] = this.embeddings.map(item => ({
|
||||
...item,
|
||||
similarity: this.cosineSimilarity(queryEmbedding, item.embedding)
|
||||
}));
|
||||
const topScore = Math.max(...similarities.map(s => s.similarity));
|
||||
const dynamicThreshold = Math.max(threshold, topScore * 0.85);
|
||||
|
||||
const topScore = Math.max(...similarities.map(s => s.similarity));
|
||||
const dynamicCutOff = Math.max(threshold, topScore * 0.85);
|
||||
const results = similarities
|
||||
.filter(item => item.similarity >= dynamicThreshold)
|
||||
.sort((a, b) => b.similarity - a.similarity)
|
||||
.slice(0, maxResults);
|
||||
|
||||
const results = similarities
|
||||
.filter(item => item.similarity >= dynamicCutOff)
|
||||
.sort((a, b) => b.similarity - a.similarity)
|
||||
.slice(0, maxResults);
|
||||
console.log(`[EMBEDDINGS-SERVICE] Found ${results.length} similar items (threshold: ${dynamicThreshold.toFixed(3)})`);
|
||||
|
||||
|
||||
const orderingValid = results.every((item, index) => {
|
||||
if (index === 0) return true;
|
||||
return item.similarity <= results[index - 1].similarity;
|
||||
if (results.length > 0) {
|
||||
console.log('[EMBEDDINGS-SERVICE] Top 5 matches:');
|
||||
results.slice(0, 5).forEach((item, idx) => {
|
||||
console.log(` ${idx + 1}. ${item.name} (${item.type}) = ${item.similarity.toFixed(4)}`);
|
||||
});
|
||||
|
||||
if (!orderingValid) {
|
||||
console.error('[EMBEDDINGS] CRITICAL: Similarity ordering is broken!');
|
||||
results.forEach((item, idx) => {
|
||||
console.error(` ${idx}: ${item.name} = ${item.similarity.toFixed(4)}`);
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`[EMBEDDINGS] Found ${results.length} similar items (threshold: ${threshold})`);
|
||||
if (results.length > 0) {
|
||||
console.log('[EMBEDDINGS] Top 10 similarity matches:');
|
||||
results.slice(0, 10).forEach((item, idx) => {
|
||||
console.log(` ${idx + 1}. ${item.name} (${item.type}) = ${item.similarity.toFixed(4)}`);
|
||||
});
|
||||
|
||||
const topSimilarity = results[0].similarity;
|
||||
const hasHigherSimilarity = results.some(item => item.similarity > topSimilarity);
|
||||
if (hasHigherSimilarity) {
|
||||
console.error('[EMBEDDINGS] CRITICAL: Top result is not actually the highest similarity!');
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
|
||||
} else {
|
||||
console.log(`[EMBEDDINGS] No embeddings data, using fallback text matching: ${query}`);
|
||||
|
||||
const { getToolsData } = await import('./dataService.js');
|
||||
const toolsData = await getToolsData();
|
||||
|
||||
const queryLower = query.toLowerCase();
|
||||
const queryWords = queryLower.split(/\s+/).filter(w => w.length > 2);
|
||||
|
||||
const similarities: SimilarityResult[] = toolsData.tools
|
||||
.map((tool: any) => {
|
||||
let similarity = 0;
|
||||
|
||||
if (tool.name.toLowerCase().includes(queryLower)) {
|
||||
similarity += 0.8;
|
||||
}
|
||||
|
||||
if (tool.description && tool.description.toLowerCase().includes(queryLower)) {
|
||||
similarity += 0.6;
|
||||
}
|
||||
|
||||
if (tool.tags && Array.isArray(tool.tags)) {
|
||||
const matchingTags = tool.tags.filter((tag: string) =>
|
||||
tag.toLowerCase().includes(queryLower) || queryLower.includes(tag.toLowerCase())
|
||||
);
|
||||
if (tool.tags.length > 0) {
|
||||
similarity += (matchingTags.length / tool.tags.length) * 0.4;
|
||||
}
|
||||
}
|
||||
|
||||
const toolText = `${tool.name} ${tool.description || ''} ${(tool.tags || []).join(' ')}`.toLowerCase();
|
||||
const matchingWords = queryWords.filter(word => toolText.includes(word));
|
||||
if (queryWords.length > 0) {
|
||||
similarity += (matchingWords.length / queryWords.length) * 0.3;
|
||||
}
|
||||
|
||||
return {
|
||||
id: `tool_${tool.name.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase()}`,
|
||||
type: 'tool' as const,
|
||||
name: tool.name,
|
||||
content: toolText,
|
||||
embedding: [],
|
||||
metadata: {
|
||||
domains: tool.domains || [],
|
||||
phases: tool.phases || [],
|
||||
tags: tool.tags || [],
|
||||
skillLevel: tool.skillLevel,
|
||||
type: tool.type
|
||||
},
|
||||
similarity: Math.min(similarity, 1.0)
|
||||
};
|
||||
})
|
||||
.filter(item => item.similarity >= threshold)
|
||||
.sort((a, b) => b.similarity - a.similarity)
|
||||
.slice(0, maxResults);
|
||||
|
||||
console.log(`[EMBEDDINGS] Fallback found ${similarities.length} similar items`);
|
||||
return similarities;
|
||||
}
|
||||
|
||||
return results;
|
||||
|
||||
} catch (error) {
|
||||
console.error('[EMBEDDINGS] Failed to find similar items:', error);
|
||||
console.error('[EMBEDDINGS-SERVICE] Similarity search failed:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
isEnabled(): boolean {
|
||||
if (!this.enabled && !this.isInitialized) {
|
||||
this.checkEnabledStatus().catch(console.error);
|
||||
}
|
||||
|
||||
return this.enabled;
|
||||
return this.config.enabled;
|
||||
}
|
||||
|
||||
getStats(): { enabled: boolean; initialized: boolean; count: number } {
|
||||
return {
|
||||
enabled: this.enabled,
|
||||
enabled: this.config.enabled,
|
||||
initialized: this.isInitialized,
|
||||
count: this.embeddings.length
|
||||
};
|
||||
}
|
||||
|
||||
getConfig(): EmbeddingsConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
}
|
||||
|
||||
const embeddingsService = new EmbeddingsService();
|
||||
|
||||
export { embeddingsService, type EmbeddingData, type SimilarityResult };
|
||||
export const embeddingsService = new EmbeddingsService();
|
||||
132
src/utils/jsonUtils.ts
Normal file
132
src/utils/jsonUtils.ts
Normal file
@ -0,0 +1,132 @@
|
||||
// src/utils/jsonUtils.ts
|
||||
export class JSONParser {
|
||||
static safeParseJSON(jsonString: string, fallback: any = null): any {
|
||||
try {
|
||||
let cleaned = jsonString.trim();
|
||||
|
||||
// Remove code block markers
|
||||
const jsonBlockPatterns = [
|
||||
/```json\s*([\s\S]*?)\s*```/i,
|
||||
/```\s*([\s\S]*?)\s*```/i,
|
||||
/\{[\s\S]*\}/,
|
||||
];
|
||||
|
||||
for (const pattern of jsonBlockPatterns) {
|
||||
const match = cleaned.match(pattern);
|
||||
if (match) {
|
||||
cleaned = match[1] || match[0];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle truncated JSON
|
||||
if (!cleaned.endsWith('}') && !cleaned.endsWith(']')) {
|
||||
console.warn('[JSON-PARSER] JSON appears truncated, attempting recovery');
|
||||
cleaned = this.repairTruncatedJSON(cleaned);
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(cleaned);
|
||||
|
||||
// Ensure proper structure for tool selection responses
|
||||
if (parsed && typeof parsed === 'object') {
|
||||
if (!parsed.selectedTools) parsed.selectedTools = [];
|
||||
if (!parsed.selectedConcepts) parsed.selectedConcepts = [];
|
||||
if (!Array.isArray(parsed.selectedTools)) parsed.selectedTools = [];
|
||||
if (!Array.isArray(parsed.selectedConcepts)) parsed.selectedConcepts = [];
|
||||
}
|
||||
|
||||
return parsed;
|
||||
|
||||
} catch (error) {
|
||||
console.warn('[JSON-PARSER] JSON parsing failed:', error.message);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
private static repairTruncatedJSON(cleaned: string): string {
|
||||
let braceCount = 0;
|
||||
let bracketCount = 0;
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
let lastCompleteStructure = '';
|
||||
|
||||
for (let i = 0; i < cleaned.length; i++) {
|
||||
const char = cleaned[i];
|
||||
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === '\\') {
|
||||
escaped = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === '"' && !escaped) {
|
||||
inString = !inString;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inString) {
|
||||
if (char === '{') braceCount++;
|
||||
if (char === '}') braceCount--;
|
||||
if (char === '[') bracketCount++;
|
||||
if (char === ']') bracketCount--;
|
||||
|
||||
if (braceCount === 0 && bracketCount === 0 && (char === '}' || char === ']')) {
|
||||
lastCompleteStructure = cleaned.substring(0, i + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (lastCompleteStructure) {
|
||||
return lastCompleteStructure;
|
||||
} else {
|
||||
if (braceCount > 0) cleaned += '}';
|
||||
if (bracketCount > 0) cleaned += ']';
|
||||
return cleaned;
|
||||
}
|
||||
}
|
||||
|
||||
static extractToolsFromMalformedJSON(jsonString: string): { selectedTools: string[]; selectedConcepts: string[] } {
|
||||
const selectedTools: string[] = [];
|
||||
const selectedConcepts: string[] = [];
|
||||
|
||||
const toolsMatch = jsonString.match(/"selectedTools"\s*:\s*\[([\s\S]*?)\]/i);
|
||||
if (toolsMatch) {
|
||||
const toolMatches = toolsMatch[1].match(/"([^"]+)"/g);
|
||||
if (toolMatches) {
|
||||
selectedTools.push(...toolMatches.map(match => match.replace(/"/g, '')));
|
||||
}
|
||||
}
|
||||
|
||||
const conceptsMatch = jsonString.match(/"selectedConcepts"\s*:\s*\[([\s\S]*?)\]/i);
|
||||
if (conceptsMatch) {
|
||||
const conceptMatches = conceptsMatch[1].match(/"([^"]+)"/g);
|
||||
if (conceptMatches) {
|
||||
selectedConcepts.push(...conceptMatches.map(match => match.replace(/"/g, '')));
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: extract any quoted strings that look like tool names
|
||||
if (selectedTools.length === 0 && selectedConcepts.length === 0) {
|
||||
const allMatches = jsonString.match(/"([^"]+)"/g);
|
||||
if (allMatches) {
|
||||
const possibleNames = allMatches
|
||||
.map(match => match.replace(/"/g, ''))
|
||||
.filter(name =>
|
||||
name.length > 2 &&
|
||||
!['selectedTools', 'selectedConcepts', 'reasoning'].includes(name) &&
|
||||
!name.includes(':') &&
|
||||
!name.match(/^\d+$/)
|
||||
)
|
||||
.slice(0, 15);
|
||||
|
||||
selectedTools.push(...possibleNames);
|
||||
}
|
||||
}
|
||||
|
||||
return { selectedTools, selectedConcepts };
|
||||
}
|
||||
}
|
||||
458
src/utils/toolSelector.ts
Normal file
458
src/utils/toolSelector.ts
Normal file
@ -0,0 +1,458 @@
|
||||
// src/utils/toolSelector.ts
|
||||
import { aiService } from './aiService.js';
|
||||
import { embeddingsService, type SimilarityResult } from './embeddings.js';
|
||||
import { confidenceScoring } from './confidenceScoring.js';
|
||||
import { getPrompt } from '../config/prompts.js';
|
||||
import 'dotenv/config';
|
||||
|
||||
export interface ToolSelectionConfig {
|
||||
maxSelectedItems: number;
|
||||
embeddingCandidates: number;
|
||||
similarityThreshold: number;
|
||||
embeddingSelectionLimit: number;
|
||||
embeddingConceptsLimit: number;
|
||||
noEmbeddingsToolLimit: number;
|
||||
noEmbeddingsConceptLimit: number;
|
||||
embeddingsMinTools: number;
|
||||
embeddingsMaxReductionRatio: number;
|
||||
methodSelectionRatio: number;
|
||||
softwareSelectionRatio: number;
|
||||
}
|
||||
|
||||
export interface SelectionContext {
|
||||
userQuery: string;
|
||||
mode: string;
|
||||
embeddingsSimilarities: Map<string, number>;
|
||||
seenToolNames: Set<string>;
|
||||
selectedTools?: Array<{
|
||||
tool: any;
|
||||
phase: string;
|
||||
priority: string;
|
||||
justification?: string;
|
||||
taskRelevance?: number;
|
||||
limitations?: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ToolSelectionResult {
|
||||
selectedTools: any[];
|
||||
selectedConcepts: any[];
|
||||
selectionMethod: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
class ToolSelector {
|
||||
private config: ToolSelectionConfig;
|
||||
|
||||
constructor() {
|
||||
this.config = {
|
||||
maxSelectedItems: this.getEnvInt('AI_MAX_SELECTED_ITEMS', 25),
|
||||
embeddingCandidates: this.getEnvInt('AI_EMBEDDING_CANDIDATES', 50),
|
||||
similarityThreshold: this.getEnvFloat('AI_SIMILARITY_THRESHOLD', 0.3),
|
||||
embeddingSelectionLimit: this.getEnvInt('AI_EMBEDDING_SELECTION_LIMIT', 30),
|
||||
embeddingConceptsLimit: this.getEnvInt('AI_EMBEDDING_CONCEPTS_LIMIT', 15),
|
||||
noEmbeddingsToolLimit: this.getEnvInt('AI_NO_EMBEDDINGS_TOOL_LIMIT', 25),
|
||||
noEmbeddingsConceptLimit: this.getEnvInt('AI_NO_EMBEDDINGS_CONCEPT_LIMIT', 10),
|
||||
embeddingsMinTools: this.getEnvInt('AI_EMBEDDINGS_MIN_TOOLS', 8),
|
||||
embeddingsMaxReductionRatio: this.getEnvFloat('AI_EMBEDDINGS_MAX_REDUCTION_RATIO', 0.75),
|
||||
methodSelectionRatio: this.getEnvFloat('AI_METHOD_SELECTION_RATIO', 0.4),
|
||||
softwareSelectionRatio: this.getEnvFloat('AI_SOFTWARE_SELECTION_RATIO', 0.5)
|
||||
};
|
||||
|
||||
console.log('[TOOL-SELECTOR] Initialized with config:', this.config);
|
||||
}
|
||||
|
||||
private getEnvInt(key: string, defaultValue: number): number {
|
||||
const value = process.env[key];
|
||||
return value ? parseInt(value, 10) : defaultValue;
|
||||
}
|
||||
|
||||
private getEnvFloat(key: string, defaultValue: number): number {
|
||||
const value = process.env[key];
|
||||
return value ? parseFloat(value) : defaultValue;
|
||||
}
|
||||
|
||||
async getIntelligentCandidates(
|
||||
userQuery: string,
|
||||
toolsData: any,
|
||||
mode: string,
|
||||
context: SelectionContext
|
||||
): Promise<{
|
||||
tools: any[];
|
||||
concepts: any[];
|
||||
domains: any[];
|
||||
phases: any[];
|
||||
'domain-agnostic-software': any[];
|
||||
selectionMethod: string;
|
||||
}> {
|
||||
console.log('[TOOL-SELECTOR] Getting intelligent candidates for query');
|
||||
|
||||
let candidateTools: any[] = [];
|
||||
let candidateConcepts: any[] = [];
|
||||
let selectionMethod = 'unknown';
|
||||
|
||||
context.embeddingsSimilarities.clear();
|
||||
|
||||
try {
|
||||
await embeddingsService.waitForInitialization();
|
||||
} catch (error) {
|
||||
console.error('[TOOL-SELECTOR] Embeddings initialization failed:', error);
|
||||
}
|
||||
|
||||
if (embeddingsService.isEnabled()) {
|
||||
console.log('[TOOL-SELECTOR] Using embeddings for candidate selection');
|
||||
|
||||
const similarItems = await embeddingsService.findSimilar(
|
||||
userQuery,
|
||||
this.config.embeddingCandidates,
|
||||
this.config.similarityThreshold
|
||||
) as SimilarityResult[];
|
||||
|
||||
console.log('[TOOL-SELECTOR] Embeddings found', similarItems.length, 'similar items');
|
||||
|
||||
// Store similarities for confidence calculation
|
||||
similarItems.forEach(item => {
|
||||
context.embeddingsSimilarities.set(item.name, item.similarity);
|
||||
});
|
||||
|
||||
const toolsMap = new Map(toolsData.tools.map((tool: any) => [tool.name, tool]));
|
||||
const conceptsMap = new Map(toolsData.concepts.map((concept: any) => [concept.name, concept]));
|
||||
|
||||
const similarTools = similarItems
|
||||
.filter((item: any) => item.type === 'tool')
|
||||
.map((item: any) => toolsMap.get(item.name))
|
||||
.filter((tool: any): tool is NonNullable<any> => tool !== undefined && tool !== null);
|
||||
|
||||
const similarConcepts = similarItems
|
||||
.filter((item: any) => item.type === 'concept')
|
||||
.map((item: any) => conceptsMap.get(item.name))
|
||||
.filter((concept: any): concept is NonNullable<any> => concept !== undefined && concept !== null);
|
||||
|
||||
const totalAvailableTools = toolsData.tools.length;
|
||||
const reductionRatio = similarTools.length / totalAvailableTools;
|
||||
|
||||
if (similarTools.length >= this.config.embeddingsMinTools && reductionRatio <= this.config.embeddingsMaxReductionRatio) {
|
||||
candidateTools = similarTools;
|
||||
candidateConcepts = similarConcepts;
|
||||
selectionMethod = 'embeddings_candidates';
|
||||
|
||||
console.log('[TOOL-SELECTOR] Using embeddings filtering:', totalAvailableTools, '→', similarTools.length, 'tools');
|
||||
} else {
|
||||
console.log('[TOOL-SELECTOR] Embeddings filtering insufficient, using full dataset');
|
||||
candidateTools = toolsData.tools;
|
||||
candidateConcepts = toolsData.concepts;
|
||||
selectionMethod = 'full_dataset';
|
||||
}
|
||||
} else {
|
||||
console.log('[TOOL-SELECTOR] Embeddings disabled, using full dataset');
|
||||
candidateTools = toolsData.tools;
|
||||
candidateConcepts = toolsData.concepts;
|
||||
selectionMethod = 'full_dataset';
|
||||
}
|
||||
|
||||
const selection = await this.performAISelection(
|
||||
userQuery,
|
||||
candidateTools,
|
||||
candidateConcepts,
|
||||
mode,
|
||||
selectionMethod,
|
||||
context
|
||||
);
|
||||
|
||||
return {
|
||||
tools: selection.selectedTools,
|
||||
concepts: selection.selectedConcepts,
|
||||
domains: toolsData.domains,
|
||||
phases: toolsData.phases,
|
||||
'domain-agnostic-software': toolsData['domain-agnostic-software'],
|
||||
selectionMethod
|
||||
};
|
||||
}
|
||||
|
||||
private async performAISelection(
|
||||
userQuery: string,
|
||||
candidateTools: any[],
|
||||
candidateConcepts: any[],
|
||||
mode: string,
|
||||
selectionMethod: string,
|
||||
context: SelectionContext
|
||||
): Promise<ToolSelectionResult> {
|
||||
console.log('[TOOL-SELECTOR] Performing AI selection');
|
||||
|
||||
const candidateMethods = candidateTools.filter((tool: any) => tool && tool.type === 'method');
|
||||
const candidateSoftware = candidateTools.filter((tool: any) => tool && tool.type === 'software');
|
||||
|
||||
console.log('[TOOL-SELECTOR] Candidates:', candidateMethods.length, 'methods,', candidateSoftware.length, 'software,', candidateConcepts.length, 'concepts');
|
||||
|
||||
const methodsWithFullData = candidateMethods.map(this.createToolData);
|
||||
const softwareWithFullData = candidateSoftware.map(this.createToolData);
|
||||
const conceptsWithFullData = candidateConcepts.map(this.createConceptData);
|
||||
|
||||
let toolsToSend: any[];
|
||||
let conceptsToSend: any[];
|
||||
|
||||
if (selectionMethod === 'embeddings_candidates') {
|
||||
const totalLimit = this.config.embeddingSelectionLimit;
|
||||
const methodLimit = Math.ceil(totalLimit * this.config.methodSelectionRatio);
|
||||
const softwareLimit = Math.floor(totalLimit * this.config.softwareSelectionRatio);
|
||||
|
||||
toolsToSend = [
|
||||
...methodsWithFullData.slice(0, methodLimit),
|
||||
...softwareWithFullData.slice(0, softwareLimit)
|
||||
];
|
||||
|
||||
const remainingCapacity = totalLimit - toolsToSend.length;
|
||||
if (remainingCapacity > 0) {
|
||||
if (methodsWithFullData.length > methodLimit) {
|
||||
toolsToSend.push(...methodsWithFullData.slice(methodLimit, methodLimit + remainingCapacity));
|
||||
} else if (softwareWithFullData.length > softwareLimit) {
|
||||
toolsToSend.push(...softwareWithFullData.slice(softwareLimit, softwareLimit + remainingCapacity));
|
||||
}
|
||||
}
|
||||
|
||||
conceptsToSend = conceptsWithFullData.slice(0, this.config.embeddingConceptsLimit);
|
||||
} else {
|
||||
const maxTools = this.config.noEmbeddingsToolLimit;
|
||||
const maxConcepts = this.config.noEmbeddingsConceptLimit;
|
||||
const methodLimit = Math.ceil(maxTools * 0.4);
|
||||
const softwareLimit = Math.floor(maxTools * 0.5);
|
||||
|
||||
toolsToSend = [
|
||||
...methodsWithFullData.slice(0, methodLimit),
|
||||
...softwareWithFullData.slice(0, softwareLimit)
|
||||
];
|
||||
|
||||
const remainingCapacity = maxTools - toolsToSend.length;
|
||||
if (remainingCapacity > 0) {
|
||||
if (methodsWithFullData.length > methodLimit) {
|
||||
toolsToSend.push(...methodsWithFullData.slice(methodLimit, methodLimit + remainingCapacity));
|
||||
} else if (softwareWithFullData.length > softwareLimit) {
|
||||
toolsToSend.push(...softwareWithFullData.slice(softwareLimit, softwareLimit + remainingCapacity));
|
||||
}
|
||||
}
|
||||
|
||||
conceptsToSend = conceptsWithFullData.slice(0, maxConcepts);
|
||||
}
|
||||
|
||||
const basePrompt = getPrompt('toolSelection', mode, userQuery, selectionMethod, this.config.maxSelectedItems);
|
||||
const prompt = getPrompt('toolSelectionWithData', basePrompt, toolsToSend, conceptsToSend);
|
||||
|
||||
// Validate prompt length
|
||||
aiService.validatePromptLength(prompt);
|
||||
|
||||
console.log('[TOOL-SELECTOR] Sending to AI:',
|
||||
toolsToSend.filter((t: any) => t.type === 'method').length, 'methods,',
|
||||
toolsToSend.filter((t: any) => t.type === 'software').length, 'software,',
|
||||
conceptsToSend.length, 'concepts'
|
||||
);
|
||||
|
||||
try {
|
||||
const response = await aiService.callAI(prompt, { maxTokens: 2500 });
|
||||
const result = this.safeParseJSON(response.content, null);
|
||||
|
||||
if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) {
|
||||
console.error('[TOOL-SELECTOR] AI selection returned invalid structure');
|
||||
throw new Error('AI selection failed to return valid tool and concept selection');
|
||||
}
|
||||
|
||||
const totalSelected = result.selectedTools.length + result.selectedConcepts.length;
|
||||
if (totalSelected === 0) {
|
||||
throw new Error('AI selection returned empty selection');
|
||||
}
|
||||
|
||||
const toolsMap = new Map(candidateTools.map((tool: any) => [tool.name, tool]));
|
||||
const conceptsMap = new Map(candidateConcepts.map((concept: any) => [concept.name, concept]));
|
||||
|
||||
const selectedTools = result.selectedTools
|
||||
.map((name: string) => toolsMap.get(name))
|
||||
.filter((tool: any): tool is NonNullable<any> => tool !== undefined && tool !== null);
|
||||
|
||||
const selectedConcepts = result.selectedConcepts
|
||||
.map((name: string) => conceptsMap.get(name))
|
||||
.filter((concept: any): concept is NonNullable<any> => concept !== undefined && concept !== null);
|
||||
|
||||
const selectedMethods = selectedTools.filter((t: any) => t && t.type === 'method');
|
||||
const selectedSoftware = selectedTools.filter((t: any) => t && t.type === 'software');
|
||||
|
||||
console.log('[TOOL-SELECTOR] AI selected:', selectedMethods.length, 'methods,', selectedSoftware.length, 'software,', selectedConcepts.length, 'concepts');
|
||||
|
||||
const confidence = confidenceScoring.calculateSelectionConfidence(result, candidateTools.length + candidateConcepts.length);
|
||||
|
||||
return {
|
||||
selectedTools,
|
||||
selectedConcepts,
|
||||
selectionMethod,
|
||||
confidence
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('[TOOL-SELECTOR] AI selection failed:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async selectToolsForPhase(
|
||||
userQuery: string,
|
||||
phase: any,
|
||||
availableTools: any[],
|
||||
context: SelectionContext
|
||||
): Promise<Array<{
|
||||
toolName: string;
|
||||
taskRelevance: number;
|
||||
justification: string;
|
||||
limitations: string[];
|
||||
}>> {
|
||||
console.log('[TOOL-SELECTOR] Selecting tools for phase:', phase.id);
|
||||
|
||||
if (availableTools.length === 0) {
|
||||
console.log('[TOOL-SELECTOR] No tools available for phase:', phase.id);
|
||||
return [];
|
||||
}
|
||||
|
||||
const prompt = getPrompt('phaseToolSelection', userQuery, phase, availableTools);
|
||||
|
||||
try {
|
||||
const response = await aiService.callMicroTaskAI(prompt, 1000);
|
||||
const selections = this.safeParseJSON(response.content, []);
|
||||
|
||||
if (Array.isArray(selections)) {
|
||||
const validSelections = selections.filter((sel: any) => {
|
||||
const matchingTool = availableTools.find((tool: any) => tool && tool.name === sel.toolName);
|
||||
if (!matchingTool) {
|
||||
console.warn('[TOOL-SELECTOR] Invalid tool selection for phase:', phase.id, sel.toolName);
|
||||
}
|
||||
return !!matchingTool;
|
||||
});
|
||||
|
||||
console.log('[TOOL-SELECTOR] Valid selections for phase:', phase.id, validSelections.length);
|
||||
return validSelections;
|
||||
}
|
||||
|
||||
return [];
|
||||
|
||||
} catch (error) {
|
||||
console.error('[TOOL-SELECTOR] Phase tool selection failed:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private createToolData = (tool: any) => ({
|
||||
name: tool.name,
|
||||
type: tool.type,
|
||||
description: tool.description,
|
||||
domains: tool.domains,
|
||||
phases: tool.phases,
|
||||
platforms: tool.platforms || [],
|
||||
tags: tool.tags || [],
|
||||
skillLevel: tool.skillLevel,
|
||||
license: tool.license,
|
||||
accessType: tool.accessType,
|
||||
projectUrl: tool.projectUrl,
|
||||
knowledgebase: tool.knowledgebase,
|
||||
related_concepts: tool.related_concepts || [],
|
||||
related_software: tool.related_software || []
|
||||
});
|
||||
|
||||
private createConceptData = (concept: any) => ({
|
||||
name: concept.name,
|
||||
type: 'concept',
|
||||
description: concept.description,
|
||||
domains: concept.domains,
|
||||
phases: concept.phases,
|
||||
tags: concept.tags || [],
|
||||
skillLevel: concept.skillLevel,
|
||||
related_concepts: concept.related_concepts || [],
|
||||
related_software: concept.related_software || []
|
||||
});
|
||||
|
||||
private safeParseJSON(jsonString: string, fallback: any = null): any {
|
||||
try {
|
||||
let cleaned = jsonString.trim();
|
||||
|
||||
// Remove code block markers
|
||||
const jsonBlockPatterns = [
|
||||
/```json\s*([\s\S]*?)\s*```/i,
|
||||
/```\s*([\s\S]*?)\s*```/i,
|
||||
/\{[\s\S]*\}/,
|
||||
];
|
||||
|
||||
for (const pattern of jsonBlockPatterns) {
|
||||
const match = cleaned.match(pattern);
|
||||
if (match) {
|
||||
cleaned = match[1] || match[0];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle truncated JSON
|
||||
if (!cleaned.endsWith('}') && !cleaned.endsWith(']')) {
|
||||
console.warn('[TOOL-SELECTOR] JSON appears truncated, attempting recovery');
|
||||
|
||||
let braceCount = 0;
|
||||
let bracketCount = 0;
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
let lastCompleteStructure = '';
|
||||
|
||||
for (let i = 0; i < cleaned.length; i++) {
|
||||
const char = cleaned[i];
|
||||
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === '\\') {
|
||||
escaped = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === '"' && !escaped) {
|
||||
inString = !inString;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inString) {
|
||||
if (char === '{') braceCount++;
|
||||
if (char === '}') braceCount--;
|
||||
if (char === '[') bracketCount++;
|
||||
if (char === ']') bracketCount--;
|
||||
|
||||
if (braceCount === 0 && bracketCount === 0 && (char === '}' || char === ']')) {
|
||||
lastCompleteStructure = cleaned.substring(0, i + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (lastCompleteStructure) {
|
||||
cleaned = lastCompleteStructure;
|
||||
} else {
|
||||
if (braceCount > 0) cleaned += '}';
|
||||
if (bracketCount > 0) cleaned += ']';
|
||||
}
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(cleaned);
|
||||
|
||||
// Ensure proper structure
|
||||
if (parsed && typeof parsed === 'object') {
|
||||
if (!parsed.selectedTools) parsed.selectedTools = [];
|
||||
if (!parsed.selectedConcepts) parsed.selectedConcepts = [];
|
||||
if (!Array.isArray(parsed.selectedTools)) parsed.selectedTools = [];
|
||||
if (!Array.isArray(parsed.selectedConcepts)) parsed.selectedConcepts = [];
|
||||
}
|
||||
|
||||
return parsed;
|
||||
|
||||
} catch (error) {
|
||||
console.warn('[TOOL-SELECTOR] JSON parsing failed:', error.message);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
getConfig(): ToolSelectionConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
}
|
||||
|
||||
export const toolSelector = new ToolSelector();
|
||||
Loading…
x
Reference in New Issue
Block a user