// src/utils/aiPipeline.ts import { getCompressedToolsDataForAI } from './dataService.js'; import { embeddingsService, type EmbeddingData } from './embeddings.js'; import { vectorIndex } from "./vectorIndex.js"; interface AIConfig { endpoint: string; apiKey: string; model: string; } interface MicroTaskResult { taskType: string; content: string; processingTimeMs: number; success: boolean; error?: string; } interface AnalysisResult { recommendation: any; processingStats: { embeddingsUsed: boolean; candidatesFromEmbeddings: number; finalSelectedItems: number; processingTimeMs: number; microTasksCompleted: number; microTasksFailed: number; contextContinuityUsed: boolean; }; } // Context object that builds up through pipeline interface AnalysisContext { userQuery: string; mode: string; filteredData: any; // Context continuity contextHistory: string[]; // Results scenarioAnalysis?: string; problemAnalysis?: string; investigationApproach?: string; criticalConsiderations?: string; selectedTools?: Array<{tool: any, phase: string, priority: string, justification?: string}>; backgroundKnowledge?: Array<{concept: any, relevance: string}>; } /** * Improved DFIR micro‑task pipeline – 2025‑08‑01 revision (bug‑fixed) */ class ImprovedMicroTaskAIPipeline { private config: AIConfig; private maxSelectedItems: number; private embeddingCandidates: number; private similarityThreshold: number; private microTaskDelay: number; constructor() { this.config = { endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'), apiKey: this.getEnv('AI_ANALYZER_API_KEY'), model: this.getEnv('AI_ANALYZER_MODEL') }; // Candidate selection tuned for higher precision this.maxSelectedItems = parseInt(process.env.AI_MAX_SELECTED_ITEMS || '60', 10); this.embeddingCandidates = parseInt(process.env.AI_EMBEDDING_CANDIDATES || '40', 10); this.similarityThreshold = parseFloat(process.env.AI_SIMILARITY_THRESHOLD || '0.5'); this.microTaskDelay = parseInt(process.env.AI_MICRO_TASK_DELAY_MS || '500', 10); } private getEnv(key: string): string { const value = process.env[key]; if (!value) throw new Error(`Missing environment variable: ${key}`); return value; } /** Embedding → LLM blended selector */ private async getIntelligentCandidates(userQuery: string, toolsData: any, mode: string) { const candidateTools = new Set(); const candidateConcepts = new Set(); if (embeddingsService.isEnabled()) { const similarItems = await vectorIndex.findSimilar(userQuery, this.embeddingCandidates); similarItems.forEach(item => { if (item.type === 'tool') candidateTools.add(item.name); if (item.type === 'concept') candidateConcepts.add(item.name); }); console.log(`[PIPELINE] Embedding hits → ${candidateTools.size} tools / ${candidateConcepts.size} concepts`); } const reducedData = { ...toolsData, tools: candidateTools.size ? toolsData.tools.filter((t: any) => candidateTools.has(t.name)) : toolsData.tools, concepts: candidateConcepts.size ? toolsData.concepts.filter((c: any) => candidateConcepts.has(c.name)) : toolsData.concepts }; return this.aiSelection(userQuery, reducedData, mode); } /** Language‑model based selector (no 50‑item cap) */ private async aiSelection(userQuery: string, toolsData: any, mode: string) { const toolsList = toolsData.tools.map((tool: any) => ({ name: tool.name, type: tool.type, description: tool.description.slice(0, 200) + '...', domains: tool.domains, phases: tool.phases, tags: tool.tags?.slice(0, 5) || [], skillLevel: tool.skillLevel })); const conceptsList = toolsData.concepts.map((concept: any) => ({ name: concept.name, type: 'concept', description: concept.description.slice(0, 200) + '...', domains: concept.domains, phases: concept.phases, tags: concept.tags?.slice(0, 5) || [] })); const modeInstruction = mode === 'workflow' ? 'The user wants a COMPREHENSIVE WORKFLOW with multiple tools/methods across different phases.' : 'The user wants SPECIFIC TOOLS/METHODS that directly solve their particular problem.'; const prompt = `You are a DFIR expert tasked with selecting the most relevant tools and concepts for a user query. ${modeInstruction} AVAILABLE TOOLS: ${JSON.stringify(toolsList, null, 2)} AVAILABLE CONCEPTS: ${JSON.stringify(conceptsList, null, 2)} USER QUERY: "${userQuery}" Select the most relevant items (max ${this.maxSelectedItems} total). For workflow mode, prioritize breadth across phases. For tool mode, prioritize specificity and direct relevance. Respond with ONLY this JSON format: { "selectedTools": ["Tool Name 1", "Tool Name 2", ...], "selectedConcepts": ["Concept Name 1", "Concept Name 2", ...], "reasoning": "Brief explanation of selection criteria and approach" }`; try { const response = await this.callAI(prompt, 1500); const cleaned = response.replace(/^```json\s*/i, '').replace(/\s*```\s*$/g, '').trim(); const result = JSON.parse(cleaned); if (!Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) { throw new Error('Invalid selection result structure'); } const totalSelected = result.selectedTools.length + result.selectedConcepts.length; if (totalSelected > this.maxSelectedItems) { console.warn(`[PIPELINE] Selection exceeded limit (${totalSelected}), truncating`); result.selectedTools = result.selectedTools.slice(0, Math.floor(this.maxSelectedItems * 0.8)); result.selectedConcepts = result.selectedConcepts.slice(0, Math.ceil(this.maxSelectedItems * 0.2)); } console.log(`[PIPELINE] LLM selector → ${result.selectedTools.length} tools / ${result.selectedConcepts.length} concepts`); return { tools: toolsData.tools.filter((tool: any) => result.selectedTools.includes(tool.name)), concepts: toolsData.concepts.filter((concept: any) => result.selectedConcepts.includes(concept.name)), domains: toolsData.domains, phases: toolsData.phases, 'domain-agnostic-software': toolsData['domain-agnostic-software'] }; } catch (err) { console.error('[PIPELINE] Failed to parse selector response'); throw new Error('Invalid JSON response from selector AI'); } } private delay(ms: number) { return new Promise(res => setTimeout(res, ms)); } private async callMicroTaskAI(prompt: string, context: AnalysisContext, maxTokens = 300): Promise { const start = Date.now(); const contextPrompt = context.contextHistory.length ? `BISHERIGE ANALYSE:\n${context.contextHistory.join('\n\n')}\n\nAKTUELLE AUFGABE:\n${prompt}` : prompt; try { const response = await this.callAI(contextPrompt, maxTokens); return { taskType: 'micro-task', content: response.trim(), processingTimeMs: Date.now() - start, success: true }; } catch (e) { return { taskType: 'micro-task', content: '', processingTimeMs: Date.now() - start, success: false, error: (e as Error).message }; } } // FIXED: Restore original micro-task structure with context continuity // MICRO-TASK 1: Scenario/Problem Analysis private async analyzeScenario(context: AnalysisContext): Promise { const isWorkflow = context.mode === 'workflow'; const prompt = `Sie sind ein erfahrener DFIR-Experte. Analysieren Sie das folgende ${isWorkflow ? 'forensische Szenario' : 'technische Problem'}. ${isWorkflow ? 'FORENSISCHES SZENARIO' : 'TECHNISCHES PROBLEM'}: "${context.userQuery}" Führen Sie eine systematische ${isWorkflow ? 'Szenario-Analyse' : 'Problem-Analyse'} durch und berücksichtigen Sie dabei: ${isWorkflow ? `- Auf das Szenario bezogene Problemstellungen` : `- konkrete problembezogene Aufgabenstellung` } WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen, Aufzählungen oder Markdown-Formatierung. Maximum 150 Wörter.`; const result = await this.callMicroTaskAI(prompt, context, 220); if (result.success) { if (isWorkflow) { context.scenarioAnalysis = result.content; } else { context.problemAnalysis = result.content; } // ADDED: Build context history context.contextHistory.push(`${isWorkflow ? 'Szenario' : 'Problem'}-Analyse: ${result.content.slice(0, 200)}...`); } return result; } // MICRO-TASK 2: Investigation/Solution Approach private async generateApproach(context: AnalysisContext): Promise { const isWorkflow = context.mode === 'workflow'; const analysis = isWorkflow ? context.scenarioAnalysis : context.problemAnalysis; const prompt = `Basierend auf der Analyse entwickeln Sie einen fundierten ${isWorkflow ? 'Untersuchungsansatz' : 'Lösungsansatz'}. ${isWorkflow ? 'SZENARIO' : 'PROBLEM'}: "${context.userQuery}" Entwickeln Sie einen systematischen ${isWorkflow ? 'Untersuchungsansatz' : 'Lösungsansatz'} unter Berücksichtigung von: ${isWorkflow ? `- Triage-Prioritäten nach forensischer Dringlichkeit (wenn zutreffend) - Phasenabfolge nach NIST SP 800-86-Methodik (Datensammlung - Auswertung - Analyse - Report)` : `- pragmatischer, zielorientierter Lösungsansatz im benehmen mit Anforderungen an die Reproduzierbarkeit` } WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 150 Wörter.`; const result = await this.callMicroTaskAI(prompt, context, 220); if (result.success) { context.investigationApproach = result.content; context.contextHistory.push(`${isWorkflow ? 'Untersuchungs' : 'Lösungs'}ansatz: ${result.content.slice(0, 200)}...`); } return result; } // MICRO-TASK 3: Critical Considerations private async generateCriticalConsiderations(context: AnalysisContext): Promise { const isWorkflow = context.mode === 'workflow'; const prompt = `Identifizieren Sie ${isWorkflow ? 'kritische forensische Überlegungen' : 'wichtige methodische Voraussetzungen'} für diesen Fall. ${isWorkflow ? 'SZENARIO' : 'PROBLEM'}: "${context.userQuery}" Berücksichtigen Sie folgende Aspekte: ${isWorkflow ? `- Szenariobezogene typische Problemstellungen, die auftreten können` : `- Problembezogene Schwierigkeiten, die das Ergebnis negativ beeinträchtigen könnten` } WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 120 Wörter.`; const result = await this.callMicroTaskAI(prompt, context, 180); if (result.success) { context.criticalConsiderations = result.content; context.contextHistory.push(`Kritische Überlegungen: ${result.content.slice(0, 200)}...`); } return result; } // MICRO-TASK 4: Tool Selection for Phase (Workflow mode) private async selectToolsForPhase(context: AnalysisContext, phase: any): Promise { const phaseTools = context.filteredData.tools.filter((tool: any) => tool.phases && tool.phases.includes(phase.id) ); if (phaseTools.length === 0) { return { taskType: 'tool-selection', content: JSON.stringify([]), processingTimeMs: 0, success: true }; } const prompt = `Wählen Sie 2-3 Methoden/Tools für die Phase "${phase.name}" basierend auf objektiven, fallbezogenen Kriterien. SZENARIO: "${context.userQuery}" VERFÜGBARE TOOLS FÜR ${phase.name.toUpperCase()}: ${phaseTools.map((tool: any) => `- ${tool.name}: ${tool.description.slice(0, 100)}...`).join('\n')} Wählen Sie Methoden/Tools nach forensischen Kriterien aus: - Eignung für die spezifische Lösung des Problems - besondere Fähigkeiten der Methode/des Tools, das sie von anderen abgrenzt - Reproduzierbarkeit und Objektivität Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format (kein zusätzlicher Text): [ { "toolName": "Exakter Methoden/Tool-Name", "priority": "high|medium|low", "justification": "Objektive Begründung warum diese Methode/Tool für das spezifische Szenario besser geeignet ist" } ]`; const result = await this.callMicroTaskAI(prompt, context, 450); if (result.success) { try { const selections = JSON.parse(result.content.replace(/^```json\s*/i, '').replace(/\s*```\s*$/g, '').trim()); const validSelections = selections.filter((sel: any) => phaseTools.some((tool: any) => tool.name === sel.toolName) ); if (!context.selectedTools) context.selectedTools = []; validSelections.forEach((sel: any) => { const tool = phaseTools.find((t: any) => t.name === sel.toolName); if (tool) { context.selectedTools!.push({ tool, phase: phase.id, priority: sel.priority, justification: sel.justification }); } }); } catch (parseError) { console.warn(`[IMPROVED PIPELINE] Failed to parse tool selection for ${phase.name}:`, result.content.slice(0, 200)); return { ...result, success: false, error: 'JSON parsing failed' }; } } return result; } // MICRO-TASK 5: Tool Evaluation (Tool mode) private async evaluateSpecificTool(context: AnalysisContext, tool: any, rank: number): Promise { const prompt = `Bewerten Sie diese Methode/Tool fallbezogen für das spezifische Problem. PROBLEM: "${context.userQuery}" TOOL: ${tool.name} BESCHREIBUNG: ${tool.description} PLATTFORMEN: ${tool.platforms?.join(', ') || 'N/A'} SKILL LEVEL: ${tool.skillLevel} Bewerten Sie nach forensischen Standards und antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format: { "suitability_score": "high|medium|low", "detailed_explanation": "Detaillierte forensische Begründung warum diese Methode/Tool das Problem löst", "implementation_approach": "Konkrete methodische Schritte zur korrekten Anwendung für dieses spezifische Problem", "pros": ["Forensischer Vorteil 1", "Validierter Vorteil 2"], "cons": ["Methodische Limitation 1", "Potenzielle Schwäche 2"], "alternatives": "Alternative Ansätze falls diese Methode/Tool nicht optimal ist" }`; const result = await this.callMicroTaskAI(prompt, context, 650); if (result.success) { try { const evaluation = JSON.parse(result.content.replace(/^```json\s*/i, '').replace(/\s*```\s*$/g, '').trim()); if (!context.selectedTools) context.selectedTools = []; context.selectedTools.push({ tool: { ...tool, evaluation: { ...evaluation, rank } }, phase: 'evaluation', priority: evaluation.suitability_score }); } catch (parseError) { console.warn(`[IMPROVED PIPELINE] Failed to parse tool evaluation for ${tool.name}:`, result.content.slice(0, 200)); return { ...result, success: false, error: 'JSON parsing failed' }; } } return result; } // MICRO-TASK 6: Background Knowledge private async selectBackgroundKnowledge(context: AnalysisContext): Promise { const availableConcepts = context.filteredData.concepts; if (availableConcepts.length === 0) { return { taskType: 'background-knowledge', content: JSON.stringify([]), processingTimeMs: 0, success: true }; } const selectedToolNames = context.selectedTools?.map(st => st.tool.name) || []; const prompt = `Wählen Sie relevante forensische Konzepte für das Verständnis der empfohlenen Methodik. ${context.mode === 'workflow' ? 'SZENARIO' : 'PROBLEM'}: "${context.userQuery}" EMPFOHLENE TOOLS: ${selectedToolNames.join(', ')} VERFÜGBARE KONZEPTE: ${availableConcepts.slice(0, 15).map((concept: any) => `- ${concept.name}: ${concept.description.slice(0, 80)}...`).join('\n')} Wählen Sie 2-4 Konzepte aus, die für die Lösung des Problems essentiell sind. Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format: [ { "conceptName": "Exakter Konzept-Name", "relevance": "Forensische Relevanz: Warum dieses Konzept für die Lösung des Problems kritisch ist" } ]`; const result = await this.callMicroTaskAI(prompt, context, 400); if (result.success) { try { const selections = JSON.parse(result.content.replace(/^```json\s*/i, '').replace(/\s*```\s*$/g, '').trim()); context.backgroundKnowledge = selections.filter((sel: any) => availableConcepts.some((concept: any) => concept.name === sel.conceptName) ).map((sel: any) => ({ concept: availableConcepts.find((c: any) => c.name === sel.conceptName), relevance: sel.relevance })); } catch (parseError) { console.warn('[IMPROVED PIPELINE] Failed to parse background knowledge selection:', result.content.slice(0, 200)); return { ...result, success: false, error: 'JSON parsing failed' }; } } return result; } // MICRO-TASK 7: Final Recommendations private async generateFinalRecommendations(context: AnalysisContext): Promise { const isWorkflow = context.mode === 'workflow'; const prompt = isWorkflow ? `Erstellen Sie eine forensisch fundierte Workflow-Empfehlung unter Anwendung der gewählten Methoden/Tools. SZENARIO: "${context.userQuery}" AUSGEWÄHLTE TOOLS: ${context.selectedTools?.map(st => st.tool.name).join(', ') || 'Keine Tools ausgewählt'} Erstellen Sie konkrete Workflow-Schritte für dieses spezifische Szenario unter Berücksichtigung von Objektivität und rechtlicher Verwertbarkeit (Reproduzierbarkeit, Transparenz). WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 120 Wörter.` : `Erstellen Sie wichtige Überlegungen für die korrekte Methoden-/Tool-Anwendung. PROBLEM: "${context.userQuery}" EMPFOHLENE TOOLS: ${context.selectedTools?.map(st => st.tool.name).join(', ') || 'Keine Methoden/Tools ausgewählt'} Geben Sie kritische Überlegungen für die korrekte Anwendung der empfohlenen Methoden/Tools. WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 100 Wörter.`; const result = await this.callMicroTaskAI(prompt, context, 180); return result; } // Helper method for AI calls private async callAI(prompt: string, maxTokens: number = 1000): Promise { const response = await fetch(`${this.config.endpoint}/v1/chat/completions`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.config.apiKey}` }, body: JSON.stringify({ model: this.config.model, messages: [{ role: 'user', content: prompt }], max_tokens: maxTokens, temperature: 0.3 }) }); if (!response.ok) { const errorText = await response.text(); throw new Error(`AI API error: ${response.status} - ${errorText}`); } const data = await response.json(); const content = data.choices?.[0]?.message?.content; if (!content) { throw new Error('No response from AI model'); } return content; } async processQuery(userQuery: string, mode: string): Promise { const startTime = Date.now(); let completedTasks = 0; let failedTasks = 0; try { const toolsData = await getCompressedToolsDataForAI(); const filteredData = await this.getIntelligentCandidates(userQuery, toolsData, mode); const context: AnalysisContext = { userQuery, mode, filteredData, contextHistory: [] }; console.log(`[IMPROVED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`); // MICRO-TASK SEQUENCE (restored original structure) // Task 1: Scenario/Problem Analysis const analysisResult = await this.analyzeScenario(context); if (analysisResult.success) completedTasks++; else failedTasks++; await this.delay(this.microTaskDelay); // Task 2: Investigation/Solution Approach const approachResult = await this.generateApproach(context); if (approachResult.success) completedTasks++; else failedTasks++; await this.delay(this.microTaskDelay); // Task 3: Critical Considerations const considerationsResult = await this.generateCriticalConsiderations(context); if (considerationsResult.success) completedTasks++; else failedTasks++; await this.delay(this.microTaskDelay); // Task 4: Tool Selection/Evaluation (mode-dependent) if (mode === 'workflow') { // Select tools for each phase const phases = toolsData.phases || []; for (const phase of phases) { const toolSelectionResult = await this.selectToolsForPhase(context, phase); if (toolSelectionResult.success) completedTasks++; else failedTasks++; await this.delay(this.microTaskDelay); } } else { const shuffled = [...filteredData.tools].sort(() => Math.random() - 0.5); // FIX const topTools = shuffled.slice(0, 3); for (let i = 0; i < topTools.length; i++) { const evaluationResult = await this.evaluateSpecificTool(context, topTools[i], i + 1); if (evaluationResult.success) completedTasks++; else failedTasks++; await this.delay(this.microTaskDelay); } } // Task 5: Background Knowledge Selection const knowledgeResult = await this.selectBackgroundKnowledge(context); if (knowledgeResult.success) completedTasks++; else failedTasks++; await this.delay(this.microTaskDelay); // Task 6: Final Recommendations const finalResult = await this.generateFinalRecommendations(context); if (finalResult.success) completedTasks++; else failedTasks++; const recommendation = this.buildRecommendation(context, mode, ''); // finalContent injected inside omitted logic const processingStats = { embeddingsUsed: embeddingsService.isEnabled(), candidatesFromEmbeddings: filteredData.tools.length, finalSelectedItems: (context.selectedTools?.length || 0) + (context.backgroundKnowledge?.length || 0), processingTimeMs: Date.now() - startTime, microTasksCompleted: completedTasks, microTasksFailed: failedTasks, contextContinuityUsed: true }; return { recommendation, processingStats }; } catch (error) { console.error('[PIPELINE] Processing failed:', error); throw error; } } // Build recommendation (same as original structure) private buildRecommendation(context: AnalysisContext, mode: string, finalContent: string): any { const isWorkflow = mode === 'workflow'; const base = { [isWorkflow ? 'scenario_analysis' : 'problem_analysis']: isWorkflow ? context.scenarioAnalysis : context.problemAnalysis, investigation_approach: context.investigationApproach, critical_considerations: context.criticalConsiderations, background_knowledge: context.backgroundKnowledge?.map(bk => ({ concept_name: bk.concept.name, relevance: bk.relevance })) || [] }; if (isWorkflow) { return { ...base, recommended_tools: context.selectedTools?.map(st => ({ name: st.tool.name, phase: st.phase, priority: st.priority, justification: st.justification || `Empfohlen für ${st.phase}` })) || [], workflow_suggestion: finalContent }; } else { return { ...base, recommended_tools: context.selectedTools?.map(st => ({ name: st.tool.name, rank: st.tool.evaluation?.rank || 1, suitability_score: st.priority, detailed_explanation: st.tool.evaluation?.detailed_explanation || '', implementation_approach: st.tool.evaluation?.implementation_approach || '', pros: st.tool.evaluation?.pros || [], cons: st.tool.evaluation?.cons || [], alternatives: st.tool.evaluation?.alternatives || '' })) || [], additional_considerations: finalContent }; } } } // Global instance const aiPipeline = new ImprovedMicroTaskAIPipeline(); export { aiPipeline, type AnalysisResult };