// src/utils/auditService.ts import 'dotenv/config'; function env(key: string, fallback: string | undefined = undefined): string | undefined { if (typeof process !== 'undefined' && process.env?.[key] !== undefined) { return process.env[key]; } if (typeof import.meta !== 'undefined' && (import.meta as any).env?.[key] !== undefined) { return (import.meta as any).env[key]; } return fallback; } export interface AuditEntry { timestamp: number; phase: string; action: string; input: any; output: any; confidence: number; processingTimeMs: number; metadata: { aiModel?: string; aiParameters?: any; promptTokens?: number; completionTokens?: number; toolsDataHash?: string; embeddingsUsed?: boolean; microTaskType?: string; confidenceFactors?: string[]; reasoning?: string; aiPrompt?: string; aiResponse?: string; toolSelectionCriteria?: string; availableToolsCount?: number; selectedToolsCount?: number; phaseId?: string; toolsAdded?: string[]; completionReasoning?: string; similarityScores?: Record; contextLength?: number; decisionBasis?: 'ai-analysis' | 'semantic-search' | 'hybrid' | 'rule-based'; inputSummary?: string; outputSummary?: string; [key: string]: any; }; } interface AuditConfig { enabled: boolean; retentionHours: number; maxEntries: number; } class AuditService { private config: AuditConfig; private activeAuditTrail: AuditEntry[] = []; constructor() { this.config = this.loadConfig(); console.log('[AUDIT-SERVICE] Initialized with meaningful audit logging'); } private loadConfig(): AuditConfig { const enabled = env('FORENSIC_AUDIT_ENABLED', 'true') === 'true'; const retentionHours = parseInt(env('FORENSIC_AUDIT_RETENTION_HOURS', '72') || '72', 10); const maxEntries = parseInt(env('FORENSIC_AUDIT_MAX_ENTRIES', '50') || '50', 10); return { enabled, retentionHours, maxEntries }; } addEntry( phase: string, action: string, input: any, output: any, confidence: number, startTime: number, metadata: Record = {} ): void { if (!this.config.enabled) return; if (action === 'pipeline-start' || action === 'pipeline-end') { return; } const enhancedMetadata = { ...metadata, inputSummary: this.createSpecificSummary(input, action, 'input'), outputSummary: this.createSpecificSummary(output, action, 'output'), decisionBasis: metadata.decisionBasis || this.inferDecisionBasis(metadata), reasoning: metadata.reasoning || this.generateSpecificReasoning(action, input, output, metadata, confidence) }; const entry: AuditEntry = { timestamp: Date.now(), phase, action, input: input, output: output, confidence: Math.round(confidence), processingTimeMs: Date.now() - startTime, metadata: enhancedMetadata }; this.activeAuditTrail.push(entry); if (this.activeAuditTrail.length > this.config.maxEntries) { this.activeAuditTrail.shift(); } console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`); } addAIDecision( phase: string, aiPrompt: string, aiResponse: string, confidence: number, reasoning: string, startTime: number, metadata: Record = {} ): void { this.addEntry( phase, 'ai-decision', { prompt: this.createPromptSummary(aiPrompt) }, { response: aiResponse }, confidence, startTime, { ...metadata, reasoning, aiPrompt: aiPrompt, aiResponse: aiResponse, decisionBasis: 'ai-analysis' } ); } addToolSelection( selectedTools: string[], availableTools: string[], confidence: number, startTime: number, metadata: Record = {} ): void { const calculatedConfidence = this.calculateSelectionConfidence( selectedTools, availableTools, metadata ); const decisionBasis = metadata.embeddingsUsed || metadata.similarityScores ? 'semantic-search' : (metadata.aiPrompt || metadata.microTaskType ? 'ai-analysis' : 'rule-based'); this.addEntry( 'tool-selection', 'selection-decision', { availableTools: availableTools.slice(0, 10), totalAvailable: availableTools.length, }, { selectedTools: selectedTools, selectionRatio: selectedTools.length / availableTools.length }, calculatedConfidence, startTime, { ...metadata, availableToolsCount: availableTools.length, selectedToolsCount: selectedTools.length, decisionBasis } ); } addPhaseCompletion( phaseId: string, addedTools: string[], reasoning: string, startTime: number, metadata: Record = {} ): void { if (!addedTools || addedTools.length === 0) { console.log(`[AUDIT-SERVICE] Skipping phase completion for ${phaseId} - no tools added`); return; } const calculatedConfidence = this.calculatePhaseCompletionConfidence(addedTools, reasoning, metadata); this.addEntry( 'phase-completion', 'phase-enhancement', { phaseId: phaseId, phaseName: this.getPhaseDisplayName(phaseId), searchStrategy: 'semantic-search-with-ai-reasoning' }, { addedTools: addedTools, toolsAddedCount: addedTools.length }, calculatedConfidence, startTime, { ...metadata, reasoning: reasoning, decisionBasis: 'hybrid' } ); } addEmbeddingsSearch( query: string, similarResults: any[], threshold: number, startTime: number, metadata: Record = {} ): void { const calculatedConfidence = this.calculateEmbeddingsConfidence(similarResults, threshold); this.addEntry( 'embeddings', 'similarity-search', { query: query, threshold: threshold }, { resultsCount: similarResults.length, topMatches: similarResults.slice(0, 5).map(r => `${r.name} (${Math.round(r.similarity * 100)}%)`) }, calculatedConfidence, startTime, { ...metadata, embeddingsUsed: true, searchThreshold: threshold, totalMatches: similarResults.length, decisionBasis: 'semantic-search' } ); } addConfidenceCalculation( toolName: string, confidence: any, startTime: number, metadata: Record = {} ): void { this.addEntry( 'confidence-scoring', 'tool-confidence', { toolName: toolName, semanticSimilarity: confidence.semanticRelevance, taskRelevance: confidence.taskSuitability }, { overallConfidence: confidence.overall, strengthIndicators: confidence.strengthIndicators?.slice(0, 2) || [], uncertaintyFactors: confidence.uncertaintyFactors?.slice(0, 2) || [] }, confidence.overall, startTime, { ...metadata, confidenceCalculation: true, decisionBasis: 'ai-analysis' } ); } private calculateSelectionConfidence( selectedTools: string[], availableTools: string[], metadata: Record ): number { let confidence = 50; const selectionRatio = selectedTools.length / availableTools.length; if (selectionRatio >= 0.05 && selectionRatio <= 0.20) { confidence += 25; } else if (selectionRatio < 0.05) { confidence += 15; } else if (selectionRatio > 0.30) { confidence -= 20; } if (selectedTools.length >= 5 && selectedTools.length <= 25) { confidence += 10; } return Math.min(95, Math.max(40, confidence)); } private calculatePhaseCompletionConfidence( addedTools: string[], reasoning: string, metadata: Record ): number { let confidence = 60; if (addedTools.length > 0) { confidence += 20; } if (reasoning && reasoning.length > 50) { confidence += 15; } if (metadata.aiReasoningUsed) { confidence += 10; } if (addedTools.length <= 2) { confidence += 5; } return Math.min(90, Math.max(50, confidence)); } private calculateEmbeddingsConfidence(similarResults: any[], threshold: number): number { let confidence = 50; if (similarResults.length > 0) { confidence += 20; } if (similarResults.length >= 5 && similarResults.length <= 30) { confidence += 15; } const avgSimilarity = similarResults.length > 0 ? similarResults.reduce((sum, r) => sum + r.similarity, 0) / similarResults.length : 0; if (avgSimilarity > 0.7) { confidence += 15; } else if (avgSimilarity > 0.5) { confidence += 10; } if (threshold >= 0.3 && threshold <= 0.5) { confidence += 5; } return Math.min(95, Math.max(30, confidence)); } private createSpecificSummary(data: any, action: string, type: 'input' | 'output'): string { if (!data) return 'Leer'; switch (action) { case 'selection-decision': if (type === 'input') { if (data.availableTools && Array.isArray(data.availableTools)) { const preview = data.availableTools.slice(0, 3).join(', '); return `${data.totalAvailable || data.availableTools.length} Tools: ${preview}${data.availableTools.length > 3 ? '...' : ''}`; } else if (data.totalAvailable) { return `${data.totalAvailable} Tools verfügbar, Methode: ${data.selectionMethod}`; } return `${data.candidateCount || 0} Kandidaten für Auswahl`; } else { if (Array.isArray(data.selectedTools)) { return `${data.selectedTools.length} ausgewählt: ${data.selectedTools.slice(0, 3).join(', ')}${data.selectedTools.length > 3 ? '...' : ''}`; } return `Auswahl: ${data.selectionRatio ? Math.round(data.selectionRatio * 100) + '%' : 'unbekannt'}`; } case 'phase-tool-selection': if (type === 'input') { if (Array.isArray(data.availableTools)) { const toolPreview = data.availableTools.slice(0, 3).join(', '); return `${data.availableTools.length} Tools für ${data.phaseName || data.phaseId}: ${toolPreview}${data.availableTools.length > 3 ? '...' : ''}`; } return `Phase: ${data.phaseName || data.phaseId} (${data.toolCount || 0} Tools)`; } else { if (Array.isArray(data.selectedTools)) { return `${data.selectedTools.length} ausgewählt: ${data.selectedTools.join(', ')}`; } return `${data.selectionCount || 0} Tools, Ø ${data.avgTaskRelevance || 0}% Relevanz`; } case 'similarity-search': if (type === 'input') { return `Suche: "${data.query}" (Min. ${data.threshold} Ähnlichkeit)`; } else { if (Array.isArray(data.topMatches)) { return `${data.resultsCount} Treffer: ${data.topMatches.slice(0, 2).join(', ')}${data.topMatches.length > 2 ? '...' : ''}`; } return `${data.resultsCount || 0} semantische Treffer gefunden`; } case 'phase-enhancement': if (type === 'input') { return `Vervollständige Phase: ${data.phaseName || data.phaseId}`; } else { if (Array.isArray(data.addedTools) && data.addedTools.length > 0) { return `${data.addedTools.length} hinzugefügt: ${data.addedTools.join(', ')}`; } return `${data.toolsAddedCount || 0} Tools für Phase hinzugefügt`; } case 'ai-decision': if (type === 'input') { if (data.prompt) { const promptPreview = data.prompt.slice(0, 80).replace(/\n/g, ' '); return `KI-Prompt: ${promptPreview}${data.prompt.length > 80 ? ' [Vorschau]' : ''}`; } return 'KI-Analyse angefordert'; } else { if (data.response) { const responsePreview = data.response.slice(0, 80).replace(/\n/g, ' '); return `KI-Antwort: ${responsePreview}${data.response.length > 80 ? ' [Vorschau]' : ''}`; } return 'KI-Analyse abgeschlossen'; } case 'tool-confidence': if (type === 'input') { return `Tool: ${data.toolName} (Sem: ${data.semanticSimilarity}%, Task: ${data.taskRelevance}%)`; } else { const strengthCount = data.strengthIndicators?.length || 0; const uncertaintyCount = data.uncertaintyFactors?.length || 0; return `${data.overallConfidence}% Vertrauen (${strengthCount} Stärken, ${uncertaintyCount} Unsicherheiten)`; } case 'tool-added-to-phase': if (type === 'output') { const justificationPreview = data.justification ? data.justification.slice(0, 60).replace(/\n/g, ' ') + (data.justification.length > 60 ? ' [Vorschau]' : '') : 'Hinzugefügt'; return `Begründung: ${justificationPreview}`; } else { const justificationPreview = data.justification ? data.justification.slice(0, 60).replace(/\n/g, ' ') + '...' : 'Hinzugefügt'; return `Begründung: ${justificationPreview}`; } case 'concept-selection': if (type === 'input') { const conceptCount = Array.isArray(data.availableConcepts) ? data.availableConcepts.length : 0; const toolContext = Array.isArray(data.selectedToolsContext) ? data.selectedToolsContext.length : 0; return `${conceptCount} Konzepte verfügbar, ${toolContext} Tools als Kontext`; } else { if (Array.isArray(data.selectedConcepts)) { return `${data.selectedConcepts.length} ausgewählt: ${data.selectedConcepts.slice(0, 2).join(', ')}${data.selectedConcepts.length > 2 ? '...' : ''}`; } return `Konzeptauswahl abgeschlossen`; } } if (typeof data === 'string') { return data.length > 100 ? data.slice(0, 100) + '...' : data; } if (Array.isArray(data)) { if (data.length === 0) return 'Leeres Array'; if (data.length <= 3) return data.join(', '); return `${data.slice(0, 3).join(', ')} + ${data.length - 3} weitere`; } if (typeof data === 'object') { const keys = Object.keys(data); if (keys.length === 0) return 'Leeres Objekt'; if (keys.length <= 2) { const pairs = keys.map(key => { const value = data[key]; if (typeof value === 'string' && value.length > 30) { return `${key}: ${value.slice(0, 30)}...`; } else if (Array.isArray(value)) { return `${key}: [${value.length} Items]`; } else { return `${key}: ${value}`; } }); return pairs.join(', '); } else { const sampleKeys = keys.slice(0, 3); const sampleValues = sampleKeys.map(key => { const value = data[key]; if (typeof value === 'string' && value.length > 20) { return `${key}: ${value.slice(0, 20)}...`; } else if (Array.isArray(value)) { return `${key}: [${value.length}]`; } else { return `${key}: ${value}`; } }); return `${sampleValues.join(', ')}${keys.length > 3 ? ` + ${keys.length - 3} weitere` : ''}`; } } return String(data); } private createPromptSummary(prompt: string): string { if (!prompt || prompt.length <= 200) return prompt; return prompt.slice(0, 200) + ' [Eingabe-Vorschau]'; } private generateSpecificReasoning( action: string, input: any, output: any, metadata: Record, confidence: number ): string { if (metadata.reasoning && metadata.reasoning.length > 20 && !metadata.reasoning.includes('completed with')) { return metadata.reasoning; } switch (action) { case 'selection-decision': const selectionRatio = metadata.selectedToolsCount / metadata.availableToolsCount; const method = metadata.selectionMethod === 'embeddings_candidates' ? 'Semantische Analyse' : 'KI-Analyse'; return `${method} wählte ${metadata.selectedToolsCount} von ${metadata.availableToolsCount} Tools (${Math.round(selectionRatio * 100)}%) - ausgewogene Auswahl für forensische Aufgabenstellung`; case 'similarity-search': { const totalMatches = typeof metadata.totalMatches === 'number' ? metadata.totalMatches : 0; const scoresObj = (metadata.similarityScores ?? {}) as Record; const scores = Object.values(scoresObj) as number[]; const denom = totalMatches > 0 ? totalMatches : scores.length; const sum = scores.reduce((acc, v) => acc + (typeof v === 'number' ? v : 0), 0); const avgSim = denom > 0 ? sum / denom : 0; return `Semantische Suche fand ${totalMatches} relevante Items mit durchschnittlicher Ähnlichkeit von ${Math.round(avgSim * 100)}%`; } case 'ai-decision': const taskType = metadata.microTaskType; if (taskType) { const typeNames = { 'scenario-analysis': 'Szenario-Analyse', 'investigation-approach': 'Untersuchungsansatz', 'critical-considerations': 'Kritische Überlegungen', 'tool-evaluation': 'Tool-Bewertung', 'background-knowledge': 'Hintergrundwissen-Auswahl', 'final-recommendations': 'Abschließende Empfehlungen' }; return `KI analysierte ${typeNames[taskType] || taskType} mit ${confidence}% Vertrauen - fundierte Empfehlung`; } return `KI-Entscheidung mit ${confidence}% Vertrauen basierend auf agentischer Expertenanalyse`; case 'phase-enhancement': const phaseData = input?.phaseName || input?.phaseId; const toolCount = output?.toolsAddedCount || 0; return `${phaseData}-Phase durch ${toolCount} zusätzliche Tools vervollständigt - ursprüngliche Auswahl war zu spezifisch und übersah wichtige Methoden`; case 'tool-confidence': return `Vertrauenswertung für ${input?.toolName}: ${confidence}% basierend auf semantischer Relevanz (${input?.semanticSimilarity}%) und Aufgabeneignung (${input?.taskRelevance}%)`; default: return `${action} mit ${confidence}% Vertrauen abgeschlossen`; } } private getPhaseDisplayName(phaseId: string): string { const phaseNames: Record = { 'preparation': 'Vorbereitung', 'acquisition': 'Datensammlung', 'examination': 'Untersuchung', 'analysis': 'Analyse', 'reporting': 'Dokumentation', 'presentation': 'Präsentation' }; return phaseNames[phaseId] || phaseId; } private inferDecisionBasis(metadata: Record): string { if (metadata.embeddingsUsed) return 'semantic-search'; if (metadata.aiPrompt || metadata.microTaskType) return 'ai-analysis'; if (metadata.semanticQuery && metadata.aiReasoningUsed) return 'hybrid'; return 'rule-based'; } getCurrentAuditTrail(): AuditEntry[] { return [...this.activeAuditTrail]; } clearAuditTrail(): void { if (this.activeAuditTrail.length > 0) { console.log(`[AUDIT-SERVICE] Cleared ${this.activeAuditTrail.length} audit entries`); this.activeAuditTrail = []; } } finalizeAuditTrail(): AuditEntry[] { const finalTrail = [...this.activeAuditTrail]; console.log(`[AUDIT-SERVICE] Finalized audit trail with ${finalTrail.length} meaningful entries`); this.clearAuditTrail(); return finalTrail; } isEnabled(): boolean { return this.config.enabled; } getConfig(): AuditConfig { return { ...this.config }; } calculateAIResponseConfidence( response: string, expectedLength: { min: number; max: number }, taskType: string ): number { let confidence = 50; if (response.length >= expectedLength.min) { confidence += 20; if (response.length <= expectedLength.max) { confidence += 10; } } else { confidence -= 20; } if (response.includes('...') || response.endsWith('...')) { confidence -= 10; } switch (taskType) { case 'scenario-analysis': case 'investigation-approach': case 'critical-considerations': const forensicTerms = ['forensisch', 'beweis', 'evidence', 'analyse', 'untersuchung', 'methodik']; const termsFound = forensicTerms.filter(term => response.toLowerCase().includes(term) ).length; confidence += Math.min(15, termsFound * 3); break; case 'tool-evaluation': if (response.includes('detailed_explanation') || response.includes('implementation_approach')) { confidence += 15; } if (response.includes('pros') && response.includes('limitations')) { confidence += 10; } break; case 'background-knowledge': try { const parsed = JSON.parse(response); if (Array.isArray(parsed) && parsed.length > 0) { confidence += 20; } } catch { confidence -= 20; } break; } return Math.min(95, Math.max(25, confidence)); } getAuditStatistics(auditTrail: AuditEntry[]): any { if (!auditTrail || auditTrail.length === 0) { return { totalTime: 0, avgConfidence: 0, stepCount: 0, highConfidenceSteps: 0, lowConfidenceSteps: 0, phaseBreakdown: {}, aiDecisionCount: 0, embeddingsUsageCount: 0, toolSelectionCount: 0, qualityMetrics: { avgProcessingTime: 0, confidenceDistribution: { high: 0, medium: 0, low: 0 } } }; } const totalTime = auditTrail.reduce((sum, entry) => sum + (entry.processingTimeMs || 0), 0); const validConfidenceEntries = auditTrail.filter(entry => typeof entry.confidence === 'number'); const avgConfidence = validConfidenceEntries.length > 0 ? Math.round(validConfidenceEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validConfidenceEntries.length) : 0; return { totalTime, avgConfidence, stepCount: auditTrail.length, highConfidenceSteps: auditTrail.filter(entry => (entry.confidence || 0) >= 80).length, lowConfidenceSteps: auditTrail.filter(entry => (entry.confidence || 0) < 60).length, phaseBreakdown: {}, aiDecisionCount: auditTrail.filter(entry => entry.action === 'ai-decision').length, embeddingsUsageCount: auditTrail.filter(entry => entry.metadata?.embeddingsUsed).length, toolSelectionCount: auditTrail.filter(entry => entry.action === 'selection-decision').length, qualityMetrics: { avgProcessingTime: auditTrail.length > 0 ? totalTime / auditTrail.length : 0, confidenceDistribution: { high: auditTrail.filter(entry => (entry.confidence || 0) >= 80).length, medium: auditTrail.filter(entry => (entry.confidence || 0) >= 60 && (entry.confidence || 0) < 80).length, low: auditTrail.filter(entry => (entry.confidence || 0) < 60).length } } }; } validateAuditTrail(auditTrail: AuditEntry[]): { isValid: boolean; issues: string[]; warnings: string[]; } { const issues: string[] = []; const warnings: string[] = []; if (!Array.isArray(auditTrail)) { issues.push('Audit trail is not an array'); return { isValid: false, issues, warnings }; } if (auditTrail.length === 0) { warnings.push('Audit trail is empty'); } auditTrail.forEach((entry, index) => { if (!entry || typeof entry !== 'object') { issues.push(`Entry ${index} is not a valid object`); return; } const requiredFields = ['timestamp', 'phase', 'action']; requiredFields.forEach(field => { if (!(field in entry)) { issues.push(`Entry ${index} missing required field: ${field}`); } }); if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) { warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`); } }); return { isValid: issues.length === 0, issues, warnings }; } } export const auditService = new AuditService();