791 lines
26 KiB
TypeScript
791 lines
26 KiB
TypeScript
// src/utils/auditService.ts - Fixed with meaningful confidence and reasoning
|
|
import 'dotenv/config';
|
|
|
|
function env(key: string, fallback: string | undefined = undefined): string | undefined {
|
|
if (typeof process !== 'undefined' && process.env?.[key] !== undefined) {
|
|
return process.env[key];
|
|
}
|
|
if (typeof import.meta !== 'undefined' && (import.meta as any).env?.[key] !== undefined) {
|
|
return (import.meta as any).env[key];
|
|
}
|
|
return fallback;
|
|
}
|
|
|
|
export interface AuditEntry {
|
|
timestamp: number;
|
|
phase: string;
|
|
action: string;
|
|
input: any;
|
|
output: any;
|
|
confidence: number;
|
|
processingTimeMs: number;
|
|
metadata: {
|
|
aiModel?: string;
|
|
aiParameters?: any;
|
|
promptTokens?: number;
|
|
completionTokens?: number;
|
|
toolsDataHash?: string;
|
|
embeddingsUsed?: boolean;
|
|
selectionMethod?: string;
|
|
microTaskType?: string;
|
|
confidenceFactors?: string[];
|
|
reasoning?: string;
|
|
aiPrompt?: string;
|
|
aiResponse?: string;
|
|
toolSelectionCriteria?: string;
|
|
availableToolsCount?: number;
|
|
selectedToolsCount?: number;
|
|
phaseId?: string;
|
|
toolsAdded?: string[];
|
|
completionReasoning?: string;
|
|
similarityScores?: Record<string, number>;
|
|
contextLength?: number;
|
|
decisionBasis?: 'ai-analysis' | 'semantic-search' | 'hybrid' | 'rule-based';
|
|
inputSummary?: string;
|
|
outputSummary?: string;
|
|
[key: string]: any;
|
|
};
|
|
}
|
|
|
|
interface AuditConfig {
|
|
enabled: boolean;
|
|
retentionHours: number;
|
|
maxEntries: number;
|
|
}
|
|
|
|
class AuditService {
|
|
private config: AuditConfig;
|
|
private activeAuditTrail: AuditEntry[] = [];
|
|
|
|
constructor() {
|
|
this.config = this.loadConfig();
|
|
console.log('[AUDIT-SERVICE] Initialized with meaningful audit logging');
|
|
}
|
|
|
|
private loadConfig(): AuditConfig {
|
|
const enabled = env('FORENSIC_AUDIT_ENABLED', 'true') === 'true';
|
|
const retentionHours = parseInt(env('FORENSIC_AUDIT_RETENTION_HOURS', '72') || '72', 10);
|
|
const maxEntries = parseInt(env('FORENSIC_AUDIT_MAX_ENTRIES', '50') || '50', 10);
|
|
|
|
return {
|
|
enabled,
|
|
retentionHours,
|
|
maxEntries
|
|
};
|
|
}
|
|
|
|
addEntry(
|
|
phase: string,
|
|
action: string,
|
|
input: any,
|
|
output: any,
|
|
confidence: number,
|
|
startTime: number,
|
|
metadata: Record<string, any> = {}
|
|
): void {
|
|
if (!this.config.enabled) return;
|
|
|
|
// Skip initialization and completion entries as they don't add transparency
|
|
if (action === 'pipeline-start' || action === 'pipeline-end') {
|
|
return;
|
|
}
|
|
|
|
const enhancedMetadata = {
|
|
...metadata,
|
|
inputSummary: this.createSpecificSummary(input, action, 'input'),
|
|
outputSummary: this.createSpecificSummary(output, action, 'output'),
|
|
decisionBasis: metadata.decisionBasis || this.inferDecisionBasis(metadata),
|
|
reasoning: metadata.reasoning || this.generateSpecificReasoning(action, input, output, metadata, confidence)
|
|
};
|
|
|
|
const entry: AuditEntry = {
|
|
timestamp: Date.now(),
|
|
phase,
|
|
action,
|
|
input: input,
|
|
output: output,
|
|
confidence: Math.round(confidence),
|
|
processingTimeMs: Date.now() - startTime,
|
|
metadata: enhancedMetadata
|
|
};
|
|
|
|
this.activeAuditTrail.push(entry);
|
|
|
|
if (this.activeAuditTrail.length > this.config.maxEntries) {
|
|
this.activeAuditTrail.shift();
|
|
}
|
|
|
|
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`);
|
|
}
|
|
|
|
addAIDecision(
|
|
phase: string,
|
|
aiPrompt: string,
|
|
aiResponse: string,
|
|
confidence: number,
|
|
reasoning: string,
|
|
startTime: number,
|
|
metadata: Record<string, any> = {}
|
|
): void {
|
|
this.addEntry(
|
|
phase,
|
|
'ai-decision',
|
|
{ prompt: this.truncatePrompt(aiPrompt) },
|
|
{ response: this.truncateResponse(aiResponse) },
|
|
confidence,
|
|
startTime,
|
|
{
|
|
...metadata,
|
|
reasoning,
|
|
aiPrompt: aiPrompt,
|
|
aiResponse: aiResponse,
|
|
decisionBasis: 'ai-analysis'
|
|
}
|
|
);
|
|
}
|
|
|
|
addToolSelection(
|
|
selectedTools: string[],
|
|
availableTools: string[],
|
|
selectionMethod: string,
|
|
confidence: number,
|
|
startTime: number,
|
|
metadata: Record<string, any> = {}
|
|
): void {
|
|
// Calculate meaningful confidence based on selection quality
|
|
const calculatedConfidence = this.calculateSelectionConfidence(
|
|
selectedTools,
|
|
availableTools,
|
|
selectionMethod,
|
|
metadata
|
|
);
|
|
|
|
this.addEntry(
|
|
'tool-selection',
|
|
'selection-decision',
|
|
{
|
|
availableTools: availableTools.slice(0, 10), // Show first 10 for context
|
|
totalAvailable: availableTools.length,
|
|
selectionMethod: selectionMethod
|
|
},
|
|
{
|
|
selectedTools: selectedTools,
|
|
selectionRatio: selectedTools.length / availableTools.length
|
|
},
|
|
calculatedConfidence,
|
|
startTime,
|
|
{
|
|
...metadata,
|
|
selectionMethod,
|
|
availableToolsCount: availableTools.length,
|
|
selectedToolsCount: selectedTools.length,
|
|
decisionBasis: selectionMethod.includes('embeddings') ? 'semantic-search' : 'ai-analysis'
|
|
}
|
|
);
|
|
}
|
|
|
|
addPhaseCompletion(
|
|
phaseId: string,
|
|
addedTools: string[],
|
|
reasoning: string,
|
|
startTime: number,
|
|
metadata: Record<string, any> = {}
|
|
): void {
|
|
// Only add if tools were actually added
|
|
if (!addedTools || addedTools.length === 0) {
|
|
console.log(`[AUDIT-SERVICE] Skipping phase completion for ${phaseId} - no tools added`);
|
|
return;
|
|
}
|
|
|
|
const calculatedConfidence = this.calculatePhaseCompletionConfidence(addedTools, reasoning, metadata);
|
|
|
|
this.addEntry(
|
|
'phase-completion',
|
|
'phase-enhancement',
|
|
{
|
|
phaseId: phaseId,
|
|
phaseName: this.getPhaseDisplayName(phaseId),
|
|
searchStrategy: 'semantic-search-with-ai-reasoning'
|
|
},
|
|
{
|
|
addedTools: addedTools,
|
|
toolsAddedCount: addedTools.length
|
|
},
|
|
calculatedConfidence,
|
|
startTime,
|
|
{
|
|
...metadata,
|
|
reasoning: reasoning,
|
|
decisionBasis: 'hybrid'
|
|
}
|
|
);
|
|
}
|
|
|
|
addEmbeddingsSearch(
|
|
query: string,
|
|
similarResults: any[],
|
|
threshold: number,
|
|
startTime: number,
|
|
metadata: Record<string, any> = {}
|
|
): void {
|
|
const calculatedConfidence = this.calculateEmbeddingsConfidence(similarResults, threshold);
|
|
|
|
this.addEntry(
|
|
'embeddings',
|
|
'similarity-search',
|
|
{
|
|
query: query,
|
|
threshold: threshold
|
|
},
|
|
{
|
|
resultsCount: similarResults.length,
|
|
topMatches: similarResults.slice(0, 5).map(r => `${r.name} (${Math.round(r.similarity * 100)}%)`)
|
|
},
|
|
calculatedConfidence,
|
|
startTime,
|
|
{
|
|
...metadata,
|
|
embeddingsUsed: true,
|
|
searchThreshold: threshold,
|
|
totalMatches: similarResults.length,
|
|
decisionBasis: 'semantic-search'
|
|
}
|
|
);
|
|
}
|
|
|
|
addConfidenceCalculation(
|
|
toolName: string,
|
|
confidence: any,
|
|
startTime: number,
|
|
metadata: Record<string, any> = {}
|
|
): void {
|
|
this.addEntry(
|
|
'confidence-scoring',
|
|
'tool-confidence',
|
|
{
|
|
toolName: toolName,
|
|
semanticSimilarity: confidence.semanticRelevance,
|
|
taskRelevance: confidence.taskSuitability
|
|
},
|
|
{
|
|
overallConfidence: confidence.overall,
|
|
strengthIndicators: confidence.strengthIndicators?.slice(0, 2) || [],
|
|
uncertaintyFactors: confidence.uncertaintyFactors?.slice(0, 2) || []
|
|
},
|
|
confidence.overall,
|
|
startTime,
|
|
{
|
|
...metadata,
|
|
confidenceCalculation: true,
|
|
decisionBasis: 'ai-analysis'
|
|
}
|
|
);
|
|
}
|
|
|
|
private calculateSelectionConfidence(
|
|
selectedTools: string[],
|
|
availableTools: string[],
|
|
selectionMethod: string,
|
|
metadata: Record<string, any>
|
|
): number {
|
|
let confidence = 50;
|
|
|
|
const selectionRatio = selectedTools.length / availableTools.length;
|
|
|
|
// Good selection ratio (5-20% of available tools)
|
|
if (selectionRatio >= 0.05 && selectionRatio <= 0.20) {
|
|
confidence += 25;
|
|
} else if (selectionRatio < 0.05) {
|
|
confidence += 15; // Very selective is good
|
|
} else if (selectionRatio > 0.30) {
|
|
confidence -= 20; // Too many tools selected
|
|
}
|
|
|
|
// Embeddings usage bonus
|
|
if (selectionMethod.includes('embeddings')) {
|
|
confidence += 15;
|
|
}
|
|
|
|
// Reasonable number of tools selected
|
|
if (selectedTools.length >= 5 && selectedTools.length <= 25) {
|
|
confidence += 10;
|
|
}
|
|
|
|
return Math.min(95, Math.max(40, confidence));
|
|
}
|
|
|
|
private calculatePhaseCompletionConfidence(
|
|
addedTools: string[],
|
|
reasoning: string,
|
|
metadata: Record<string, any>
|
|
): number {
|
|
let confidence = 60;
|
|
|
|
// Tools actually added
|
|
if (addedTools.length > 0) {
|
|
confidence += 20;
|
|
}
|
|
|
|
// Good reasoning provided
|
|
if (reasoning && reasoning.length > 50) {
|
|
confidence += 15;
|
|
}
|
|
|
|
// AI reasoning was used successfully
|
|
if (metadata.aiReasoningUsed) {
|
|
confidence += 10;
|
|
}
|
|
|
|
// Not too many tools added (indicates thoughtful selection)
|
|
if (addedTools.length <= 2) {
|
|
confidence += 5;
|
|
}
|
|
|
|
return Math.min(90, Math.max(50, confidence));
|
|
}
|
|
|
|
private calculateEmbeddingsConfidence(similarResults: any[], threshold: number): number {
|
|
let confidence = 50;
|
|
|
|
// Found relevant results
|
|
if (similarResults.length > 0) {
|
|
confidence += 20;
|
|
}
|
|
|
|
// Good number of results (not too few, not too many)
|
|
if (similarResults.length >= 5 && similarResults.length <= 30) {
|
|
confidence += 15;
|
|
}
|
|
|
|
// High similarity scores
|
|
const avgSimilarity = similarResults.length > 0 ?
|
|
similarResults.reduce((sum, r) => sum + r.similarity, 0) / similarResults.length : 0;
|
|
|
|
if (avgSimilarity > 0.7) {
|
|
confidence += 15;
|
|
} else if (avgSimilarity > 0.5) {
|
|
confidence += 10;
|
|
}
|
|
|
|
// Reasonable threshold
|
|
if (threshold >= 0.3 && threshold <= 0.5) {
|
|
confidence += 5;
|
|
}
|
|
|
|
return Math.min(95, Math.max(30, confidence));
|
|
}
|
|
|
|
private createSpecificSummary(data: any, action: string, type: 'input' | 'output'): string {
|
|
if (!data) return 'Leer';
|
|
|
|
// Action-specific summaries that show actual meaningful data
|
|
switch (action) {
|
|
case 'selection-decision':
|
|
if (type === 'input') {
|
|
if (data.availableTools && Array.isArray(data.availableTools)) {
|
|
const preview = data.availableTools.slice(0, 3).join(', ');
|
|
return `${data.totalAvailable || data.availableTools.length} Tools: ${preview}${data.availableTools.length > 3 ? '...' : ''}`;
|
|
} else if (data.totalAvailable) {
|
|
return `${data.totalAvailable} Tools verfügbar, Methode: ${data.selectionMethod}`;
|
|
}
|
|
return `${data.candidateCount || 0} Kandidaten für Auswahl`;
|
|
} else {
|
|
if (Array.isArray(data.selectedTools)) {
|
|
return `${data.selectedTools.length} ausgewählt: ${data.selectedTools.slice(0, 3).join(', ')}${data.selectedTools.length > 3 ? '...' : ''}`;
|
|
}
|
|
return `Auswahl: ${data.selectionRatio ? Math.round(data.selectionRatio * 100) + '%' : 'unbekannt'}`;
|
|
}
|
|
|
|
case 'phase-tool-selection':
|
|
if (type === 'input') {
|
|
if (Array.isArray(data.availableTools)) {
|
|
const toolPreview = data.availableTools.slice(0, 3).join(', ');
|
|
return `${data.availableTools.length} Tools für ${data.phaseName || data.phaseId}: ${toolPreview}${data.availableTools.length > 3 ? '...' : ''}`;
|
|
}
|
|
return `Phase: ${data.phaseName || data.phaseId} (${data.toolCount || 0} Tools)`;
|
|
} else {
|
|
if (Array.isArray(data.selectedTools)) {
|
|
return `${data.selectedTools.length} ausgewählt: ${data.selectedTools.join(', ')}`;
|
|
}
|
|
return `${data.selectionCount || 0} Tools, Ø ${data.avgTaskRelevance || 0}% Relevanz`;
|
|
}
|
|
|
|
case 'similarity-search':
|
|
if (type === 'input') {
|
|
return `Suche: "${data.query}" (Min. ${data.threshold} Ähnlichkeit)`;
|
|
} else {
|
|
if (Array.isArray(data.topMatches)) {
|
|
return `${data.resultsCount} Treffer: ${data.topMatches.slice(0, 2).join(', ')}${data.topMatches.length > 2 ? '...' : ''}`;
|
|
}
|
|
return `${data.resultsCount || 0} semantische Treffer gefunden`;
|
|
}
|
|
|
|
case 'phase-enhancement':
|
|
if (type === 'input') {
|
|
return `Vervollständige Phase: ${data.phaseName || data.phaseId}`;
|
|
} else {
|
|
if (Array.isArray(data.addedTools) && data.addedTools.length > 0) {
|
|
return `${data.addedTools.length} hinzugefügt: ${data.addedTools.join(', ')}`;
|
|
}
|
|
return `${data.toolsAddedCount || 0} Tools für Phase hinzugefügt`;
|
|
}
|
|
|
|
case 'ai-decision':
|
|
if (type === 'input') {
|
|
if (data.prompt) {
|
|
const promptPreview = data.prompt.slice(0, 80).replace(/\n/g, ' ');
|
|
return `KI-Prompt: ${promptPreview}...`;
|
|
}
|
|
return 'KI-Analyse angefordert';
|
|
} else {
|
|
if (data.response) {
|
|
const responsePreview = data.response.slice(0, 80).replace(/\n/g, ' ');
|
|
return `KI-Antwort: ${responsePreview}...`;
|
|
}
|
|
return 'KI-Analyse abgeschlossen';
|
|
}
|
|
|
|
case 'tool-confidence':
|
|
if (type === 'input') {
|
|
return `Tool: ${data.toolName} (Sem: ${data.semanticSimilarity}%, Task: ${data.taskRelevance}%)`;
|
|
} else {
|
|
const strengthCount = data.strengthIndicators?.length || 0;
|
|
const uncertaintyCount = data.uncertaintyFactors?.length || 0;
|
|
return `${data.overallConfidence}% Vertrauen (${strengthCount} Stärken, ${uncertaintyCount} Unsicherheiten)`;
|
|
}
|
|
|
|
case 'tool-added-to-phase':
|
|
if (type === 'input') {
|
|
return `${data.toolName} → ${data.phaseId} (${data.taskRelevance}% Relevanz, ${data.priority})`;
|
|
} else {
|
|
const justificationPreview = data.justification ?
|
|
data.justification.slice(0, 60).replace(/\n/g, ' ') + '...' : 'Hinzugefügt';
|
|
return `Begründung: ${justificationPreview}`;
|
|
}
|
|
|
|
case 'concept-selection':
|
|
if (type === 'input') {
|
|
const conceptCount = Array.isArray(data.availableConcepts) ? data.availableConcepts.length : 0;
|
|
const toolContext = Array.isArray(data.selectedToolsContext) ? data.selectedToolsContext.length : 0;
|
|
return `${conceptCount} Konzepte verfügbar, ${toolContext} Tools als Kontext`;
|
|
} else {
|
|
if (Array.isArray(data.selectedConcepts)) {
|
|
return `${data.selectedConcepts.length} ausgewählt: ${data.selectedConcepts.slice(0, 2).join(', ')}${data.selectedConcepts.length > 2 ? '...' : ''}`;
|
|
}
|
|
return `Konzeptauswahl abgeschlossen`;
|
|
}
|
|
}
|
|
|
|
// Enhanced fallback that shows actual key-value content instead of just "X Eigenschaften"
|
|
if (typeof data === 'string') {
|
|
return data.length > 100 ? data.slice(0, 100) + '...' : data;
|
|
}
|
|
|
|
if (Array.isArray(data)) {
|
|
if (data.length === 0) return 'Leeres Array';
|
|
if (data.length <= 3) return data.join(', ');
|
|
return `${data.slice(0, 3).join(', ')} + ${data.length - 3} weitere`;
|
|
}
|
|
|
|
if (typeof data === 'object') {
|
|
const keys = Object.keys(data);
|
|
if (keys.length === 0) return 'Leeres Objekt';
|
|
|
|
// Show actual key-value pairs for small objects instead of just counting properties
|
|
if (keys.length <= 2) {
|
|
const pairs = keys.map(key => {
|
|
const value = data[key];
|
|
if (typeof value === 'string' && value.length > 30) {
|
|
return `${key}: ${value.slice(0, 30)}...`;
|
|
} else if (Array.isArray(value)) {
|
|
return `${key}: [${value.length} Items]`;
|
|
} else {
|
|
return `${key}: ${value}`;
|
|
}
|
|
});
|
|
return pairs.join(', ');
|
|
} else {
|
|
// For larger objects, show key names and some sample values
|
|
const sampleKeys = keys.slice(0, 3);
|
|
const sampleValues = sampleKeys.map(key => {
|
|
const value = data[key];
|
|
if (typeof value === 'string' && value.length > 20) {
|
|
return `${key}: ${value.slice(0, 20)}...`;
|
|
} else if (Array.isArray(value)) {
|
|
return `${key}: [${value.length}]`;
|
|
} else {
|
|
return `${key}: ${value}`;
|
|
}
|
|
});
|
|
return `${sampleValues.join(', ')}${keys.length > 3 ? ` + ${keys.length - 3} weitere` : ''}`;
|
|
}
|
|
}
|
|
|
|
return String(data);
|
|
}
|
|
|
|
private generateSpecificReasoning(
|
|
action: string,
|
|
input: any,
|
|
output: any,
|
|
metadata: Record<string, any>,
|
|
confidence: number
|
|
): string {
|
|
// Use provided reasoning if available and meaningful
|
|
if (metadata.reasoning && metadata.reasoning.length > 20 && !metadata.reasoning.includes('completed with')) {
|
|
return metadata.reasoning;
|
|
}
|
|
|
|
switch (action) {
|
|
case 'selection-decision':
|
|
const selectionRatio = metadata.selectedToolsCount / metadata.availableToolsCount;
|
|
const method = metadata.selectionMethod === 'embeddings_candidates' ? 'Semantische Analyse' : 'KI-Analyse';
|
|
return `${method} wählte ${metadata.selectedToolsCount} von ${metadata.availableToolsCount} Tools (${Math.round(selectionRatio * 100)}%) - ausgewogene Auswahl für forensische Aufgabenstellung`;
|
|
|
|
case 'similarity-search': {
|
|
const totalMatches =
|
|
typeof metadata.totalMatches === 'number' ? metadata.totalMatches : 0;
|
|
|
|
// Safely narrow & cast similarityScores to a number map
|
|
const scoresObj = (metadata.similarityScores ?? {}) as Record<string, number>;
|
|
const scores = Object.values(scoresObj) as number[];
|
|
|
|
// Use totalMatches if it looks sensible; otherwise fall back to scores.length
|
|
const denom = totalMatches > 0 ? totalMatches : scores.length;
|
|
|
|
const sum = scores.reduce((acc, v) => acc + (typeof v === 'number' ? v : 0), 0);
|
|
const avgSim = denom > 0 ? sum / denom : 0;
|
|
|
|
return `Semantische Suche fand ${totalMatches} relevante Items mit durchschnittlicher Ähnlichkeit von ${Math.round(avgSim * 100)}%`;
|
|
}
|
|
|
|
case 'ai-decision':
|
|
const taskType = metadata.microTaskType;
|
|
if (taskType) {
|
|
const typeNames = {
|
|
'scenario-analysis': 'Szenario-Analyse',
|
|
'investigation-approach': 'Untersuchungsansatz',
|
|
'critical-considerations': 'Kritische Überlegungen',
|
|
'tool-evaluation': 'Tool-Bewertung',
|
|
'background-knowledge': 'Hintergrundwissen-Auswahl',
|
|
'final-recommendations': 'Abschließende Empfehlungen'
|
|
};
|
|
return `KI analysierte ${typeNames[taskType] || taskType} mit ${confidence}% Vertrauen - fundierte forensische Methodikempfehlung`;
|
|
}
|
|
return `KI-Entscheidung mit ${confidence}% Vertrauen basierend auf forensischer Expertenanalyse`;
|
|
|
|
case 'phase-enhancement':
|
|
const phaseData = input?.phaseName || input?.phaseId;
|
|
const toolCount = output?.toolsAddedCount || 0;
|
|
return `${phaseData}-Phase durch ${toolCount} zusätzliche Tools vervollständigt - ursprüngliche Auswahl war zu spezifisch und übersah wichtige Methoden`;
|
|
|
|
case 'tool-confidence':
|
|
return `Vertrauenswertung für ${input?.toolName}: ${confidence}% basierend auf semantischer Relevanz (${input?.semanticSimilarity}%) und Aufgabeneignung (${input?.taskRelevance}%)`;
|
|
|
|
default:
|
|
return `${action} mit ${confidence}% Vertrauen abgeschlossen`;
|
|
}
|
|
}
|
|
|
|
private truncatePrompt(prompt: string): string {
|
|
if (!prompt || prompt.length <= 200) return prompt;
|
|
return prompt.slice(0, 200) + '...[gekürzt]';
|
|
}
|
|
|
|
private truncateResponse(response: string): string {
|
|
if (!response || response.length <= 300) return response;
|
|
return response.slice(0, 300) + '...[gekürzt]';
|
|
}
|
|
|
|
private getPhaseDisplayName(phaseId: string): string {
|
|
const phaseNames: Record<string, string> = {
|
|
'preparation': 'Vorbereitung',
|
|
'acquisition': 'Datensammlung',
|
|
'examination': 'Untersuchung',
|
|
'analysis': 'Analyse',
|
|
'reporting': 'Dokumentation',
|
|
'presentation': 'Präsentation'
|
|
};
|
|
return phaseNames[phaseId] || phaseId;
|
|
}
|
|
|
|
private inferDecisionBasis(metadata: Record<string, any>): string {
|
|
if (metadata.embeddingsUsed || metadata.selectionMethod?.includes('embeddings')) return 'semantic-search';
|
|
if (metadata.aiPrompt || metadata.microTaskType) return 'ai-analysis';
|
|
if (metadata.semanticQuery && metadata.aiReasoningUsed) return 'hybrid';
|
|
return 'rule-based';
|
|
}
|
|
|
|
getCurrentAuditTrail(): AuditEntry[] {
|
|
return [...this.activeAuditTrail];
|
|
}
|
|
|
|
clearAuditTrail(): void {
|
|
if (this.activeAuditTrail.length > 0) {
|
|
console.log(`[AUDIT-SERVICE] Cleared ${this.activeAuditTrail.length} audit entries`);
|
|
this.activeAuditTrail = [];
|
|
}
|
|
}
|
|
|
|
finalizeAuditTrail(): AuditEntry[] {
|
|
const finalTrail = [...this.activeAuditTrail];
|
|
console.log(`[AUDIT-SERVICE] Finalized audit trail with ${finalTrail.length} meaningful entries`);
|
|
this.clearAuditTrail();
|
|
return finalTrail;
|
|
}
|
|
|
|
isEnabled(): boolean {
|
|
return this.config.enabled;
|
|
}
|
|
|
|
getConfig(): AuditConfig {
|
|
return { ...this.config };
|
|
}
|
|
|
|
calculateAIResponseConfidence(
|
|
response: string,
|
|
expectedLength: { min: number; max: number },
|
|
taskType: string
|
|
): number {
|
|
let confidence = 50;
|
|
|
|
if (response.length >= expectedLength.min) {
|
|
confidence += 20;
|
|
if (response.length <= expectedLength.max) {
|
|
confidence += 10;
|
|
}
|
|
} else {
|
|
confidence -= 20;
|
|
}
|
|
|
|
if (response.includes('...') || response.endsWith('...')) {
|
|
confidence -= 10;
|
|
}
|
|
|
|
switch (taskType) {
|
|
case 'scenario-analysis':
|
|
case 'investigation-approach':
|
|
case 'critical-considerations':
|
|
const forensicTerms = ['forensisch', 'beweis', 'evidence', 'analyse', 'untersuchung', 'methodik'];
|
|
const termsFound = forensicTerms.filter(term =>
|
|
response.toLowerCase().includes(term)
|
|
).length;
|
|
confidence += Math.min(15, termsFound * 3);
|
|
break;
|
|
|
|
case 'tool-evaluation':
|
|
if (response.includes('detailed_explanation') || response.includes('implementation_approach')) {
|
|
confidence += 15;
|
|
}
|
|
if (response.includes('pros') && response.includes('limitations')) {
|
|
confidence += 10;
|
|
}
|
|
break;
|
|
|
|
case 'background-knowledge':
|
|
try {
|
|
const parsed = JSON.parse(response);
|
|
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
confidence += 20;
|
|
}
|
|
} catch {
|
|
confidence -= 20;
|
|
}
|
|
break;
|
|
}
|
|
|
|
return Math.min(95, Math.max(25, confidence));
|
|
}
|
|
|
|
// Additional utility methods remain the same...
|
|
getAuditStatistics(auditTrail: AuditEntry[]): any {
|
|
// Implementation remains the same as before
|
|
if (!auditTrail || auditTrail.length === 0) {
|
|
return {
|
|
totalTime: 0,
|
|
avgConfidence: 0,
|
|
stepCount: 0,
|
|
highConfidenceSteps: 0,
|
|
lowConfidenceSteps: 0,
|
|
phaseBreakdown: {},
|
|
aiDecisionCount: 0,
|
|
embeddingsUsageCount: 0,
|
|
toolSelectionCount: 0,
|
|
qualityMetrics: {
|
|
avgProcessingTime: 0,
|
|
confidenceDistribution: { high: 0, medium: 0, low: 0 }
|
|
}
|
|
};
|
|
}
|
|
|
|
const totalTime = auditTrail.reduce((sum, entry) => sum + (entry.processingTimeMs || 0), 0);
|
|
const validConfidenceEntries = auditTrail.filter(entry => typeof entry.confidence === 'number');
|
|
const avgConfidence = validConfidenceEntries.length > 0
|
|
? Math.round(validConfidenceEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validConfidenceEntries.length)
|
|
: 0;
|
|
|
|
return {
|
|
totalTime,
|
|
avgConfidence,
|
|
stepCount: auditTrail.length,
|
|
highConfidenceSteps: auditTrail.filter(entry => (entry.confidence || 0) >= 80).length,
|
|
lowConfidenceSteps: auditTrail.filter(entry => (entry.confidence || 0) < 60).length,
|
|
phaseBreakdown: {},
|
|
aiDecisionCount: auditTrail.filter(entry => entry.action === 'ai-decision').length,
|
|
embeddingsUsageCount: auditTrail.filter(entry => entry.metadata?.embeddingsUsed).length,
|
|
toolSelectionCount: auditTrail.filter(entry => entry.action === 'selection-decision').length,
|
|
qualityMetrics: {
|
|
avgProcessingTime: auditTrail.length > 0 ? totalTime / auditTrail.length : 0,
|
|
confidenceDistribution: {
|
|
high: auditTrail.filter(entry => (entry.confidence || 0) >= 80).length,
|
|
medium: auditTrail.filter(entry => (entry.confidence || 0) >= 60 && (entry.confidence || 0) < 80).length,
|
|
low: auditTrail.filter(entry => (entry.confidence || 0) < 60).length
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
validateAuditTrail(auditTrail: AuditEntry[]): {
|
|
isValid: boolean;
|
|
issues: string[];
|
|
warnings: string[];
|
|
} {
|
|
const issues: string[] = [];
|
|
const warnings: string[] = [];
|
|
|
|
if (!Array.isArray(auditTrail)) {
|
|
issues.push('Audit trail is not an array');
|
|
return { isValid: false, issues, warnings };
|
|
}
|
|
|
|
if (auditTrail.length === 0) {
|
|
warnings.push('Audit trail is empty');
|
|
}
|
|
|
|
auditTrail.forEach((entry, index) => {
|
|
if (!entry || typeof entry !== 'object') {
|
|
issues.push(`Entry ${index} is not a valid object`);
|
|
return;
|
|
}
|
|
|
|
const requiredFields = ['timestamp', 'phase', 'action'];
|
|
requiredFields.forEach(field => {
|
|
if (!(field in entry)) {
|
|
issues.push(`Entry ${index} missing required field: ${field}`);
|
|
}
|
|
});
|
|
|
|
if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) {
|
|
warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`);
|
|
}
|
|
});
|
|
|
|
return {
|
|
isValid: issues.length === 0,
|
|
issues,
|
|
warnings
|
|
};
|
|
}
|
|
}
|
|
|
|
export const auditService = new AuditService(); |