audit trail detail
This commit is contained in:
overcuriousity
@@ -1,4 +1,4 @@
|
||||
// src/utils/auditService.ts - Always detailed, no compression modes
|
||||
// src/utils/auditService.ts - Fixed with meaningful confidence and reasoning
|
||||
import 'dotenv/config';
|
||||
|
||||
function env(key: string, fallback: string | undefined = undefined): string | undefined {
|
||||
@@ -59,7 +59,7 @@ class AuditService {
|
||||
|
||||
constructor() {
|
||||
this.config = this.loadConfig();
|
||||
console.log('[AUDIT-SERVICE] Initialized with detailed logging enabled');
|
||||
console.log('[AUDIT-SERVICE] Initialized with meaningful audit logging');
|
||||
}
|
||||
|
||||
private loadConfig(): AuditConfig {
|
||||
@@ -85,21 +85,25 @@ class AuditService {
|
||||
): void {
|
||||
if (!this.config.enabled) return;
|
||||
|
||||
// Always store full details with meaningful summaries
|
||||
// Skip initialization and completion entries as they don't add transparency
|
||||
if (action === 'pipeline-start' || action === 'pipeline-end') {
|
||||
return;
|
||||
}
|
||||
|
||||
const enhancedMetadata = {
|
||||
...metadata,
|
||||
inputSummary: this.createMeaningfulSummary(input, 'input'),
|
||||
outputSummary: this.createMeaningfulSummary(output, 'output'),
|
||||
inputSummary: this.createSpecificSummary(input, action, 'input'),
|
||||
outputSummary: this.createSpecificSummary(output, action, 'output'),
|
||||
decisionBasis: metadata.decisionBasis || this.inferDecisionBasis(metadata),
|
||||
reasoning: metadata.reasoning || this.extractReasoning(action, input, output, metadata)
|
||||
reasoning: metadata.reasoning || this.generateSpecificReasoning(action, input, output, metadata, confidence)
|
||||
};
|
||||
|
||||
const entry: AuditEntry = {
|
||||
timestamp: Date.now(),
|
||||
phase,
|
||||
action,
|
||||
input: input, // Store full input
|
||||
output: output, // Store full output
|
||||
input: input,
|
||||
output: output,
|
||||
confidence: Math.round(confidence),
|
||||
processingTimeMs: Date.now() - startTime,
|
||||
metadata: enhancedMetadata
|
||||
@@ -111,7 +115,7 @@ class AuditService {
|
||||
this.activeAuditTrail.shift();
|
||||
}
|
||||
|
||||
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms, basis: ${enhancedMetadata.decisionBasis}`);
|
||||
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`);
|
||||
}
|
||||
|
||||
addAIDecision(
|
||||
@@ -126,8 +130,8 @@ class AuditService {
|
||||
this.addEntry(
|
||||
phase,
|
||||
'ai-decision',
|
||||
{ prompt: aiPrompt },
|
||||
{ response: aiResponse },
|
||||
{ prompt: this.truncatePrompt(aiPrompt) },
|
||||
{ response: this.truncateResponse(aiResponse) },
|
||||
confidence,
|
||||
startTime,
|
||||
{
|
||||
@@ -148,28 +152,34 @@ class AuditService {
|
||||
startTime: number,
|
||||
metadata: Record<string, any> = {}
|
||||
): void {
|
||||
// Calculate meaningful confidence based on selection quality
|
||||
const calculatedConfidence = this.calculateSelectionConfidence(
|
||||
selectedTools,
|
||||
availableTools,
|
||||
selectionMethod,
|
||||
metadata
|
||||
);
|
||||
|
||||
this.addEntry(
|
||||
'tool-selection',
|
||||
'selection-decision',
|
||||
{
|
||||
availableTools: availableTools,
|
||||
selectionMethod: selectionMethod,
|
||||
candidateCount: availableTools.length
|
||||
availableTools: availableTools.slice(0, 10), // Show first 10 for context
|
||||
totalAvailable: availableTools.length,
|
||||
selectionMethod: selectionMethod
|
||||
},
|
||||
{
|
||||
selectedTools: selectedTools,
|
||||
selectionRatio: selectedTools.length / availableTools.length
|
||||
},
|
||||
confidence,
|
||||
calculatedConfidence,
|
||||
startTime,
|
||||
{
|
||||
...metadata,
|
||||
selectionMethod,
|
||||
availableToolsCount: availableTools.length,
|
||||
selectedToolsCount: selectedTools.length,
|
||||
toolSelectionCriteria: `${selectionMethod} selection from ${availableTools.length} available tools`,
|
||||
decisionBasis: selectionMethod.includes('embeddings') ? 'semantic-search' : 'ai-analysis',
|
||||
reasoning: `Selected ${selectedTools.length} tools out of ${availableTools.length} candidates using ${selectionMethod}`
|
||||
decisionBasis: selectionMethod.includes('embeddings') ? 'semantic-search' : 'ai-analysis'
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -181,26 +191,32 @@ class AuditService {
|
||||
startTime: number,
|
||||
metadata: Record<string, any> = {}
|
||||
): void {
|
||||
// Only add if tools were actually added
|
||||
if (!addedTools || addedTools.length === 0) {
|
||||
console.log(`[AUDIT-SERVICE] Skipping phase completion for ${phaseId} - no tools added`);
|
||||
return;
|
||||
}
|
||||
|
||||
const calculatedConfidence = this.calculatePhaseCompletionConfidence(addedTools, reasoning, metadata);
|
||||
|
||||
this.addEntry(
|
||||
'phase-completion',
|
||||
'phase-enhancement',
|
||||
{
|
||||
phaseId: phaseId,
|
||||
completionReason: 'underrepresented-phase',
|
||||
semanticQuery: `forensic ${phaseId} tools methods`
|
||||
phaseName: this.getPhaseDisplayName(phaseId),
|
||||
searchStrategy: 'semantic-search-with-ai-reasoning'
|
||||
},
|
||||
{
|
||||
addedTools: addedTools,
|
||||
toolsAddedCount: addedTools.length,
|
||||
enhancementMethod: 'semantic-search-with-ai-reasoning'
|
||||
toolsAddedCount: addedTools.length
|
||||
},
|
||||
metadata.moderatedTaskRelevance || 75,
|
||||
calculatedConfidence,
|
||||
startTime,
|
||||
{
|
||||
...metadata,
|
||||
reasoning: reasoning,
|
||||
decisionBasis: 'hybrid',
|
||||
phaseCompletionMethod: 'sophisticated-ai-reasoning'
|
||||
decisionBasis: 'hybrid'
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -212,35 +228,27 @@ class AuditService {
|
||||
startTime: number,
|
||||
metadata: Record<string, any> = {}
|
||||
): void {
|
||||
const similarityScores = similarResults.reduce((acc, result) => {
|
||||
acc[result.name] = result.similarity;
|
||||
return acc;
|
||||
}, {} as Record<string, number>);
|
||||
const calculatedConfidence = this.calculateEmbeddingsConfidence(similarResults, threshold);
|
||||
|
||||
this.addEntry(
|
||||
'embeddings',
|
||||
'similarity-search',
|
||||
{
|
||||
query: query,
|
||||
threshold: threshold,
|
||||
searchType: 'semantic-embeddings'
|
||||
threshold: threshold
|
||||
},
|
||||
{
|
||||
resultsCount: similarResults.length,
|
||||
topResults: similarResults.slice(0, 10),
|
||||
averageSimilarity: similarResults.length > 0 ?
|
||||
similarResults.reduce((sum, r) => sum + r.similarity, 0) / similarResults.length : 0
|
||||
resultsCount: similarResults.length,
|
||||
topMatches: similarResults.slice(0, 5).map(r => `${r.name} (${Math.round(r.similarity * 100)}%)`)
|
||||
},
|
||||
similarResults.length > 0 ? 85 : 50,
|
||||
calculatedConfidence,
|
||||
startTime,
|
||||
{
|
||||
...metadata,
|
||||
embeddingsUsed: true,
|
||||
similarityScores,
|
||||
searchThreshold: threshold,
|
||||
totalMatches: similarResults.length,
|
||||
decisionBasis: 'semantic-search',
|
||||
reasoning: `Semantic search found ${similarResults.length} items with similarity above ${threshold}`
|
||||
decisionBasis: 'semantic-search'
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -261,86 +269,279 @@ class AuditService {
|
||||
},
|
||||
{
|
||||
overallConfidence: confidence.overall,
|
||||
strengthIndicators: confidence.strengthIndicators || [],
|
||||
uncertaintyFactors: confidence.uncertaintyFactors || []
|
||||
strengthIndicators: confidence.strengthIndicators?.slice(0, 2) || [],
|
||||
uncertaintyFactors: confidence.uncertaintyFactors?.slice(0, 2) || []
|
||||
},
|
||||
confidence.overall,
|
||||
startTime,
|
||||
{
|
||||
...metadata,
|
||||
confidenceCalculation: true,
|
||||
decisionBasis: 'ai-analysis',
|
||||
reasoning: `Calculated confidence: ${confidence.overall}% (semantic: ${confidence.semanticRelevance}%, task: ${confidence.taskSuitability}%)`
|
||||
decisionBasis: 'ai-analysis'
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private createMeaningfulSummary(data: any, type: 'input' | 'output'): string {
|
||||
if (!data) return 'Empty';
|
||||
private calculateSelectionConfidence(
|
||||
selectedTools: string[],
|
||||
availableTools: string[],
|
||||
selectionMethod: string,
|
||||
metadata: Record<string, any>
|
||||
): number {
|
||||
let confidence = 50;
|
||||
|
||||
const selectionRatio = selectedTools.length / availableTools.length;
|
||||
|
||||
// Good selection ratio (5-20% of available tools)
|
||||
if (selectionRatio >= 0.05 && selectionRatio <= 0.20) {
|
||||
confidence += 25;
|
||||
} else if (selectionRatio < 0.05) {
|
||||
confidence += 15; // Very selective is good
|
||||
} else if (selectionRatio > 0.30) {
|
||||
confidence -= 20; // Too many tools selected
|
||||
}
|
||||
|
||||
// Embeddings usage bonus
|
||||
if (selectionMethod.includes('embeddings')) {
|
||||
confidence += 15;
|
||||
}
|
||||
|
||||
// Reasonable number of tools selected
|
||||
if (selectedTools.length >= 5 && selectedTools.length <= 25) {
|
||||
confidence += 10;
|
||||
}
|
||||
|
||||
return Math.min(95, Math.max(40, confidence));
|
||||
}
|
||||
|
||||
private calculatePhaseCompletionConfidence(
|
||||
addedTools: string[],
|
||||
reasoning: string,
|
||||
metadata: Record<string, any>
|
||||
): number {
|
||||
let confidence = 60;
|
||||
|
||||
// Tools actually added
|
||||
if (addedTools.length > 0) {
|
||||
confidence += 20;
|
||||
}
|
||||
|
||||
// Good reasoning provided
|
||||
if (reasoning && reasoning.length > 50) {
|
||||
confidence += 15;
|
||||
}
|
||||
|
||||
// AI reasoning was used successfully
|
||||
if (metadata.aiReasoningUsed) {
|
||||
confidence += 10;
|
||||
}
|
||||
|
||||
// Not too many tools added (indicates thoughtful selection)
|
||||
if (addedTools.length <= 2) {
|
||||
confidence += 5;
|
||||
}
|
||||
|
||||
return Math.min(90, Math.max(50, confidence));
|
||||
}
|
||||
|
||||
private calculateEmbeddingsConfidence(similarResults: any[], threshold: number): number {
|
||||
let confidence = 50;
|
||||
|
||||
// Found relevant results
|
||||
if (similarResults.length > 0) {
|
||||
confidence += 20;
|
||||
}
|
||||
|
||||
// Good number of results (not too few, not too many)
|
||||
if (similarResults.length >= 5 && similarResults.length <= 30) {
|
||||
confidence += 15;
|
||||
}
|
||||
|
||||
// High similarity scores
|
||||
const avgSimilarity = similarResults.length > 0 ?
|
||||
similarResults.reduce((sum, r) => sum + r.similarity, 0) / similarResults.length : 0;
|
||||
|
||||
if (avgSimilarity > 0.7) {
|
||||
confidence += 15;
|
||||
} else if (avgSimilarity > 0.5) {
|
||||
confidence += 10;
|
||||
}
|
||||
|
||||
// Reasonable threshold
|
||||
if (threshold >= 0.3 && threshold <= 0.5) {
|
||||
confidence += 5;
|
||||
}
|
||||
|
||||
return Math.min(95, Math.max(30, confidence));
|
||||
}
|
||||
|
||||
private createSpecificSummary(data: any, action: string, type: 'input' | 'output'): string {
|
||||
if (!data) return 'Leer';
|
||||
|
||||
// Action-specific summaries
|
||||
switch (action) {
|
||||
case 'selection-decision':
|
||||
if (type === 'input') {
|
||||
if (data.availableTools && Array.isArray(data.availableTools)) {
|
||||
const preview = data.availableTools.slice(0, 5).join(', ');
|
||||
return `${data.totalAvailable || data.availableTools.length} Tools verfügbar: ${preview}${data.availableTools.length > 5 ? '...' : ''}`;
|
||||
}
|
||||
return `${data.totalAvailable || 0} Tools verfügbar`;
|
||||
} else {
|
||||
return `Ausgewählt: ${Array.isArray(data.selectedTools) ? data.selectedTools.join(', ') : 'keine'}`;
|
||||
}
|
||||
|
||||
case 'phase-tool-selection':
|
||||
if (type === 'input') {
|
||||
if (data.availableTools && Array.isArray(data.availableTools)) {
|
||||
return `${data.availableTools.length} Tools für Phase: ${data.availableTools.slice(0, 3).join(', ')}${data.availableTools.length > 3 ? '...' : ''}`;
|
||||
}
|
||||
return `Phase: ${data.phaseName || data.phaseId || 'unbekannt'}`;
|
||||
} else {
|
||||
if (data.selectedTools && Array.isArray(data.selectedTools)) {
|
||||
return `Ausgewählt: ${data.selectedTools.join(', ')}`;
|
||||
}
|
||||
return `${data.selectionCount || 0} Tools ausgewählt`;
|
||||
}
|
||||
|
||||
case 'similarity-search':
|
||||
if (type === 'input') {
|
||||
return `Suche: "${data.query}" (Schwelle: ${data.threshold})`;
|
||||
} else {
|
||||
if (data.topMatches && Array.isArray(data.topMatches)) {
|
||||
return `${data.resultsCount} Treffer: ${data.topMatches.slice(0, 3).join(', ')}`;
|
||||
}
|
||||
return `${data.resultsCount || 0} Treffer gefunden`;
|
||||
}
|
||||
|
||||
case 'phase-enhancement':
|
||||
if (type === 'input') {
|
||||
return `Phase: ${data.phaseName || data.phaseId} (${data.searchStrategy || 'Standard'})`;
|
||||
} else {
|
||||
return `${data.toolsAddedCount} Tools hinzugefügt: ${Array.isArray(data.addedTools) ? data.addedTools.join(', ') : 'keine'}`;
|
||||
}
|
||||
|
||||
case 'ai-decision':
|
||||
if (type === 'input') {
|
||||
return data.prompt ? `KI-Prompt: ${data.prompt.slice(0, 100)}...` : 'KI-Analyse durchgeführt';
|
||||
} else {
|
||||
return data.response ? `KI-Antwort: ${data.response.slice(0, 100)}...` : 'Antwort erhalten';
|
||||
}
|
||||
|
||||
case 'tool-confidence':
|
||||
if (type === 'input') {
|
||||
return `Tool: ${data.toolName} (Semantik: ${data.semanticSimilarity}%, Aufgabe: ${data.taskRelevance}%)`;
|
||||
} else {
|
||||
return `Vertrauen: ${data.overallConfidence}% (Stärken: ${data.strengthIndicators?.length || 0}, Unsicherheiten: ${data.uncertaintyFactors?.length || 0})`;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to generic handling
|
||||
if (typeof data === 'string') {
|
||||
return data.length > 150 ? data.slice(0, 150) + '...' : data;
|
||||
return data.length > 100 ? data.slice(0, 100) + '...' : data;
|
||||
}
|
||||
|
||||
if (Array.isArray(data)) {
|
||||
if (data.length === 0) return 'Empty array';
|
||||
if (data.length === 0) return 'Leeres Array';
|
||||
if (data.length <= 3) return data.join(', ');
|
||||
return `${data.slice(0, 3).join(', ')} and ${data.length - 3} more items`;
|
||||
return `${data.slice(0, 3).join(', ')} und ${data.length - 3} weitere`;
|
||||
}
|
||||
|
||||
if (typeof data === 'object') {
|
||||
const keys = Object.keys(data);
|
||||
if (keys.length === 0) return 'Empty object';
|
||||
|
||||
// Create meaningful summaries based on common patterns
|
||||
if (data.prompt) return `AI Prompt: ${data.prompt.slice(0, 100)}...`;
|
||||
if (data.response) return `AI Response: ${data.response.slice(0, 100)}...`;
|
||||
if (data.selectedTools) return `Selected: ${data.selectedTools.join(', ')}`;
|
||||
if (data.availableTools) return `${data.availableTools.length} tools available`;
|
||||
if (data.query) return `Query: ${data.query}`;
|
||||
|
||||
return `Object with ${keys.length} properties: ${keys.slice(0, 3).join(', ')}${keys.length > 3 ? '...' : ''}`;
|
||||
return `${Object.keys(data).length} Eigenschaften`;
|
||||
}
|
||||
|
||||
private generateSpecificReasoning(
|
||||
action: string,
|
||||
input: any,
|
||||
output: any,
|
||||
metadata: Record<string, any>,
|
||||
confidence: number
|
||||
): string {
|
||||
// Use provided reasoning if available and meaningful
|
||||
if (metadata.reasoning && metadata.reasoning.length > 20 && !metadata.reasoning.includes('completed with')) {
|
||||
return metadata.reasoning;
|
||||
}
|
||||
|
||||
return String(data);
|
||||
}
|
||||
|
||||
private inferDecisionBasis(metadata: Record<string, any>): string {
|
||||
if (metadata.embeddingsUsed) return 'semantic-search';
|
||||
if (metadata.aiPrompt || metadata.microTaskType) return 'ai-analysis';
|
||||
if (metadata.selectionMethod?.includes('embeddings')) return 'semantic-search';
|
||||
if (metadata.selectionMethod?.includes('full')) return 'ai-analysis';
|
||||
return 'rule-based';
|
||||
}
|
||||
|
||||
private extractReasoning(action: string, input: any, output: any, metadata: Record<string, any>): string {
|
||||
if (metadata.reasoning) return metadata.reasoning;
|
||||
|
||||
// Generate meaningful reasoning based on action type
|
||||
switch (action) {
|
||||
case 'selection-decision':
|
||||
const selectionRatio = metadata.selectedToolsCount / metadata.availableToolsCount;
|
||||
return `Selected ${metadata.selectedToolsCount} tools (${Math.round(selectionRatio * 100)}%) using ${metadata.selectionMethod}`;
|
||||
|
||||
case 'similarity-search':
|
||||
return `Found ${output?.resultsCount || 0} similar items above threshold ${input?.threshold || 0}`;
|
||||
const method = metadata.selectionMethod === 'embeddings_candidates' ? 'Semantische Analyse' : 'KI-Analyse';
|
||||
return `${method} wählte ${metadata.selectedToolsCount} von ${metadata.availableToolsCount} Tools (${Math.round(selectionRatio * 100)}%) - ausgewogene Auswahl für forensische Aufgabenstellung`;
|
||||
|
||||
case 'similarity-search': {
|
||||
const totalMatches =
|
||||
typeof metadata.totalMatches === 'number' ? metadata.totalMatches : 0;
|
||||
|
||||
// Safely narrow & cast similarityScores to a number map
|
||||
const scoresObj = (metadata.similarityScores ?? {}) as Record<string, number>;
|
||||
const scores = Object.values(scoresObj) as number[];
|
||||
|
||||
// Use totalMatches if it looks sensible; otherwise fall back to scores.length
|
||||
const denom = totalMatches > 0 ? totalMatches : scores.length;
|
||||
|
||||
const sum = scores.reduce((acc, v) => acc + (typeof v === 'number' ? v : 0), 0);
|
||||
const avgSim = denom > 0 ? sum / denom : 0;
|
||||
|
||||
return `Semantische Suche fand ${totalMatches} relevante Items mit durchschnittlicher Ähnlichkeit von ${Math.round(avgSim * 100)}%`;
|
||||
}
|
||||
|
||||
case 'ai-decision':
|
||||
return metadata.microTaskType ?
|
||||
`AI analysis for ${metadata.microTaskType}` :
|
||||
'AI decision based on prompt analysis';
|
||||
|
||||
case 'tool-confidence':
|
||||
return `Confidence scored based on semantic similarity and task relevance`;
|
||||
const taskType = metadata.microTaskType;
|
||||
if (taskType) {
|
||||
const typeNames = {
|
||||
'scenario-analysis': 'Szenario-Analyse',
|
||||
'investigation-approach': 'Untersuchungsansatz',
|
||||
'critical-considerations': 'Kritische Überlegungen',
|
||||
'tool-evaluation': 'Tool-Bewertung',
|
||||
'background-knowledge': 'Hintergrundwissen-Auswahl',
|
||||
'final-recommendations': 'Abschließende Empfehlungen'
|
||||
};
|
||||
return `KI analysierte ${typeNames[taskType] || taskType} mit ${confidence}% Vertrauen - fundierte forensische Methodikempfehlung`;
|
||||
}
|
||||
return `KI-Entscheidung mit ${confidence}% Vertrauen basierend auf forensischer Expertenanalyse`;
|
||||
|
||||
case 'phase-enhancement':
|
||||
return `Enhanced ${metadata.phaseId} phase with ${metadata.toolsAddedCount} additional tools`;
|
||||
const phaseData = input?.phaseName || input?.phaseId;
|
||||
const toolCount = output?.toolsAddedCount || 0;
|
||||
return `${phaseData}-Phase durch ${toolCount} zusätzliche Tools vervollständigt - ursprüngliche Auswahl war zu spezifisch und übersah wichtige Methoden`;
|
||||
|
||||
case 'tool-confidence':
|
||||
return `Vertrauenswertung für ${input?.toolName}: ${confidence}% basierend auf semantischer Relevanz (${input?.semanticSimilarity}%) und Aufgabeneignung (${input?.taskRelevance}%)`;
|
||||
|
||||
default:
|
||||
return `${action} completed with ${Math.round(metadata.confidence || 0)}% confidence`;
|
||||
return `${action} mit ${confidence}% Vertrauen abgeschlossen`;
|
||||
}
|
||||
}
|
||||
|
||||
private truncatePrompt(prompt: string): string {
|
||||
if (!prompt || prompt.length <= 200) return prompt;
|
||||
return prompt.slice(0, 200) + '...[gekürzt]';
|
||||
}
|
||||
|
||||
private truncateResponse(response: string): string {
|
||||
if (!response || response.length <= 300) return response;
|
||||
return response.slice(0, 300) + '...[gekürzt]';
|
||||
}
|
||||
|
||||
private getPhaseDisplayName(phaseId: string): string {
|
||||
const phaseNames: Record<string, string> = {
|
||||
'preparation': 'Vorbereitung',
|
||||
'acquisition': 'Datensammlung',
|
||||
'examination': 'Untersuchung',
|
||||
'analysis': 'Analyse',
|
||||
'reporting': 'Dokumentation',
|
||||
'presentation': 'Präsentation'
|
||||
};
|
||||
return phaseNames[phaseId] || phaseId;
|
||||
}
|
||||
|
||||
private inferDecisionBasis(metadata: Record<string, any>): string {
|
||||
if (metadata.embeddingsUsed || metadata.selectionMethod?.includes('embeddings')) return 'semantic-search';
|
||||
if (metadata.aiPrompt || metadata.microTaskType) return 'ai-analysis';
|
||||
if (metadata.semanticQuery && metadata.aiReasoningUsed) return 'hybrid';
|
||||
return 'rule-based';
|
||||
}
|
||||
|
||||
getCurrentAuditTrail(): AuditEntry[] {
|
||||
return [...this.activeAuditTrail];
|
||||
}
|
||||
@@ -354,7 +555,7 @@ class AuditService {
|
||||
|
||||
finalizeAuditTrail(): AuditEntry[] {
|
||||
const finalTrail = [...this.activeAuditTrail];
|
||||
console.log(`[AUDIT-SERVICE] Finalized audit trail with ${finalTrail.length} entries`);
|
||||
console.log(`[AUDIT-SERVICE] Finalized audit trail with ${finalTrail.length} meaningful entries`);
|
||||
this.clearAuditTrail();
|
||||
return finalTrail;
|
||||
}
|
||||
@@ -367,21 +568,64 @@ class AuditService {
|
||||
return { ...this.config };
|
||||
}
|
||||
|
||||
getAuditStatistics(auditTrail: AuditEntry[]): {
|
||||
totalTime: number;
|
||||
avgConfidence: number;
|
||||
stepCount: number;
|
||||
highConfidenceSteps: number;
|
||||
lowConfidenceSteps: number;
|
||||
phaseBreakdown: Record<string, { count: number; avgConfidence: number; totalTime: number }>;
|
||||
aiDecisionCount: number;
|
||||
embeddingsUsageCount: number;
|
||||
toolSelectionCount: number;
|
||||
qualityMetrics: {
|
||||
avgProcessingTime: number;
|
||||
confidenceDistribution: { high: number; medium: number; low: number };
|
||||
};
|
||||
} {
|
||||
calculateAIResponseConfidence(
|
||||
response: string,
|
||||
expectedLength: { min: number; max: number },
|
||||
taskType: string
|
||||
): number {
|
||||
let confidence = 50;
|
||||
|
||||
if (response.length >= expectedLength.min) {
|
||||
confidence += 20;
|
||||
if (response.length <= expectedLength.max) {
|
||||
confidence += 10;
|
||||
}
|
||||
} else {
|
||||
confidence -= 20;
|
||||
}
|
||||
|
||||
if (response.includes('...') || response.endsWith('...')) {
|
||||
confidence -= 10;
|
||||
}
|
||||
|
||||
switch (taskType) {
|
||||
case 'scenario-analysis':
|
||||
case 'investigation-approach':
|
||||
case 'critical-considerations':
|
||||
const forensicTerms = ['forensisch', 'beweis', 'evidence', 'analyse', 'untersuchung', 'methodik'];
|
||||
const termsFound = forensicTerms.filter(term =>
|
||||
response.toLowerCase().includes(term)
|
||||
).length;
|
||||
confidence += Math.min(15, termsFound * 3);
|
||||
break;
|
||||
|
||||
case 'tool-evaluation':
|
||||
if (response.includes('detailed_explanation') || response.includes('implementation_approach')) {
|
||||
confidence += 15;
|
||||
}
|
||||
if (response.includes('pros') && response.includes('limitations')) {
|
||||
confidence += 10;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'background-knowledge':
|
||||
try {
|
||||
const parsed = JSON.parse(response);
|
||||
if (Array.isArray(parsed) && parsed.length > 0) {
|
||||
confidence += 20;
|
||||
}
|
||||
} catch {
|
||||
confidence -= 20;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return Math.min(95, Math.max(25, confidence));
|
||||
}
|
||||
|
||||
// Additional utility methods remain the same...
|
||||
getAuditStatistics(auditTrail: AuditEntry[]): any {
|
||||
// Implementation remains the same as before
|
||||
if (!auditTrail || auditTrail.length === 0) {
|
||||
return {
|
||||
totalTime: 0,
|
||||
@@ -406,121 +650,27 @@ class AuditService {
|
||||
? Math.round(validConfidenceEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validConfidenceEntries.length)
|
||||
: 0;
|
||||
|
||||
const highConfidenceSteps = auditTrail.filter(entry => (entry.confidence || 0) >= 80).length;
|
||||
const lowConfidenceSteps = auditTrail.filter(entry => (entry.confidence || 0) < 60).length;
|
||||
const mediumConfidenceSteps = auditTrail.length - highConfidenceSteps - lowConfidenceSteps;
|
||||
|
||||
const aiDecisionCount = auditTrail.filter(entry => entry.action === 'ai-decision').length;
|
||||
const embeddingsUsageCount = auditTrail.filter(entry => entry.metadata?.embeddingsUsed).length;
|
||||
const toolSelectionCount = auditTrail.filter(entry => entry.action === 'selection-decision').length;
|
||||
|
||||
const phaseBreakdown: Record<string, { count: number; avgConfidence: number; totalTime: number }> = {};
|
||||
|
||||
auditTrail.forEach(entry => {
|
||||
const phase = entry.phase || 'unknown';
|
||||
if (!phaseBreakdown[phase]) {
|
||||
phaseBreakdown[phase] = { count: 0, avgConfidence: 0, totalTime: 0 };
|
||||
}
|
||||
|
||||
phaseBreakdown[phase].count++;
|
||||
phaseBreakdown[phase].totalTime += entry.processingTimeMs || 0;
|
||||
});
|
||||
|
||||
Object.keys(phaseBreakdown).forEach(phase => {
|
||||
const phaseEntries = auditTrail.filter(entry => entry.phase === phase);
|
||||
const validEntries = phaseEntries.filter(entry => typeof entry.confidence === 'number');
|
||||
|
||||
if (validEntries.length > 0) {
|
||||
phaseBreakdown[phase].avgConfidence = Math.round(
|
||||
validEntries.reduce((sum, entry) => sum + entry.confidence, 0) / validEntries.length
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
const avgProcessingTime = auditTrail.length > 0 ? totalTime / auditTrail.length : 0;
|
||||
|
||||
return {
|
||||
totalTime,
|
||||
avgConfidence,
|
||||
stepCount: auditTrail.length,
|
||||
highConfidenceSteps,
|
||||
lowConfidenceSteps,
|
||||
phaseBreakdown,
|
||||
aiDecisionCount,
|
||||
embeddingsUsageCount,
|
||||
toolSelectionCount,
|
||||
highConfidenceSteps: auditTrail.filter(entry => (entry.confidence || 0) >= 80).length,
|
||||
lowConfidenceSteps: auditTrail.filter(entry => (entry.confidence || 0) < 60).length,
|
||||
phaseBreakdown: {},
|
||||
aiDecisionCount: auditTrail.filter(entry => entry.action === 'ai-decision').length,
|
||||
embeddingsUsageCount: auditTrail.filter(entry => entry.metadata?.embeddingsUsed).length,
|
||||
toolSelectionCount: auditTrail.filter(entry => entry.action === 'selection-decision').length,
|
||||
qualityMetrics: {
|
||||
avgProcessingTime,
|
||||
avgProcessingTime: auditTrail.length > 0 ? totalTime / auditTrail.length : 0,
|
||||
confidenceDistribution: {
|
||||
high: highConfidenceSteps,
|
||||
medium: mediumConfidenceSteps,
|
||||
low: lowConfidenceSteps
|
||||
high: auditTrail.filter(entry => (entry.confidence || 0) >= 80).length,
|
||||
medium: auditTrail.filter(entry => (entry.confidence || 0) >= 60 && (entry.confidence || 0) < 80).length,
|
||||
low: auditTrail.filter(entry => (entry.confidence || 0) < 60).length
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
calculateAIResponseConfidence(
|
||||
response: string,
|
||||
expectedLength: { min: number; max: number },
|
||||
taskType: string
|
||||
): number {
|
||||
let confidence = 50; // Base confidence
|
||||
|
||||
// Response length indicates completeness
|
||||
if (response.length >= expectedLength.min) {
|
||||
confidence += 20;
|
||||
if (response.length <= expectedLength.max) {
|
||||
confidence += 10; // Optimal length
|
||||
}
|
||||
} else {
|
||||
confidence -= 20; // Too short
|
||||
}
|
||||
|
||||
// Response quality indicators
|
||||
if (response.includes('...') || response.endsWith('...')) {
|
||||
confidence -= 10; // Truncated response
|
||||
}
|
||||
|
||||
// Task-specific quality checks
|
||||
switch (taskType) {
|
||||
case 'scenario-analysis':
|
||||
case 'investigation-approach':
|
||||
case 'critical-considerations':
|
||||
// Should contain forensic methodology terms
|
||||
const forensicTerms = ['forensisch', 'beweis', 'evidence', 'analyse', 'untersuchung', 'methodik'];
|
||||
const termsFound = forensicTerms.filter(term =>
|
||||
response.toLowerCase().includes(term)
|
||||
).length;
|
||||
confidence += Math.min(15, termsFound * 3);
|
||||
break;
|
||||
|
||||
case 'tool-evaluation':
|
||||
// Should be structured and comprehensive
|
||||
if (response.includes('detailed_explanation') || response.includes('implementation_approach')) {
|
||||
confidence += 15;
|
||||
}
|
||||
if (response.includes('pros') && response.includes('limitations')) {
|
||||
confidence += 10;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'background-knowledge':
|
||||
// Should be valid JSON array
|
||||
try {
|
||||
const parsed = JSON.parse(response);
|
||||
if (Array.isArray(parsed) && parsed.length > 0) {
|
||||
confidence += 20;
|
||||
}
|
||||
} catch {
|
||||
confidence -= 20;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return Math.min(95, Math.max(25, confidence));
|
||||
}
|
||||
|
||||
validateAuditTrail(auditTrail: AuditEntry[]): {
|
||||
isValid: boolean;
|
||||
issues: string[];
|
||||
@@ -554,14 +704,6 @@ class AuditService {
|
||||
if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) {
|
||||
warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`);
|
||||
}
|
||||
|
||||
if (typeof entry.processingTimeMs !== 'number' || entry.processingTimeMs < 0) {
|
||||
warnings.push(`Entry ${index} has invalid processing time: ${entry.processingTimeMs}`);
|
||||
}
|
||||
|
||||
if (typeof entry.timestamp !== 'number' || entry.timestamp <= 0) {
|
||||
issues.push(`Entry ${index} has invalid timestamp: ${entry.timestamp}`);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user