mstoeck3/forensic-pathways
2
0
Fork 0
Code Issues 17 Pull Requests Projects Releases Activity

audit trail details

Browse Source
This commit is contained in:
overcuriousity
2025-08-17 15:45:40 +02:00
parent dd26d45a21
commit 5c3c308225
3 changed files with 448 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

282
src/utils/auditService.ts
View File

@@ -1,4 +1,4 @@
// src/utils/auditService.ts - Enhanced for forensic-grade transparency
// src/utils/auditService.ts - Always detailed, no compression modes
import 'dotenv/config';
function env(key: string, fallback: string | undefined = undefined): string | undefined {
@@ -40,13 +40,15 @@ export interface AuditEntry {
completionReasoning?: string;
similarityScores?: Record<string, number>;
contextLength?: number;
decisionBasis?: 'ai-analysis' | 'semantic-search' | 'hybrid' | 'rule-based';
inputSummary?: string;
outputSummary?: string;
[key: string]: any;
};
}
interface AuditConfig {
enabled: boolean;
detailLevel: 'minimal' | 'standard' | 'verbose';
retentionHours: number;
maxEntries: number;
}
@@ -57,21 +59,16 @@ class AuditService {
constructor() {
this.config = this.loadConfig();
console.log('[AUDIT-SERVICE] Initialized:', {
enabled: this.config.enabled,
detailLevel: this.config.detailLevel
});
console.log('[AUDIT-SERVICE] Initialized with detailed logging enabled');
}
private loadConfig(): AuditConfig {
const enabledFlag = env('FORENSIC_AUDIT_ENABLED', 'false');
const detailLevel = env('FORENSIC_AUDIT_DETAIL_LEVEL', 'standard') as 'minimal' | 'standard' | 'verbose';
const enabled = env('FORENSIC_AUDIT_ENABLED', 'true') === 'true';
const retentionHours = parseInt(env('FORENSIC_AUDIT_RETENTION_HOURS', '72') || '72', 10);
const maxEntries = parseInt(env('FORENSIC_AUDIT_MAX_ENTRIES', '50') || '50', 10);
return {
enabled: enabledFlag === 'true',
detailLevel,
enabled,
retentionHours,
maxEntries
};
@@ -88,15 +85,24 @@ class AuditService {
): void {
if (!this.config.enabled) return;
// Always store full details with meaningful summaries
const enhancedMetadata = {
...metadata,
inputSummary: this.createMeaningfulSummary(input, 'input'),
outputSummary: this.createMeaningfulSummary(output, 'output'),
decisionBasis: metadata.decisionBasis || this.inferDecisionBasis(metadata),
reasoning: metadata.reasoning || this.extractReasoning(action, input, output, metadata)
};
const entry: AuditEntry = {
timestamp: Date.now(),
phase,
action,
input: this.compressData(input),
output: this.compressData(output),
input: input, // Store full input
output: output, // Store full output
confidence: Math.round(confidence),
processingTimeMs: Date.now() - startTime,
metadata
metadata: enhancedMetadata
};
this.activeAuditTrail.push(entry);
@@ -105,7 +111,7 @@ class AuditService {
this.activeAuditTrail.shift();
}
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`);
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms, basis: ${enhancedMetadata.decisionBasis}`);
}
addAIDecision(
@@ -120,15 +126,16 @@ class AuditService {
this.addEntry(
phase,
'ai-decision',
{ prompt: this.truncateForAudit(aiPrompt) },
{ response: this.truncateForAudit(aiResponse) },
{ prompt: aiPrompt },
{ response: aiResponse },
confidence,
startTime,
{
...metadata,
reasoning,
aiPrompt: this.config.detailLevel === 'verbose' ? aiPrompt : this.truncateForAudit(aiPrompt),
aiResponse: this.config.detailLevel === 'verbose' ? aiResponse : this.truncateForAudit(aiResponse)
aiPrompt: aiPrompt,
aiResponse: aiResponse,
decisionBasis: 'ai-analysis'
}
);
}
@@ -144,8 +151,15 @@ class AuditService {
this.addEntry(
'tool-selection',
'selection-decision',
{ availableTools: availableTools.length > 10 ? availableTools.slice(0, 10) : availableTools },
{ selectedTools },
{
availableTools: availableTools,
selectionMethod: selectionMethod,
candidateCount: availableTools.length
},
{
selectedTools: selectedTools,
selectionRatio: selectedTools.length / availableTools.length
},
confidence,
startTime,
{
@@ -153,7 +167,9 @@ class AuditService {
selectionMethod,
availableToolsCount: availableTools.length,
selectedToolsCount: selectedTools.length,
toolSelectionCriteria: `${selectionMethod} selection from ${availableTools.length} available tools`
toolSelectionCriteria: `${selectionMethod} selection from ${availableTools.length} available tools`,
decisionBasis: selectionMethod.includes('embeddings') ? 'semantic-search' : 'ai-analysis',
reasoning: `Selected ${selectedTools.length} tools out of ${availableTools.length} candidates using ${selectionMethod}`
}
);
}
@@ -169,11 +185,12 @@ class AuditService {
'phase-completion',
'phase-enhancement',
{
phaseId,
addedTools,
reasoning: reasoning.slice(0, 200)
phaseId: phaseId,
completionReason: 'underrepresented-phase',
semanticQuery: `forensic ${phaseId} tools methods`
},
{
addedTools: addedTools,
toolsAddedCount: addedTools.length,
enhancementMethod: 'semantic-search-with-ai-reasoning'
},
@@ -181,6 +198,8 @@ class AuditService {
startTime,
{
...metadata,
reasoning: reasoning,
decisionBasis: 'hybrid',
phaseCompletionMethod: 'sophisticated-ai-reasoning'
}
);
@@ -201,8 +220,17 @@ class AuditService {
this.addEntry(
'embeddings',
'similarity-search',
{ query: this.truncateForAudit(query), threshold },
{ resultsCount: similarResults.length, topResults: similarResults.slice(0, 5).map(r => r.name) },
{
query: query,
threshold: threshold,
searchType: 'semantic-embeddings'
},
{
resultsCount: similarResults.length,
topResults: similarResults.slice(0, 10),
averageSimilarity: similarResults.length > 0 ?
similarResults.reduce((sum, r) => sum + r.similarity, 0) / similarResults.length : 0
},
similarResults.length > 0 ? 85 : 50,
startTime,
{
@@ -210,7 +238,9 @@ class AuditService {
embeddingsUsed: true,
similarityScores,
searchThreshold: threshold,
totalMatches: similarResults.length
totalMatches: similarResults.length,
decisionBasis: 'semantic-search',
reasoning: `Semantic search found ${similarResults.length} items with similarity above ${threshold}`
}
);
}
@@ -225,26 +255,92 @@ class AuditService {
'confidence-scoring',
'tool-confidence',
{
toolName,
confidence: {
overall: confidence.overall,
semantic: confidence.semanticRelevance,
task: confidence.taskSuitability
}
toolName: toolName,
semanticSimilarity: confidence.semanticRelevance,
taskRelevance: confidence.taskSuitability
},
{
uncertaintyFactorsCount: confidence.uncertaintyFactors?.length || 0,
strengthIndicatorsCount: confidence.strengthIndicators?.length || 0
overallConfidence: confidence.overall,
strengthIndicators: confidence.strengthIndicators || [],
uncertaintyFactors: confidence.uncertaintyFactors || []
},
confidence.overall,
startTime,
{
...metadata,
confidenceCalculation: true
confidenceCalculation: true,
decisionBasis: 'ai-analysis',
reasoning: `Calculated confidence: ${confidence.overall}% (semantic: ${confidence.semanticRelevance}%, task: ${confidence.taskSuitability}%)`
}
);
}
private createMeaningfulSummary(data: any, type: 'input' | 'output'): string {
if (!data) return 'Empty';
if (typeof data === 'string') {
return data.length > 150 ? data.slice(0, 150) + '...' : data;
}
if (Array.isArray(data)) {
if (data.length === 0) return 'Empty array';
if (data.length <= 3) return data.join(', ');
return `${data.slice(0, 3).join(', ')} and ${data.length - 3} more items`;
}
if (typeof data === 'object') {
const keys = Object.keys(data);
if (keys.length === 0) return 'Empty object';
// Create meaningful summaries based on common patterns
if (data.prompt) return `AI Prompt: ${data.prompt.slice(0, 100)}...`;
if (data.response) return `AI Response: ${data.response.slice(0, 100)}...`;
if (data.selectedTools) return `Selected: ${data.selectedTools.join(', ')}`;
if (data.availableTools) return `${data.availableTools.length} tools available`;
if (data.query) return `Query: ${data.query}`;
return `Object with ${keys.length} properties: ${keys.slice(0, 3).join(', ')}${keys.length > 3 ? '...' : ''}`;
}
return String(data);
}
private inferDecisionBasis(metadata: Record<string, any>): string {
if (metadata.embeddingsUsed) return 'semantic-search';
if (metadata.aiPrompt || metadata.microTaskType) return 'ai-analysis';
if (metadata.selectionMethod?.includes('embeddings')) return 'semantic-search';
if (metadata.selectionMethod?.includes('full')) return 'ai-analysis';
return 'rule-based';
}
private extractReasoning(action: string, input: any, output: any, metadata: Record<string, any>): string {
if (metadata.reasoning) return metadata.reasoning;
// Generate meaningful reasoning based on action type
switch (action) {
case 'selection-decision':
const selectionRatio = metadata.selectedToolsCount / metadata.availableToolsCount;
return `Selected ${metadata.selectedToolsCount} tools (${Math.round(selectionRatio * 100)}%) using ${metadata.selectionMethod}`;
case 'similarity-search':
return `Found ${output?.resultsCount || 0} similar items above threshold ${input?.threshold || 0}`;
case 'ai-decision':
return metadata.microTaskType ?
`AI analysis for ${metadata.microTaskType}` :
'AI decision based on prompt analysis';
case 'tool-confidence':
return `Confidence scored based on semantic similarity and task relevance`;
case 'phase-enhancement':
return `Enhanced ${metadata.phaseId} phase with ${metadata.toolsAddedCount} additional tools`;
default:
return `${action} completed with ${Math.round(metadata.confidence || 0)}% confidence`;
}
}
getCurrentAuditTrail(): AuditEntry[] {
return [...this.activeAuditTrail];
}
@@ -263,42 +359,6 @@ class AuditService {
return finalTrail;
}
private compressData(data: any): any {
if (this.config.detailLevel === 'verbose') {
return data;
} else if (this.config.detailLevel === 'standard') {
return this.summarizeForStorage(data);
} else {
return this.minimalSummary(data);
}
}
private summarizeForStorage(data: any): any {
if (typeof data === 'string' && data.length > 500) {
return data.slice(0, 500) + '...[truncated]';
}
if (Array.isArray(data) && data.length > 10) {
return [...data.slice(0, 10), `...[${data.length - 10} more items]`];
}
return data;
}
private minimalSummary(data: any): any {
if (typeof data === 'string' && data.length > 100) {
return data.slice(0, 100) + '...[truncated]';
}
if (Array.isArray(data) && data.length > 3) {
return [...data.slice(0, 3), `...[${data.length - 3} more items]`];
}
return data;
}
private truncateForAudit(text: string, maxLength: number = 300): string {
if (typeof text !== 'string') return String(text);
if (text.length <= maxLength) return text;
return text.slice(0, maxLength) + '...[truncated for audit]';
}
isEnabled(): boolean {
return this.config.enabled;
}
@@ -320,7 +380,6 @@ class AuditService {
qualityMetrics: {
avgProcessingTime: number;
confidenceDistribution: { high: number; medium: number; low: number };
aiTransparency: number;
};
} {
if (!auditTrail || auditTrail.length === 0) {
@@ -336,8 +395,7 @@ class AuditService {
toolSelectionCount: 0,
qualityMetrics: {
avgProcessingTime: 0,
confidenceDistribution: { high: 0, medium: 0, low: 0 },
aiTransparency: 0
confidenceDistribution: { high: 0, medium: 0, low: 0 }
}
};
}
@@ -380,8 +438,6 @@ class AuditService {
});
const avgProcessingTime = auditTrail.length > 0 ? totalTime / auditTrail.length : 0;
const aiTransparency = auditTrail.length > 0 ?
(auditTrail.filter(entry => entry.metadata?.aiPrompt || entry.metadata?.reasoning).length / auditTrail.length) * 100 : 0;
return {
totalTime,
@@ -399,12 +455,72 @@ class AuditService {
high: highConfidenceSteps,
medium: mediumConfidenceSteps,
low: lowConfidenceSteps
},
aiTransparency: Math.round(aiTransparency)
}
}
};
}
calculateAIResponseConfidence(
response: string,
expectedLength: { min: number; max: number },
taskType: string
): number {
let confidence = 50; // Base confidence
// Response length indicates completeness
if (response.length >= expectedLength.min) {
confidence += 20;
if (response.length <= expectedLength.max) {
confidence += 10; // Optimal length
}
} else {
confidence -= 20; // Too short
}
// Response quality indicators
if (response.includes('...') || response.endsWith('...')) {
confidence -= 10; // Truncated response
}
// Task-specific quality checks
switch (taskType) {
case 'scenario-analysis':
case 'investigation-approach':
case 'critical-considerations':
// Should contain forensic methodology terms
const forensicTerms = ['forensisch', 'beweis', 'evidence', 'analyse', 'untersuchung', 'methodik'];
const termsFound = forensicTerms.filter(term =>
response.toLowerCase().includes(term)
).length;
confidence += Math.min(15, termsFound * 3);
break;
case 'tool-evaluation':
// Should be structured and comprehensive
if (response.includes('detailed_explanation') || response.includes('implementation_approach')) {
confidence += 15;
}
if (response.includes('pros') && response.includes('limitations')) {
confidence += 10;
}
break;
case 'background-knowledge':
// Should be valid JSON array
try {
const parsed = JSON.parse(response);
if (Array.isArray(parsed) && parsed.length > 0) {
confidence += 20;
}
} catch {
confidence -= 20;
}
break;
}
return Math.min(95, Math.max(25, confidence));
}
validateAuditTrail(auditTrail: AuditEntry[]): {
isValid: boolean;
issues: string[];
@@ -435,14 +551,6 @@ class AuditService {
}
});
if (entry.action === 'ai-decision' && !entry.metadata?.aiPrompt && !entry.metadata?.reasoning) {
warnings.push(`Entry ${index}: AI decision lacks transparency (no prompt or reasoning)`);
}
if (entry.action === 'selection-decision' && !entry.metadata?.selectionMethod) {
warnings.push(`Entry ${index}: Tool selection lacks methodology info`);
}
if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) {
warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`);
}