mstoeck3/forensic-pathways
2
0
Fork 0
Code Issues 17 Pull Requests Projects Releases Activity

phase 2

Browse Source
This commit is contained in:
overcuriousity
2025-08-16 22:03:40 +02:00
parent 0c7c502b03
commit 77f09ed399
8 changed files with 1392 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
src/utils/auditService.ts
View File

@@ -1,4 +1,4 @@
// src/utils/auditService.ts - Refactored
// src/utils/auditService.ts - Enhanced for forensic-grade transparency
import 'dotenv/config';
function env(key: string, fallback: string | undefined = undefined): string | undefined {
@@ -19,7 +19,29 @@ export interface AuditEntry {
output: any;
confidence: number;
processingTimeMs: number;
metadata: Record<string, any>;
metadata: {
aiModel?: string;
aiParameters?: any;
promptTokens?: number;
completionTokens?: number;
toolsDataHash?: string;
embeddingsUsed?: boolean;
selectionMethod?: string;
microTaskType?: string;
confidenceFactors?: string[];
reasoning?: string;
aiPrompt?: string;
aiResponse?: string;
toolSelectionCriteria?: string;
availableToolsCount?: number;
selectedToolsCount?: number;
phaseId?: string;
toolsAdded?: string[];
completionReasoning?: string;
similarityScores?: Record<string, number>;
contextLength?: number;
[key: string]: any;
};
}
interface AuditConfig {
@@ -87,6 +109,135 @@ class AuditService {
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`);
}
// NEW: Specialized audit methods for forensic transparency
addAIDecision(
phase: string,
aiPrompt: string,
aiResponse: string,
confidence: number,
reasoning: string,
startTime: number,
metadata: Record<string, any> = {}
): void {
this.addEntry(
phase,
'ai-decision',
{ prompt: this.truncateForAudit(aiPrompt) },
{ response: this.truncateForAudit(aiResponse) },
confidence,
startTime,
{
...metadata,
reasoning,
aiPrompt: this.config.detailLevel === 'verbose' ? aiPrompt : this.truncateForAudit(aiPrompt),
aiResponse: this.config.detailLevel === 'verbose' ? aiResponse : this.truncateForAudit(aiResponse)
}
);
}
addToolSelection(
selectedTools: string[],
availableTools: string[],
selectionMethod: string,
confidence: number,
startTime: number,
metadata: Record<string, any> = {}
): void {
this.addEntry(
'tool-selection',
'selection-decision',
{ availableTools: availableTools.length > 10 ? availableTools.slice(0, 10) : availableTools },
{ selectedTools },
confidence,
startTime,
{
...metadata,
selectionMethod,
availableToolsCount: availableTools.length,
selectedToolsCount: selectedTools.length,
toolSelectionCriteria: `${selectionMethod} selection from ${availableTools.length} available tools`
}
);
}
addPhaseCompletion(
phaseId: string,
toolsAdded: string[],
completionReasoning: string,
startTime: number,
metadata: Record<string, any> = {}
): void {
this.addEntry(
'phase-completion',
'phase-enhancement',
{ phaseId, underrepresentedPhase: true },
{ toolsAdded },
75, // Default confidence for phase completion
startTime,
{
...metadata,
phaseId,
toolsAdded,
completionReasoning,
enhancementType: 'semantic-phase-completion'
}
);
}
addEmbeddingsSearch(
query: string,
similarResults: any[],
threshold: number,
startTime: number,
metadata: Record<string, any> = {}
): void {
const similarityScores = similarResults.reduce((acc, result) => {
acc[result.name] = result.similarity;
return acc;
}, {} as Record<string, number>);
this.addEntry(
'embeddings',
'similarity-search',
{ query: this.truncateForAudit(query), threshold },
{ resultsCount: similarResults.length, topResults: similarResults.slice(0, 5).map(r => r.name) },
similarResults.length > 0 ? 85 : 50,
startTime,
{
...metadata,
embeddingsUsed: true,
similarityScores,
searchThreshold: threshold,
totalMatches: similarResults.length
}
);
}
addConfidenceCalculation(
toolName: string,
confidenceBreakdown: any,
startTime: number,
metadata: Record<string, any> = {}
): void {
this.addEntry(
'confidence-scoring',
'tool-confidence',
{ toolName },
{ confidenceBreakdown },
confidenceBreakdown.overall || 50,
startTime,
{
...metadata,
confidenceFactors: [
...(confidenceBreakdown.strengthIndicators || []),
...(confidenceBreakdown.uncertaintyFactors || [])
],
semanticRelevance: confidenceBreakdown.semanticRelevance,
taskSuitability: confidenceBreakdown.taskSuitability
}
);
}
getCurrentAuditTrail(): AuditEntry[] {
return [...this.activeAuditTrail];
}
@@ -135,6 +286,12 @@ class AuditService {
return data;
}
private truncateForAudit(text: string, maxLength: number = 300): string {
if (typeof text !== 'string') return String(text);
if (text.length <= maxLength) return text;
return text.slice(0, maxLength) + '...[truncated for audit]';
}
isEnabled(): boolean {
return this.config.enabled;
}
@@ -143,7 +300,7 @@ class AuditService {
return { ...this.config };
}
// Statistics and analysis methods
// Statistics and analysis methods (enhanced)
getAuditStatistics(auditTrail: AuditEntry[]): {
totalTime: number;
avgConfidence: number;
@@ -151,6 +308,14 @@ class AuditService {
highConfidenceSteps: number;
lowConfidenceSteps: number;
phaseBreakdown: Record<string, { count: number; avgConfidence: number; totalTime: number }>;
aiDecisionCount: number;
embeddingsUsageCount: number;
toolSelectionCount: number;
qualityMetrics: {
avgProcessingTime: number;
confidenceDistribution: { high: number; medium: number; low: number };
aiTransparency: number;
};
} {
if (!auditTrail || auditTrail.length === 0) {
return {
@@ -159,7 +324,15 @@ class AuditService {
stepCount: 0,
highConfidenceSteps: 0,
lowConfidenceSteps: 0,
phaseBreakdown: {}
phaseBreakdown: {},
aiDecisionCount: 0,
embeddingsUsageCount: 0,
toolSelectionCount: 0,
qualityMetrics: {
avgProcessingTime: 0,
confidenceDistribution: { high: 0, medium: 0, low: 0 },
aiTransparency: 0
}
};
}
@@ -171,6 +344,12 @@ class AuditService {
const highConfidenceSteps = auditTrail.filter(entry => (entry.confidence || 0) >= 80).length;
const lowConfidenceSteps = auditTrail.filter(entry => (entry.confidence || 0) < 60).length;
const mediumConfidenceSteps = auditTrail.length - highConfidenceSteps - lowConfidenceSteps;
// Enhanced metrics
const aiDecisionCount = auditTrail.filter(entry => entry.action === 'ai-decision').length;
const embeddingsUsageCount = auditTrail.filter(entry => entry.metadata?.embeddingsUsed).length;
const toolSelectionCount = auditTrail.filter(entry => entry.action === 'selection-decision').length;
// Phase breakdown
const phaseBreakdown: Record<string, { count: number; avgConfidence: number; totalTime: number }> = {};
@@ -197,13 +376,29 @@ class AuditService {
}
});
const avgProcessingTime = auditTrail.length > 0 ? totalTime / auditTrail.length : 0;
const aiTransparency = auditTrail.length > 0 ?
(auditTrail.filter(entry => entry.metadata?.aiPrompt || entry.metadata?.reasoning).length / auditTrail.length) * 100 : 0;
return {
totalTime,
avgConfidence,
stepCount: auditTrail.length,
highConfidenceSteps,
lowConfidenceSteps,
phaseBreakdown
phaseBreakdown,
aiDecisionCount,
embeddingsUsageCount,
toolSelectionCount,
qualityMetrics: {
avgProcessingTime,
confidenceDistribution: {
high: highConfidenceSteps,
medium: mediumConfidenceSteps,
low: lowConfidenceSteps
},
aiTransparency: Math.round(aiTransparency)
}
};
}
@@ -238,6 +433,15 @@ class AuditService {
}
});
// Enhanced validation for audit quality
if (entry.action === 'ai-decision' && !entry.metadata?.aiPrompt && !entry.metadata?.reasoning) {
warnings.push(`Entry ${index}: AI decision lacks transparency (no prompt or reasoning)`);
}
if (entry.action === 'selection-decision' && !entry.metadata?.selectionMethod) {
warnings.push(`Entry ${index}: Tool selection lacks methodology info`);
}
// Data type validation
if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) {
warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`);