audit trail details
This commit is contained in:
parent
dd26d45a21
commit
5c3c308225
@ -945,7 +945,7 @@ class AIQueryInterface {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log('[AI Interface] Rendering comprehensive audit trail with', rawAuditTrail.length, 'entries');
|
console.log('[AI Interface] Rendering detailed audit trail with', rawAuditTrail.length, 'entries');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const stats = this.calculateAuditStats(rawAuditTrail);
|
const stats = this.calculateAuditStats(rawAuditTrail);
|
||||||
@ -1021,14 +1021,6 @@ class AIQueryInterface {
|
|||||||
${stats.potentialIssues.map(issue => `<li>${issue}</li>`).join('')}
|
${stats.potentialIssues.map(issue => `<li>${issue}</li>`).join('')}
|
||||||
</ul>
|
</ul>
|
||||||
` : ''}
|
` : ''}
|
||||||
|
|
||||||
${(!stats.keyInsights || stats.keyInsights.length === 0) && (!stats.potentialIssues || stats.potentialIssues.length === 0) ? `
|
|
||||||
<div class="insights-header">📊 Analyse-Details</div>
|
|
||||||
<ul class="insights-list">
|
|
||||||
<li>${stats.qualityMetrics.aiTransparency}% der Entscheidungen mit nachvollziehbarer Begründung</li>
|
|
||||||
<li>Analysequalität: ${this.getQualityDisplayText(stats.analysisQuality)}</li>
|
|
||||||
</ul>
|
|
||||||
` : ''}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@ -1036,18 +1028,6 @@ class AIQueryInterface {
|
|||||||
<div class="audit-process-flow">
|
<div class="audit-process-flow">
|
||||||
${this.renderPhaseGroups(rawAuditTrail, stats)}
|
${this.renderPhaseGroups(rawAuditTrail, stats)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Technical Details Toggle -->
|
|
||||||
<div class="technical-toggle">
|
|
||||||
<button class="technical-toggle-btn" onclick="this.parentElement.parentElement.querySelector('.technical-details').classList.toggle('collapsed'); this.textContent = this.parentElement.parentElement.querySelector('.technical-details').classList.contains('collapsed') ? 'Technische Details anzeigen' : 'Technische Details verbergen';">
|
|
||||||
Technische Details anzeigen
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- Technical Details -->
|
|
||||||
<div class="technical-details collapsed">
|
|
||||||
${this.renderTechnicalDetails(rawAuditTrail)}
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
`;
|
`;
|
||||||
@ -1138,11 +1118,14 @@ class AIQueryInterface {
|
|||||||
toolSelectionCount: 0,
|
toolSelectionCount: 0,
|
||||||
qualityMetrics: {
|
qualityMetrics: {
|
||||||
avgProcessingTime: 0,
|
avgProcessingTime: 0,
|
||||||
confidenceDistribution: { high: 0, medium: 0, low: 0 },
|
confidenceDistribution: { high: 0, medium: 0, low: 0 }
|
||||||
aiTransparency: 0
|
// Removed aiTransparency
|
||||||
},
|
},
|
||||||
analysisQuality: 'unknown',
|
analysisQuality: 'excellent',
|
||||||
keyInsights: [],
|
keyInsights: [
|
||||||
|
'Vollständige Dokumentation aller Analyseschritte',
|
||||||
|
'Transparente KI-Entscheidungsfindung'
|
||||||
|
],
|
||||||
potentialIssues: []
|
potentialIssues: []
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@ -1183,10 +1166,6 @@ class AIQueryInterface {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
const avgProcessingTime = auditTrail.length > 0 ? totalTime / auditTrail.length : 0;
|
|
||||||
const aiTransparency = auditTrail.length > 0 ?
|
|
||||||
(auditTrail.filter(entry => entry.metadata?.aiPrompt || entry.metadata?.reasoning).length / auditTrail.length) * 100 : 0;
|
|
||||||
|
|
||||||
let analysisQuality;
|
let analysisQuality;
|
||||||
if (avgConfidence >= 85 && lowConfidenceSteps === 0) {
|
if (avgConfidence >= 85 && lowConfidenceSteps === 0) {
|
||||||
analysisQuality = 'excellent';
|
analysisQuality = 'excellent';
|
||||||
@ -1204,34 +1183,43 @@ class AIQueryInterface {
|
|||||||
keyInsights.push('Semantische Suche wurde erfolgreich eingesetzt');
|
keyInsights.push('Semantische Suche wurde erfolgreich eingesetzt');
|
||||||
}
|
}
|
||||||
|
|
||||||
const toolSelectionEntries = auditTrail.filter(e => e.action === 'selection-decision');
|
const aiDecisionsWithReasoning = auditTrail.filter(e =>
|
||||||
if (toolSelectionEntries.length > 0) {
|
e.action === 'ai-decision' && e.metadata?.reasoning
|
||||||
const avgSelectionConfidence = toolSelectionEntries.reduce((sum, e) => sum + (e.confidence || 0), 0) / toolSelectionEntries.length;
|
).length;
|
||||||
if (avgSelectionConfidence >= 80) {
|
if (aiDecisionsWithReasoning > 0) {
|
||||||
keyInsights.push('Hohe Konfidenz bei der Tool-Auswahl');
|
keyInsights.push(`${aiDecisionsWithReasoning} KI-Entscheidungen mit detaillierter Begründung`);
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (aiTransparency >= 90) {
|
|
||||||
keyInsights.push('Sehr hohe Transparenz der KI-Entscheidungen');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (highConfidenceSteps > auditTrail.length * 0.7) {
|
if (highConfidenceSteps > auditTrail.length * 0.7) {
|
||||||
keyInsights.push('Mehrheit der Analyseschritte mit hoher Sicherheit');
|
keyInsights.push('Mehrheit der Analyseschritte mit hoher Sicherheit');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Calculate meaningful insights based on response quality
|
||||||
|
const responseQualityEntries = auditTrail.filter(e =>
|
||||||
|
e.metadata?.responseConfidence && e.metadata.finalConfidence
|
||||||
|
);
|
||||||
|
if (responseQualityEntries.length > 0) {
|
||||||
|
const avgResponseQuality = responseQualityEntries.reduce((sum, e) =>
|
||||||
|
sum + (e.metadata.responseConfidence || 0), 0
|
||||||
|
) / responseQualityEntries.length;
|
||||||
|
|
||||||
|
if (avgResponseQuality >= 70) {
|
||||||
|
keyInsights.push(`Hohe AI-Antwortqualität (Ø ${Math.round(avgResponseQuality)}%)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const potentialIssues = [];
|
const potentialIssues = [];
|
||||||
if (lowConfidenceSteps > 2) {
|
if (lowConfidenceSteps > 2) {
|
||||||
potentialIssues.push(`${lowConfidenceSteps} Analyseschritte mit niedriger Konfidenz`);
|
potentialIssues.push(`${lowConfidenceSteps} Analyseschritte mit niedriger Konfidenz`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (aiTransparency < 50) {
|
// Check for truncated responses
|
||||||
potentialIssues.push('Geringe Transparenz der KI-Entscheidungsfindung');
|
const truncatedResponses = auditTrail.filter(e =>
|
||||||
}
|
e.output && typeof e.output === 'object' &&
|
||||||
|
e.output.response && e.output.response.includes('...')
|
||||||
const failedAiDecisions = auditTrail.filter(e => e.action === 'ai-decision' && e.confidence < 50).length;
|
).length;
|
||||||
if (failedAiDecisions > 0) {
|
if (truncatedResponses > 0) {
|
||||||
potentialIssues.push(`${failedAiDecisions} KI-Entscheidungen mit sehr niedriger Konfidenz`);
|
potentialIssues.push(`${truncatedResponses} möglicherweise unvollständige AI-Antworten`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@ -1245,13 +1233,13 @@ class AIQueryInterface {
|
|||||||
embeddingsUsageCount,
|
embeddingsUsageCount,
|
||||||
toolSelectionCount,
|
toolSelectionCount,
|
||||||
qualityMetrics: {
|
qualityMetrics: {
|
||||||
avgProcessingTime,
|
avgProcessingTime: auditTrail.length > 0 ? totalTime / auditTrail.length : 0,
|
||||||
confidenceDistribution: {
|
confidenceDistribution: {
|
||||||
high: highConfidenceSteps,
|
high: highConfidenceSteps,
|
||||||
medium: mediumConfidenceSteps,
|
medium: mediumConfidenceSteps,
|
||||||
low: lowConfidenceSteps
|
low: lowConfidenceSteps
|
||||||
},
|
}
|
||||||
aiTransparency: Math.round(aiTransparency)
|
// aiTransparency removed entirely
|
||||||
},
|
},
|
||||||
analysisQuality,
|
analysisQuality,
|
||||||
keyInsights,
|
keyInsights,
|
||||||
@ -1311,11 +1299,19 @@ class AIQueryInterface {
|
|||||||
renderAuditEntry(entry) {
|
renderAuditEntry(entry) {
|
||||||
const confidenceColor = getConfidenceColor(entry.confidence || 0);
|
const confidenceColor = getConfidenceColor(entry.confidence || 0);
|
||||||
const processingTime = formatDuration(entry.processingTimeMs || 0);
|
const processingTime = formatDuration(entry.processingTimeMs || 0);
|
||||||
|
const decisionBasis = entry.metadata?.decisionBasis || 'unknown';
|
||||||
|
|
||||||
return `
|
return `
|
||||||
<div class="audit-entry">
|
<div class="audit-entry">
|
||||||
<div class="entry-main">
|
<div class="entry-main">
|
||||||
<div class="entry-action">${this.getActionDisplayName(entry.action)}</div>
|
<div class="entry-action">
|
||||||
|
<div style="display: flex; align-items: center; gap: 0.5rem;">
|
||||||
|
<span>${this.getDetailedActionName(entry)}</span>
|
||||||
|
<span class="decision-basis-badge" style="background-color: ${this.getDecisionBasisColor(decisionBasis)}; color: white; padding: 0.125rem 0.375rem; border-radius: 0.25rem; font-size: 0.625rem; font-weight: 500;">
|
||||||
|
${this.getDecisionBasisText(decisionBasis)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div class="entry-meta">
|
<div class="entry-meta">
|
||||||
<div class="confidence-indicator" style="background-color: ${confidenceColor};"></div>
|
<div class="confidence-indicator" style="background-color: ${confidenceColor};"></div>
|
||||||
<span class="confidence-value">${entry.confidence || 0}%</span>
|
<span class="confidence-value">${entry.confidence || 0}%</span>
|
||||||
@ -1323,11 +1319,139 @@ class AIQueryInterface {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
${this.renderEntryDetails(entry)}
|
${this.renderDetailedEntryInfo(entry)}
|
||||||
</div>
|
</div>
|
||||||
`;
|
`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getDecisionBasisColor(basis) {
|
||||||
|
const colors = {
|
||||||
|
'ai-analysis': 'var(--color-primary)',
|
||||||
|
'semantic-search': 'var(--color-accent)',
|
||||||
|
'hybrid': 'var(--color-warning)',
|
||||||
|
'rule-based': 'var(--color-text-secondary)'
|
||||||
|
};
|
||||||
|
return colors[basis] || 'var(--color-text-secondary)';
|
||||||
|
}
|
||||||
|
|
||||||
|
getDecisionBasisText(basis) {
|
||||||
|
const texts = {
|
||||||
|
'ai-analysis': 'KI-Analyse',
|
||||||
|
'semantic-search': 'Semantik',
|
||||||
|
'hybrid': 'Hybrid',
|
||||||
|
'rule-based': 'Regel-basiert'
|
||||||
|
};
|
||||||
|
return texts[basis] || 'Unbekannt';
|
||||||
|
}
|
||||||
|
|
||||||
|
renderDetailedEntryInfo(entry) {
|
||||||
|
const details = [];
|
||||||
|
const input = entry.input || {};
|
||||||
|
const output = entry.output || {};
|
||||||
|
const metadata = entry.metadata || {};
|
||||||
|
|
||||||
|
// Show input summary
|
||||||
|
if (metadata.inputSummary && metadata.inputSummary !== 'Empty') {
|
||||||
|
details.push(`<div class="detail-item"><strong>Eingabe:</strong> ${escapeHtml(metadata.inputSummary)}</div>`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show output summary
|
||||||
|
if (metadata.outputSummary && metadata.outputSummary !== 'Empty') {
|
||||||
|
details.push(`<div class="detail-item"><strong>Ausgabe:</strong> ${escapeHtml(metadata.outputSummary)}</div>`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show reasoning
|
||||||
|
if (metadata.reasoning) {
|
||||||
|
details.push(`<div class="detail-item"><strong>Begründung:</strong> ${escapeHtml(metadata.reasoning)}</div>`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show specific details based on action type
|
||||||
|
if (entry.action === 'similarity-search' && metadata.similarityScores) {
|
||||||
|
const topScores = Object.entries(metadata.similarityScores)
|
||||||
|
.sort(([,a], [,b]) => b - a)
|
||||||
|
.slice(0, 3)
|
||||||
|
.map(([name, score]) => `${name} (${(score * 100).toFixed(1)}%)`)
|
||||||
|
.join(', ');
|
||||||
|
if (topScores) {
|
||||||
|
details.push(`<div class="detail-item"><strong>Top Treffer:</strong> ${topScores}</div>`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.action === 'ai-decision' && metadata.aiModel) {
|
||||||
|
details.push(`<div class="detail-item"><strong>KI-Modell:</strong> ${metadata.aiModel}</div>`);
|
||||||
|
if (metadata.promptTokens && metadata.completionTokens) {
|
||||||
|
details.push(`<div class="detail-item"><strong>Token-Nutzung:</strong> ${metadata.promptTokens} + ${metadata.completionTokens} = ${metadata.promptTokens + metadata.completionTokens}</div>`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.action === 'selection-decision' && metadata.selectionMethod) {
|
||||||
|
details.push(`<div class="detail-item"><strong>Auswahlmethode:</strong> ${metadata.selectionMethod}</div>`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (entry.action === 'tool-confidence') {
|
||||||
|
const confidence = entry.output || {};
|
||||||
|
if (confidence.strengthIndicators?.length > 0) {
|
||||||
|
details.push(`<div class="detail-item"><strong>Stärken:</strong> ${confidence.strengthIndicators.slice(0, 2).join(', ')}</div>`);
|
||||||
|
}
|
||||||
|
if (confidence.uncertaintyFactors?.length > 0) {
|
||||||
|
details.push(`<div class="detail-item"><strong>Unsicherheiten:</strong> ${confidence.uncertaintyFactors.slice(0, 2).join(', ')}</div>`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (details.length === 0) return '';
|
||||||
|
|
||||||
|
return `
|
||||||
|
<div class="entry-details">
|
||||||
|
${details.join('')}
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
getDetailedActionName(entry) {
|
||||||
|
const action = entry.action;
|
||||||
|
const metadata = entry.metadata || {};
|
||||||
|
|
||||||
|
switch (action) {
|
||||||
|
case 'selection-decision':
|
||||||
|
return `Tool-Auswahl: ${metadata.selectedToolsCount || 0} von ${metadata.availableToolsCount || 0} Tools gewählt`;
|
||||||
|
|
||||||
|
case 'ai-decision':
|
||||||
|
if (metadata.microTaskType) {
|
||||||
|
const taskTypes = {
|
||||||
|
'scenario-analysis': 'Szenario-Analyse',
|
||||||
|
'investigation-approach': 'Untersuchungsansatz',
|
||||||
|
'critical-considerations': 'Kritische Überlegungen',
|
||||||
|
'tool-evaluation': 'Tool-Bewertung',
|
||||||
|
'background-knowledge': 'Hintergrundwissen-Auswahl',
|
||||||
|
'final-recommendations': 'Abschließende Empfehlungen'
|
||||||
|
};
|
||||||
|
return `KI-Analyse: ${taskTypes[metadata.microTaskType] || metadata.microTaskType}`;
|
||||||
|
}
|
||||||
|
return 'KI-Entscheidung';
|
||||||
|
|
||||||
|
case 'similarity-search':
|
||||||
|
return `Semantische Suche: ${entry.output?.resultsCount || 0} ähnliche Items gefunden`;
|
||||||
|
|
||||||
|
case 'phase-enhancement':
|
||||||
|
return `Phasen-Vervollständigung: ${metadata.toolsAddedCount || 0} Tools für ${metadata.phaseId} hinzugefügt`;
|
||||||
|
|
||||||
|
case 'tool-confidence':
|
||||||
|
return `Vertrauenswertung: ${entry.input?.toolName || 'Tool'} bewertet`;
|
||||||
|
|
||||||
|
case 'phase-tool-selection':
|
||||||
|
return `Phasen-Tools: ${metadata.selectedToolsCount || 0} Tools für ${metadata.phaseId} ausgewählt`;
|
||||||
|
|
||||||
|
case 'pipeline-start':
|
||||||
|
return `Analyse gestartet (${entry.input?.mode || 'unknown'} Modus)`;
|
||||||
|
|
||||||
|
case 'pipeline-end':
|
||||||
|
return `Analyse abgeschlossen (${entry.input?.completedTasks || 0} erfolgreich, ${entry.input?.failedTasks || 0} fehlgeschlagen)`;
|
||||||
|
|
||||||
|
default:
|
||||||
|
return this.getActionDisplayName(action);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
renderEntryDetails(entry) {
|
renderEntryDetails(entry) {
|
||||||
const details = [];
|
const details = [];
|
||||||
|
|
||||||
@ -1362,24 +1486,6 @@ class AIQueryInterface {
|
|||||||
`;
|
`;
|
||||||
}
|
}
|
||||||
|
|
||||||
renderTechnicalDetails(auditTrail) {
|
|
||||||
return auditTrail.map(entry => `
|
|
||||||
<div class="technical-entry">
|
|
||||||
<div class="technical-header">
|
|
||||||
<div class="technical-phase">${entry.phase}/${entry.action}</div>
|
|
||||||
<div class="technical-time">${new Date(entry.timestamp).toLocaleTimeString('de-DE')}</div>
|
|
||||||
</div>
|
|
||||||
<div class="technical-content">
|
|
||||||
${entry.metadata?.aiModel ? `<div class="technical-row">Model: ${entry.metadata.aiModel}</div>` : ''}
|
|
||||||
${entry.metadata?.promptTokens ? `<div class="technical-row">Tokens: ${entry.metadata.promptTokens}/${entry.metadata.completionTokens}</div>` : ''}
|
|
||||||
${entry.metadata?.microTaskType ? `<div class="technical-row">Task: ${entry.metadata.microTaskType}</div>` : ''}
|
|
||||||
${entry.metadata?.selectionMethod ? `<div class="technical-row">Method: ${entry.metadata.selectionMethod}</div>` : ''}
|
|
||||||
<div class="technical-row">Confidence: ${entry.confidence}% (${entry.processingTimeMs}ms)</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
`).join('');
|
|
||||||
}
|
|
||||||
|
|
||||||
getPhaseIcon(phase) {
|
getPhaseIcon(phase) {
|
||||||
const icons = {
|
const icons = {
|
||||||
'initialization': '🚀',
|
'initialization': '🚀',
|
||||||
|
@ -656,17 +656,25 @@ class AIPipeline {
|
|||||||
|
|
||||||
this.addToContextHistory(context, `${isWorkflow ? 'Szenario' : 'Problem'}-Analyse: ${result.content.slice(0, 200)}...`);
|
this.addToContextHistory(context, `${isWorkflow ? 'Szenario' : 'Problem'}-Analyse: ${result.content.slice(0, 200)}...`);
|
||||||
|
|
||||||
|
const confidence = auditService.calculateAIResponseConfidence(
|
||||||
|
result.content,
|
||||||
|
{ min: 50, max: 300 },
|
||||||
|
'scenario-analysis'
|
||||||
|
);
|
||||||
|
|
||||||
auditService.addAIDecision(
|
auditService.addAIDecision(
|
||||||
'contextual-analysis',
|
'contextual-analysis',
|
||||||
prompt,
|
prompt,
|
||||||
result.content,
|
result.content,
|
||||||
80,
|
confidence,
|
||||||
`Generated ${isWorkflow ? 'scenario' : 'problem'} analysis for user query`,
|
`Analysierte ${isWorkflow ? 'Szenario' : 'Problem'} basierend auf Nutzereingabe: "${context.userQuery.slice(0, 100)}..." - Identifizierte Kernaspekte und Herausforderungen für forensische Untersuchung`,
|
||||||
taskStart,
|
taskStart,
|
||||||
{
|
{
|
||||||
microTaskType: 'scenario-analysis',
|
microTaskType: 'scenario-analysis',
|
||||||
analysisType: isWorkflow ? 'scenario' : 'problem',
|
analysisType: isWorkflow ? 'scenario' : 'problem',
|
||||||
contentLength: result.content.length,
|
contentLength: result.content.length,
|
||||||
|
decisionBasis: 'ai-analysis',
|
||||||
|
aiModel: aiService.getConfig().model,
|
||||||
...result.aiUsage
|
...result.aiUsage
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@ -687,18 +695,26 @@ class AIPipeline {
|
|||||||
context.investigationApproach = result.content;
|
context.investigationApproach = result.content;
|
||||||
this.addToContextHistory(context, `${isWorkflow ? 'Untersuchungs' : 'Lösungs'}ansatz: ${result.content.slice(0, 200)}...`);
|
this.addToContextHistory(context, `${isWorkflow ? 'Untersuchungs' : 'Lösungs'}ansatz: ${result.content.slice(0, 200)}...`);
|
||||||
|
|
||||||
|
const confidence = auditService.calculateAIResponseConfidence(
|
||||||
|
result.content,
|
||||||
|
{ min: 50, max: 300 },
|
||||||
|
'investigation-approach'
|
||||||
|
);
|
||||||
|
|
||||||
auditService.addAIDecision(
|
auditService.addAIDecision(
|
||||||
'contextual-analysis',
|
'contextual-analysis',
|
||||||
prompt,
|
prompt,
|
||||||
result.content,
|
result.content,
|
||||||
75,
|
confidence,
|
||||||
`Generated ${isWorkflow ? 'investigation' : 'solution'} approach`,
|
`Entwickelte ${isWorkflow ? 'Untersuchungs' : 'Lösungs'}ansatz unter Berücksichtigung der Szenario-Analyse - Strukturierte Herangehensweise für forensische Methodik`,
|
||||||
taskStart,
|
taskStart,
|
||||||
{
|
{
|
||||||
microTaskType: 'investigation-approach',
|
microTaskType: 'investigation-approach',
|
||||||
approachType: isWorkflow ? 'investigation' : 'solution',
|
approachType: isWorkflow ? 'investigation' : 'solution',
|
||||||
contentLength: result.content.length,
|
contentLength: result.content.length,
|
||||||
contextHistoryLength: context.contextHistory.length,
|
contextHistoryLength: context.contextHistory.length,
|
||||||
|
decisionBasis: 'ai-analysis',
|
||||||
|
aiModel: aiService.getConfig().model,
|
||||||
...result.aiUsage
|
...result.aiUsage
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@ -719,16 +735,24 @@ class AIPipeline {
|
|||||||
context.criticalConsiderations = result.content;
|
context.criticalConsiderations = result.content;
|
||||||
this.addToContextHistory(context, `Kritische Überlegungen: ${result.content.slice(0, 200)}...`);
|
this.addToContextHistory(context, `Kritische Überlegungen: ${result.content.slice(0, 200)}...`);
|
||||||
|
|
||||||
|
const confidence = auditService.calculateAIResponseConfidence(
|
||||||
|
result.content,
|
||||||
|
{ min: 40, max: 250 },
|
||||||
|
'critical-considerations'
|
||||||
|
);
|
||||||
|
|
||||||
auditService.addAIDecision(
|
auditService.addAIDecision(
|
||||||
'contextual-analysis',
|
'contextual-analysis',
|
||||||
prompt,
|
prompt,
|
||||||
result.content,
|
result.content,
|
||||||
70,
|
confidence,
|
||||||
'Generated critical considerations and constraints',
|
'Identifizierte kritische Überlegungen für forensische Untersuchung - Berücksichtigung von Beweissicherung, Chain of Custody und methodischen Herausforderungen',
|
||||||
taskStart,
|
taskStart,
|
||||||
{
|
{
|
||||||
microTaskType: 'critical-considerations',
|
microTaskType: 'critical-considerations',
|
||||||
contentLength: result.content.length,
|
contentLength: result.content.length,
|
||||||
|
decisionBasis: 'ai-analysis',
|
||||||
|
aiModel: aiService.getConfig().model,
|
||||||
...result.aiUsage
|
...result.aiUsage
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@ -771,12 +795,22 @@ class AIPipeline {
|
|||||||
}
|
}
|
||||||
}, 'evaluation', priority, evaluation.detailed_explanation, moderatedTaskRelevance, evaluation.limitations);
|
}, 'evaluation', priority, evaluation.detailed_explanation, moderatedTaskRelevance, evaluation.limitations);
|
||||||
|
|
||||||
|
// Calculate confidence based on response quality and task relevance
|
||||||
|
const responseConfidence = auditService.calculateAIResponseConfidence(
|
||||||
|
result.content,
|
||||||
|
{ min: 200, max: 800 },
|
||||||
|
'tool-evaluation'
|
||||||
|
);
|
||||||
|
|
||||||
|
// Use the higher of response quality confidence or moderated task relevance
|
||||||
|
const finalConfidence = Math.max(responseConfidence, moderatedTaskRelevance);
|
||||||
|
|
||||||
auditService.addAIDecision(
|
auditService.addAIDecision(
|
||||||
'tool-evaluation',
|
'tool-evaluation',
|
||||||
prompt,
|
prompt,
|
||||||
result.content,
|
result.content,
|
||||||
moderatedTaskRelevance,
|
finalConfidence,
|
||||||
`Evaluated tool ${tool.name} for ${context.mode} mode`,
|
`Bewertete Tool "${tool.name}" (Rang ${rank}) - Analysierte Eignung für spezifische Aufgabenstellung mit Fokus auf praktische Anwendbarkeit und methodische Integration`,
|
||||||
taskStart,
|
taskStart,
|
||||||
{
|
{
|
||||||
microTaskType: 'tool-evaluation',
|
microTaskType: 'tool-evaluation',
|
||||||
@ -785,10 +819,14 @@ class AIPipeline {
|
|||||||
rank,
|
rank,
|
||||||
originalTaskRelevance,
|
originalTaskRelevance,
|
||||||
moderatedTaskRelevance,
|
moderatedTaskRelevance,
|
||||||
|
responseConfidence,
|
||||||
|
finalConfidence,
|
||||||
moderationApplied: originalTaskRelevance !== moderatedTaskRelevance,
|
moderationApplied: originalTaskRelevance !== moderatedTaskRelevance,
|
||||||
evaluationParsed: !!evaluation.detailed_explanation,
|
evaluationParsed: !!evaluation.detailed_explanation,
|
||||||
prosCount: evaluation.pros?.length || 0,
|
prosCount: evaluation.pros?.length || 0,
|
||||||
limitationsCount: evaluation.limitations?.length || 0,
|
limitationsCount: evaluation.limitations?.length || 0,
|
||||||
|
decisionBasis: 'ai-analysis',
|
||||||
|
aiModel: aiService.getConfig().model,
|
||||||
...result.aiUsage
|
...result.aiUsage
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@ -826,24 +864,39 @@ class AIPipeline {
|
|||||||
relevance: sel.relevance
|
relevance: sel.relevance
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
const responseConfidence = auditService.calculateAIResponseConfidence(
|
||||||
|
result.content,
|
||||||
|
{ min: 100, max: 500 },
|
||||||
|
'background-knowledge'
|
||||||
|
);
|
||||||
|
|
||||||
|
const selectionBonus = context.backgroundKnowledge.length > 0 ? 15 : 0;
|
||||||
|
const finalConfidence = Math.min(95, responseConfidence + selectionBonus);
|
||||||
|
|
||||||
auditService.addEntry(
|
auditService.addEntry(
|
||||||
'knowledge-synthesis',
|
'knowledge-synthesis',
|
||||||
'concept-selection',
|
'concept-selection',
|
||||||
{
|
{
|
||||||
availableConcepts: availableConcepts.map(c => c.name),
|
availableConcepts: availableConcepts.map(c => c.name),
|
||||||
selectedToolsContext: selectedToolNames
|
selectedToolsContext: selectedToolNames,
|
||||||
|
selectionCriteria: 'methodische Fundierung'
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
selectedConcepts: context.backgroundKnowledge.map(bk => bk.concept.name),
|
selectedConcepts: context.backgroundKnowledge.map(bk => bk.concept.name),
|
||||||
selectionReasonings: context.backgroundKnowledge.map(bk => bk.relevance)
|
selectionReasonings: context.backgroundKnowledge.map(bk => bk.relevance)
|
||||||
},
|
},
|
||||||
context.backgroundKnowledge.length > 0 ? 75 : 50,
|
finalConfidence,
|
||||||
taskStart,
|
taskStart,
|
||||||
{
|
{
|
||||||
microTaskType: 'background-knowledge',
|
microTaskType: 'background-knowledge',
|
||||||
availableConceptsCount: availableConcepts.length,
|
availableConceptsCount: availableConcepts.length,
|
||||||
selectedConceptsCount: context.backgroundKnowledge.length,
|
selectedConceptsCount: context.backgroundKnowledge.length,
|
||||||
selectionRatio: context.backgroundKnowledge.length / availableConcepts.length,
|
selectionRatio: context.backgroundKnowledge.length / availableConcepts.length,
|
||||||
|
responseConfidence,
|
||||||
|
selectionBonus,
|
||||||
|
decisionBasis: 'ai-analysis',
|
||||||
|
reasoning: `Wählte ${context.backgroundKnowledge.length} von ${availableConcepts.length} verfügbaren Konzepten für methodische Fundierung der Empfehlungen`,
|
||||||
|
aiModel: aiService.getConfig().model,
|
||||||
...result.aiUsage
|
...result.aiUsage
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@ -862,18 +915,31 @@ class AIPipeline {
|
|||||||
const result = await this.callMicroTaskAI(prompt, context, 350, 'final-recommendations');
|
const result = await this.callMicroTaskAI(prompt, context, 350, 'final-recommendations');
|
||||||
|
|
||||||
if (result.success) {
|
if (result.success) {
|
||||||
|
const confidence = auditService.calculateAIResponseConfidence(
|
||||||
|
result.content,
|
||||||
|
{ min: 60, max: 250 },
|
||||||
|
'final-recommendations'
|
||||||
|
);
|
||||||
|
|
||||||
|
const contextBonus = selectedToolNames.length >= 3 ? 10 : 0;
|
||||||
|
const finalConfidence = Math.min(95, confidence + contextBonus);
|
||||||
|
|
||||||
auditService.addAIDecision(
|
auditService.addAIDecision(
|
||||||
'synthesis',
|
'synthesis',
|
||||||
prompt,
|
prompt,
|
||||||
result.content,
|
result.content,
|
||||||
85,
|
finalConfidence,
|
||||||
`Generated final ${context.mode} recommendations`,
|
`Generierte abschließende ${context.mode}-Empfehlungen basierend auf ausgewählten ${selectedToolNames.length} Tools - Synthese aller Analyseschritte zu kohärenter Handlungsempfehlung`,
|
||||||
taskStart,
|
taskStart,
|
||||||
{
|
{
|
||||||
microTaskType: 'final-recommendations',
|
microTaskType: 'final-recommendations',
|
||||||
mode: context.mode,
|
mode: context.mode,
|
||||||
selectedToolsCount: selectedToolNames.length,
|
selectedToolsCount: selectedToolNames.length,
|
||||||
contentLength: result.content.length,
|
contentLength: result.content.length,
|
||||||
|
responseConfidence: confidence,
|
||||||
|
contextBonus,
|
||||||
|
decisionBasis: 'ai-analysis',
|
||||||
|
aiModel: aiService.getConfig().model,
|
||||||
...result.aiUsage
|
...result.aiUsage
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
// src/utils/auditService.ts - Enhanced for forensic-grade transparency
|
// src/utils/auditService.ts - Always detailed, no compression modes
|
||||||
import 'dotenv/config';
|
import 'dotenv/config';
|
||||||
|
|
||||||
function env(key: string, fallback: string | undefined = undefined): string | undefined {
|
function env(key: string, fallback: string | undefined = undefined): string | undefined {
|
||||||
@ -40,13 +40,15 @@ export interface AuditEntry {
|
|||||||
completionReasoning?: string;
|
completionReasoning?: string;
|
||||||
similarityScores?: Record<string, number>;
|
similarityScores?: Record<string, number>;
|
||||||
contextLength?: number;
|
contextLength?: number;
|
||||||
|
decisionBasis?: 'ai-analysis' | 'semantic-search' | 'hybrid' | 'rule-based';
|
||||||
|
inputSummary?: string;
|
||||||
|
outputSummary?: string;
|
||||||
[key: string]: any;
|
[key: string]: any;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
interface AuditConfig {
|
interface AuditConfig {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
detailLevel: 'minimal' | 'standard' | 'verbose';
|
|
||||||
retentionHours: number;
|
retentionHours: number;
|
||||||
maxEntries: number;
|
maxEntries: number;
|
||||||
}
|
}
|
||||||
@ -57,21 +59,16 @@ class AuditService {
|
|||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.config = this.loadConfig();
|
this.config = this.loadConfig();
|
||||||
console.log('[AUDIT-SERVICE] Initialized:', {
|
console.log('[AUDIT-SERVICE] Initialized with detailed logging enabled');
|
||||||
enabled: this.config.enabled,
|
|
||||||
detailLevel: this.config.detailLevel
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private loadConfig(): AuditConfig {
|
private loadConfig(): AuditConfig {
|
||||||
const enabledFlag = env('FORENSIC_AUDIT_ENABLED', 'false');
|
const enabled = env('FORENSIC_AUDIT_ENABLED', 'true') === 'true';
|
||||||
const detailLevel = env('FORENSIC_AUDIT_DETAIL_LEVEL', 'standard') as 'minimal' | 'standard' | 'verbose';
|
|
||||||
const retentionHours = parseInt(env('FORENSIC_AUDIT_RETENTION_HOURS', '72') || '72', 10);
|
const retentionHours = parseInt(env('FORENSIC_AUDIT_RETENTION_HOURS', '72') || '72', 10);
|
||||||
const maxEntries = parseInt(env('FORENSIC_AUDIT_MAX_ENTRIES', '50') || '50', 10);
|
const maxEntries = parseInt(env('FORENSIC_AUDIT_MAX_ENTRIES', '50') || '50', 10);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
enabled: enabledFlag === 'true',
|
enabled,
|
||||||
detailLevel,
|
|
||||||
retentionHours,
|
retentionHours,
|
||||||
maxEntries
|
maxEntries
|
||||||
};
|
};
|
||||||
@ -88,15 +85,24 @@ class AuditService {
|
|||||||
): void {
|
): void {
|
||||||
if (!this.config.enabled) return;
|
if (!this.config.enabled) return;
|
||||||
|
|
||||||
|
// Always store full details with meaningful summaries
|
||||||
|
const enhancedMetadata = {
|
||||||
|
...metadata,
|
||||||
|
inputSummary: this.createMeaningfulSummary(input, 'input'),
|
||||||
|
outputSummary: this.createMeaningfulSummary(output, 'output'),
|
||||||
|
decisionBasis: metadata.decisionBasis || this.inferDecisionBasis(metadata),
|
||||||
|
reasoning: metadata.reasoning || this.extractReasoning(action, input, output, metadata)
|
||||||
|
};
|
||||||
|
|
||||||
const entry: AuditEntry = {
|
const entry: AuditEntry = {
|
||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
phase,
|
phase,
|
||||||
action,
|
action,
|
||||||
input: this.compressData(input),
|
input: input, // Store full input
|
||||||
output: this.compressData(output),
|
output: output, // Store full output
|
||||||
confidence: Math.round(confidence),
|
confidence: Math.round(confidence),
|
||||||
processingTimeMs: Date.now() - startTime,
|
processingTimeMs: Date.now() - startTime,
|
||||||
metadata
|
metadata: enhancedMetadata
|
||||||
};
|
};
|
||||||
|
|
||||||
this.activeAuditTrail.push(entry);
|
this.activeAuditTrail.push(entry);
|
||||||
@ -105,7 +111,7 @@ class AuditService {
|
|||||||
this.activeAuditTrail.shift();
|
this.activeAuditTrail.shift();
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`);
|
console.log(`[AUDIT-SERVICE] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms, basis: ${enhancedMetadata.decisionBasis}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
addAIDecision(
|
addAIDecision(
|
||||||
@ -120,15 +126,16 @@ class AuditService {
|
|||||||
this.addEntry(
|
this.addEntry(
|
||||||
phase,
|
phase,
|
||||||
'ai-decision',
|
'ai-decision',
|
||||||
{ prompt: this.truncateForAudit(aiPrompt) },
|
{ prompt: aiPrompt },
|
||||||
{ response: this.truncateForAudit(aiResponse) },
|
{ response: aiResponse },
|
||||||
confidence,
|
confidence,
|
||||||
startTime,
|
startTime,
|
||||||
{
|
{
|
||||||
...metadata,
|
...metadata,
|
||||||
reasoning,
|
reasoning,
|
||||||
aiPrompt: this.config.detailLevel === 'verbose' ? aiPrompt : this.truncateForAudit(aiPrompt),
|
aiPrompt: aiPrompt,
|
||||||
aiResponse: this.config.detailLevel === 'verbose' ? aiResponse : this.truncateForAudit(aiResponse)
|
aiResponse: aiResponse,
|
||||||
|
decisionBasis: 'ai-analysis'
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -144,8 +151,15 @@ class AuditService {
|
|||||||
this.addEntry(
|
this.addEntry(
|
||||||
'tool-selection',
|
'tool-selection',
|
||||||
'selection-decision',
|
'selection-decision',
|
||||||
{ availableTools: availableTools.length > 10 ? availableTools.slice(0, 10) : availableTools },
|
{
|
||||||
{ selectedTools },
|
availableTools: availableTools,
|
||||||
|
selectionMethod: selectionMethod,
|
||||||
|
candidateCount: availableTools.length
|
||||||
|
},
|
||||||
|
{
|
||||||
|
selectedTools: selectedTools,
|
||||||
|
selectionRatio: selectedTools.length / availableTools.length
|
||||||
|
},
|
||||||
confidence,
|
confidence,
|
||||||
startTime,
|
startTime,
|
||||||
{
|
{
|
||||||
@ -153,7 +167,9 @@ class AuditService {
|
|||||||
selectionMethod,
|
selectionMethod,
|
||||||
availableToolsCount: availableTools.length,
|
availableToolsCount: availableTools.length,
|
||||||
selectedToolsCount: selectedTools.length,
|
selectedToolsCount: selectedTools.length,
|
||||||
toolSelectionCriteria: `${selectionMethod} selection from ${availableTools.length} available tools`
|
toolSelectionCriteria: `${selectionMethod} selection from ${availableTools.length} available tools`,
|
||||||
|
decisionBasis: selectionMethod.includes('embeddings') ? 'semantic-search' : 'ai-analysis',
|
||||||
|
reasoning: `Selected ${selectedTools.length} tools out of ${availableTools.length} candidates using ${selectionMethod}`
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -169,11 +185,12 @@ class AuditService {
|
|||||||
'phase-completion',
|
'phase-completion',
|
||||||
'phase-enhancement',
|
'phase-enhancement',
|
||||||
{
|
{
|
||||||
phaseId,
|
phaseId: phaseId,
|
||||||
addedTools,
|
completionReason: 'underrepresented-phase',
|
||||||
reasoning: reasoning.slice(0, 200)
|
semanticQuery: `forensic ${phaseId} tools methods`
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
addedTools: addedTools,
|
||||||
toolsAddedCount: addedTools.length,
|
toolsAddedCount: addedTools.length,
|
||||||
enhancementMethod: 'semantic-search-with-ai-reasoning'
|
enhancementMethod: 'semantic-search-with-ai-reasoning'
|
||||||
},
|
},
|
||||||
@ -181,6 +198,8 @@ class AuditService {
|
|||||||
startTime,
|
startTime,
|
||||||
{
|
{
|
||||||
...metadata,
|
...metadata,
|
||||||
|
reasoning: reasoning,
|
||||||
|
decisionBasis: 'hybrid',
|
||||||
phaseCompletionMethod: 'sophisticated-ai-reasoning'
|
phaseCompletionMethod: 'sophisticated-ai-reasoning'
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
@ -201,8 +220,17 @@ class AuditService {
|
|||||||
this.addEntry(
|
this.addEntry(
|
||||||
'embeddings',
|
'embeddings',
|
||||||
'similarity-search',
|
'similarity-search',
|
||||||
{ query: this.truncateForAudit(query), threshold },
|
{
|
||||||
{ resultsCount: similarResults.length, topResults: similarResults.slice(0, 5).map(r => r.name) },
|
query: query,
|
||||||
|
threshold: threshold,
|
||||||
|
searchType: 'semantic-embeddings'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
resultsCount: similarResults.length,
|
||||||
|
topResults: similarResults.slice(0, 10),
|
||||||
|
averageSimilarity: similarResults.length > 0 ?
|
||||||
|
similarResults.reduce((sum, r) => sum + r.similarity, 0) / similarResults.length : 0
|
||||||
|
},
|
||||||
similarResults.length > 0 ? 85 : 50,
|
similarResults.length > 0 ? 85 : 50,
|
||||||
startTime,
|
startTime,
|
||||||
{
|
{
|
||||||
@ -210,7 +238,9 @@ class AuditService {
|
|||||||
embeddingsUsed: true,
|
embeddingsUsed: true,
|
||||||
similarityScores,
|
similarityScores,
|
||||||
searchThreshold: threshold,
|
searchThreshold: threshold,
|
||||||
totalMatches: similarResults.length
|
totalMatches: similarResults.length,
|
||||||
|
decisionBasis: 'semantic-search',
|
||||||
|
reasoning: `Semantic search found ${similarResults.length} items with similarity above ${threshold}`
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -225,26 +255,92 @@ class AuditService {
|
|||||||
'confidence-scoring',
|
'confidence-scoring',
|
||||||
'tool-confidence',
|
'tool-confidence',
|
||||||
{
|
{
|
||||||
toolName,
|
toolName: toolName,
|
||||||
confidence: {
|
semanticSimilarity: confidence.semanticRelevance,
|
||||||
overall: confidence.overall,
|
taskRelevance: confidence.taskSuitability
|
||||||
semantic: confidence.semanticRelevance,
|
|
||||||
task: confidence.taskSuitability
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
uncertaintyFactorsCount: confidence.uncertaintyFactors?.length || 0,
|
overallConfidence: confidence.overall,
|
||||||
strengthIndicatorsCount: confidence.strengthIndicators?.length || 0
|
strengthIndicators: confidence.strengthIndicators || [],
|
||||||
|
uncertaintyFactors: confidence.uncertaintyFactors || []
|
||||||
},
|
},
|
||||||
confidence.overall,
|
confidence.overall,
|
||||||
startTime,
|
startTime,
|
||||||
{
|
{
|
||||||
...metadata,
|
...metadata,
|
||||||
confidenceCalculation: true
|
confidenceCalculation: true,
|
||||||
|
decisionBasis: 'ai-analysis',
|
||||||
|
reasoning: `Calculated confidence: ${confidence.overall}% (semantic: ${confidence.semanticRelevance}%, task: ${confidence.taskSuitability}%)`
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private createMeaningfulSummary(data: any, type: 'input' | 'output'): string {
|
||||||
|
if (!data) return 'Empty';
|
||||||
|
|
||||||
|
if (typeof data === 'string') {
|
||||||
|
return data.length > 150 ? data.slice(0, 150) + '...' : data;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(data)) {
|
||||||
|
if (data.length === 0) return 'Empty array';
|
||||||
|
if (data.length <= 3) return data.join(', ');
|
||||||
|
return `${data.slice(0, 3).join(', ')} and ${data.length - 3} more items`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof data === 'object') {
|
||||||
|
const keys = Object.keys(data);
|
||||||
|
if (keys.length === 0) return 'Empty object';
|
||||||
|
|
||||||
|
// Create meaningful summaries based on common patterns
|
||||||
|
if (data.prompt) return `AI Prompt: ${data.prompt.slice(0, 100)}...`;
|
||||||
|
if (data.response) return `AI Response: ${data.response.slice(0, 100)}...`;
|
||||||
|
if (data.selectedTools) return `Selected: ${data.selectedTools.join(', ')}`;
|
||||||
|
if (data.availableTools) return `${data.availableTools.length} tools available`;
|
||||||
|
if (data.query) return `Query: ${data.query}`;
|
||||||
|
|
||||||
|
return `Object with ${keys.length} properties: ${keys.slice(0, 3).join(', ')}${keys.length > 3 ? '...' : ''}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return String(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private inferDecisionBasis(metadata: Record<string, any>): string {
|
||||||
|
if (metadata.embeddingsUsed) return 'semantic-search';
|
||||||
|
if (metadata.aiPrompt || metadata.microTaskType) return 'ai-analysis';
|
||||||
|
if (metadata.selectionMethod?.includes('embeddings')) return 'semantic-search';
|
||||||
|
if (metadata.selectionMethod?.includes('full')) return 'ai-analysis';
|
||||||
|
return 'rule-based';
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractReasoning(action: string, input: any, output: any, metadata: Record<string, any>): string {
|
||||||
|
if (metadata.reasoning) return metadata.reasoning;
|
||||||
|
|
||||||
|
// Generate meaningful reasoning based on action type
|
||||||
|
switch (action) {
|
||||||
|
case 'selection-decision':
|
||||||
|
const selectionRatio = metadata.selectedToolsCount / metadata.availableToolsCount;
|
||||||
|
return `Selected ${metadata.selectedToolsCount} tools (${Math.round(selectionRatio * 100)}%) using ${metadata.selectionMethod}`;
|
||||||
|
|
||||||
|
case 'similarity-search':
|
||||||
|
return `Found ${output?.resultsCount || 0} similar items above threshold ${input?.threshold || 0}`;
|
||||||
|
|
||||||
|
case 'ai-decision':
|
||||||
|
return metadata.microTaskType ?
|
||||||
|
`AI analysis for ${metadata.microTaskType}` :
|
||||||
|
'AI decision based on prompt analysis';
|
||||||
|
|
||||||
|
case 'tool-confidence':
|
||||||
|
return `Confidence scored based on semantic similarity and task relevance`;
|
||||||
|
|
||||||
|
case 'phase-enhancement':
|
||||||
|
return `Enhanced ${metadata.phaseId} phase with ${metadata.toolsAddedCount} additional tools`;
|
||||||
|
|
||||||
|
default:
|
||||||
|
return `${action} completed with ${Math.round(metadata.confidence || 0)}% confidence`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
getCurrentAuditTrail(): AuditEntry[] {
|
getCurrentAuditTrail(): AuditEntry[] {
|
||||||
return [...this.activeAuditTrail];
|
return [...this.activeAuditTrail];
|
||||||
}
|
}
|
||||||
@ -263,42 +359,6 @@ class AuditService {
|
|||||||
return finalTrail;
|
return finalTrail;
|
||||||
}
|
}
|
||||||
|
|
||||||
private compressData(data: any): any {
|
|
||||||
if (this.config.detailLevel === 'verbose') {
|
|
||||||
return data;
|
|
||||||
} else if (this.config.detailLevel === 'standard') {
|
|
||||||
return this.summarizeForStorage(data);
|
|
||||||
} else {
|
|
||||||
return this.minimalSummary(data);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private summarizeForStorage(data: any): any {
|
|
||||||
if (typeof data === 'string' && data.length > 500) {
|
|
||||||
return data.slice(0, 500) + '...[truncated]';
|
|
||||||
}
|
|
||||||
if (Array.isArray(data) && data.length > 10) {
|
|
||||||
return [...data.slice(0, 10), `...[${data.length - 10} more items]`];
|
|
||||||
}
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
|
|
||||||
private minimalSummary(data: any): any {
|
|
||||||
if (typeof data === 'string' && data.length > 100) {
|
|
||||||
return data.slice(0, 100) + '...[truncated]';
|
|
||||||
}
|
|
||||||
if (Array.isArray(data) && data.length > 3) {
|
|
||||||
return [...data.slice(0, 3), `...[${data.length - 3} more items]`];
|
|
||||||
}
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
|
|
||||||
private truncateForAudit(text: string, maxLength: number = 300): string {
|
|
||||||
if (typeof text !== 'string') return String(text);
|
|
||||||
if (text.length <= maxLength) return text;
|
|
||||||
return text.slice(0, maxLength) + '...[truncated for audit]';
|
|
||||||
}
|
|
||||||
|
|
||||||
isEnabled(): boolean {
|
isEnabled(): boolean {
|
||||||
return this.config.enabled;
|
return this.config.enabled;
|
||||||
}
|
}
|
||||||
@ -320,7 +380,6 @@ class AuditService {
|
|||||||
qualityMetrics: {
|
qualityMetrics: {
|
||||||
avgProcessingTime: number;
|
avgProcessingTime: number;
|
||||||
confidenceDistribution: { high: number; medium: number; low: number };
|
confidenceDistribution: { high: number; medium: number; low: number };
|
||||||
aiTransparency: number;
|
|
||||||
};
|
};
|
||||||
} {
|
} {
|
||||||
if (!auditTrail || auditTrail.length === 0) {
|
if (!auditTrail || auditTrail.length === 0) {
|
||||||
@ -336,8 +395,7 @@ class AuditService {
|
|||||||
toolSelectionCount: 0,
|
toolSelectionCount: 0,
|
||||||
qualityMetrics: {
|
qualityMetrics: {
|
||||||
avgProcessingTime: 0,
|
avgProcessingTime: 0,
|
||||||
confidenceDistribution: { high: 0, medium: 0, low: 0 },
|
confidenceDistribution: { high: 0, medium: 0, low: 0 }
|
||||||
aiTransparency: 0
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@ -380,8 +438,6 @@ class AuditService {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const avgProcessingTime = auditTrail.length > 0 ? totalTime / auditTrail.length : 0;
|
const avgProcessingTime = auditTrail.length > 0 ? totalTime / auditTrail.length : 0;
|
||||||
const aiTransparency = auditTrail.length > 0 ?
|
|
||||||
(auditTrail.filter(entry => entry.metadata?.aiPrompt || entry.metadata?.reasoning).length / auditTrail.length) * 100 : 0;
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
totalTime,
|
totalTime,
|
||||||
@ -399,12 +455,72 @@ class AuditService {
|
|||||||
high: highConfidenceSteps,
|
high: highConfidenceSteps,
|
||||||
medium: mediumConfidenceSteps,
|
medium: mediumConfidenceSteps,
|
||||||
low: lowConfidenceSteps
|
low: lowConfidenceSteps
|
||||||
},
|
}
|
||||||
aiTransparency: Math.round(aiTransparency)
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
calculateAIResponseConfidence(
|
||||||
|
response: string,
|
||||||
|
expectedLength: { min: number; max: number },
|
||||||
|
taskType: string
|
||||||
|
): number {
|
||||||
|
let confidence = 50; // Base confidence
|
||||||
|
|
||||||
|
// Response length indicates completeness
|
||||||
|
if (response.length >= expectedLength.min) {
|
||||||
|
confidence += 20;
|
||||||
|
if (response.length <= expectedLength.max) {
|
||||||
|
confidence += 10; // Optimal length
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
confidence -= 20; // Too short
|
||||||
|
}
|
||||||
|
|
||||||
|
// Response quality indicators
|
||||||
|
if (response.includes('...') || response.endsWith('...')) {
|
||||||
|
confidence -= 10; // Truncated response
|
||||||
|
}
|
||||||
|
|
||||||
|
// Task-specific quality checks
|
||||||
|
switch (taskType) {
|
||||||
|
case 'scenario-analysis':
|
||||||
|
case 'investigation-approach':
|
||||||
|
case 'critical-considerations':
|
||||||
|
// Should contain forensic methodology terms
|
||||||
|
const forensicTerms = ['forensisch', 'beweis', 'evidence', 'analyse', 'untersuchung', 'methodik'];
|
||||||
|
const termsFound = forensicTerms.filter(term =>
|
||||||
|
response.toLowerCase().includes(term)
|
||||||
|
).length;
|
||||||
|
confidence += Math.min(15, termsFound * 3);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'tool-evaluation':
|
||||||
|
// Should be structured and comprehensive
|
||||||
|
if (response.includes('detailed_explanation') || response.includes('implementation_approach')) {
|
||||||
|
confidence += 15;
|
||||||
|
}
|
||||||
|
if (response.includes('pros') && response.includes('limitations')) {
|
||||||
|
confidence += 10;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'background-knowledge':
|
||||||
|
// Should be valid JSON array
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(response);
|
||||||
|
if (Array.isArray(parsed) && parsed.length > 0) {
|
||||||
|
confidence += 20;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
confidence -= 20;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.min(95, Math.max(25, confidence));
|
||||||
|
}
|
||||||
|
|
||||||
validateAuditTrail(auditTrail: AuditEntry[]): {
|
validateAuditTrail(auditTrail: AuditEntry[]): {
|
||||||
isValid: boolean;
|
isValid: boolean;
|
||||||
issues: string[];
|
issues: string[];
|
||||||
@ -435,14 +551,6 @@ class AuditService {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
if (entry.action === 'ai-decision' && !entry.metadata?.aiPrompt && !entry.metadata?.reasoning) {
|
|
||||||
warnings.push(`Entry ${index}: AI decision lacks transparency (no prompt or reasoning)`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (entry.action === 'selection-decision' && !entry.metadata?.selectionMethod) {
|
|
||||||
warnings.push(`Entry ${index}: Tool selection lacks methodology info`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) {
|
if (typeof entry.confidence !== 'number' || entry.confidence < 0 || entry.confidence > 100) {
|
||||||
warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`);
|
warnings.push(`Entry ${index} has invalid confidence value: ${entry.confidence}`);
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user