This commit is contained in:
overcuriousity
2025-08-17 17:27:08 +02:00
parent 5ecbabea90
commit bcd92af8a0
10 changed files with 18 additions and 77 deletions

View File

@@ -1,4 +1,4 @@
// src/utils/auditService.ts - Fixed with meaningful confidence and reasoning
// src/utils/auditService.ts
import 'dotenv/config';
function env(key: string, fallback: string | undefined = undefined): string | undefined {
@@ -85,7 +85,6 @@ class AuditService {
): void {
if (!this.config.enabled) return;
// Skip initialization and completion entries as they don't add transparency
if (action === 'pipeline-start' || action === 'pipeline-end') {
return;
}
@@ -152,7 +151,6 @@ class AuditService {
startTime: number,
metadata: Record<string, any> = {}
): void {
// Calculate meaningful confidence based on selection quality
const calculatedConfidence = this.calculateSelectionConfidence(
selectedTools,
availableTools,
@@ -164,7 +162,7 @@ class AuditService {
'tool-selection',
'selection-decision',
{
availableTools: availableTools.slice(0, 10), // Show first 10 for context
availableTools: availableTools.slice(0, 10),
totalAvailable: availableTools.length,
selectionMethod: selectionMethod
},
@@ -191,7 +189,6 @@ class AuditService {
startTime: number,
metadata: Record<string, any> = {}
): void {
// Only add if tools were actually added
if (!addedTools || addedTools.length === 0) {
console.log(`[AUDIT-SERVICE] Skipping phase completion for ${phaseId} - no tools added`);
return;
@@ -292,21 +289,18 @@ class AuditService {
const selectionRatio = selectedTools.length / availableTools.length;
// Good selection ratio (5-20% of available tools)
if (selectionRatio >= 0.05 && selectionRatio <= 0.20) {
confidence += 25;
} else if (selectionRatio < 0.05) {
confidence += 15; // Very selective is good
confidence += 15;
} else if (selectionRatio > 0.30) {
confidence -= 20; // Too many tools selected
confidence -= 20;
}
// Embeddings usage bonus
if (selectionMethod.includes('embeddings')) {
confidence += 15;
}
// Reasonable number of tools selected
if (selectedTools.length >= 5 && selectedTools.length <= 25) {
confidence += 10;
}
@@ -321,22 +315,18 @@ class AuditService {
): number {
let confidence = 60;
// Tools actually added
if (addedTools.length > 0) {
confidence += 20;
}
// Good reasoning provided
if (reasoning && reasoning.length > 50) {
confidence += 15;
}
// AI reasoning was used successfully
if (metadata.aiReasoningUsed) {
confidence += 10;
}
// Not too many tools added (indicates thoughtful selection)
if (addedTools.length <= 2) {
confidence += 5;
}
@@ -347,17 +337,14 @@ class AuditService {
private calculateEmbeddingsConfidence(similarResults: any[], threshold: number): number {
let confidence = 50;
// Found relevant results
if (similarResults.length > 0) {
confidence += 20;
}
// Good number of results (not too few, not too many)
if (similarResults.length >= 5 && similarResults.length <= 30) {
confidence += 15;
}
// High similarity scores
const avgSimilarity = similarResults.length > 0 ?
similarResults.reduce((sum, r) => sum + r.similarity, 0) / similarResults.length : 0;
@@ -367,7 +354,6 @@ class AuditService {
confidence += 10;
}
// Reasonable threshold
if (threshold >= 0.3 && threshold <= 0.5) {
confidence += 5;
}
@@ -378,7 +364,6 @@ class AuditService {
private createSpecificSummary(data: any, action: string, type: 'input' | 'output'): string {
if (!data) return 'Leer';
// Action-specific summaries that show actual meaningful data
switch (action) {
case 'selection-decision':
if (type === 'input') {
@@ -476,7 +461,6 @@ class AuditService {
}
}
// Enhanced fallback that shows actual key-value content instead of just "X Eigenschaften"
if (typeof data === 'string') {
return data.length > 100 ? data.slice(0, 100) + '...' : data;
}
@@ -491,7 +475,6 @@ class AuditService {
const keys = Object.keys(data);
if (keys.length === 0) return 'Leeres Objekt';
// Show actual key-value pairs for small objects instead of just counting properties
if (keys.length <= 2) {
const pairs = keys.map(key => {
const value = data[key];
@@ -505,7 +488,6 @@ class AuditService {
});
return pairs.join(', ');
} else {
// For larger objects, show key names and some sample values
const sampleKeys = keys.slice(0, 3);
const sampleValues = sampleKeys.map(key => {
const value = data[key];
@@ -531,7 +513,6 @@ class AuditService {
metadata: Record<string, any>,
confidence: number
): string {
// Use provided reasoning if available and meaningful
if (metadata.reasoning && metadata.reasoning.length > 20 && !metadata.reasoning.includes('completed with')) {
return metadata.reasoning;
}
@@ -546,11 +527,9 @@ class AuditService {
const totalMatches =
typeof metadata.totalMatches === 'number' ? metadata.totalMatches : 0;
// Safely narrow & cast similarityScores to a number map
const scoresObj = (metadata.similarityScores ?? {}) as Record<string, number>;
const scores = Object.values(scoresObj) as number[];
// Use totalMatches if it looks sensible; otherwise fall back to scores.length
const denom = totalMatches > 0 ? totalMatches : scores.length;
const sum = scores.reduce((acc, v) => acc + (typeof v === 'number' ? v : 0), 0);
@@ -697,9 +676,7 @@ class AuditService {
return Math.min(95, Math.max(25, confidence));
}
// Additional utility methods remain the same...
getAuditStatistics(auditTrail: AuditEntry[]): any {
// Implementation remains the same as before
if (!auditTrail || auditTrail.length === 0) {
return {
totalTime: 0,