mstoeck3/forensic-pathways
2
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Activity

enhancement 1: audit trail

Browse Source
This commit is contained in:
overcuriousity 2025-08-03 12:41:02 +02:00
parent 57c507915f
commit 6308c03709
6 changed files with 639 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
.env.example
View File

@ -1,79 +1,154 @@
# =========================================== # ============================================================================
# ForensicPathways Environment Configuration # ForensicPathways Environment Configuration
# =========================================== # ============================================================================
# Copy this file to .env and adjust the values below.
# Settings are ordered by likelihood of needing adjustment during setup.
# === Authentication Configuration === # ============================================================================
# 1. CORE APPLICATION SETTINGS (REQUIRED - ADJUST FOR YOUR SETUP)
# ============================================================================
# Your application's public URL (used for redirects and links)
PUBLIC_BASE_URL=http://localhost:4321
# Application environment (development, production, staging)
NODE_ENV=development
# Secret key for session encryption (CHANGE IN PRODUCTION!)
AUTH_SECRET=your-secret-key-change-in-production-please
# ============================================================================
# 2. AI SERVICES CONFIGURATION (REQUIRED FOR AI FEATURES)
# ============================================================================
# Main AI Analysis Service (for query processing and recommendations)
# Example uses Mistral AI - adjust endpoint/model as needed
AI_ANALYZER_ENDPOINT=https://api.mistral.ai/v1
AI_ANALYZER_API_KEY=your-mistral-api-key-here
AI_ANALYZER_MODEL=mistral-small-latest
# Vector Embeddings Service (for semantic search - can use same provider)
AI_EMBEDDINGS_ENABLED=true
AI_EMBEDDINGS_ENDPOINT=https://api.mistral.ai/v1/embeddings
AI_EMBEDDINGS_API_KEY=your-mistral-api-key-here
AI_EMBEDDINGS_MODEL=mistral-embed
# ============================================================================
# 3. AUTHENTICATION (OPTIONAL - SET TO 'true' IF NEEDED)
# ============================================================================
# Enable authentication for different features
AUTHENTICATION_NECESSARY=false AUTHENTICATION_NECESSARY=false
AUTHENTICATION_NECESSARY_CONTRIBUTIONS=false AUTHENTICATION_NECESSARY_CONTRIBUTIONS=false
AUTHENTICATION_NECESSARY_AI=false AUTHENTICATION_NECESSARY_AI=false
AUTH_SECRET=your-secret-key-change-in-production
# OIDC Configuration (if authentication enabled) # OIDC Provider Settings (only needed if authentication enabled)
OIDC_ENDPOINT=https://your-oidc-provider.com OIDC_ENDPOINT=https://your-oidc-provider.com
OIDC_CLIENT_ID=your-client-id OIDC_CLIENT_ID=your-client-id
OIDC_CLIENT_SECRET=your-client-secret OIDC_CLIENT_SECRET=your-client-secret
# =================================================================== # ============================================================================
# AI CONFIGURATION - Complete Reference for Improved Pipeline # 4. ADVANCED AI CONFIGURATION (FINE-TUNING - DEFAULT VALUES USUALLY WORK)
# =================================================================== # ============================================================================
# === CORE AI ENDPOINTS & MODELS === # Pipeline Performance Settings
AI_API_ENDPOINT=https://llm.mikoshi.de AI_MAX_SELECTED_ITEMS=60 # Tools analyzed per micro-task
AI_API_KEY=sREDACTED3w AI_EMBEDDING_CANDIDATES=60 # Vector search candidates
AI_MODEL='mistral/mistral-small-latest' AI_MICRO_TASK_DELAY_MS=500 # Delay between AI micro-tasks
# === IMPROVED PIPELINE: Use separate analyzer model (mistral-small is fine) === # Rate Limiting (requests per minute)
AI_ANALYZER_ENDPOINT=https://llm.mikoshi.de AI_RATE_LIMIT_MAX_REQUESTS=6 # Main query rate limit
AI_ANALYZER_API_KEY=skREDACTEDw3w AI_MICRO_TASK_RATE_LIMIT=15 # Micro-task rate limit
AI_ANALYZER_MODEL='mistral/mistral-small-latest' AI_RATE_LIMIT_DELAY_MS=3000 # Delay between rate-limited calls
# === EMBEDDINGS CONFIGURATION === # Embeddings Batch Processing
AI_EMBEDDINGS_ENABLED=true AI_EMBEDDINGS_BATCH_SIZE=20 # Embeddings processed per batch
AI_EMBEDDINGS_ENDPOINT=https://api.mistral.ai/v1/embeddings AI_EMBEDDINGS_BATCH_DELAY_MS=1000 # Delay between embedding batches
AI_EMBEDDINGS_API_KEY=ZREDACTED3wL
AI_EMBEDDINGS_MODEL=mistral-embed
AI_EMBEDDINGS_BATCH_SIZE=20
AI_EMBEDDINGS_BATCH_DELAY_MS=1000
# === PIPELINE: VectorIndex (HNSW) Configuration === # Timeouts and Limits
AI_MAX_SELECTED_ITEMS=60 # Tools visible to each micro-task AI_MICRO_TASK_TIMEOUT_MS=25000 # Max time per micro-task
AI_EMBEDDING_CANDIDATES=60 # VectorIndex candidates (HNSW is more efficient) AI_QUEUE_MAX_SIZE=50 # Max queued requests
AI_SIMILARITY_THRESHOLD=0.3 # Not used by VectorIndex (uses cosine distance internally) AI_SIMILARITY_THRESHOLD=0.3 # Vector similarity threshold
# === MICRO-TASK CONFIGURATION === # ============================================================================
AI_MICRO_TASK_DELAY_MS=500 # Delay between micro-tasks # 5. FORENSIC AUDIT SYSTEM (OPTIONAL - FOR TRANSPARENCY AND DEBUGGING)
AI_MICRO_TASK_TIMEOUT_MS=25000 # Timeout per micro-task (increased for full context) # ============================================================================
# === RATE LIMITING === # Enable detailed audit trail of AI decision-making
AI_RATE_LIMIT_DELAY_MS=3000 # Main rate limit delay FORENSIC_AUDIT_ENABLED=false
AI_RATE_LIMIT_MAX_REQUESTS=6 # Main requests per minute (reduced - fewer but richer calls)
AI_MICRO_TASK_RATE_LIMIT=15 # Micro-task requests per minute (was 30)
# === QUEUE MANAGEMENT === # Audit detail level: minimal, standard, verbose
AI_QUEUE_MAX_SIZE=50 FORENSIC_AUDIT_DETAIL_LEVEL=standard
AI_QUEUE_CLEANUP_INTERVAL_MS=300000
# === PERFORMANCE & MONITORING === # Audit retention and limits
AI_MICRO_TASK_DEBUG=true FORENSIC_AUDIT_RETENTION_HOURS=72 # Keep audit data for 3 days
AI_PERFORMANCE_METRICS=true FORENSIC_AUDIT_MAX_ENTRIES=50 # Max entries per request
AI_RESPONSE_CACHE_TTL_MS=3600000
# =================================================================== # ============================================================================
# LEGACY VARIABLES (still used but less important) # 6. QUALITY CONTROL AND BIAS DETECTION (OPTIONAL - ADVANCED FEATURES)
# =================================================================== # ============================================================================
# These are still used by other parts of the system: # Confidence Scoring Weights (must sum to 1.0)
AI_RESPONSE_CACHE_TTL_MS=3600000 # For caching responses CONFIDENCE_EMBEDDINGS_WEIGHT=0.3
AI_QUEUE_MAX_SIZE=50 # Queue management CONFIDENCE_CONSENSUS_WEIGHT=0.25
AI_QUEUE_CLEANUP_INTERVAL_MS=300000 # Queue cleanup CONFIDENCE_DOMAIN_MATCH_WEIGHT=0.25
CONFIDENCE_FRESHNESS_WEIGHT=0.2
# === Application Configuration === # Confidence Thresholds (0-100)
PUBLIC_BASE_URL=http://localhost:4321 CONFIDENCE_MINIMUM_THRESHOLD=40
NODE_ENV=development CONFIDENCE_MEDIUM_THRESHOLD=60
CONFIDENCE_HIGH_THRESHOLD=80
# Nextcloud Integration (Optional) # Bias Detection Settings
NEXTCLOUD_ENDPOINT=https://your-nextcloud.com BIAS_DETECTION_ENABLED=false
NEXTCLOUD_USERNAME=your-username BIAS_POPULARITY_THRESHOLD=0.7 # Detect over-popular tools
NEXTCLOUD_PASSWORD=your-password BIAS_DIVERSITY_MINIMUM=0.6 # Require recommendation diversity
NEXTCLOUD_UPLOAD_PATH=/kb-media BIAS_CELEBRITY_TOOLS="Volatility 3,Wireshark,Autopsy,Maltego"
NEXTCLOUD_PUBLIC_URL=https://your-nextcloud.com/s/
# Quality Control Thresholds
QUALITY_MIN_RESPONSE_LENGTH=50 # Minimum AI response length
QUALITY_MIN_SELECTION_COUNT=1 # Minimum tools selected
QUALITY_MAX_PROCESSING_TIME_MS=30000 # Max processing time
# ============================================================================
# 7. USER INTERFACE PREFERENCES (OPTIONAL - UI DEFAULTS)
# ============================================================================
# Default UI behavior (users can override)
UI_SHOW_AUDIT_TRAIL_DEFAULT=false
UI_SHOW_CONFIDENCE_SCORES=true
UI_SHOW_BIAS_WARNINGS=true
UI_AUDIT_TRAIL_COLLAPSIBLE=true
# ============================================================================
# 8. EXTERNAL INTEGRATIONS (OPTIONAL - ONLY IF USING THESE SERVICES)
# ============================================================================
# Nextcloud Integration (for file uploads)
# NEXTCLOUD_ENDPOINT=https://your-nextcloud.com
# NEXTCLOUD_USERNAME=your-username
# NEXTCLOUD_PASSWORD=your-password
# NEXTCLOUD_UPLOAD_PATH=/kb-media
# NEXTCLOUD_PUBLIC_URL=https://your-nextcloud.com/s/
# ============================================================================
# 9. PERFORMANCE AND MONITORING (OPTIONAL - FOR PRODUCTION OPTIMIZATION)
# ============================================================================
# Caching and Queue Management
AI_RESPONSE_CACHE_TTL_MS=3600000 # Cache responses for 1 hour
AI_QUEUE_CLEANUP_INTERVAL_MS=300000 # Cleanup queue every 5 minutes
# Debug and Monitoring
AI_MICRO_TASK_DEBUG=false # Enable detailed micro-task logging
AI_PERFORMANCE_METRICS=false # Enable performance tracking
# ============================================================================
# SETUP CHECKLIST:
# ============================================================================
# 1. Set PUBLIC_BASE_URL to your domain
# 2. Change AUTH_SECRET to a secure random string
# 3. Configure AI service endpoints and API keys
# 4. Set authentication options if needed
# 5. Test with default advanced settings before adjusting
# ============================================================================

8
README.md
View File

@ -62,7 +62,7 @@ Ein kuratiertes Verzeichnis für Digital Forensics und Incident Response (DFIR)
### AI Service (Mistral/OpenAI-kompatibel) ### AI Service (Mistral/OpenAI-kompatibel)
- **Zweck:** KI-gestützte Tool-Empfehlungen - **Zweck:** KI-gestützte Tool-Empfehlungen
- **Konfiguration:** `AI_API_ENDPOINT`, `AI_API_KEY`, `AI_MODEL` - **Konfiguration:** `AI_ANALYZER_ENDPOINT`, `AI_ANALYZER_API_KEY`, `AI_ANALYZER_MODEL`
### Uptime Kuma ### Uptime Kuma
- **Zweck:** Status-Monitoring für gehostete Services - **Zweck:** Status-Monitoring für gehostete Services
@ -157,9 +157,9 @@ PUBLIC_BASE_URL=https://your-domain.com
NODE_ENV=production NODE_ENV=production
# AI Service Configuration (Required for AI features) # AI Service Configuration (Required for AI features)
AI_MODEL=mistral-large-latest AI_ANALYZER_MODEL=mistral-large-latest
AI_API_ENDPOINT=https://api.mistral.ai AI_ANALYZER_ENDPOINT=https://api.mistral.ai
AI_API_KEY=your-mistral-api-key AI_ANALYZER_API_KEY=your-mistral-api-key
AI_RATE_LIMIT_DELAY_MS=1000 AI_RATE_LIMIT_DELAY_MS=1000
# Git Integration (Required for contributions) # Git Integration (Required for contributions)

94
src/components/AIQueryInterface.astro
View File

@ -711,6 +711,7 @@ class AIQueryInterface {
${this.renderBackgroundKnowledge(recommendation.background_knowledge)} ${this.renderBackgroundKnowledge(recommendation.background_knowledge)}
${this.renderWorkflowPhases(toolsByPhase, phaseOrder, phaseNames)} ${this.renderWorkflowPhases(toolsByPhase, phaseOrder, phaseNames)}
${this.renderWorkflowSuggestion(recommendation.workflow_suggestion)} ${this.renderWorkflowSuggestion(recommendation.workflow_suggestion)}
${this.renderAuditTrail(recommendation.auditTrail)}
</div> </div>
`; `;
@ -725,12 +726,105 @@ class AIQueryInterface {
${this.renderBackgroundKnowledge(recommendation.background_knowledge)} ${this.renderBackgroundKnowledge(recommendation.background_knowledge)}
${this.renderToolRecommendations(recommendation.recommended_tools)} ${this.renderToolRecommendations(recommendation.recommended_tools)}
${this.renderAdditionalConsiderations(recommendation.additional_considerations)} ${this.renderAdditionalConsiderations(recommendation.additional_considerations)}
${this.renderAuditTrail(recommendation.auditTrail)}
</div> </div>
`; `;
this.elements.results.innerHTML = html; this.elements.results.innerHTML = html;
} }
// NEW: Audit Trail Rendering Functions
renderAuditTrail(auditTrail) {
if (!auditTrail || !Array.isArray(auditTrail) || auditTrail.length === 0) {
return '';
}
// Reuse existing card and info styles from global.css
return `
<div class="card-info-sm mt-4" style="border-left: 4px solid var(--color-text-secondary);">
<div class="flex items-center gap-2 mb-3">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/>
<polyline points="14 2 14 8 20 8"/>
<line x1="16" y1="13" x2="8" y2="13"/>
<line x1="16" y1="17" x2="8" y2="17"/>
</svg>
<h4 class="text-sm font-semibold text-secondary">Forensic Audit Trail (${auditTrail.length} Entries)</h4>
<button class="btn-icon btn-sm" onclick="this.nextElementSibling.style.display = this.nextElementSibling.style.display === 'none' ? 'block' : 'none'; this.innerHTML = this.nextElementSibling.style.display === 'none' ? this.innerHTML.replace('▼', '▶') : this.innerHTML.replace('▶', '▼');">
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<polyline points="6 9 12 15 18 9"/>
</svg>
</button>
</div>
<div style="display: none;" class="audit-trail-details">
<div class="text-xs text-muted mb-2">
<strong>Transparency Note:</strong> This trail documents every decision step for forensic verification and reproducibility.
</div>
${auditTrail.map(entry => this.renderAuditEntry(entry)).join('')}
</div>
</div>
`;
}
renderAuditEntry(entry) {
const confidenceColor = entry.confidence >= 80 ? 'var(--color-accent)' :
entry.confidence >= 60 ? 'var(--color-warning)' : 'var(--color-error)';
const formattedTime = new Date(entry.timestamp).toLocaleTimeString('de-DE', {
hour: '2-digit',
minute: '2-digit',
second: '2-digit'
});
// Reuse existing grid and text utilities
return `
<div class="border-l-2 pl-3 py-2 mb-2" style="border-left-color: ${confidenceColor};">
<div class="flex justify-between items-center mb-1">
<span class="text-xs font-medium">${entry.phase} → ${entry.action}</span>
<div class="flex items-center gap-2">
<span class="badge badge-mini" style="background-color: ${confidenceColor}; color: white;">
${entry.confidence}% confidence
</span>
<span class="text-xs text-muted">${entry.processingTimeMs}ms</span>
<span class="text-xs text-muted">${formattedTime}</span>
</div>
</div>
<div class="text-xs text-muted grid-cols-2 gap-2" style="display: grid;">
<div><strong>Input:</strong> ${this.formatAuditData(entry.input)}</div>
<div><strong>Output:</strong> ${this.formatAuditData(entry.output)}</div>
</div>
${entry.metadata && Object.keys(entry.metadata).length > 0 ? `
<div class="text-xs text-muted mt-1 pt-1 border-t border-dashed">
<strong>Metadata:</strong> ${this.formatAuditData(entry.metadata)}
</div>
` : ''}
</div>
`;
}
formatAuditData(data) {
if (data === null || data === undefined) return 'null';
if (typeof data === 'string') {
return data.length > 100 ? data.slice(0, 100) + '...' : data;
}
if (typeof data === 'number') return data.toString();
if (typeof data === 'boolean') return data.toString();
if (Array.isArray(data)) {
if (data.length === 0) return '[]';
if (data.length <= 3) return JSON.stringify(data);
return `[${data.slice(0, 3).map(i => typeof i === 'string' ? i : JSON.stringify(i)).join(', ')}, ...+${data.length - 3}]`;
}
if (typeof data === 'object') {
const keys = Object.keys(data);
if (keys.length === 0) return '{}';
if (keys.length <= 3) {
return '{' + keys.map(k => `${k}: ${typeof data[k] === 'string' ? data[k].slice(0, 20) + (data[k].length > 20 ? '...' : '') : JSON.stringify(data[k])}`).join(', ') + '}';
}
return `{${keys.slice(0, 3).join(', ')}, ...+${keys.length - 3} keys}`;
}
return String(data);
}
renderHeader(title, query) { renderHeader(title, query) {
return ` return `
<div style="text-align: center; margin-bottom: 2rem; padding: 1.5rem; background: linear-gradient(135deg, var(--color-primary) 0%, #525252 100%); color: white; border-radius: 0.75rem;"> <div style="text-align: center; margin-bottom: 2rem; padding: 1.5rem; background: linear-gradient(135deg, var(--color-primary) 0%, #525252 100%); color: white; border-radius: 0.75rem;">

126
src/config/forensic.config.ts Normal file
View File

@ -0,0 +1,126 @@
// src/config/forensic.config.ts
// Centralized configuration for forensic RAG enhancements
export const FORENSIC_CONFIG = {
audit: {
enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true',
detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as 'minimal' | 'standard' | 'verbose') || 'standard',
retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10),
maxEntriesPerRequest: parseInt(process.env.FORENSIC_AUDIT_MAX_ENTRIES || '50', 10)
},
confidence: {
embeddingsWeight: parseFloat(process.env.CONFIDENCE_EMBEDDINGS_WEIGHT || '0.3'),
consensusWeight: parseFloat(process.env.CONFIDENCE_CONSENSUS_WEIGHT || '0.25'),
domainMatchWeight: parseFloat(process.env.CONFIDENCE_DOMAIN_MATCH_WEIGHT || '0.25'),
freshnessWeight: parseFloat(process.env.CONFIDENCE_FRESHNESS_WEIGHT || '0.2'),
minimumThreshold: parseInt(process.env.CONFIDENCE_MINIMUM_THRESHOLD || '40', 10),
highThreshold: parseInt(process.env.CONFIDENCE_HIGH_THRESHOLD || '80', 10),
mediumThreshold: parseInt(process.env.CONFIDENCE_MEDIUM_THRESHOLD || '60', 10)
},
bias: {
enabled: process.env.BIAS_DETECTION_ENABLED === 'true',
popularityThreshold: parseFloat(process.env.BIAS_POPULARITY_THRESHOLD || '0.7'),
diversityMinimum: parseFloat(process.env.BIAS_DIVERSITY_MINIMUM || '0.6'),
domainMismatchThreshold: parseFloat(process.env.BIAS_DOMAIN_MISMATCH_THRESHOLD || '0.3'),
warningThreshold: parseInt(process.env.BIAS_WARNING_THRESHOLD || '3', 10),
celebrityTools: (process.env.BIAS_CELEBRITY_TOOLS || 'Volatility 3,Wireshark,Autopsy,Maltego').split(',').map(t => t.trim())
},
// Quality thresholds for various metrics
quality: {
minResponseLength: parseInt(process.env.QUALITY_MIN_RESPONSE_LENGTH || '50', 10),
minSelectionCount: parseInt(process.env.QUALITY_MIN_SELECTION_COUNT || '1', 10),
maxProcessingTime: parseInt(process.env.QUALITY_MAX_PROCESSING_TIME_MS || '30000', 10)
},
// Display preferences
ui: {
showAuditTrailByDefault: process.env.UI_SHOW_AUDIT_TRAIL_DEFAULT === 'true',
showConfidenceScores: process.env.UI_SHOW_CONFIDENCE_SCORES !== 'false',
showBiasWarnings: process.env.UI_SHOW_BIAS_WARNINGS !== 'false',
auditTrailCollapsible: process.env.UI_AUDIT_TRAIL_COLLAPSIBLE !== 'false'
}
};
// Validation function to ensure configuration is valid
export function validateForensicConfig(): { valid: boolean; errors: string[] } {
const errors: string[] = [];
// Validate audit configuration
if (FORENSIC_CONFIG.audit.retentionHours < 1 || FORENSIC_CONFIG.audit.retentionHours > 168) {
errors.push('FORENSIC_AUDIT_RETENTION_HOURS must be between 1 and 168 (1 week)');
}
if (!['minimal', 'standard', 'verbose'].includes(FORENSIC_CONFIG.audit.detailLevel)) {
errors.push('FORENSIC_AUDIT_DETAIL_LEVEL must be one of: minimal, standard, verbose');
}
// Validate confidence weights sum to approximately 1.0
const weightSum = FORENSIC_CONFIG.confidence.embeddingsWeight +
FORENSIC_CONFIG.confidence.consensusWeight +
FORENSIC_CONFIG.confidence.domainMatchWeight +
FORENSIC_CONFIG.confidence.freshnessWeight;
if (Math.abs(weightSum - 1.0) > 0.05) {
errors.push(`Confidence weights must sum to 1.0 (currently ${weightSum.toFixed(3)})`);
}
// Validate threshold ranges
if (FORENSIC_CONFIG.confidence.minimumThreshold < 0 || FORENSIC_CONFIG.confidence.minimumThreshold > 100) {
errors.push('CONFIDENCE_MINIMUM_THRESHOLD must be between 0 and 100');
}
if (FORENSIC_CONFIG.confidence.highThreshold <= FORENSIC_CONFIG.confidence.mediumThreshold) {
errors.push('CONFIDENCE_HIGH_THRESHOLD must be greater than CONFIDENCE_MEDIUM_THRESHOLD');
}
// Validate bias thresholds
if (FORENSIC_CONFIG.bias.popularityThreshold < 0 || FORENSIC_CONFIG.bias.popularityThreshold > 1) {
errors.push('BIAS_POPULARITY_THRESHOLD must be between 0 and 1');
}
if (FORENSIC_CONFIG.bias.diversityMinimum < 0 || FORENSIC_CONFIG.bias.diversityMinimum > 1) {
errors.push('BIAS_DIVERSITY_MINIMUM must be between 0 and 1');
}
return {
valid: errors.length === 0,
errors
};
}
// Helper functions for configuration access
export function isAuditEnabled(): boolean {
return FORENSIC_CONFIG.audit.enabled;
}
export function getAuditDetailLevel(): 'minimal' | 'standard' | 'verbose' {
return FORENSIC_CONFIG.audit.detailLevel;
}
export function getConfidenceThresholds() {
return {
minimum: FORENSIC_CONFIG.confidence.minimumThreshold,
medium: FORENSIC_CONFIG.confidence.mediumThreshold,
high: FORENSIC_CONFIG.confidence.highThreshold
};
}
export function isBiasDetectionEnabled(): boolean {
return FORENSIC_CONFIG.bias.enabled;
}
// Initialize and validate configuration on module load
const configValidation = validateForensicConfig();
if (!configValidation.valid) {
console.warn('[FORENSIC CONFIG] Configuration validation failed:', configValidation.errors);
// In development, we might want to throw an error
if (process.env.NODE_ENV === 'development') {
throw new Error(`Forensic configuration invalid: ${configValidation.errors.join(', ')}`);
}
}
console.log('[FORENSIC CONFIG] Configuration loaded:', {
auditEnabled: FORENSIC_CONFIG.audit.enabled,
confidenceEnabled: true, // Always enabled
biasDetectionEnabled: FORENSIC_CONFIG.bias.enabled,
detailLevel: FORENSIC_CONFIG.audit.detailLevel
});

8
src/pages/api/ai/enhance-input.ts
View File

@ -15,8 +15,8 @@ function getEnv(key: string): string {
} }
const AI_ENDPOINT = getEnv('AI_ANALYZER_ENDPOINT'); const AI_ENDPOINT = getEnv('AI_ANALYZER_ENDPOINT');
const AI_API_KEY = getEnv('AI_ANALYZER_API_KEY'); const AI_ANALYZER_API_KEY = getEnv('AI_ANALYZER_API_KEY');
const AI_MODEL = getEnv('AI_ANALYZER_MODEL'); const AI_ANALYZER_MODEL = getEnv('AI_ANALYZER_MODEL');
const rateLimitStore = new Map<string, { count: number; resetTime: number }>(); const rateLimitStore = new Map<string, { count: number; resetTime: number }>();
const RATE_LIMIT_WINDOW = 60 * 1000; // 1 minute const RATE_LIMIT_WINDOW = 60 * 1000; // 1 minute
@ -126,10 +126,10 @@ export const POST: APIRoute = async ({ request }) => {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Authorization': `Bearer ${AI_API_KEY}` 'Authorization': `Bearer ${AI_ANALYZER_API_KEY}`
}, },
body: JSON.stringify({ body: JSON.stringify({
model: AI_MODEL, model: AI_ANALYZER_MODEL,
messages: [ messages: [
{ {
role: 'user', role: 'user',

285
src/utils/aiPipeline.ts
View File

@ -1,4 +1,4 @@
// src/utils/aiPipeline.ts // src/utils/aiPipeline.ts - Enhanced with Audit Trail System
import { getCompressedToolsDataForAI } from './dataService.js'; import { getCompressedToolsDataForAI } from './dataService.js';
import { embeddingsService, type EmbeddingData } from './embeddings.js'; import { embeddingsService, type EmbeddingData } from './embeddings.js';
@ -30,6 +30,19 @@ interface AnalysisResult {
}; };
} }
// NEW: Audit Trail Types
interface AuditEntry {
timestamp: number;
phase: string; // 'retrieval', 'selection', 'micro-task-N'
action: string; // 'embeddings-search', 'ai-selection', 'tool-evaluation'
input: any; // What went into this step
output: any; // What came out of this step
confidence: number; // 0-100: How confident we are in this step
processingTimeMs: number;
metadata: Record<string, any>; // Additional context
}
// Enhanced AnalysisContext with Audit Trail
interface AnalysisContext { interface AnalysisContext {
userQuery: string; userQuery: string;
mode: string; mode: string;
@ -47,6 +60,9 @@ interface AnalysisContext {
backgroundKnowledge?: Array<{concept: any, relevance: string}>; backgroundKnowledge?: Array<{concept: any, relevance: string}>;
seenToolNames: Set<string>; seenToolNames: Set<string>;
// NEW: Audit Trail
auditTrail: AuditEntry[];
} }
class ImprovedMicroTaskAIPipeline { class ImprovedMicroTaskAIPipeline {
@ -59,6 +75,16 @@ class ImprovedMicroTaskAIPipeline {
private maxContextTokens: number; private maxContextTokens: number;
private maxPromptTokens: number; private maxPromptTokens: number;
// NEW: Audit Configuration
private auditConfig: {
enabled: boolean;
detailLevel: 'minimal' | 'standard' | 'verbose';
retentionHours: number;
};
// NEW: Temporary audit storage for pre-context operations
private tempAuditEntries: AuditEntry[] = [];
constructor() { constructor() {
this.config = { this.config = {
endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'), endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'),
@ -73,6 +99,13 @@ class ImprovedMicroTaskAIPipeline {
this.maxContextTokens = parseInt(process.env.AI_MAX_CONTEXT_TOKENS || '4000', 10); this.maxContextTokens = parseInt(process.env.AI_MAX_CONTEXT_TOKENS || '4000', 10);
this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10); this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10);
// NEW: Initialize Audit Configuration
this.auditConfig = {
enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true',
detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as any) || 'standard',
retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10)
};
} }
private getEnv(key: string): string { private getEnv(key: string): string {
@ -83,6 +116,94 @@ class ImprovedMicroTaskAIPipeline {
return value; return value;
} }
// NEW: Audit Trail Utility Functions
private addAuditEntry(
context: AnalysisContext | null,
phase: string,
action: string,
input: any,
output: any,
confidence: number,
startTime: number,
metadata: Record<string, any> = {}
): void {
if (!this.auditConfig.enabled) return;
const auditEntry: AuditEntry = {
timestamp: Date.now(),
phase,
action,
input: this.auditConfig.detailLevel === 'verbose' ? input : this.summarizeForAudit(input),
output: this.auditConfig.detailLevel === 'verbose' ? output : this.summarizeForAudit(output),
confidence,
processingTimeMs: Date.now() - startTime,
metadata
};
if (context) {
context.auditTrail.push(auditEntry);
} else {
// Store in temporary array for later merging
this.tempAuditEntries.push(auditEntry);
}
// Log for debugging when audit is enabled
console.log(`[AUDIT] ${phase}/${action}: ${confidence}% confidence, ${Date.now() - startTime}ms`);
}
// NEW: Merge temporary audit entries into context
private mergeTemporaryAuditEntries(context: AnalysisContext): void {
if (!this.auditConfig.enabled || this.tempAuditEntries.length === 0) return;
const entryCount = this.tempAuditEntries.length;
// Add temp entries to the beginning of the context audit trail
context.auditTrail.unshift(...this.tempAuditEntries);
this.tempAuditEntries = []; // Clear temp storage
console.log(`[AUDIT] Merged ${entryCount} temporary audit entries into context`);
}
private summarizeForAudit(data: any): any {
if (this.auditConfig.detailLevel === 'minimal') {
if (typeof data === 'string' && data.length > 100) {
return data.slice(0, 100) + '...[truncated]';
}
if (Array.isArray(data) && data.length > 3) {
return [...data.slice(0, 3), `...[${data.length - 3} more items]`];
}
} else if (this.auditConfig.detailLevel === 'standard') {
if (typeof data === 'string' && data.length > 500) {
return data.slice(0, 500) + '...[truncated]';
}
if (Array.isArray(data) && data.length > 10) {
return [...data.slice(0, 10), `...[${data.length - 10} more items]`];
}
}
return data;
}
private calculateSelectionConfidence(result: any, candidateCount: number): number {
if (!result || !result.selectedTools) return 30;
const selectionRatio = result.selectedTools.length / candidateCount;
const hasReasoning = result.reasoning && result.reasoning.length > 50;
let confidence = 60; // Base confidence
// Good selection ratio (not too many, not too few)
if (selectionRatio > 0.05 && selectionRatio < 0.3) confidence += 20;
else if (selectionRatio <= 0.05) confidence -= 10; // Too few
else confidence -= 15; // Too many
// Has detailed reasoning
if (hasReasoning) confidence += 15;
// Selected tools have good distribution
if (result.selectedConcepts && result.selectedConcepts.length > 0) confidence += 5;
return Math.min(95, Math.max(25, confidence));
}
private estimateTokens(text: string): number { private estimateTokens(text: string): number {
return Math.ceil(text.length / 4); return Math.ceil(text.length / 4);
} }
@ -140,6 +261,7 @@ class ImprovedMicroTaskAIPipeline {
let selectionMethod = 'unknown'; let selectionMethod = 'unknown';
if (embeddingsService.isEnabled()) { if (embeddingsService.isEnabled()) {
const embeddingsStart = Date.now();
const similarItems = await embeddingsService.findSimilar( const similarItems = await embeddingsService.findSimilar(
userQuery, userQuery,
this.embeddingCandidates, this.embeddingCandidates,
@ -168,6 +290,17 @@ class ImprovedMicroTaskAIPipeline {
candidateConcepts = toolsData.concepts; candidateConcepts = toolsData.concepts;
selectionMethod = 'full_dataset'; selectionMethod = 'full_dataset';
} }
// NEW: Add Audit Entry for Embeddings Search
if (this.auditConfig.enabled) {
this.addAuditEntry(null, 'retrieval', 'embeddings-search',
{ query: userQuery, threshold: this.similarityThreshold, candidates: this.embeddingCandidates },
{ candidatesFound: similarItems.length, toolNames: Array.from(toolNames), conceptNames: Array.from(conceptNames) },
similarItems.length >= 15 ? 85 : 60, // Confidence based on result quality
embeddingsStart,
{ selectionMethod, embeddingsEnabled: true }
);
}
} else { } else {
console.log(`[IMPROVED PIPELINE] Embeddings disabled, using full dataset`); console.log(`[IMPROVED PIPELINE] Embeddings disabled, using full dataset`);
candidateTools = toolsData.tools; candidateTools = toolsData.tools;
@ -194,6 +327,8 @@ class ImprovedMicroTaskAIPipeline {
mode: string, mode: string,
selectionMethod: string selectionMethod: string
) { ) {
const selectionStart = Date.now();
const modeInstruction = mode === 'workflow' const modeInstruction = mode === 'workflow'
? 'The user wants a COMPREHENSIVE WORKFLOW with multiple tools/methods across different phases. Select 15-25 tools that cover the full investigation lifecycle.' ? 'The user wants a COMPREHENSIVE WORKFLOW with multiple tools/methods across different phases. Select 15-25 tools that cover the full investigation lifecycle.'
: 'The user wants SPECIFIC TOOLS/METHODS that directly solve their particular problem. Select 3-8 tools that are most relevant and effective.'; : 'The user wants SPECIFIC TOOLS/METHODS that directly solve their particular problem. Select 3-8 tools that are most relevant and effective.';
@ -298,6 +433,18 @@ Respond with ONLY this JSON format:
if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) { if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) {
console.error('[IMPROVED PIPELINE] AI selection returned invalid structure:', response.slice(0, 200)); console.error('[IMPROVED PIPELINE] AI selection returned invalid structure:', response.slice(0, 200));
// NEW: Add Audit Entry for Failed Selection
if (this.auditConfig.enabled) {
this.addAuditEntry(null, 'selection', 'ai-tool-selection-failed',
{ candidateCount: candidateTools.length, mode, prompt: prompt.slice(0, 200) },
{ error: 'Invalid JSON structure', response: response.slice(0, 200) },
10, // Very low confidence
selectionStart,
{ aiModel: this.config.model, selectionMethod }
);
}
throw new Error('AI selection failed to return valid tool selection'); throw new Error('AI selection failed to return valid tool selection');
} }
@ -315,6 +462,24 @@ Respond with ONLY this JSON format:
console.log(`[IMPROVED PIPELINE] Final selection: ${selectedTools.length} tools with bias prevention applied`); console.log(`[IMPROVED PIPELINE] Final selection: ${selectedTools.length} tools with bias prevention applied`);
// NEW: Add Audit Entry for Successful Selection
if (this.auditConfig.enabled) {
const confidence = this.calculateSelectionConfidence(result, candidateTools.length);
this.addAuditEntry(null, 'selection', 'ai-tool-selection',
{ candidateCount: candidateTools.length, mode, promptLength: prompt.length },
{
selectedToolCount: result.selectedTools.length,
selectedConceptCount: result.selectedConcepts.length,
reasoning: result.reasoning?.slice(0, 200) + '...',
finalToolNames: selectedTools.map(t => t.name)
},
confidence,
selectionStart,
{ aiModel: this.config.model, selectionMethod, promptTokens: this.estimateTokens(prompt) }
);
}
return { return {
selectedTools, selectedTools,
selectedConcepts selectedConcepts
@ -323,12 +488,25 @@ Respond with ONLY this JSON format:
} catch (error) { } catch (error) {
console.error('[IMPROVED PIPELINE] AI selection failed:', error); console.error('[IMPROVED PIPELINE] AI selection failed:', error);
// NEW: Add Audit Entry for Selection Error
if (this.auditConfig.enabled) {
this.addAuditEntry(null, 'selection', 'ai-tool-selection-error',
{ candidateCount: candidateTools.length, mode },
{ error: error.message },
5, // Very low confidence
selectionStart,
{ aiModel: this.config.model, selectionMethod }
);
}
console.log('[IMPROVED PIPELINE] Using emergency keyword-based selection'); console.log('[IMPROVED PIPELINE] Using emergency keyword-based selection');
return this.emergencyKeywordSelection(userQuery, candidateTools, candidateConcepts, mode); return this.emergencyKeywordSelection(userQuery, candidateTools, candidateConcepts, mode);
} }
} }
private emergencyKeywordSelection(userQuery: string, candidateTools: any[], candidateConcepts: any[], mode: string) { private emergencyKeywordSelection(userQuery: string, candidateTools: any[], candidateConcepts: any[], mode: string) {
const emergencyStart = Date.now();
const queryLower = userQuery.toLowerCase(); const queryLower = userQuery.toLowerCase();
const keywords = queryLower.split(/\s+/).filter(word => word.length > 3); const keywords = queryLower.split(/\s+/).filter(word => word.length > 3);
@ -354,6 +532,17 @@ Respond with ONLY this JSON format:
console.log(`[IMPROVED PIPELINE] Emergency selection: ${selectedTools.length} tools, keywords: ${keywords.slice(0, 5).join(', ')}`); console.log(`[IMPROVED PIPELINE] Emergency selection: ${selectedTools.length} tools, keywords: ${keywords.slice(0, 5).join(', ')}`);
// NEW: Add Audit Entry for Emergency Selection
if (this.auditConfig.enabled) {
this.addAuditEntry(null, 'selection', 'emergency-keyword-selection',
{ keywords: keywords.slice(0, 10), candidateCount: candidateTools.length },
{ selectedCount: selectedTools.length, topScores: scoredTools.slice(0, 5).map(s => ({ name: s.tool.name, score: s.score })) },
40, // Moderate confidence for emergency selection
emergencyStart,
{ selectionMethod: 'emergency_keyword' }
);
}
return { return {
selectedTools, selectedTools,
selectedConcepts: candidateConcepts.slice(0, 3) selectedConcepts: candidateConcepts.slice(0, 3)
@ -382,21 +571,43 @@ Respond with ONLY this JSON format:
try { try {
const response = await this.callAI(contextPrompt, maxTokens); const response = await this.callAI(contextPrompt, maxTokens);
return { const result = {
taskType: 'micro-task', taskType: 'micro-task',
content: response.trim(), content: response.trim(),
processingTimeMs: Date.now() - startTime, processingTimeMs: Date.now() - startTime,
success: true success: true
}; };
// NEW: Add Audit Entry for Successful Micro-Task
this.addAuditEntry(context, 'micro-task', 'ai-analysis',
{ promptLength: contextPrompt.length, maxTokens },
{ responseLength: response.length, contentPreview: response.slice(0, 100) },
response.length > 50 ? 80 : 60, // Confidence based on response quality
startTime,
{ aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 }
);
return result;
} catch (error) { } catch (error) {
return { const result = {
taskType: 'micro-task', taskType: 'micro-task',
content: '', content: '',
processingTimeMs: Date.now() - startTime, processingTimeMs: Date.now() - startTime,
success: false, success: false,
error: error.message error: error.message
}; };
// NEW: Add Audit Entry for Failed Micro-Task
this.addAuditEntry(context, 'micro-task', 'ai-analysis-failed',
{ promptLength: contextPrompt.length, maxTokens },
{ error: error.message },
5, // Very low confidence
startTime,
{ aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 }
);
return result;
} }
} }
@ -550,6 +761,15 @@ Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format (kein zusätzlicher Text):
this.addToolToSelection(context, tool, phase.id, sel.priority, sel.justification); this.addToolToSelection(context, tool, phase.id, sel.priority, sel.justification);
} }
}); });
// NEW: Add audit entry for tool selection
this.addAuditEntry(context, 'micro-task', 'phase-tool-selection',
{ phase: phase.id, availableTools: phaseTools.length },
{ validSelections: validSelections.length, selectedTools: validSelections.map(s => s.toolName) },
validSelections.length > 0 ? 75 : 30,
Date.now() - result.processingTimeMs,
{ phaseName: phase.name }
);
} }
} }
@ -595,6 +815,15 @@ Bewerten Sie nach forensischen Standards und antworten Sie AUSSCHLIESSLICH mit d
rank rank
} }
}, 'evaluation', evaluation.suitability_score); }, 'evaluation', evaluation.suitability_score);
// NEW: Add audit entry for tool evaluation
this.addAuditEntry(context, 'micro-task', 'tool-evaluation',
{ toolName: tool.name, rank },
{ suitabilityScore: evaluation.suitability_score, hasExplanation: !!evaluation.detailed_explanation },
evaluation.suitability_score === 'high' ? 85 : evaluation.suitability_score === 'medium' ? 70 : 50,
Date.now() - result.processingTimeMs,
{ toolType: tool.type }
);
} }
return result; return result;
@ -644,6 +873,15 @@ Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format:
concept: availableConcepts.find((c: any) => c.name === sel.conceptName), concept: availableConcepts.find((c: any) => c.name === sel.conceptName),
relevance: sel.relevance relevance: sel.relevance
})); }));
// NEW: Add audit entry for background knowledge selection
this.addAuditEntry(context, 'micro-task', 'background-knowledge-selection',
{ availableConcepts: availableConcepts.length },
{ selectedConcepts: context.backgroundKnowledge?.length || 0 },
context.backgroundKnowledge && context.backgroundKnowledge.length > 0 ? 75 : 40,
Date.now() - result.processingTimeMs,
{}
);
} }
} }
@ -711,7 +949,10 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
let completedTasks = 0; let completedTasks = 0;
let failedTasks = 0; let failedTasks = 0;
console.log(`[IMPROVED PIPELINE] Starting ${mode} query processing with context continuity`); // NEW: Clear any previous temporary audit entries
this.tempAuditEntries = [];
console.log(`[IMPROVED PIPELINE] Starting ${mode} query processing with context continuity and audit trail`);
try { try {
// Stage 1: Get intelligent candidates (embeddings + AI selection) // Stage 1: Get intelligent candidates (embeddings + AI selection)
@ -725,11 +966,25 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
contextHistory: [], contextHistory: [],
maxContextLength: this.maxContextTokens, maxContextLength: this.maxContextTokens,
currentContextLength: 0, currentContextLength: 0,
seenToolNames: new Set<string>() seenToolNames: new Set<string>(),
// NEW: Initialize audit trail
auditTrail: []
}; };
// NEW: Merge any temporary audit entries from pre-context operations
this.mergeTemporaryAuditEntries(context);
console.log(`[IMPROVED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`); console.log(`[IMPROVED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`);
// NEW: Add initial audit entry
this.addAuditEntry(context, 'initialization', 'pipeline-start',
{ userQuery, mode, toolsDataLoaded: !!toolsData },
{ candidateTools: filteredData.tools.length, candidateConcepts: filteredData.concepts.length },
90, // High confidence for initialization
startTime,
{ auditEnabled: this.auditConfig.enabled }
);
// MICRO-TASK SEQUENCE // MICRO-TASK SEQUENCE
// Task 1: Scenario/Problem Analysis // Task 1: Scenario/Problem Analysis
@ -776,6 +1031,15 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
// Build final recommendation // Build final recommendation
const recommendation = this.buildRecommendation(context, mode, finalResult.content); const recommendation = this.buildRecommendation(context, mode, finalResult.content);
// NEW: Add final audit entry
this.addAuditEntry(context, 'completion', 'pipeline-end',
{ completedTasks, failedTasks },
{ finalRecommendation: !!recommendation, auditEntriesGenerated: context.auditTrail.length },
completedTasks > failedTasks ? 85 : 60,
startTime,
{ totalProcessingTimeMs: Date.now() - startTime }
);
const processingStats = { const processingStats = {
embeddingsUsed: embeddingsService.isEnabled(), embeddingsUsed: embeddingsService.isEnabled(),
candidatesFromEmbeddings: filteredData.tools.length, candidatesFromEmbeddings: filteredData.tools.length,
@ -789,14 +1053,23 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
console.log(`[IMPROVED PIPELINE] Completed: ${completedTasks} tasks, Failed: ${failedTasks} tasks`); console.log(`[IMPROVED PIPELINE] Completed: ${completedTasks} tasks, Failed: ${failedTasks} tasks`);
console.log(`[IMPROVED PIPELINE] Unique tools selected: ${context.seenToolNames.size}`); console.log(`[IMPROVED PIPELINE] Unique tools selected: ${context.seenToolNames.size}`);
console.log(`[IMPROVED PIPELINE] Audit trail entries: ${context.auditTrail.length}`);
return { return {
recommendation, recommendation: {
...recommendation,
// NEW: Include audit trail in response
auditTrail: this.auditConfig.enabled ? context.auditTrail : undefined
},
processingStats processingStats
}; };
} catch (error) { } catch (error) {
console.error('[IMPROVED PIPELINE] Processing failed:', error); console.error('[IMPROVED PIPELINE] Processing failed:', error);
// NEW: Ensure temp audit entries are cleared even on error
this.tempAuditEntries = [];
throw error; throw error;
} }
} }