diff --git a/.env.example b/.env.example index cf2ad4b..db539be 100644 --- a/.env.example +++ b/.env.example @@ -1,79 +1,154 @@ -# =========================================== +# ============================================================================ # ForensicPathways Environment Configuration -# =========================================== +# ============================================================================ +# Copy this file to .env and adjust the values below. +# Settings are ordered by likelihood of needing adjustment during setup. -# === Authentication Configuration === +# ============================================================================ +# 1. CORE APPLICATION SETTINGS (REQUIRED - ADJUST FOR YOUR SETUP) +# ============================================================================ + +# Your application's public URL (used for redirects and links) +PUBLIC_BASE_URL=http://localhost:4321 + +# Application environment (development, production, staging) +NODE_ENV=development + +# Secret key for session encryption (CHANGE IN PRODUCTION!) +AUTH_SECRET=your-secret-key-change-in-production-please + +# ============================================================================ +# 2. AI SERVICES CONFIGURATION (REQUIRED FOR AI FEATURES) +# ============================================================================ + +# Main AI Analysis Service (for query processing and recommendations) +# Example uses Mistral AI - adjust endpoint/model as needed +AI_ANALYZER_ENDPOINT=https://api.mistral.ai/v1 +AI_ANALYZER_API_KEY=your-mistral-api-key-here +AI_ANALYZER_MODEL=mistral-small-latest + +# Vector Embeddings Service (for semantic search - can use same provider) +AI_EMBEDDINGS_ENABLED=true +AI_EMBEDDINGS_ENDPOINT=https://api.mistral.ai/v1/embeddings +AI_EMBEDDINGS_API_KEY=your-mistral-api-key-here +AI_EMBEDDINGS_MODEL=mistral-embed + +# ============================================================================ +# 3. AUTHENTICATION (OPTIONAL - SET TO 'true' IF NEEDED) +# ============================================================================ + +# Enable authentication for different features AUTHENTICATION_NECESSARY=false AUTHENTICATION_NECESSARY_CONTRIBUTIONS=false AUTHENTICATION_NECESSARY_AI=false -AUTH_SECRET=your-secret-key-change-in-production -# OIDC Configuration (if authentication enabled) +# OIDC Provider Settings (only needed if authentication enabled) OIDC_ENDPOINT=https://your-oidc-provider.com OIDC_CLIENT_ID=your-client-id OIDC_CLIENT_SECRET=your-client-secret -# =================================================================== -# AI CONFIGURATION - Complete Reference for Improved Pipeline -# =================================================================== +# ============================================================================ +# 4. ADVANCED AI CONFIGURATION (FINE-TUNING - DEFAULT VALUES USUALLY WORK) +# ============================================================================ -# === CORE AI ENDPOINTS & MODELS === -AI_API_ENDPOINT=https://llm.mikoshi.de -AI_API_KEY=sREDACTED3w -AI_MODEL='mistral/mistral-small-latest' +# Pipeline Performance Settings +AI_MAX_SELECTED_ITEMS=60 # Tools analyzed per micro-task +AI_EMBEDDING_CANDIDATES=60 # Vector search candidates +AI_MICRO_TASK_DELAY_MS=500 # Delay between AI micro-tasks -# === IMPROVED PIPELINE: Use separate analyzer model (mistral-small is fine) === -AI_ANALYZER_ENDPOINT=https://llm.mikoshi.de -AI_ANALYZER_API_KEY=skREDACTEDw3w -AI_ANALYZER_MODEL='mistral/mistral-small-latest' +# Rate Limiting (requests per minute) +AI_RATE_LIMIT_MAX_REQUESTS=6 # Main query rate limit +AI_MICRO_TASK_RATE_LIMIT=15 # Micro-task rate limit +AI_RATE_LIMIT_DELAY_MS=3000 # Delay between rate-limited calls -# === EMBEDDINGS CONFIGURATION === -AI_EMBEDDINGS_ENABLED=true -AI_EMBEDDINGS_ENDPOINT=https://api.mistral.ai/v1/embeddings -AI_EMBEDDINGS_API_KEY=ZREDACTED3wL -AI_EMBEDDINGS_MODEL=mistral-embed -AI_EMBEDDINGS_BATCH_SIZE=20 -AI_EMBEDDINGS_BATCH_DELAY_MS=1000 +# Embeddings Batch Processing +AI_EMBEDDINGS_BATCH_SIZE=20 # Embeddings processed per batch +AI_EMBEDDINGS_BATCH_DELAY_MS=1000 # Delay between embedding batches -# === PIPELINE: VectorIndex (HNSW) Configuration === -AI_MAX_SELECTED_ITEMS=60 # Tools visible to each micro-task -AI_EMBEDDING_CANDIDATES=60 # VectorIndex candidates (HNSW is more efficient) -AI_SIMILARITY_THRESHOLD=0.3 # Not used by VectorIndex (uses cosine distance internally) +# Timeouts and Limits +AI_MICRO_TASK_TIMEOUT_MS=25000 # Max time per micro-task +AI_QUEUE_MAX_SIZE=50 # Max queued requests +AI_SIMILARITY_THRESHOLD=0.3 # Vector similarity threshold -# === MICRO-TASK CONFIGURATION === -AI_MICRO_TASK_DELAY_MS=500 # Delay between micro-tasks -AI_MICRO_TASK_TIMEOUT_MS=25000 # Timeout per micro-task (increased for full context) +# ============================================================================ +# 5. FORENSIC AUDIT SYSTEM (OPTIONAL - FOR TRANSPARENCY AND DEBUGGING) +# ============================================================================ -# === RATE LIMITING === -AI_RATE_LIMIT_DELAY_MS=3000 # Main rate limit delay -AI_RATE_LIMIT_MAX_REQUESTS=6 # Main requests per minute (reduced - fewer but richer calls) -AI_MICRO_TASK_RATE_LIMIT=15 # Micro-task requests per minute (was 30) +# Enable detailed audit trail of AI decision-making +FORENSIC_AUDIT_ENABLED=false -# === QUEUE MANAGEMENT === -AI_QUEUE_MAX_SIZE=50 -AI_QUEUE_CLEANUP_INTERVAL_MS=300000 +# Audit detail level: minimal, standard, verbose +FORENSIC_AUDIT_DETAIL_LEVEL=standard -# === PERFORMANCE & MONITORING === -AI_MICRO_TASK_DEBUG=true -AI_PERFORMANCE_METRICS=true -AI_RESPONSE_CACHE_TTL_MS=3600000 +# Audit retention and limits +FORENSIC_AUDIT_RETENTION_HOURS=72 # Keep audit data for 3 days +FORENSIC_AUDIT_MAX_ENTRIES=50 # Max entries per request -# =================================================================== -# LEGACY VARIABLES (still used but less important) -# =================================================================== +# ============================================================================ +# 6. QUALITY CONTROL AND BIAS DETECTION (OPTIONAL - ADVANCED FEATURES) +# ============================================================================ -# These are still used by other parts of the system: -AI_RESPONSE_CACHE_TTL_MS=3600000 # For caching responses -AI_QUEUE_MAX_SIZE=50 # Queue management -AI_QUEUE_CLEANUP_INTERVAL_MS=300000 # Queue cleanup +# Confidence Scoring Weights (must sum to 1.0) +CONFIDENCE_EMBEDDINGS_WEIGHT=0.3 +CONFIDENCE_CONSENSUS_WEIGHT=0.25 +CONFIDENCE_DOMAIN_MATCH_WEIGHT=0.25 +CONFIDENCE_FRESHNESS_WEIGHT=0.2 -# === Application Configuration === -PUBLIC_BASE_URL=http://localhost:4321 -NODE_ENV=development +# Confidence Thresholds (0-100) +CONFIDENCE_MINIMUM_THRESHOLD=40 +CONFIDENCE_MEDIUM_THRESHOLD=60 +CONFIDENCE_HIGH_THRESHOLD=80 -# Nextcloud Integration (Optional) -NEXTCLOUD_ENDPOINT=https://your-nextcloud.com -NEXTCLOUD_USERNAME=your-username -NEXTCLOUD_PASSWORD=your-password -NEXTCLOUD_UPLOAD_PATH=/kb-media -NEXTCLOUD_PUBLIC_URL=https://your-nextcloud.com/s/ \ No newline at end of file +# Bias Detection Settings +BIAS_DETECTION_ENABLED=false +BIAS_POPULARITY_THRESHOLD=0.7 # Detect over-popular tools +BIAS_DIVERSITY_MINIMUM=0.6 # Require recommendation diversity +BIAS_CELEBRITY_TOOLS="Volatility 3,Wireshark,Autopsy,Maltego" + +# Quality Control Thresholds +QUALITY_MIN_RESPONSE_LENGTH=50 # Minimum AI response length +QUALITY_MIN_SELECTION_COUNT=1 # Minimum tools selected +QUALITY_MAX_PROCESSING_TIME_MS=30000 # Max processing time + +# ============================================================================ +# 7. USER INTERFACE PREFERENCES (OPTIONAL - UI DEFAULTS) +# ============================================================================ + +# Default UI behavior (users can override) +UI_SHOW_AUDIT_TRAIL_DEFAULT=false +UI_SHOW_CONFIDENCE_SCORES=true +UI_SHOW_BIAS_WARNINGS=true +UI_AUDIT_TRAIL_COLLAPSIBLE=true + +# ============================================================================ +# 8. EXTERNAL INTEGRATIONS (OPTIONAL - ONLY IF USING THESE SERVICES) +# ============================================================================ + +# Nextcloud Integration (for file uploads) +# NEXTCLOUD_ENDPOINT=https://your-nextcloud.com +# NEXTCLOUD_USERNAME=your-username +# NEXTCLOUD_PASSWORD=your-password +# NEXTCLOUD_UPLOAD_PATH=/kb-media +# NEXTCLOUD_PUBLIC_URL=https://your-nextcloud.com/s/ + +# ============================================================================ +# 9. PERFORMANCE AND MONITORING (OPTIONAL - FOR PRODUCTION OPTIMIZATION) +# ============================================================================ + +# Caching and Queue Management +AI_RESPONSE_CACHE_TTL_MS=3600000 # Cache responses for 1 hour +AI_QUEUE_CLEANUP_INTERVAL_MS=300000 # Cleanup queue every 5 minutes + +# Debug and Monitoring +AI_MICRO_TASK_DEBUG=false # Enable detailed micro-task logging +AI_PERFORMANCE_METRICS=false # Enable performance tracking + +# ============================================================================ +# SETUP CHECKLIST: +# ============================================================================ +# 1. Set PUBLIC_BASE_URL to your domain +# 2. Change AUTH_SECRET to a secure random string +# 3. Configure AI service endpoints and API keys +# 4. Set authentication options if needed +# 5. Test with default advanced settings before adjusting +# ============================================================================ \ No newline at end of file diff --git a/README.md b/README.md index 89aac12..b6f72a2 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Ein kuratiertes Verzeichnis für Digital Forensics und Incident Response (DFIR) ### AI Service (Mistral/OpenAI-kompatibel) - **Zweck:** KI-gestützte Tool-Empfehlungen -- **Konfiguration:** `AI_API_ENDPOINT`, `AI_API_KEY`, `AI_MODEL` +- **Konfiguration:** `AI_ANALYZER_ENDPOINT`, `AI_ANALYZER_API_KEY`, `AI_ANALYZER_MODEL` ### Uptime Kuma - **Zweck:** Status-Monitoring für gehostete Services @@ -157,9 +157,9 @@ PUBLIC_BASE_URL=https://your-domain.com NODE_ENV=production # AI Service Configuration (Required for AI features) -AI_MODEL=mistral-large-latest -AI_API_ENDPOINT=https://api.mistral.ai -AI_API_KEY=your-mistral-api-key +AI_ANALYZER_MODEL=mistral-large-latest +AI_ANALYZER_ENDPOINT=https://api.mistral.ai +AI_ANALYZER_API_KEY=your-mistral-api-key AI_RATE_LIMIT_DELAY_MS=1000 # Git Integration (Required for contributions) diff --git a/src/components/AIQueryInterface.astro b/src/components/AIQueryInterface.astro index 2061bed..585b4cf 100644 --- a/src/components/AIQueryInterface.astro +++ b/src/components/AIQueryInterface.astro @@ -711,6 +711,7 @@ class AIQueryInterface { ${this.renderBackgroundKnowledge(recommendation.background_knowledge)} ${this.renderWorkflowPhases(toolsByPhase, phaseOrder, phaseNames)} ${this.renderWorkflowSuggestion(recommendation.workflow_suggestion)} + ${this.renderAuditTrail(recommendation.auditTrail)} `; @@ -725,12 +726,105 @@ class AIQueryInterface { ${this.renderBackgroundKnowledge(recommendation.background_knowledge)} ${this.renderToolRecommendations(recommendation.recommended_tools)} ${this.renderAdditionalConsiderations(recommendation.additional_considerations)} + ${this.renderAuditTrail(recommendation.auditTrail)} `; this.elements.results.innerHTML = html; } + // NEW: Audit Trail Rendering Functions + renderAuditTrail(auditTrail) { + if (!auditTrail || !Array.isArray(auditTrail) || auditTrail.length === 0) { + return ''; + } + + // Reuse existing card and info styles from global.css + return ` +
+
+ + + + + + +

Forensic Audit Trail (${auditTrail.length} Entries)

+ +
+ +
+ `; + } + + renderAuditEntry(entry) { + const confidenceColor = entry.confidence >= 80 ? 'var(--color-accent)' : + entry.confidence >= 60 ? 'var(--color-warning)' : 'var(--color-error)'; + + const formattedTime = new Date(entry.timestamp).toLocaleTimeString('de-DE', { + hour: '2-digit', + minute: '2-digit', + second: '2-digit' + }); + + // Reuse existing grid and text utilities + return ` +
+
+ ${entry.phase} → ${entry.action} +
+ + ${entry.confidence}% confidence + + ${entry.processingTimeMs}ms + ${formattedTime} +
+
+
+
Input: ${this.formatAuditData(entry.input)}
+
Output: ${this.formatAuditData(entry.output)}
+
+ ${entry.metadata && Object.keys(entry.metadata).length > 0 ? ` +
+ Metadata: ${this.formatAuditData(entry.metadata)} +
+ ` : ''} +
+ `; + } + + formatAuditData(data) { + if (data === null || data === undefined) return 'null'; + if (typeof data === 'string') { + return data.length > 100 ? data.slice(0, 100) + '...' : data; + } + if (typeof data === 'number') return data.toString(); + if (typeof data === 'boolean') return data.toString(); + if (Array.isArray(data)) { + if (data.length === 0) return '[]'; + if (data.length <= 3) return JSON.stringify(data); + return `[${data.slice(0, 3).map(i => typeof i === 'string' ? i : JSON.stringify(i)).join(', ')}, ...+${data.length - 3}]`; + } + if (typeof data === 'object') { + const keys = Object.keys(data); + if (keys.length === 0) return '{}'; + if (keys.length <= 3) { + return '{' + keys.map(k => `${k}: ${typeof data[k] === 'string' ? data[k].slice(0, 20) + (data[k].length > 20 ? '...' : '') : JSON.stringify(data[k])}`).join(', ') + '}'; + } + return `{${keys.slice(0, 3).join(', ')}, ...+${keys.length - 3} keys}`; + } + return String(data); + } + renderHeader(title, query) { return `
diff --git a/src/config/forensic.config.ts b/src/config/forensic.config.ts new file mode 100644 index 0000000..5723854 --- /dev/null +++ b/src/config/forensic.config.ts @@ -0,0 +1,126 @@ +// src/config/forensic.config.ts +// Centralized configuration for forensic RAG enhancements + +export const FORENSIC_CONFIG = { + audit: { + enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true', + detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as 'minimal' | 'standard' | 'verbose') || 'standard', + retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10), + maxEntriesPerRequest: parseInt(process.env.FORENSIC_AUDIT_MAX_ENTRIES || '50', 10) + }, + confidence: { + embeddingsWeight: parseFloat(process.env.CONFIDENCE_EMBEDDINGS_WEIGHT || '0.3'), + consensusWeight: parseFloat(process.env.CONFIDENCE_CONSENSUS_WEIGHT || '0.25'), + domainMatchWeight: parseFloat(process.env.CONFIDENCE_DOMAIN_MATCH_WEIGHT || '0.25'), + freshnessWeight: parseFloat(process.env.CONFIDENCE_FRESHNESS_WEIGHT || '0.2'), + minimumThreshold: parseInt(process.env.CONFIDENCE_MINIMUM_THRESHOLD || '40', 10), + highThreshold: parseInt(process.env.CONFIDENCE_HIGH_THRESHOLD || '80', 10), + mediumThreshold: parseInt(process.env.CONFIDENCE_MEDIUM_THRESHOLD || '60', 10) + }, + bias: { + enabled: process.env.BIAS_DETECTION_ENABLED === 'true', + popularityThreshold: parseFloat(process.env.BIAS_POPULARITY_THRESHOLD || '0.7'), + diversityMinimum: parseFloat(process.env.BIAS_DIVERSITY_MINIMUM || '0.6'), + domainMismatchThreshold: parseFloat(process.env.BIAS_DOMAIN_MISMATCH_THRESHOLD || '0.3'), + warningThreshold: parseInt(process.env.BIAS_WARNING_THRESHOLD || '3', 10), + celebrityTools: (process.env.BIAS_CELEBRITY_TOOLS || 'Volatility 3,Wireshark,Autopsy,Maltego').split(',').map(t => t.trim()) + }, + // Quality thresholds for various metrics + quality: { + minResponseLength: parseInt(process.env.QUALITY_MIN_RESPONSE_LENGTH || '50', 10), + minSelectionCount: parseInt(process.env.QUALITY_MIN_SELECTION_COUNT || '1', 10), + maxProcessingTime: parseInt(process.env.QUALITY_MAX_PROCESSING_TIME_MS || '30000', 10) + }, + // Display preferences + ui: { + showAuditTrailByDefault: process.env.UI_SHOW_AUDIT_TRAIL_DEFAULT === 'true', + showConfidenceScores: process.env.UI_SHOW_CONFIDENCE_SCORES !== 'false', + showBiasWarnings: process.env.UI_SHOW_BIAS_WARNINGS !== 'false', + auditTrailCollapsible: process.env.UI_AUDIT_TRAIL_COLLAPSIBLE !== 'false' + } +}; + +// Validation function to ensure configuration is valid +export function validateForensicConfig(): { valid: boolean; errors: string[] } { + const errors: string[] = []; + + // Validate audit configuration + if (FORENSIC_CONFIG.audit.retentionHours < 1 || FORENSIC_CONFIG.audit.retentionHours > 168) { + errors.push('FORENSIC_AUDIT_RETENTION_HOURS must be between 1 and 168 (1 week)'); + } + + if (!['minimal', 'standard', 'verbose'].includes(FORENSIC_CONFIG.audit.detailLevel)) { + errors.push('FORENSIC_AUDIT_DETAIL_LEVEL must be one of: minimal, standard, verbose'); + } + + // Validate confidence weights sum to approximately 1.0 + const weightSum = FORENSIC_CONFIG.confidence.embeddingsWeight + + FORENSIC_CONFIG.confidence.consensusWeight + + FORENSIC_CONFIG.confidence.domainMatchWeight + + FORENSIC_CONFIG.confidence.freshnessWeight; + + if (Math.abs(weightSum - 1.0) > 0.05) { + errors.push(`Confidence weights must sum to 1.0 (currently ${weightSum.toFixed(3)})`); + } + + // Validate threshold ranges + if (FORENSIC_CONFIG.confidence.minimumThreshold < 0 || FORENSIC_CONFIG.confidence.minimumThreshold > 100) { + errors.push('CONFIDENCE_MINIMUM_THRESHOLD must be between 0 and 100'); + } + + if (FORENSIC_CONFIG.confidence.highThreshold <= FORENSIC_CONFIG.confidence.mediumThreshold) { + errors.push('CONFIDENCE_HIGH_THRESHOLD must be greater than CONFIDENCE_MEDIUM_THRESHOLD'); + } + + // Validate bias thresholds + if (FORENSIC_CONFIG.bias.popularityThreshold < 0 || FORENSIC_CONFIG.bias.popularityThreshold > 1) { + errors.push('BIAS_POPULARITY_THRESHOLD must be between 0 and 1'); + } + + if (FORENSIC_CONFIG.bias.diversityMinimum < 0 || FORENSIC_CONFIG.bias.diversityMinimum > 1) { + errors.push('BIAS_DIVERSITY_MINIMUM must be between 0 and 1'); + } + + return { + valid: errors.length === 0, + errors + }; +} + +// Helper functions for configuration access +export function isAuditEnabled(): boolean { + return FORENSIC_CONFIG.audit.enabled; +} + +export function getAuditDetailLevel(): 'minimal' | 'standard' | 'verbose' { + return FORENSIC_CONFIG.audit.detailLevel; +} + +export function getConfidenceThresholds() { + return { + minimum: FORENSIC_CONFIG.confidence.minimumThreshold, + medium: FORENSIC_CONFIG.confidence.mediumThreshold, + high: FORENSIC_CONFIG.confidence.highThreshold + }; +} + +export function isBiasDetectionEnabled(): boolean { + return FORENSIC_CONFIG.bias.enabled; +} + +// Initialize and validate configuration on module load +const configValidation = validateForensicConfig(); +if (!configValidation.valid) { + console.warn('[FORENSIC CONFIG] Configuration validation failed:', configValidation.errors); + // In development, we might want to throw an error + if (process.env.NODE_ENV === 'development') { + throw new Error(`Forensic configuration invalid: ${configValidation.errors.join(', ')}`); + } +} + +console.log('[FORENSIC CONFIG] Configuration loaded:', { + auditEnabled: FORENSIC_CONFIG.audit.enabled, + confidenceEnabled: true, // Always enabled + biasDetectionEnabled: FORENSIC_CONFIG.bias.enabled, + detailLevel: FORENSIC_CONFIG.audit.detailLevel +}); \ No newline at end of file diff --git a/src/pages/api/ai/enhance-input.ts b/src/pages/api/ai/enhance-input.ts index d21c317..0d12a09 100644 --- a/src/pages/api/ai/enhance-input.ts +++ b/src/pages/api/ai/enhance-input.ts @@ -15,8 +15,8 @@ function getEnv(key: string): string { } const AI_ENDPOINT = getEnv('AI_ANALYZER_ENDPOINT'); -const AI_API_KEY = getEnv('AI_ANALYZER_API_KEY'); -const AI_MODEL = getEnv('AI_ANALYZER_MODEL'); +const AI_ANALYZER_API_KEY = getEnv('AI_ANALYZER_API_KEY'); +const AI_ANALYZER_MODEL = getEnv('AI_ANALYZER_MODEL'); const rateLimitStore = new Map(); const RATE_LIMIT_WINDOW = 60 * 1000; // 1 minute @@ -126,10 +126,10 @@ export const POST: APIRoute = async ({ request }) => { method: 'POST', headers: { 'Content-Type': 'application/json', - 'Authorization': `Bearer ${AI_API_KEY}` + 'Authorization': `Bearer ${AI_ANALYZER_API_KEY}` }, body: JSON.stringify({ - model: AI_MODEL, + model: AI_ANALYZER_MODEL, messages: [ { role: 'user', diff --git a/src/utils/aiPipeline.ts b/src/utils/aiPipeline.ts index ac727f7..f608077 100644 --- a/src/utils/aiPipeline.ts +++ b/src/utils/aiPipeline.ts @@ -1,4 +1,4 @@ -// src/utils/aiPipeline.ts +// src/utils/aiPipeline.ts - Enhanced with Audit Trail System import { getCompressedToolsDataForAI } from './dataService.js'; import { embeddingsService, type EmbeddingData } from './embeddings.js'; @@ -30,6 +30,19 @@ interface AnalysisResult { }; } +// NEW: Audit Trail Types +interface AuditEntry { + timestamp: number; + phase: string; // 'retrieval', 'selection', 'micro-task-N' + action: string; // 'embeddings-search', 'ai-selection', 'tool-evaluation' + input: any; // What went into this step + output: any; // What came out of this step + confidence: number; // 0-100: How confident we are in this step + processingTimeMs: number; + metadata: Record; // Additional context +} + +// Enhanced AnalysisContext with Audit Trail interface AnalysisContext { userQuery: string; mode: string; @@ -47,6 +60,9 @@ interface AnalysisContext { backgroundKnowledge?: Array<{concept: any, relevance: string}>; seenToolNames: Set; + + // NEW: Audit Trail + auditTrail: AuditEntry[]; } class ImprovedMicroTaskAIPipeline { @@ -58,6 +74,16 @@ class ImprovedMicroTaskAIPipeline { private maxContextTokens: number; private maxPromptTokens: number; + + // NEW: Audit Configuration + private auditConfig: { + enabled: boolean; + detailLevel: 'minimal' | 'standard' | 'verbose'; + retentionHours: number; + }; + + // NEW: Temporary audit storage for pre-context operations + private tempAuditEntries: AuditEntry[] = []; constructor() { this.config = { @@ -73,6 +99,13 @@ class ImprovedMicroTaskAIPipeline { this.maxContextTokens = parseInt(process.env.AI_MAX_CONTEXT_TOKENS || '4000', 10); this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10); + + // NEW: Initialize Audit Configuration + this.auditConfig = { + enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true', + detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as any) || 'standard', + retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10) + }; } private getEnv(key: string): string { @@ -83,6 +116,94 @@ class ImprovedMicroTaskAIPipeline { return value; } + // NEW: Audit Trail Utility Functions + private addAuditEntry( + context: AnalysisContext | null, + phase: string, + action: string, + input: any, + output: any, + confidence: number, + startTime: number, + metadata: Record = {} + ): void { + if (!this.auditConfig.enabled) return; + + const auditEntry: AuditEntry = { + timestamp: Date.now(), + phase, + action, + input: this.auditConfig.detailLevel === 'verbose' ? input : this.summarizeForAudit(input), + output: this.auditConfig.detailLevel === 'verbose' ? output : this.summarizeForAudit(output), + confidence, + processingTimeMs: Date.now() - startTime, + metadata + }; + + if (context) { + context.auditTrail.push(auditEntry); + } else { + // Store in temporary array for later merging + this.tempAuditEntries.push(auditEntry); + } + + // Log for debugging when audit is enabled + console.log(`[AUDIT] ${phase}/${action}: ${confidence}% confidence, ${Date.now() - startTime}ms`); + } + + // NEW: Merge temporary audit entries into context + private mergeTemporaryAuditEntries(context: AnalysisContext): void { + if (!this.auditConfig.enabled || this.tempAuditEntries.length === 0) return; + + const entryCount = this.tempAuditEntries.length; + // Add temp entries to the beginning of the context audit trail + context.auditTrail.unshift(...this.tempAuditEntries); + this.tempAuditEntries = []; // Clear temp storage + + console.log(`[AUDIT] Merged ${entryCount} temporary audit entries into context`); + } + + private summarizeForAudit(data: any): any { + if (this.auditConfig.detailLevel === 'minimal') { + if (typeof data === 'string' && data.length > 100) { + return data.slice(0, 100) + '...[truncated]'; + } + if (Array.isArray(data) && data.length > 3) { + return [...data.slice(0, 3), `...[${data.length - 3} more items]`]; + } + } else if (this.auditConfig.detailLevel === 'standard') { + if (typeof data === 'string' && data.length > 500) { + return data.slice(0, 500) + '...[truncated]'; + } + if (Array.isArray(data) && data.length > 10) { + return [...data.slice(0, 10), `...[${data.length - 10} more items]`]; + } + } + return data; + } + + private calculateSelectionConfidence(result: any, candidateCount: number): number { + if (!result || !result.selectedTools) return 30; + + const selectionRatio = result.selectedTools.length / candidateCount; + const hasReasoning = result.reasoning && result.reasoning.length > 50; + + let confidence = 60; // Base confidence + + // Good selection ratio (not too many, not too few) + if (selectionRatio > 0.05 && selectionRatio < 0.3) confidence += 20; + else if (selectionRatio <= 0.05) confidence -= 10; // Too few + else confidence -= 15; // Too many + + // Has detailed reasoning + if (hasReasoning) confidence += 15; + + // Selected tools have good distribution + if (result.selectedConcepts && result.selectedConcepts.length > 0) confidence += 5; + + return Math.min(95, Math.max(25, confidence)); + } + private estimateTokens(text: string): number { return Math.ceil(text.length / 4); } @@ -140,6 +261,7 @@ class ImprovedMicroTaskAIPipeline { let selectionMethod = 'unknown'; if (embeddingsService.isEnabled()) { + const embeddingsStart = Date.now(); const similarItems = await embeddingsService.findSimilar( userQuery, this.embeddingCandidates, @@ -168,6 +290,17 @@ class ImprovedMicroTaskAIPipeline { candidateConcepts = toolsData.concepts; selectionMethod = 'full_dataset'; } + + // NEW: Add Audit Entry for Embeddings Search + if (this.auditConfig.enabled) { + this.addAuditEntry(null, 'retrieval', 'embeddings-search', + { query: userQuery, threshold: this.similarityThreshold, candidates: this.embeddingCandidates }, + { candidatesFound: similarItems.length, toolNames: Array.from(toolNames), conceptNames: Array.from(conceptNames) }, + similarItems.length >= 15 ? 85 : 60, // Confidence based on result quality + embeddingsStart, + { selectionMethod, embeddingsEnabled: true } + ); + } } else { console.log(`[IMPROVED PIPELINE] Embeddings disabled, using full dataset`); candidateTools = toolsData.tools; @@ -194,6 +327,8 @@ class ImprovedMicroTaskAIPipeline { mode: string, selectionMethod: string ) { + const selectionStart = Date.now(); + const modeInstruction = mode === 'workflow' ? 'The user wants a COMPREHENSIVE WORKFLOW with multiple tools/methods across different phases. Select 15-25 tools that cover the full investigation lifecycle.' : 'The user wants SPECIFIC TOOLS/METHODS that directly solve their particular problem. Select 3-8 tools that are most relevant and effective.'; @@ -298,6 +433,18 @@ Respond with ONLY this JSON format: if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) { console.error('[IMPROVED PIPELINE] AI selection returned invalid structure:', response.slice(0, 200)); + + // NEW: Add Audit Entry for Failed Selection + if (this.auditConfig.enabled) { + this.addAuditEntry(null, 'selection', 'ai-tool-selection-failed', + { candidateCount: candidateTools.length, mode, prompt: prompt.slice(0, 200) }, + { error: 'Invalid JSON structure', response: response.slice(0, 200) }, + 10, // Very low confidence + selectionStart, + { aiModel: this.config.model, selectionMethod } + ); + } + throw new Error('AI selection failed to return valid tool selection'); } @@ -315,6 +462,24 @@ Respond with ONLY this JSON format: console.log(`[IMPROVED PIPELINE] Final selection: ${selectedTools.length} tools with bias prevention applied`); + // NEW: Add Audit Entry for Successful Selection + if (this.auditConfig.enabled) { + const confidence = this.calculateSelectionConfidence(result, candidateTools.length); + + this.addAuditEntry(null, 'selection', 'ai-tool-selection', + { candidateCount: candidateTools.length, mode, promptLength: prompt.length }, + { + selectedToolCount: result.selectedTools.length, + selectedConceptCount: result.selectedConcepts.length, + reasoning: result.reasoning?.slice(0, 200) + '...', + finalToolNames: selectedTools.map(t => t.name) + }, + confidence, + selectionStart, + { aiModel: this.config.model, selectionMethod, promptTokens: this.estimateTokens(prompt) } + ); + } + return { selectedTools, selectedConcepts @@ -323,12 +488,25 @@ Respond with ONLY this JSON format: } catch (error) { console.error('[IMPROVED PIPELINE] AI selection failed:', error); + // NEW: Add Audit Entry for Selection Error + if (this.auditConfig.enabled) { + this.addAuditEntry(null, 'selection', 'ai-tool-selection-error', + { candidateCount: candidateTools.length, mode }, + { error: error.message }, + 5, // Very low confidence + selectionStart, + { aiModel: this.config.model, selectionMethod } + ); + } + console.log('[IMPROVED PIPELINE] Using emergency keyword-based selection'); return this.emergencyKeywordSelection(userQuery, candidateTools, candidateConcepts, mode); } } private emergencyKeywordSelection(userQuery: string, candidateTools: any[], candidateConcepts: any[], mode: string) { + const emergencyStart = Date.now(); + const queryLower = userQuery.toLowerCase(); const keywords = queryLower.split(/\s+/).filter(word => word.length > 3); @@ -354,6 +532,17 @@ Respond with ONLY this JSON format: console.log(`[IMPROVED PIPELINE] Emergency selection: ${selectedTools.length} tools, keywords: ${keywords.slice(0, 5).join(', ')}`); + // NEW: Add Audit Entry for Emergency Selection + if (this.auditConfig.enabled) { + this.addAuditEntry(null, 'selection', 'emergency-keyword-selection', + { keywords: keywords.slice(0, 10), candidateCount: candidateTools.length }, + { selectedCount: selectedTools.length, topScores: scoredTools.slice(0, 5).map(s => ({ name: s.tool.name, score: s.score })) }, + 40, // Moderate confidence for emergency selection + emergencyStart, + { selectionMethod: 'emergency_keyword' } + ); + } + return { selectedTools, selectedConcepts: candidateConcepts.slice(0, 3) @@ -382,21 +571,43 @@ Respond with ONLY this JSON format: try { const response = await this.callAI(contextPrompt, maxTokens); - return { + const result = { taskType: 'micro-task', content: response.trim(), processingTimeMs: Date.now() - startTime, success: true }; + + // NEW: Add Audit Entry for Successful Micro-Task + this.addAuditEntry(context, 'micro-task', 'ai-analysis', + { promptLength: contextPrompt.length, maxTokens }, + { responseLength: response.length, contentPreview: response.slice(0, 100) }, + response.length > 50 ? 80 : 60, // Confidence based on response quality + startTime, + { aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 } + ); + + return result; } catch (error) { - return { + const result = { taskType: 'micro-task', content: '', processingTimeMs: Date.now() - startTime, success: false, error: error.message }; + + // NEW: Add Audit Entry for Failed Micro-Task + this.addAuditEntry(context, 'micro-task', 'ai-analysis-failed', + { promptLength: contextPrompt.length, maxTokens }, + { error: error.message }, + 5, // Very low confidence + startTime, + { aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 } + ); + + return result; } } @@ -550,6 +761,15 @@ Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format (kein zusätzlicher Text): this.addToolToSelection(context, tool, phase.id, sel.priority, sel.justification); } }); + + // NEW: Add audit entry for tool selection + this.addAuditEntry(context, 'micro-task', 'phase-tool-selection', + { phase: phase.id, availableTools: phaseTools.length }, + { validSelections: validSelections.length, selectedTools: validSelections.map(s => s.toolName) }, + validSelections.length > 0 ? 75 : 30, + Date.now() - result.processingTimeMs, + { phaseName: phase.name } + ); } } @@ -595,6 +815,15 @@ Bewerten Sie nach forensischen Standards und antworten Sie AUSSCHLIESSLICH mit d rank } }, 'evaluation', evaluation.suitability_score); + + // NEW: Add audit entry for tool evaluation + this.addAuditEntry(context, 'micro-task', 'tool-evaluation', + { toolName: tool.name, rank }, + { suitabilityScore: evaluation.suitability_score, hasExplanation: !!evaluation.detailed_explanation }, + evaluation.suitability_score === 'high' ? 85 : evaluation.suitability_score === 'medium' ? 70 : 50, + Date.now() - result.processingTimeMs, + { toolType: tool.type } + ); } return result; @@ -644,6 +873,15 @@ Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format: concept: availableConcepts.find((c: any) => c.name === sel.conceptName), relevance: sel.relevance })); + + // NEW: Add audit entry for background knowledge selection + this.addAuditEntry(context, 'micro-task', 'background-knowledge-selection', + { availableConcepts: availableConcepts.length }, + { selectedConcepts: context.backgroundKnowledge?.length || 0 }, + context.backgroundKnowledge && context.backgroundKnowledge.length > 0 ? 75 : 40, + Date.now() - result.processingTimeMs, + {} + ); } } @@ -711,7 +949,10 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo let completedTasks = 0; let failedTasks = 0; - console.log(`[IMPROVED PIPELINE] Starting ${mode} query processing with context continuity`); + // NEW: Clear any previous temporary audit entries + this.tempAuditEntries = []; + + console.log(`[IMPROVED PIPELINE] Starting ${mode} query processing with context continuity and audit trail`); try { // Stage 1: Get intelligent candidates (embeddings + AI selection) @@ -725,11 +966,25 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo contextHistory: [], maxContextLength: this.maxContextTokens, currentContextLength: 0, - seenToolNames: new Set() + seenToolNames: new Set(), + // NEW: Initialize audit trail + auditTrail: [] }; + // NEW: Merge any temporary audit entries from pre-context operations + this.mergeTemporaryAuditEntries(context); + console.log(`[IMPROVED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`); + // NEW: Add initial audit entry + this.addAuditEntry(context, 'initialization', 'pipeline-start', + { userQuery, mode, toolsDataLoaded: !!toolsData }, + { candidateTools: filteredData.tools.length, candidateConcepts: filteredData.concepts.length }, + 90, // High confidence for initialization + startTime, + { auditEnabled: this.auditConfig.enabled } + ); + // MICRO-TASK SEQUENCE // Task 1: Scenario/Problem Analysis @@ -776,6 +1031,15 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo // Build final recommendation const recommendation = this.buildRecommendation(context, mode, finalResult.content); + // NEW: Add final audit entry + this.addAuditEntry(context, 'completion', 'pipeline-end', + { completedTasks, failedTasks }, + { finalRecommendation: !!recommendation, auditEntriesGenerated: context.auditTrail.length }, + completedTasks > failedTasks ? 85 : 60, + startTime, + { totalProcessingTimeMs: Date.now() - startTime } + ); + const processingStats = { embeddingsUsed: embeddingsService.isEnabled(), candidatesFromEmbeddings: filteredData.tools.length, @@ -789,14 +1053,23 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo console.log(`[IMPROVED PIPELINE] Completed: ${completedTasks} tasks, Failed: ${failedTasks} tasks`); console.log(`[IMPROVED PIPELINE] Unique tools selected: ${context.seenToolNames.size}`); + console.log(`[IMPROVED PIPELINE] Audit trail entries: ${context.auditTrail.length}`); return { - recommendation, + recommendation: { + ...recommendation, + // NEW: Include audit trail in response + auditTrail: this.auditConfig.enabled ? context.auditTrail : undefined + }, processingStats }; } catch (error) { console.error('[IMPROVED PIPELINE] Processing failed:', error); + + // NEW: Ensure temp audit entries are cleared even on error + this.tempAuditEntries = []; + throw error; } }