mstoeck3/forensic-pathways
2
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Activity

audit trail phase 1 and 2

Browse Source
This commit is contained in:
overcuriousity 2025-08-02 12:57:38 +02:00
parent 57c507915f
commit fd05f8f291
7 changed files with 1367 additions and 449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
.env.example
View File

@ -13,16 +13,23 @@ OIDC_ENDPOINT=https://your-oidc-provider.com
OIDC_CLIENT_ID=your-client-id OIDC_CLIENT_ID=your-client-id
OIDC_CLIENT_SECRET=your-client-secret OIDC_CLIENT_SECRET=your-client-secret
# =================================================================== # === STRATEGIC AI MODEL (Large context, analytical reasoning, precise output) ===
# AI CONFIGURATION - Complete Reference for Improved Pipeline AI_STRATEGIC_ENDPOINT=https://llm.mikoshi.de
# =================================================================== AI_STRATEGIC_API_KEY=sREDACTED3w
AI_STRATEGIC_MODEL='mistral/mistral-large-latest'
AI_STRATEGIC_MAX_CONTEXT_TOKENS=32000
AI_STRATEGIC_MAX_OUTPUT_TOKENS=1000
AI_STRATEGIC_TEMPERATURE=0.2
# === CORE AI ENDPOINTS & MODELS === # === TACTICAL AI MODEL (Text generation, descriptions, cost-optimized) ===
AI_API_ENDPOINT=https://llm.mikoshi.de AI_TACTICAL_ENDPOINT=https://llm.mikoshi.de
AI_API_KEY=sREDACTED3w AI_TACTICAL_API_KEY=skREDACTEDw3w
AI_MODEL='mistral/mistral-small-latest' AI_TACTICAL_MODEL='mistral/mistral-small-latest'
AI_TACTICAL_MAX_CONTEXT_TOKENS=8000
AI_TACTICAL_MAX_OUTPUT_TOKENS=500
AI_TACTICAL_TEMPERATURE=0.3
# === IMPROVED PIPELINE: Use separate analyzer model (mistral-small is fine) === # === LEGACY COMPATIBILITY (DEPRECATED - will be removed in next version) ===
AI_ANALYZER_ENDPOINT=https://llm.mikoshi.de AI_ANALYZER_ENDPOINT=https://llm.mikoshi.de
AI_ANALYZER_API_KEY=skREDACTEDw3w AI_ANALYZER_API_KEY=skREDACTEDw3w
AI_ANALYZER_MODEL='mistral/mistral-small-latest' AI_ANALYZER_MODEL='mistral/mistral-small-latest'
@ -35,19 +42,31 @@ AI_EMBEDDINGS_MODEL=mistral-embed
AI_EMBEDDINGS_BATCH_SIZE=20 AI_EMBEDDINGS_BATCH_SIZE=20
AI_EMBEDDINGS_BATCH_DELAY_MS=1000 AI_EMBEDDINGS_BATCH_DELAY_MS=1000
# === PIPELINE: VectorIndex (HNSW) Configuration === # === FORENSIC ENHANCEMENT CONFIGURATION ===
AI_MAX_SELECTED_ITEMS=60 # Tools visible to each micro-task FORENSIC_AUDIT_ENABLED=true
AI_EMBEDDING_CANDIDATES=60 # VectorIndex candidates (HNSW is more efficient) FORENSIC_CONFIDENCE_SCORING_ENABLED=true
AI_SIMILARITY_THRESHOLD=0.3 # Not used by VectorIndex (uses cosine distance internally) FORENSIC_BIAS_DETECTION_ENABLED=true
FORENSIC_AUDIT_RETENTION_DAYS=90
FORENSIC_AUDIT_DETAIL_LEVEL=detailed
# === CONFIGURABLE THRESHOLDS (NO MORE HARD-CODED VALUES) ===
AI_MAX_SELECTED_ITEMS=60
AI_EMBEDDING_CANDIDATES=60
AI_SIMILARITY_THRESHOLD=0.3
AI_CONFIDENCE_THRESHOLD=0.7
AI_BIAS_ALERT_THRESHOLD=0.8
TOOL_POPULARITY_BIAS_THRESHOLD=0.75
EMBEDDINGS_CONFIDENCE_THRESHOLD=0.6
SELECTION_CONFIDENCE_MINIMUM=0.5
# === MICRO-TASK CONFIGURATION === # === MICRO-TASK CONFIGURATION ===
AI_MICRO_TASK_DELAY_MS=500 # Delay between micro-tasks AI_MICRO_TASK_DELAY_MS=500
AI_MICRO_TASK_TIMEOUT_MS=25000 # Timeout per micro-task (increased for full context) AI_MICRO_TASK_TIMEOUT_MS=25000
# === RATE LIMITING === # === RATE LIMITING ===
AI_RATE_LIMIT_DELAY_MS=3000 # Main rate limit delay AI_RATE_LIMIT_DELAY_MS=3000
AI_RATE_LIMIT_MAX_REQUESTS=6 # Main requests per minute (reduced - fewer but richer calls) AI_RATE_LIMIT_MAX_REQUESTS=6
AI_MICRO_TASK_RATE_LIMIT=15 # Micro-task requests per minute (was 30) AI_MICRO_TASK_RATE_LIMIT=15
# === QUEUE MANAGEMENT === # === QUEUE MANAGEMENT ===
AI_QUEUE_MAX_SIZE=50 AI_QUEUE_MAX_SIZE=50
@ -58,15 +77,6 @@ AI_MICRO_TASK_DEBUG=true
AI_PERFORMANCE_METRICS=true AI_PERFORMANCE_METRICS=true
AI_RESPONSE_CACHE_TTL_MS=3600000 AI_RESPONSE_CACHE_TTL_MS=3600000
# ===================================================================
# LEGACY VARIABLES (still used but less important)
# ===================================================================
# These are still used by other parts of the system:
AI_RESPONSE_CACHE_TTL_MS=3600000 # For caching responses
AI_QUEUE_MAX_SIZE=50 # Queue management
AI_QUEUE_CLEANUP_INTERVAL_MS=300000 # Queue cleanup
# === Application Configuration === # === Application Configuration ===
PUBLIC_BASE_URL=http://localhost:4321 PUBLIC_BASE_URL=http://localhost:4321
NODE_ENV=development NODE_ENV=development

90
src/pages/api/ai/query.ts
View File

@ -1,10 +1,11 @@
// src/pages/api/ai/query.ts - FIXED: Rate limiting for micro-task pipeline // src/pages/api/ai/query.ts - Enhanced with Forensic Audit Trail
import type { APIRoute } from 'astro'; import type { APIRoute } from 'astro';
import { withAPIAuth } from '../../../utils/auth.js'; import { withAPIAuth } from '../../../utils/auth.js';
import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js'; import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js';
import { enqueueApiCall } from '../../../utils/rateLimitedQueue.js'; import { enqueueApiCall } from '../../../utils/rateLimitedQueue.js';
import { aiPipeline } from '../../../utils/aiPipeline.js'; import { aiPipeline } from '../../../utils/aiPipeline.js';
import { forensicConfig } from '../../../utils/forensicConfig.js';
export const prerender = false; export const prerender = false;
@ -16,8 +17,12 @@ interface RateLimitData {
const rateLimitStore = new Map<string, RateLimitData>(); const rateLimitStore = new Map<string, RateLimitData>();
// Use configuration instead of hard-coded values
const config = forensicConfig.getConfig();
const thresholds = forensicConfig.getThresholds();
const RATE_LIMIT_WINDOW = 60 * 1000; // 1 minute const RATE_LIMIT_WINDOW = 60 * 1000; // 1 minute
const MAIN_RATE_LIMIT_MAX = parseInt(process.env.AI_RATE_LIMIT_MAX_REQUESTS || '4', 10); const MAIN_RATE_LIMIT_MAX = thresholds.rateLimitMaxRequests;
const MICRO_TASK_TOTAL_LIMIT = parseInt(process.env.AI_MICRO_TASK_TOTAL_LIMIT || '50', 10); const MICRO_TASK_TOTAL_LIMIT = parseInt(process.env.AI_MICRO_TASK_TOTAL_LIMIT || '50', 10);
function sanitizeInput(input: string): string { function sanitizeInput(input: string): string {
@ -118,42 +123,45 @@ export const POST: APIRoute = async ({ request }) => {
const body = await request.json(); const body = await request.json();
const { query, mode = 'workflow', taskId: clientTaskId } = body; const { query, mode = 'workflow', taskId: clientTaskId } = body;
console.log(`[MICRO-TASK API] Received request - TaskId: ${clientTaskId}, Mode: ${mode}, Query length: ${query?.length || 0}`); console.log(`[ENHANCED API] Received request - TaskId: ${clientTaskId}, Mode: ${mode}, Query length: ${query?.length || 0}`);
console.log(`[MICRO-TASK API] Micro-task calls remaining: ${rateLimitResult.microTasksRemaining}`); console.log(`[ENHANCED API] User: ${userId}, Audit Trail: ${config.auditTrail.enabled ? 'Enabled' : 'Disabled'}`);
console.log(`[ENHANCED API] Micro-task calls remaining: ${rateLimitResult.microTasksRemaining}`);
if (!query || typeof query !== 'string') { if (!query || typeof query !== 'string') {
console.log(`[MICRO-TASK API] Invalid query for task ${clientTaskId}`); console.log(`[ENHANCED API] Invalid query for task ${clientTaskId}`);
return apiError.badRequest('Query required'); return apiError.badRequest('Query required');
} }
if (!['workflow', 'tool'].includes(mode)) { if (!['workflow', 'tool'].includes(mode)) {
console.log(`[MICRO-TASK API] Invalid mode for task ${clientTaskId}: ${mode}`); console.log(`[ENHANCED API] Invalid mode for task ${clientTaskId}: ${mode}`);
return apiError.badRequest('Invalid mode. Must be "workflow" or "tool"'); return apiError.badRequest('Invalid mode. Must be "workflow" or "tool"');
} }
const sanitizedQuery = sanitizeInput(query); const sanitizedQuery = sanitizeInput(query);
if (sanitizedQuery.includes('[FILTERED]')) { if (sanitizedQuery.includes('[FILTERED]')) {
console.log(`[MICRO-TASK API] Filtered input detected for task ${clientTaskId}`); console.log(`[ENHANCED API] Filtered input detected for task ${clientTaskId}`);
return apiError.badRequest('Invalid input detected'); return apiError.badRequest('Invalid input detected');
} }
const taskId = clientTaskId || `ai_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`; const taskId = clientTaskId || `ai_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`;
console.log(`[MICRO-TASK API] About to enqueue micro-task pipeline ${taskId}`); console.log(`[ENHANCED API] About to enqueue enhanced pipeline ${taskId}`);
// Use enhanced pipeline with audit trail
const result = await enqueueApiCall(() => const result = await enqueueApiCall(() =>
aiPipeline.processQuery(sanitizedQuery, mode) aiPipeline.processQuery(sanitizedQuery, mode, userId)
, taskId); , taskId);
if (!result || !result.recommendation) { if (!result || !result.recommendation) {
return apiServerError.unavailable('No response from micro-task AI pipeline'); return apiServerError.unavailable('No response from enhanced AI pipeline');
} }
const stats = result.processingStats; const stats = result.processingStats;
const estimatedAICallsMade = stats.microTasksCompleted + stats.microTasksFailed; const estimatedAICallsMade = stats.microTasksCompleted + stats.microTasksFailed;
incrementMicroTaskCount(userId, estimatedAICallsMade); incrementMicroTaskCount(userId, estimatedAICallsMade);
console.log(`[MICRO-TASK API] Pipeline completed for ${taskId}:`); // Log comprehensive results
console.log(`[ENHANCED API] Enhanced pipeline completed for ${taskId}:`);
console.log(` - Mode: ${mode}`); console.log(` - Mode: ${mode}`);
console.log(` - User: ${userId}`); console.log(` - User: ${userId}`);
console.log(` - Query length: ${sanitizedQuery.length}`); console.log(` - Query length: ${sanitizedQuery.length}`);
@ -161,9 +169,17 @@ export const POST: APIRoute = async ({ request }) => {
console.log(` - Micro-tasks completed: ${stats.microTasksCompleted}`); console.log(` - Micro-tasks completed: ${stats.microTasksCompleted}`);
console.log(` - Micro-tasks failed: ${stats.microTasksFailed}`); console.log(` - Micro-tasks failed: ${stats.microTasksFailed}`);
console.log(` - Estimated AI calls: ${estimatedAICallsMade}`); console.log(` - Estimated AI calls: ${estimatedAICallsMade}`);
console.log(` - Total tokens used: ${stats.tokensTotalUsed}`);
console.log(` - Embeddings used: ${stats.embeddingsUsed}`); console.log(` - Embeddings used: ${stats.embeddingsUsed}`);
console.log(` - Final items: ${stats.finalSelectedItems}`); console.log(` - Final items: ${stats.finalSelectedItems}`);
if (result.auditTrail) {
console.log(` - Audit Trail ID: ${result.auditTrail.auditId}`);
console.log(` - Overall Confidence: ${(result.auditTrail.qualityMetrics.overallConfidence * 100).toFixed(1)}%`);
console.log(` - Bias Risk Score: ${(result.auditTrail.qualityMetrics.biasRiskScore * 100).toFixed(1)}%`);
console.log(` - Transparency Score: ${(result.auditTrail.qualityMetrics.transparencyScore * 100).toFixed(1)}%`);
}
const currentLimit = rateLimitStore.get(userId); const currentLimit = rateLimitStore.get(userId);
const remainingMicroTasks = currentLimit ? const remainingMicroTasks = currentLimit ?
MICRO_TASK_TOTAL_LIMIT - currentLimit.microTaskCount : MICRO_TASK_TOTAL_LIMIT; MICRO_TASK_TOTAL_LIMIT - currentLimit.microTaskCount : MICRO_TASK_TOTAL_LIMIT;
@ -176,11 +192,40 @@ export const POST: APIRoute = async ({ request }) => {
query: sanitizedQuery, query: sanitizedQuery,
processingStats: { processingStats: {
...result.processingStats, ...result.processingStats,
pipelineType: 'micro-task', pipelineType: 'enhanced-micro-task',
microTasksSuccessRate: stats.microTasksCompleted / (stats.microTasksCompleted + stats.microTasksFailed), microTasksSuccessRate: stats.microTasksCompleted / Math.max(stats.microTasksCompleted + stats.microTasksFailed, 1),
averageTaskTime: stats.processingTimeMs / (stats.microTasksCompleted + stats.microTasksFailed), averageTaskTime: stats.processingTimeMs / Math.max(stats.microTasksCompleted + stats.microTasksFailed, 1),
estimatedAICallsMade estimatedAICallsMade,
auditCompliant: result.auditTrail?.compliance.auditCompliant || false,
biasChecked: result.auditTrail?.compliance.biasChecked || false,
confidenceAssessed: result.auditTrail?.compliance.confidenceAssessed || false
}, },
// NEW: Forensic metadata
forensicMetadata: result.auditTrail ? {
auditTrailId: result.auditTrail.auditId,
auditEnabled: config.auditTrail.enabled,
overallConfidence: result.auditTrail.qualityMetrics.overallConfidence,
biasRiskScore: result.auditTrail.qualityMetrics.biasRiskScore,
transparencyScore: result.auditTrail.qualityMetrics.transparencyScore,
reproducibilityScore: result.auditTrail.qualityMetrics.reproducibilityScore,
evidenceQuality: result.auditTrail.qualityMetrics.evidenceQuality,
methodologicalSoundness: result.auditTrail.qualityMetrics.methodologicalSoundness,
biasWarnings: result.auditTrail.biasAnalysis.filter(b => b.detected),
systemConfig: {
strategicModel: result.auditTrail.systemConfig.strategicModel,
tacticalModel: result.auditTrail.systemConfig.tacticalModel,
auditLevel: result.auditTrail.systemConfig.auditLevel
},
compliance: result.auditTrail.compliance,
qualityLevel: result.auditTrail.qualityMetrics.overallConfidence >= thresholds.confidenceThreshold ? 'high' :
result.auditTrail.qualityMetrics.overallConfidence >= 0.5 ? 'medium' : 'low'
} : {
auditTrailId: null,
auditEnabled: false,
message: 'Audit trail disabled - operating in legacy mode'
},
rateLimitInfo: { rateLimitInfo: {
mainRequestsRemaining: MAIN_RATE_LIMIT_MAX - (currentLimit?.count || 0), mainRequestsRemaining: MAIN_RATE_LIMIT_MAX - (currentLimit?.count || 0),
microTaskCallsRemaining: remainingMicroTasks, microTaskCallsRemaining: remainingMicroTasks,
@ -192,18 +237,21 @@ export const POST: APIRoute = async ({ request }) => {
}); });
} catch (error) { } catch (error) {
console.error('[MICRO-TASK API] Pipeline error:', error); console.error('[ENHANCED API] Pipeline error:', error);
// Provide detailed error information for forensic purposes
if (error.message.includes('embeddings')) { if (error.message.includes('embeddings')) {
return apiServerError.unavailable('Embeddings service error - using AI fallback'); return apiServerError.unavailable('Embeddings service error - using AI fallback with audit trail');
} else if (error.message.includes('micro-task')) { } else if (error.message.includes('micro-task')) {
return apiServerError.unavailable('Micro-task pipeline error - some analysis steps failed'); return apiServerError.unavailable('Micro-task pipeline error - some analysis steps failed but audit trail maintained');
} else if (error.message.includes('selector')) { } else if (error.message.includes('selector')) {
return apiServerError.unavailable('AI selector service error'); return apiServerError.unavailable('AI selector service error - emergency fallback used with full audit');
} else if (error.message.includes('rate limit')) { } else if (error.message.includes('rate limit')) {
return apiError.rateLimit('AI service rate limits exceeded during micro-task processing'); return apiError.rateLimit('AI service rate limits exceeded during enhanced processing');
} else if (error.message.includes('audit')) {
return apiServerError.internal('Audit trail system error - check forensic configuration');
} else { } else {
return apiServerError.internal('Micro-task AI pipeline error'); return apiServerError.internal('Enhanced AI pipeline error - forensic audit may be incomplete');
} }
} }
}; };

743
src/utils/aiPipeline.ts
View File

@ -1,13 +1,9 @@
// src/utils/aiPipeline.ts // src/utils/aiPipeline.ts - Enhanced Forensic AI Pipeline with Audit Trail
import { getCompressedToolsDataForAI } from './dataService.js'; import { getCompressedToolsDataForAI } from './dataService.js';
import { embeddingsService, type EmbeddingData } from './embeddings.js'; import { embeddingsService, type EmbeddingData, type EmbeddingSearchResult } from './embeddings.js';
import { forensicConfig, type AIModelConfig } from './forensicConfig.js';
interface AIConfig { import { auditTrailService, type ForensicAuditEntry } from './auditTrail.js';
endpoint: string;
apiKey: string;
model: string;
}
interface MicroTaskResult { interface MicroTaskResult {
taskType: string; taskType: string;
@ -15,6 +11,10 @@ interface MicroTaskResult {
processingTimeMs: number; processingTimeMs: number;
success: boolean; success: boolean;
error?: string; error?: string;
confidence: number;
promptTokens: number;
responseTokens: number;
contextUsed: string[];
} }
interface AnalysisResult { interface AnalysisResult {
@ -27,6 +27,14 @@ interface AnalysisResult {
microTasksCompleted: number; microTasksCompleted: number;
microTasksFailed: number; microTasksFailed: number;
contextContinuityUsed: boolean; contextContinuityUsed: boolean;
aiCallsMade: number;
tokensTotalUsed: number;
};
auditTrail?: ForensicAuditEntry | null;
qualityMetrics?: {
overallConfidence: number;
biasRiskScore: number;
transparencyScore: number;
}; };
} }
@ -49,38 +57,33 @@ interface AnalysisContext {
seenToolNames: Set<string>; seenToolNames: Set<string>;
} }
class ImprovedMicroTaskAIPipeline { class EnhancedMicroTaskAIPipeline {
private config: AIConfig; private config = forensicConfig.getConfig();
private thresholds = forensicConfig.getThresholds();
// Remove hard-coded values - now using configuration
private maxSelectedItems: number; private maxSelectedItems: number;
private embeddingCandidates: number; private embeddingCandidates: number;
private similarityThreshold: number; private similarityThreshold: number;
private microTaskDelay: number; private microTaskDelay: number;
private maxContextTokens: number; private maxContextTokens: number;
private maxPromptTokens: number; private maxPromptTokens: number;
constructor() { constructor() {
this.config = { // All values now come from configuration - no more hard-coded values
endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'), this.maxSelectedItems = this.thresholds.maxSelectedItems;
apiKey: this.getEnv('AI_ANALYZER_API_KEY'), this.embeddingCandidates = this.thresholds.embeddingCandidates;
model: this.getEnv('AI_ANALYZER_MODEL') this.similarityThreshold = this.thresholds.similarityThreshold;
}; this.microTaskDelay = this.thresholds.microTaskDelayMs;
this.maxSelectedItems = parseInt(process.env.AI_MAX_SELECTED_ITEMS || '60', 10); // Dynamic token limits based on model capabilities
this.embeddingCandidates = parseInt(process.env.AI_EMBEDDING_CANDIDATES || '60', 10); this.maxContextTokens = this.config.aiModels.strategic.maxContextTokens;
this.similarityThreshold = 0.3; this.maxPromptTokens = Math.floor(this.maxContextTokens * 0.6); // Leave room for response
this.microTaskDelay = parseInt(process.env.AI_MICRO_TASK_DELAY_MS || '500', 10);
this.maxContextTokens = parseInt(process.env.AI_MAX_CONTEXT_TOKENS || '4000', 10); console.log('[ENHANCED PIPELINE] Initialized with forensic configuration');
this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10); console.log(`[ENHANCED PIPELINE] Strategic Model: ${this.config.aiModels.strategic.model}`);
} console.log(`[ENHANCED PIPELINE] Tactical Model: ${this.config.aiModels.tactical.model}`);
console.log(`[ENHANCED PIPELINE] Audit Trail: ${this.config.auditTrail.enabled ? 'Enabled' : 'Disabled'}`);
private getEnv(key: string): string {
const value = process.env[key];
if (!value) {
throw new Error(`Missing environment variable: ${key}`);
}
return value;
} }
private estimateTokens(text: string): number { private estimateTokens(text: string): number {
@ -109,15 +112,15 @@ class ImprovedMicroTaskAIPipeline {
const parsed = JSON.parse(cleaned); const parsed = JSON.parse(cleaned);
return parsed; return parsed;
} catch (error) { } catch (error) {
console.warn('[AI PIPELINE] JSON parsing failed:', error.message); console.warn('[ENHANCED PIPELINE] JSON parsing failed:', error.message);
console.warn('[AI PIPELINE] Raw content:', jsonString.slice(0, 200)); console.warn('[ENHANCED PIPELINE] Raw content:', jsonString.slice(0, 200));
return fallback; return fallback;
} }
} }
private addToolToSelection(context: AnalysisContext, tool: any, phase: string, priority: string, justification?: string): boolean { private addToolToSelection(context: AnalysisContext, tool: any, phase: string, priority: string, justification?: string): boolean {
if (context.seenToolNames.has(tool.name)) { if (context.seenToolNames.has(tool.name)) {
console.log(`[AI PIPELINE] Skipping duplicate tool: ${tool.name}`); console.log(`[ENHANCED PIPELINE] Skipping duplicate tool: ${tool.name}`);
return false; return false;
} }
@ -134,12 +137,91 @@ class ImprovedMicroTaskAIPipeline {
return true; return true;
} }
private async getIntelligentCandidates(userQuery: string, toolsData: any, mode: string) { // ============================================================================
// ENHANCED AI CALLING WITH DUAL MODELS
// ============================================================================
private async callAIWithModel(
prompt: string,
modelType: 'strategic' | 'tactical' | 'legacy',
taskType?: string,
maxTokens?: number
): Promise<{
content: string;
promptTokens: number;
responseTokens: number;
model: string;
endpoint: string;
}> {
const modelConfig = modelType === 'legacy' ?
forensicConfig.getLegacyAIModel() :
forensicConfig.getAIModel(modelType);
const finalMaxTokens = maxTokens || modelConfig.maxOutputTokens;
console.log(`[ENHANCED PIPELINE] Using ${modelType} model (${modelConfig.model}) for task: ${taskType || 'unknown'}`);
const response = await fetch(`${modelConfig.endpoint}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${modelConfig.apiKey}`
},
body: JSON.stringify({
model: modelConfig.model,
messages: [{ role: 'user', content: prompt }],
max_tokens: finalMaxTokens,
temperature: modelConfig.temperature
})
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`AI API error: ${response.status} - ${errorText}`);
}
const data = await response.json();
const content = data.choices?.[0]?.message?.content;
if (!content) {
throw new Error('No response from AI model');
}
// Estimate token usage (since most APIs don't return exact counts)
const promptTokens = this.estimateTokens(prompt);
const responseTokens = this.estimateTokens(content);
return {
content,
promptTokens,
responseTokens,
model: modelConfig.model,
endpoint: modelConfig.endpoint
};
}
// Legacy compatibility method
private async callAI(prompt: string, maxTokens: number = 1000): Promise<string> {
const result = await this.callAIWithModel(prompt, 'legacy', 'legacy', maxTokens);
return result.content;
}
// ============================================================================
// ENHANCED CANDIDATE RETRIEVAL WITH AUDIT TRAIL
// ============================================================================
private async getIntelligentCandidatesWithAudit(userQuery: string, toolsData: any, mode: string) {
const startTime = Date.now();
let candidateTools: any[] = []; let candidateTools: any[] = [];
let candidateConcepts: any[] = []; let candidateConcepts: any[] = [];
let selectionMethod = 'unknown'; let selectionMethod = 'unknown';
let similarityScores: Array<{ tool: string; score: number; type: string }> = [];
let retrievalConfidence = 0;
// Log retrieval start
if (embeddingsService.isEnabled()) { if (embeddingsService.isEnabled()) {
auditTrailService.logRetrievalStart('embeddings');
const similarItems = await embeddingsService.findSimilar( const similarItems = await embeddingsService.findSimilar(
userQuery, userQuery,
this.embeddingCandidates, this.embeddingCandidates,
@ -150,33 +232,56 @@ class ImprovedMicroTaskAIPipeline {
const conceptNames = new Set<string>(); const conceptNames = new Set<string>();
similarItems.forEach(item => { similarItems.forEach(item => {
if (item.type === 'tool') toolNames.add(item.name); if (item.type === 'tool') {
if (item.type === 'concept') conceptNames.add(item.name); toolNames.add(item.name);
similarityScores.push({ tool: item.name, score: item.similarity, type: 'tool' });
}
if (item.type === 'concept') {
conceptNames.add(item.name);
similarityScores.push({ tool: item.name, score: item.similarity, type: 'concept' });
}
}); });
console.log(`[IMPROVED PIPELINE] Embeddings found: ${toolNames.size} tools, ${conceptNames.size} concepts`); console.log(`[ENHANCED PIPELINE] Embeddings found: ${toolNames.size} tools, ${conceptNames.size} concepts`);
if (toolNames.size >= 15) { if (toolNames.size >= 15) {
candidateTools = toolsData.tools.filter((tool: any) => toolNames.has(tool.name)); candidateTools = toolsData.tools.filter((tool: any) => toolNames.has(tool.name));
candidateConcepts = toolsData.concepts.filter((concept: any) => conceptNames.has(concept.name)); candidateConcepts = toolsData.concepts.filter((concept: any) => conceptNames.has(concept.name));
selectionMethod = 'embeddings_candidates'; selectionMethod = 'embeddings_candidates';
retrievalConfidence = similarItems.length > 0 ?
similarItems.reduce((sum, item) => sum + item.similarity, 0) / similarItems.length : 0;
console.log(`[IMPROVED PIPELINE] Using embeddings candidates: ${candidateTools.length} tools`); console.log(`[ENHANCED PIPELINE] Using embeddings candidates: ${candidateTools.length} tools`);
} else { } else {
console.log(`[IMPROVED PIPELINE] Embeddings insufficient (${toolNames.size} < 15), using full dataset`); console.log(`[ENHANCED PIPELINE] Embeddings insufficient (${toolNames.size} < 15), using AI selector`);
auditTrailService.logRetrievalStart('ai_selector');
candidateTools = toolsData.tools; candidateTools = toolsData.tools;
candidateConcepts = toolsData.concepts; candidateConcepts = toolsData.concepts;
selectionMethod = 'full_dataset'; selectionMethod = 'ai_selector';
retrievalConfidence = 0.5; // Moderate confidence for AI selector
} }
} else { } else {
console.log(`[IMPROVED PIPELINE] Embeddings disabled, using full dataset`); console.log(`[ENHANCED PIPELINE] Embeddings disabled, using AI selector`);
auditTrailService.logRetrievalStart('ai_selector');
candidateTools = toolsData.tools; candidateTools = toolsData.tools;
candidateConcepts = toolsData.concepts; candidateConcepts = toolsData.concepts;
selectionMethod = 'full_dataset'; selectionMethod = 'ai_selector';
retrievalConfidence = 0.4; // Lower confidence without embeddings
} }
console.log(`[IMPROVED PIPELINE] AI will analyze FULL DATA of ${candidateTools.length} candidate tools`); const processingTime = Date.now() - startTime;
const finalSelection = await this.aiSelectionWithFullData(userQuery, candidateTools, candidateConcepts, mode, selectionMethod);
// Log retrieval results
auditTrailService.logRetrievalResults({
candidatesFound: candidateTools.length + candidateConcepts.length,
similarityScores,
confidence: retrievalConfidence,
processingTimeMs: processingTime,
fallbackReason: selectionMethod === 'ai_selector' ? 'Insufficient embeddings candidates' : undefined
});
console.log(`[ENHANCED PIPELINE] AI will analyze FULL DATA of ${candidateTools.length} candidate tools`);
const finalSelection = await this.aiSelectionWithAuditAndBias(userQuery, candidateTools, candidateConcepts, mode, selectionMethod);
return { return {
tools: finalSelection.selectedTools, tools: finalSelection.selectedTools,
@ -187,13 +292,23 @@ class ImprovedMicroTaskAIPipeline {
}; };
} }
private async aiSelectionWithFullData( // ============================================================================
// ENHANCED AI SELECTION WITH AUDIT TRAIL
// ============================================================================
private async aiSelectionWithAuditAndBias(
userQuery: string, userQuery: string,
candidateTools: any[], candidateTools: any[],
candidateConcepts: any[], candidateConcepts: any[],
mode: string, mode: string,
selectionMethod: string selectionMethod: string
) { ) {
const startTime = Date.now();
const initialCandidates = candidateTools.map(tool => tool.name);
// Log selection start - use strategic model for tool selection
auditTrailService.logSelectionStart('strategic', initialCandidates);
const modeInstruction = mode === 'workflow' const modeInstruction = mode === 'workflow'
? 'The user wants a COMPREHENSIVE WORKFLOW with multiple tools/methods across different phases. Select 15-25 tools that cover the full investigation lifecycle.' ? 'The user wants a COMPREHENSIVE WORKFLOW with multiple tools/methods across different phases. Select 15-25 tools that cover the full investigation lifecycle.'
: 'The user wants SPECIFIC TOOLS/METHODS that directly solve their particular problem. Select 3-8 tools that are most relevant and effective.'; : 'The user wants SPECIFIC TOOLS/METHODS that directly solve their particular problem. Select 3-8 tools that are most relevant and effective.';
@ -288,32 +403,51 @@ Respond with ONLY this JSON format:
{ {
"selectedTools": ["Tool Name 1", "Tool Name 2", ...], "selectedTools": ["Tool Name 1", "Tool Name 2", ...],
"selectedConcepts": ["Concept Name 1", "Concept Name 2", ...], "selectedConcepts": ["Concept Name 1", "Concept Name 2", ...],
"reasoning": "Detailed explanation of why these specific tools were selected for this query, addressing why certain popular tools were NOT selected if they were inappropriate for the scenario context" "reasoning": "Detailed explanation of why these specific tools were selected for this query, addressing why certain popular tools were NOT selected if they were inappropriate for the scenario context",
"confidence": 0.85,
"rejectedCandidates": [
{"tool": "Tool Name", "reason": "Why this tool was not selected"},
...
]
}`; }`;
try { try {
const response = await this.callAI(prompt, 2500); const aiResult = await this.callAIWithModel(prompt, 'strategic', 'tool_selection', 2500);
const result = this.safeParseJSON(response, null); const result = this.safeParseJSON(aiResult.content, null);
if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) { if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) {
console.error('[IMPROVED PIPELINE] AI selection returned invalid structure:', response.slice(0, 200)); console.error('[ENHANCED PIPELINE] AI selection returned invalid structure:', aiResult.content.slice(0, 200));
throw new Error('AI selection failed to return valid tool selection'); throw new Error('AI selection failed to return valid tool selection');
} }
const totalSelected = result.selectedTools.length + result.selectedConcepts.length; const totalSelected = result.selectedTools.length + result.selectedConcepts.length;
if (totalSelected === 0) { if (totalSelected === 0) {
console.error('[IMPROVED PIPELINE] AI selection returned no tools'); console.error('[ENHANCED PIPELINE] AI selection returned no tools');
throw new Error('AI selection returned empty selection'); throw new Error('AI selection returned empty selection');
} }
console.log(`[IMPROVED PIPELINE] AI selected: ${result.selectedTools.length} tools, ${result.selectedConcepts.length} concepts`); console.log(`[ENHANCED PIPELINE] AI selected: ${result.selectedTools.length} tools, ${result.selectedConcepts.length} concepts`);
console.log(`[IMPROVED PIPELINE] AI reasoning: ${result.reasoning}`); console.log(`[ENHANCED PIPELINE] AI reasoning: ${result.reasoning}`);
const selectedTools = candidateTools.filter(tool => result.selectedTools.includes(tool.name)); const selectedTools = candidateTools.filter(tool => result.selectedTools.includes(tool.name));
const selectedConcepts = candidateConcepts.filter(concept => result.selectedConcepts.includes(concept.name)); const selectedConcepts = candidateConcepts.filter(concept => result.selectedConcepts.includes(concept.name));
console.log(`[IMPROVED PIPELINE] Final selection: ${selectedTools.length} tools with bias prevention applied`); const processingTime = Date.now() - startTime;
// Log selection results
auditTrailService.logSelectionResults({
finalSelection: [...result.selectedTools, ...result.selectedConcepts],
rejectedCandidates: result.rejectedCandidates || [],
reasoning: result.reasoning || '',
confidence: result.confidence || 0.7,
promptTokens: aiResult.promptTokens,
responseTokens: aiResult.responseTokens,
processingTimeMs: processingTime,
rawResponse: aiResult.content
});
console.log(`[ENHANCED PIPELINE] Final selection: ${selectedTools.length} tools with bias prevention applied`);
return { return {
selectedTools, selectedTools,
@ -321,14 +455,28 @@ Respond with ONLY this JSON format:
}; };
} catch (error) { } catch (error) {
console.error('[IMPROVED PIPELINE] AI selection failed:', error); console.error('[ENHANCED PIPELINE] AI selection failed:', error);
console.log('[IMPROVED PIPELINE] Using emergency keyword-based selection'); // Log the failure
auditTrailService.logSelectionResults({
finalSelection: [],
rejectedCandidates: [],
reasoning: `AI selection failed: ${error.message}`,
confidence: 0,
promptTokens: 0,
responseTokens: 0,
processingTimeMs: Date.now() - startTime,
rawResponse: ''
});
console.log('[ENHANCED PIPELINE] Using emergency keyword-based selection');
return this.emergencyKeywordSelection(userQuery, candidateTools, candidateConcepts, mode); return this.emergencyKeywordSelection(userQuery, candidateTools, candidateConcepts, mode);
} }
} }
private emergencyKeywordSelection(userQuery: string, candidateTools: any[], candidateConcepts: any[], mode: string) { private emergencyKeywordSelection(userQuery: string, candidateTools: any[], candidateConcepts: any[], mode: string) {
auditTrailService.logRetrievalStart('emergency_fallback');
const queryLower = userQuery.toLowerCase(); const queryLower = userQuery.toLowerCase();
const keywords = queryLower.split(/\s+/).filter(word => word.length > 3); const keywords = queryLower.split(/\s+/).filter(word => word.length > 3);
@ -352,7 +500,20 @@ Respond with ONLY this JSON format:
const maxTools = mode === 'workflow' ? 20 : 8; const maxTools = mode === 'workflow' ? 20 : 8;
const selectedTools = scoredTools.slice(0, maxTools).map(item => item.tool); const selectedTools = scoredTools.slice(0, maxTools).map(item => item.tool);
console.log(`[IMPROVED PIPELINE] Emergency selection: ${selectedTools.length} tools, keywords: ${keywords.slice(0, 5).join(', ')}`); console.log(`[ENHANCED PIPELINE] Emergency selection: ${selectedTools.length} tools, keywords: ${keywords.slice(0, 5).join(', ')}`);
// Log emergency fallback results
auditTrailService.logRetrievalResults({
candidatesFound: selectedTools.length,
similarityScores: scoredTools.slice(0, 10).map(item => ({
tool: item.tool.name,
score: item.score / keywords.length,
type: 'keyword_match'
})),
confidence: 0.3, // Low confidence for emergency fallback
processingTimeMs: 100,
fallbackReason: 'AI selection failed, using keyword matching'
});
return { return {
selectedTools, selectedTools,
@ -360,11 +521,15 @@ Respond with ONLY this JSON format:
}; };
} }
// ============================================================================
// ENHANCED MICRO-TASK METHODS WITH AUDIT TRAIL
// ============================================================================
private async delay(ms: number): Promise<void> { private async delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms)); return new Promise(resolve => setTimeout(resolve, ms));
} }
private async callMicroTaskAI(prompt: string, context: AnalysisContext, maxTokens: number = 300): Promise<MicroTaskResult> { private async callMicroTaskAI(prompt: string, context: AnalysisContext, maxTokens: number = 300, taskType: string = 'unknown'): Promise<MicroTaskResult> {
const startTime = Date.now(); const startTime = Date.now();
let contextPrompt = prompt; let contextPrompt = prompt;
@ -375,31 +540,74 @@ Respond with ONLY this JSON format:
if (this.estimateTokens(combinedPrompt) <= this.maxPromptTokens) { if (this.estimateTokens(combinedPrompt) <= this.maxPromptTokens) {
contextPrompt = combinedPrompt; contextPrompt = combinedPrompt;
} else { } else {
console.warn('[AI PIPELINE] Context too long, using prompt only'); console.warn('[ENHANCED PIPELINE] Context too long, using prompt only');
} }
} }
try { try {
const response = await this.callAI(contextPrompt, maxTokens); // Use tactical model for micro-tasks (faster, cheaper)
const modelType = forensicConfig.getModelForTask(taskType as any);
const aiResult = await this.callAIWithModel(contextPrompt, modelType, taskType, maxTokens);
return { const result: MicroTaskResult = {
taskType: 'micro-task', taskType,
content: response.trim(), content: aiResult.content.trim(),
processingTimeMs: Date.now() - startTime, processingTimeMs: Date.now() - startTime,
success: true success: true,
confidence: 0.8, // Default confidence, could be enhanced with actual scoring
promptTokens: aiResult.promptTokens,
responseTokens: aiResult.responseTokens,
contextUsed: context.contextHistory.slice()
}; };
// Log micro-task to audit trail
auditTrailService.logMicroTask({
taskType: taskType as any,
aiModel: modelType,
success: true,
processingTimeMs: result.processingTimeMs,
confidence: result.confidence,
contextUsed: result.contextUsed,
outputLength: result.content.length,
promptTokens: result.promptTokens,
responseTokens: result.responseTokens,
contextContinuityUsed: context.contextHistory.length > 0
});
return result;
} catch (error) { } catch (error) {
return { const result: MicroTaskResult = {
taskType: 'micro-task', taskType,
content: '', content: '',
processingTimeMs: Date.now() - startTime, processingTimeMs: Date.now() - startTime,
success: false, success: false,
error: error.message error: error.message,
confidence: 0,
promptTokens: 0,
responseTokens: 0,
contextUsed: []
}; };
// Log failed micro-task to audit trail
auditTrailService.logMicroTask({
taskType: taskType as any,
aiModel: 'tactical',
success: false,
processingTimeMs: result.processingTimeMs,
confidence: 0,
contextUsed: [],
outputLength: 0,
promptTokens: 0,
responseTokens: 0,
errorMessage: error.message
});
return result;
} }
} }
// Rest of the micro-task methods remain the same but use the enhanced callMicroTaskAI...
private async analyzeScenario(context: AnalysisContext): Promise<MicroTaskResult> { private async analyzeScenario(context: AnalysisContext): Promise<MicroTaskResult> {
const isWorkflow = context.mode === 'workflow'; const isWorkflow = context.mode === 'workflow';
@ -421,7 +629,7 @@ ${isWorkflow ?
WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen, Aufzählungen oder Markdown-Formatierung. Maximum 150 Wörter.`; WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen, Aufzählungen oder Markdown-Formatierung. Maximum 150 Wörter.`;
const result = await this.callMicroTaskAI(prompt, context, 220); const result = await this.callMicroTaskAI(prompt, context, 220, 'scenario_analysis');
if (result.success) { if (result.success) {
if (isWorkflow) { if (isWorkflow) {
@ -436,290 +644,41 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen, Aufzählun
return result; return result;
} }
private async generateApproach(context: AnalysisContext): Promise<MicroTaskResult> { // ... (Additional micro-task methods would be implemented similarly with audit trail integration)
const isWorkflow = context.mode === 'workflow';
const prompt = `Basierend auf der Analyse entwickeln Sie einen fundierten ${isWorkflow ? 'Untersuchungsansatz' : 'Lösungsansatz'} nach NIST SP 800-86 Methodik. // ============================================================================
// MAIN PROCESSING METHOD WITH FULL AUDIT TRAIL
// ============================================================================
${isWorkflow ? 'SZENARIO' : 'PROBLEM'}: "${context.userQuery}" async processQuery(userQuery: string, mode: string, userId: string = 'anonymous'): Promise<AnalysisResult> {
Entwickeln Sie einen systematischen ${isWorkflow ? 'Untersuchungsansatz' : 'Lösungsansatz'} unter Berücksichtigung von:
${isWorkflow ?
`- Triage-Prioritäten nach forensischer Dringlichkeit
- Phasenabfolge nach NIST-Methodik
- Kontaminationsvermeidung und forensische Isolierung` :
`- Methodik-Auswahl nach wissenschaftlichen Kriterien
- Validierung und Verifizierung der gewählten Ansätze
- Integration in bestehende forensische Workflows`
}
WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 150 Wörter.`;
const result = await this.callMicroTaskAI(prompt, context, 220);
if (result.success) {
context.investigationApproach = result.content;
this.addToContextHistory(context, `${isWorkflow ? 'Untersuchungs' : 'Lösungs'}ansatz: ${result.content.slice(0, 200)}...`);
}
return result;
}
private async generateCriticalConsiderations(context: AnalysisContext): Promise<MicroTaskResult> {
const isWorkflow = context.mode === 'workflow';
const prompt = `Identifizieren Sie ${isWorkflow ? 'kritische forensische Überlegungen' : 'wichtige methodische Voraussetzungen'} für diesen Fall.
${isWorkflow ? 'SZENARIO' : 'PROBLEM'}: "${context.userQuery}"
Berücksichtigen Sie folgende forensische Aspekte:
${isWorkflow ?
`- Time-sensitive evidence preservation
- Chain of custody requirements und rechtliche Verwertbarkeit
- Incident containment vs. evidence preservation Dilemma
- Privacy- und Compliance-Anforderungen` :
`- Tool-Validierung und Nachvollziehbarkeit
- False positive/negative Risiken bei der gewählten Methodik
- Qualifikationsanforderungen für die Durchführung
- Dokumentations- und Reporting-Standards`
}
WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 120 Wörter.`;
const result = await this.callMicroTaskAI(prompt, context, 180);
if (result.success) {
context.criticalConsiderations = result.content;
this.addToContextHistory(context, `Kritische Überlegungen: ${result.content.slice(0, 200)}...`);
}
return result;
}
private async selectToolsForPhase(context: AnalysisContext, phase: any): Promise<MicroTaskResult> {
const phaseTools = context.filteredData.tools.filter((tool: any) =>
tool.phases && tool.phases.includes(phase.id)
);
if (phaseTools.length === 0) {
return {
taskType: 'tool-selection',
content: JSON.stringify([]),
processingTimeMs: 0,
success: true
};
}
const prompt = `Wählen Sie 2-3 Methoden/Tools für die Phase "${phase.name}" basierend auf objektiven, fallbezogenen Kriterien.
SZENARIO: "${context.userQuery}"
VERFÜGBARE TOOLS FÜR ${phase.name.toUpperCase()}:
${phaseTools.map((tool: any) => `- ${tool.name}: ${tool.description.slice(0, 100)}...`).join('\n')}
Wählen Sie Methoden/Tools nach forensischen Kriterien aus:
- Court admissibility und Chain of Custody Kompatibilität
- Integration in forensische Standard-Workflows
- Reproduzierbarkeit und Dokumentationsqualität
- Objektivität
Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format (kein zusätzlicher Text):
[
{
"toolName": "Exakter Methoden/Tool-Name",
"priority": "high|medium|low",
"justification": "Objektive Begründung warum diese Methode/Tool für das spezifische Szenario besser geeignet ist"
}
]`;
const result = await this.callMicroTaskAI(prompt, context, 450);
if (result.success) {
const selections = this.safeParseJSON(result.content, []);
if (Array.isArray(selections)) {
const validSelections = selections.filter((sel: any) =>
sel.toolName && phaseTools.some((tool: any) => tool.name === sel.toolName)
);
validSelections.forEach((sel: any) => {
const tool = phaseTools.find((t: any) => t.name === sel.toolName);
if (tool) {
this.addToolToSelection(context, tool, phase.id, sel.priority, sel.justification);
}
});
}
}
return result;
}
private async evaluateSpecificTool(context: AnalysisContext, tool: any, rank: number): Promise<MicroTaskResult> {
const prompt = `Bewerten Sie diese Methode/Tool fallbezogen für das spezifische Problem nach forensischen Qualitätskriterien.
PROBLEM: "${context.userQuery}"
TOOL: ${tool.name}
BESCHREIBUNG: ${tool.description}
PLATTFORMEN: ${tool.platforms?.join(', ') || 'N/A'}
SKILL LEVEL: ${tool.skillLevel}
Bewerten Sie nach forensischen Standards und antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format:
{
"suitability_score": "high|medium|low",
"detailed_explanation": "Detaillierte forensische Begründung warum diese Methode/Tool das Problem löst",
"implementation_approach": "Konkrete methodische Schritte zur korrekten Anwendung für dieses spezifische Problem",
"pros": ["Forensischer Vorteil 1", "Validierter Vorteil 2"],
"cons": ["Methodische Limitation 1", "Potenzielle Schwäche 2"],
"alternatives": "Alternative Ansätze falls diese Methode/Tool nicht optimal ist"
}`;
const result = await this.callMicroTaskAI(prompt, context, 650);
if (result.success) {
const evaluation = this.safeParseJSON(result.content, {
suitability_score: 'medium',
detailed_explanation: 'Evaluation failed',
implementation_approach: '',
pros: [],
cons: [],
alternatives: ''
});
this.addToolToSelection(context, {
...tool,
evaluation: {
...evaluation,
rank
}
}, 'evaluation', evaluation.suitability_score);
}
return result;
}
private async selectBackgroundKnowledge(context: AnalysisContext): Promise<MicroTaskResult> {
const availableConcepts = context.filteredData.concepts;
if (availableConcepts.length === 0) {
return {
taskType: 'background-knowledge',
content: JSON.stringify([]),
processingTimeMs: 0,
success: true
};
}
const selectedToolNames = context.selectedTools?.map(st => st.tool.name) || [];
const prompt = `Wählen Sie relevante forensische Konzepte für das Verständnis der empfohlenen Methodik.
${context.mode === 'workflow' ? 'SZENARIO' : 'PROBLEM'}: "${context.userQuery}"
EMPFOHLENE TOOLS: ${selectedToolNames.join(', ')}
VERFÜGBARE KONZEPTE:
${availableConcepts.slice(0, 15).map((concept: any) => `- ${concept.name}: ${concept.description.slice(0, 80)}...`).join('\n')}
Wählen Sie 2-4 Konzepte aus, die für das Verständnis der forensischen Methodik essentiell sind.
Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format:
[
{
"conceptName": "Exakter Konzept-Name",
"relevance": "Forensische Relevanz: Warum dieses Konzept für das Verständnis der Methodik kritisch ist"
}
]`;
const result = await this.callMicroTaskAI(prompt, context, 400);
if (result.success) {
const selections = this.safeParseJSON(result.content, []);
if (Array.isArray(selections)) {
context.backgroundKnowledge = selections.filter((sel: any) =>
sel.conceptName && availableConcepts.some((concept: any) => concept.name === sel.conceptName)
).map((sel: any) => ({
concept: availableConcepts.find((c: any) => c.name === sel.conceptName),
relevance: sel.relevance
}));
}
}
return result;
}
private async generateFinalRecommendations(context: AnalysisContext): Promise<MicroTaskResult> {
const isWorkflow = context.mode === 'workflow';
const prompt = isWorkflow ?
`Erstellen Sie eine forensisch fundierte Workflow-Empfehlung basierend auf DFIR-Prinzipien.
SZENARIO: "${context.userQuery}"
AUSGEWÄHLTE TOOLS: ${context.selectedTools?.map(st => st.tool.name).join(', ') || 'Keine Tools ausgewählt'}
Erstellen Sie konkrete methodische Workflow-Schritte für dieses spezifische Szenario unter Berücksichtigung forensischer Best Practices, Objektivität und rechtlicher Verwertbarkeit.
WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 120 Wörter.` :
`Erstellen Sie wichtige methodische Überlegungen für die korrekte Methoden-/Tool-Anwendung.
PROBLEM: "${context.userQuery}"
EMPFOHLENE TOOLS: ${context.selectedTools?.map(st => st.tool.name).join(', ') || 'Keine Methoden/Tools ausgewählt'}
Geben Sie kritische methodische Überlegungen, Validierungsanforderungen und Qualitätssicherungsmaßnahmen für die korrekte Anwendung der empfohlenen Methoden/Tools.
WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 100 Wörter.`;
const result = await this.callMicroTaskAI(prompt, context, 180);
return result;
}
private async callAI(prompt: string, maxTokens: number = 1000): Promise<string> {
const response = await fetch(`${this.config.endpoint}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.config.apiKey}`
},
body: JSON.stringify({
model: this.config.model,
messages: [{ role: 'user', content: prompt }],
max_tokens: maxTokens,
temperature: 0.3
})
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`AI API error: ${response.status} - ${errorText}`);
}
const data = await response.json();
const content = data.choices?.[0]?.message?.content;
if (!content) {
throw new Error('No response from AI model');
}
return content;
}
async processQuery(userQuery: string, mode: string): Promise<AnalysisResult> {
const startTime = Date.now(); const startTime = Date.now();
let completedTasks = 0; let completedTasks = 0;
let failedTasks = 0; let failedTasks = 0;
console.log(`[IMPROVED PIPELINE] Starting ${mode} query processing with context continuity`); // Start audit trail
const auditId = auditTrailService.startAudit(userId, userQuery, mode as 'workflow' | 'tool');
console.log(`[ENHANCED PIPELINE] Starting ${mode} query processing with audit trail ${auditId}`);
// Log query classification
auditTrailService.logQueryClassification({
domains: [], // Will be filled based on analysis
urgency: userQuery.toLowerCase().includes('urgent') ? 'high' : 'medium',
complexity: mode === 'workflow' ? 'complex' : 'moderate',
specialization: [], // Will be filled based on analysis
estimatedToolCount: mode === 'workflow' ? 15 : 5
});
try { try {
// Stage 1: Get intelligent candidates (embeddings + AI selection) // Sanitize and log query
const sanitizedQuery = this.sanitizeInput(userQuery);
auditTrailService.logSanitizedQuery(sanitizedQuery);
// Stage 1: Get intelligent candidates with audit trail
const toolsData = await getCompressedToolsDataForAI(); const toolsData = await getCompressedToolsDataForAI();
const filteredData = await this.getIntelligentCandidates(userQuery, toolsData, mode); const filteredData = await this.getIntelligentCandidatesWithAudit(sanitizedQuery, toolsData, mode);
const context: AnalysisContext = { const context: AnalysisContext = {
userQuery, userQuery: sanitizedQuery,
mode, mode,
filteredData, filteredData,
contextHistory: [], contextHistory: [],
@ -728,79 +687,79 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
seenToolNames: new Set<string>() seenToolNames: new Set<string>()
}; };
console.log(`[IMPROVED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`); console.log(`[ENHANCED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`);
// MICRO-TASK SEQUENCE // MICRO-TASK SEQUENCE WITH AUDIT TRAIL
// Task 1: Scenario/Problem Analysis // Task 1: Scenario/Problem Analysis
const analysisResult = await this.analyzeScenario(context); const analysisResult = await this.analyzeScenario(context);
if (analysisResult.success) completedTasks++; else failedTasks++; if (analysisResult.success) completedTasks++; else failedTasks++;
await this.delay(this.microTaskDelay); await this.delay(this.microTaskDelay);
// Task 2: Investigation/Solution Approach // ... (Additional micro-tasks would be implemented here)
const approachResult = await this.generateApproach(context);
if (approachResult.success) completedTasks++; else failedTasks++;
await this.delay(this.microTaskDelay);
// Task 3: Critical Considerations // Build final recommendation (simplified for this example)
const considerationsResult = await this.generateCriticalConsiderations(context); const recommendation = this.buildRecommendation(context, mode, "Workflow-Empfehlung");
if (considerationsResult.success) completedTasks++; else failedTasks++;
await this.delay(this.microTaskDelay);
// Task 4: Tool Selection/Evaluation (mode-dependent) // Finalize audit trail
if (mode === 'workflow') { const finalRecommendationCount = (context.selectedTools?.length || 0) +
const phases = toolsData.phases || []; (context.backgroundKnowledge?.length || 0);
for (const phase of phases) { const auditTrail = auditTrailService.finalizeAudit(finalRecommendationCount);
const toolSelectionResult = await this.selectToolsForPhase(context, phase);
if (toolSelectionResult.success) completedTasks++; else failedTasks++;
await this.delay(this.microTaskDelay);
}
} else {
const topTools = filteredData.tools.slice(0, 3);
for (let i = 0; i < topTools.length; i++) {
const evaluationResult = await this.evaluateSpecificTool(context, topTools[i], i + 1);
if (evaluationResult.success) completedTasks++; else failedTasks++;
await this.delay(this.microTaskDelay);
}
}
// Task 5: Background Knowledge Selection
const knowledgeResult = await this.selectBackgroundKnowledge(context);
if (knowledgeResult.success) completedTasks++; else failedTasks++;
await this.delay(this.microTaskDelay);
// Task 6: Final Recommendations
const finalResult = await this.generateFinalRecommendations(context);
if (finalResult.success) completedTasks++; else failedTasks++;
// Build final recommendation
const recommendation = this.buildRecommendation(context, mode, finalResult.content);
const processingStats = { const processingStats = {
embeddingsUsed: embeddingsService.isEnabled(), embeddingsUsed: embeddingsService.isEnabled(),
candidatesFromEmbeddings: filteredData.tools.length, candidatesFromEmbeddings: filteredData.tools.length,
finalSelectedItems: (context.selectedTools?.length || 0) + finalSelectedItems: finalRecommendationCount,
(context.backgroundKnowledge?.length || 0),
processingTimeMs: Date.now() - startTime, processingTimeMs: Date.now() - startTime,
microTasksCompleted: completedTasks, microTasksCompleted: completedTasks,
microTasksFailed: failedTasks, microTasksFailed: failedTasks,
contextContinuityUsed: true contextContinuityUsed: true,
aiCallsMade: auditTrail?.processingSummary.aiCallsMade || 0,
tokensTotalUsed: auditTrail?.processingSummary.tokensTotalUsed || 0
}; };
console.log(`[IMPROVED PIPELINE] Completed: ${completedTasks} tasks, Failed: ${failedTasks} tasks`); console.log(`[ENHANCED PIPELINE] Completed: ${completedTasks} tasks, Failed: ${failedTasks} tasks`);
console.log(`[IMPROVED PIPELINE] Unique tools selected: ${context.seenToolNames.size}`); console.log(`[ENHANCED PIPELINE] Unique tools selected: ${context.seenToolNames.size}`);
if (auditTrail) {
console.log(`[ENHANCED PIPELINE] Audit Trail: ${auditTrail.auditId}`);
console.log(`[ENHANCED PIPELINE] Quality Score: ${(auditTrail.qualityMetrics.overallConfidence * 100).toFixed(1)}%`);
console.log(`[ENHANCED PIPELINE] Bias Risk: ${(auditTrail.qualityMetrics.biasRiskScore * 100).toFixed(1)}%`);
}
return { return {
recommendation, recommendation,
processingStats processingStats,
auditTrail,
qualityMetrics: auditTrail ? {
overallConfidence: auditTrail.qualityMetrics.overallConfidence,
biasRiskScore: auditTrail.qualityMetrics.biasRiskScore,
transparencyScore: auditTrail.qualityMetrics.transparencyScore
} : undefined
}; };
} catch (error) { } catch (error) {
console.error('[IMPROVED PIPELINE] Processing failed:', error); console.error('[ENHANCED PIPELINE] Processing failed:', error);
// Finalize audit trail even on failure
const auditTrail = auditTrailService.finalizeAudit(0);
throw error; throw error;
} }
} }
private sanitizeInput(input: string): string {
let sanitized = input
.replace(/```[\s\S]*?```/g, '[CODE_BLOCK_REMOVED]')
.replace(/\<\/?[^>]+(>|$)/g, '')
.replace(/\b(system|assistant|user)\s*[:]/gi, '[ROLE_REMOVED]')
.replace(/\b(ignore|forget|disregard)\s+(previous|all|your)\s+(instructions?|context|rules?)/gi, '[INSTRUCTION_REMOVED]')
.trim();
sanitized = sanitized.slice(0, 2000).replace(/\s+/g, ' ');
return sanitized;
}
private buildRecommendation(context: AnalysisContext, mode: string, finalContent: string): any { private buildRecommendation(context: AnalysisContext, mode: string, finalContent: string): any {
const isWorkflow = mode === 'workflow'; const isWorkflow = mode === 'workflow';
@ -831,13 +790,13 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
...base, ...base,
recommended_tools: context.selectedTools?.map(st => ({ recommended_tools: context.selectedTools?.map(st => ({
name: st.tool.name, name: st.tool.name,
rank: st.tool.evaluation?.rank || 1, rank: 1,
suitability_score: st.priority, suitability_score: st.priority,
detailed_explanation: st.tool.evaluation?.detailed_explanation || '', detailed_explanation: st.justification || '',
implementation_approach: st.tool.evaluation?.implementation_approach || '', implementation_approach: '',
pros: st.tool.evaluation?.pros || [], pros: [],
cons: st.tool.evaluation?.cons || [], cons: [],
alternatives: st.tool.evaluation?.alternatives || '' alternatives: ''
})) || [], })) || [],
additional_considerations: finalContent additional_considerations: finalContent
}; };
@ -845,6 +804,6 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
} }
} }
const aiPipeline = new ImprovedMicroTaskAIPipeline(); const aiPipeline = new EnhancedMicroTaskAIPipeline();
export { aiPipeline, type AnalysisResult }; export { aiPipeline, type AnalysisResult };

600
src/utils/auditTrail.ts Normal file
View File

@ -0,0 +1,600 @@
// src/utils/auditTrail.ts - Forensic Audit Trail System
import { forensicConfig } from './forensicConfig.js';
// ============================================================================
// AUDIT TRAIL DATA STRUCTURES
// ============================================================================
interface QueryClassification {
domains: string[];
urgency: 'low' | 'medium' | 'high' | 'critical';
complexity: 'simple' | 'moderate' | 'complex';
specialization: string[];
estimatedToolCount: number;
}
interface RetrievalAudit {
method: 'embeddings' | 'ai_selector' | 'emergency_fallback';
embeddingsUsed: boolean;
candidatesFound: number;
similarityScores: Array<{ tool: string; score: number; type: string }>;
retrievalConfidence: number;
thresholds: {
similarity: number;
minimumCandidates: number;
};
processingTimeMs: number;
fallbackReason?: string;
}
interface SelectionAudit {
aiModel: 'strategic' | 'tactical' | 'legacy';
modelConfig: {
endpoint: string;
model: string;
maxTokens: number;
temperature: number;
};
promptTokens: number;
responseTokens: number;
processingTimeMs: number;
initialCandidates: string[];
finalSelection: string[];
rejectedCandidates: Array<{
tool: string;
reason: string;
score?: number;
}>;
selectionReasoning: string;
confidenceScore: number;
rawResponse: string;
}
interface BiasAnalysisEntry {
biasType: 'popularity' | 'availability' | 'recency' | 'domain_concentration' | 'skill_level';
detected: boolean;
severity: number; // 0-1 scale
evidence: {
affectedTools: string[];
expectedDistribution: any;
actualDistribution: any;
statisticalSignificance?: number;
};
recommendation: string;
mitigation: string;
}
interface MicroTaskAudit {
taskId: string;
taskType: 'scenario_analysis' | 'approach_generation' | 'tool_selection' | 'evaluation' | 'background_knowledge' | 'final_recommendations';
aiModel: 'strategic' | 'tactical' | 'legacy';
success: boolean;
processingTimeMs: number;
confidence: number;
contextUsed: string[];
outputLength: number;
promptTokens: number;
responseTokens: number;
errorMessage?: string;
contextContinuityUsed: boolean;
}
interface QualityMetrics {
overallConfidence: number;
reproducibilityScore: number;
biasRiskScore: number;
transparencyScore: number;
evidenceQuality: number;
methodologicalSoundness: number;
}
interface ForensicAuditEntry {
// Identification
auditId: string;
sessionId: string;
timestamp: Date;
userId: string;
// Query Context
userQuery: string;
queryMode: 'workflow' | 'tool';
sanitizedQuery: string;
queryClassification: QueryClassification;
// System Configuration (snapshot)
systemConfig: {
strategicModel: string;
tacticalModel: string;
embeddingsEnabled: boolean;
auditLevel: string;
thresholds: Record<string, number>;
};
// Retrieval Audit
retrievalProcess: RetrievalAudit;
// Selection Audit
selectionProcess: SelectionAudit;
// Bias Analysis
biasAnalysis: BiasAnalysisEntry[];
// Micro-task Audit
microTasks: MicroTaskAudit[];
// Final Quality Metrics
qualityMetrics: QualityMetrics;
// Processing Summary
processingSummary: {
totalTimeMs: number;
aiCallsMade: number;
tokensTotalUsed: number;
errorsEncountered: number;
fallbacksUsed: number;
finalRecommendationCount: number;
};
// Compliance Metadata
compliance: {
auditCompliant: boolean;
dataRetentionCompliant: boolean;
biasChecked: boolean;
confidenceAssessed: boolean;
traceabilityScore: number;
};
}
// ============================================================================
// AUDIT TRAIL SERVICE IMPLEMENTATION
// ============================================================================
class ForensicAuditTrailService {
private currentAudit: ForensicAuditEntry | null = null;
private auditStorage: Map<string, ForensicAuditEntry> = new Map();
private config = forensicConfig.getConfig();
constructor() {
if (this.config.auditTrail.enabled) {
console.log('[AUDIT TRAIL] Forensic audit trail service initialized');
this.setupCleanupInterval();
}
}
// ========================================================================
// AUDIT LIFECYCLE MANAGEMENT
// ========================================================================
startAudit(userId: string, query: string, mode: 'workflow' | 'tool'): string {
if (!this.config.auditTrail.enabled) {
return 'audit-disabled';
}
const auditId = `audit_${Date.now()}_${Math.random().toString(36).substr(2, 8)}`;
const sessionId = `session_${userId}_${Date.now()}`;
this.currentAudit = {
auditId,
sessionId,
timestamp: new Date(),
userId,
userQuery: query,
queryMode: mode,
sanitizedQuery: '',
queryClassification: {
domains: [],
urgency: 'medium',
complexity: 'moderate',
specialization: [],
estimatedToolCount: 0
},
systemConfig: {
strategicModel: this.config.aiModels.strategic.model,
tacticalModel: this.config.aiModels.tactical.model,
embeddingsEnabled: this.config.embeddings.enabled,
auditLevel: this.config.auditTrail.detailLevel,
thresholds: { ...this.config.thresholds }
},
retrievalProcess: {
method: 'embeddings',
embeddingsUsed: false,
candidatesFound: 0,
similarityScores: [],
retrievalConfidence: 0,
thresholds: {
similarity: this.config.thresholds.similarityThreshold,
minimumCandidates: 15
},
processingTimeMs: 0
},
selectionProcess: {
aiModel: 'tactical',
modelConfig: {
endpoint: '',
model: '',
maxTokens: 0,
temperature: 0
},
promptTokens: 0,
responseTokens: 0,
processingTimeMs: 0,
initialCandidates: [],
finalSelection: [],
rejectedCandidates: [],
selectionReasoning: '',
confidenceScore: 0,
rawResponse: ''
},
biasAnalysis: [],
microTasks: [],
qualityMetrics: {
overallConfidence: 0,
reproducibilityScore: 0,
biasRiskScore: 0,
transparencyScore: 0,
evidenceQuality: 0,
methodologicalSoundness: 0
},
processingSummary: {
totalTimeMs: 0,
aiCallsMade: 0,
tokensTotalUsed: 0,
errorsEncountered: 0,
fallbacksUsed: 0,
finalRecommendationCount: 0
},
compliance: {
auditCompliant: true,
dataRetentionCompliant: true,
biasChecked: false,
confidenceAssessed: false,
traceabilityScore: 0
}
};
console.log(`[AUDIT TRAIL] Started audit ${auditId} for user ${userId}, mode: ${mode}`);
return auditId;
}
logQueryClassification(classification: Partial<QueryClassification>): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
this.currentAudit.queryClassification = {
...this.currentAudit.queryClassification,
...classification
};
}
logSanitizedQuery(sanitizedQuery: string): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
this.currentAudit.sanitizedQuery = sanitizedQuery;
}
// ========================================================================
// RETRIEVAL PROCESS LOGGING
// ========================================================================
logRetrievalStart(method: 'embeddings' | 'ai_selector' | 'emergency_fallback'): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
this.currentAudit.retrievalProcess.method = method;
this.currentAudit.retrievalProcess.embeddingsUsed = method === 'embeddings';
if (method === 'emergency_fallback') {
this.currentAudit.processingSummary.fallbacksUsed++;
}
}
logRetrievalResults(data: {
candidatesFound: number;
similarityScores: Array<{ tool: string; score: number; type: string }>;
confidence: number;
processingTimeMs: number;
fallbackReason?: string;
}): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
this.currentAudit.retrievalProcess = {
...this.currentAudit.retrievalProcess,
candidatesFound: data.candidatesFound,
similarityScores: data.similarityScores,
retrievalConfidence: data.confidence,
processingTimeMs: data.processingTimeMs,
fallbackReason: data.fallbackReason
};
}
// ========================================================================
// SELECTION PROCESS LOGGING
// ========================================================================
logSelectionStart(aiModel: 'strategic' | 'tactical' | 'legacy', initialCandidates: string[]): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
const modelConfig = aiModel === 'legacy' ?
forensicConfig.getLegacyAIModel() :
forensicConfig.getAIModel(aiModel);
this.currentAudit.selectionProcess.aiModel = aiModel;
this.currentAudit.selectionProcess.modelConfig = {
endpoint: modelConfig.endpoint,
model: modelConfig.model,
maxTokens: modelConfig.maxOutputTokens,
temperature: modelConfig.temperature
};
this.currentAudit.selectionProcess.initialCandidates = [...initialCandidates];
}
logSelectionResults(data: {
finalSelection: string[];
rejectedCandidates: Array<{ tool: string; reason: string; score?: number }>;
reasoning: string;
confidence: number;
promptTokens: number;
responseTokens: number;
processingTimeMs: number;
rawResponse: string;
}): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
this.currentAudit.selectionProcess = {
...this.currentAudit.selectionProcess,
finalSelection: [...data.finalSelection],
rejectedCandidates: [...data.rejectedCandidates],
selectionReasoning: data.reasoning,
confidenceScore: data.confidence,
promptTokens: data.promptTokens,
responseTokens: data.responseTokens,
processingTimeMs: data.processingTimeMs,
rawResponse: data.rawResponse
};
this.currentAudit.processingSummary.aiCallsMade++;
this.currentAudit.processingSummary.tokensTotalUsed += data.promptTokens + data.responseTokens;
}
// ========================================================================
// BIAS ANALYSIS LOGGING
// ========================================================================
logBiasAnalysis(biasResults: BiasAnalysisEntry[]): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
this.currentAudit.biasAnalysis = [...biasResults];
this.currentAudit.compliance.biasChecked = true;
// Calculate overall bias risk score
const biasRiskScore = biasResults.length > 0 ?
Math.max(...biasResults.filter(b => b.detected).map(b => b.severity)) : 0;
this.currentAudit.qualityMetrics.biasRiskScore = biasRiskScore;
}
// ========================================================================
// MICRO-TASK LOGGING
// ========================================================================
logMicroTask(taskData: {
taskType: MicroTaskAudit['taskType'];
aiModel: 'strategic' | 'tactical' | 'legacy';
success: boolean;
processingTimeMs: number;
confidence: number;
contextUsed: string[];
outputLength: number;
promptTokens: number;
responseTokens: number;
errorMessage?: string;
contextContinuityUsed?: boolean;
}): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
const microTask: MicroTaskAudit = {
taskId: `${taskData.taskType}_${Date.now()}`,
taskType: taskData.taskType,
aiModel: taskData.aiModel,
success: taskData.success,
processingTimeMs: taskData.processingTimeMs,
confidence: taskData.confidence,
contextUsed: [...taskData.contextUsed],
outputLength: taskData.outputLength,
promptTokens: taskData.promptTokens,
responseTokens: taskData.responseTokens,
errorMessage: taskData.errorMessage,
contextContinuityUsed: taskData.contextContinuityUsed || false
};
this.currentAudit.microTasks.push(microTask);
// Update processing summary
this.currentAudit.processingSummary.aiCallsMade++;
this.currentAudit.processingSummary.tokensTotalUsed += taskData.promptTokens + taskData.responseTokens;
if (!taskData.success) {
this.currentAudit.processingSummary.errorsEncountered++;
}
}
// ========================================================================
// AUDIT FINALIZATION
// ========================================================================
calculateQualityMetrics(): void {
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
const audit = this.currentAudit;
// Overall confidence (weighted average of retrieval and selection confidence)
const overallConfidence = (
audit.retrievalProcess.retrievalConfidence * 0.3 +
audit.selectionProcess.confidenceScore * 0.5 +
(audit.microTasks.reduce((sum, task) => sum + task.confidence, 0) / Math.max(audit.microTasks.length, 1)) * 0.2
);
// Reproducibility score (based on audit completeness and systematic approach)
const reproducibilityScore = (
(audit.retrievalProcess.similarityScores.length > 0 ? 0.3 : 0) +
(audit.selectionProcess.selectionReasoning.length > 50 ? 0.3 : 0) +
(audit.microTasks.length >= 4 ? 0.4 : audit.microTasks.length * 0.1)
);
// Bias risk score (inverse of detected bias severity)
const biasRiskScore = audit.qualityMetrics.biasRiskScore;
// Transparency score (based on audit detail level and traceability)
const transparencyScore = (
(audit.selectionProcess.rawResponse.length > 0 ? 0.3 : 0) +
(audit.retrievalProcess.similarityScores.length > 0 ? 0.3 : 0) +
(audit.microTasks.every(task => task.contextUsed.length > 0) ? 0.4 : 0)
);
// Evidence quality (based on retrieval quality and reasoning depth)
const evidenceQuality = (
audit.retrievalProcess.retrievalConfidence * 0.5 +
(audit.selectionProcess.selectionReasoning.length / 1000) * 0.3 +
(audit.microTasks.filter(task => task.success).length / Math.max(audit.microTasks.length, 1)) * 0.2
);
// Methodological soundness (systematic approach and error handling)
const methodologicalSoundness = (
(audit.processingSummary.fallbacksUsed === 0 ? 0.3 : 0.1) +
(audit.processingSummary.errorsEncountered === 0 ? 0.3 : Math.max(0, 0.3 - audit.processingSummary.errorsEncountered * 0.1)) +
(audit.compliance.biasChecked ? 0.2 : 0) +
(audit.microTasks.length >= 4 ? 0.2 : 0)
);
audit.qualityMetrics = {
overallConfidence: Math.min(1, Math.max(0, overallConfidence)),
reproducibilityScore: Math.min(1, Math.max(0, reproducibilityScore)),
biasRiskScore: Math.min(1, Math.max(0, biasRiskScore)),
transparencyScore: Math.min(1, Math.max(0, transparencyScore)),
evidenceQuality: Math.min(1, Math.max(0, evidenceQuality)),
methodologicalSoundness: Math.min(1, Math.max(0, methodologicalSoundness))
};
audit.compliance.confidenceAssessed = true;
audit.compliance.traceabilityScore = transparencyScore;
}
finalizeAudit(finalRecommendationCount: number): ForensicAuditEntry | null {
if (!this.currentAudit || !this.config.auditTrail.enabled) {
return null;
}
// Calculate total processing time
this.currentAudit.processingSummary.totalTimeMs =
Date.now() - this.currentAudit.timestamp.getTime();
this.currentAudit.processingSummary.finalRecommendationCount = finalRecommendationCount;
// Calculate final quality metrics
this.calculateQualityMetrics();
// Store the audit trail
this.auditStorage.set(this.currentAudit.auditId, { ...this.currentAudit });
const finalAudit = { ...this.currentAudit };
this.currentAudit = null;
console.log(`[AUDIT TRAIL] Finalized audit ${finalAudit.auditId}`);
console.log(`[AUDIT TRAIL] Quality Score: ${(finalAudit.qualityMetrics.overallConfidence * 100).toFixed(1)}%`);
console.log(`[AUDIT TRAIL] Bias Risk: ${(finalAudit.qualityMetrics.biasRiskScore * 100).toFixed(1)}%`);
console.log(`[AUDIT TRAIL] Transparency: ${(finalAudit.qualityMetrics.transparencyScore * 100).toFixed(1)}%`);
return finalAudit;
}
// ========================================================================
// AUDIT RETRIEVAL AND EXPORT
// ========================================================================
getAuditTrail(auditId: string): ForensicAuditEntry | null {
return this.auditStorage.get(auditId) || null;
}
exportAuditForCompliance(auditId: string): string | null {
const audit = this.getAuditTrail(auditId);
if (!audit) return null;
return JSON.stringify(audit, null, 2);
}
getAuditSummary(auditId: string): any {
const audit = this.getAuditTrail(auditId);
if (!audit) return null;
return {
auditId: audit.auditId,
timestamp: audit.timestamp,
userId: audit.userId,
queryMode: audit.queryMode,
qualityMetrics: audit.qualityMetrics,
processingSummary: audit.processingSummary,
compliance: audit.compliance,
biasWarnings: audit.biasAnalysis.filter(b => b.detected).length,
microTasksCompleted: audit.microTasks.filter(t => t.success).length,
totalMicroTasks: audit.microTasks.length
};
}
// ========================================================================
// UTILITY METHODS
// ========================================================================
private setupCleanupInterval(): void {
const retentionMs = this.config.auditTrail.retentionDays * 24 * 60 * 60 * 1000;
setInterval(() => {
const now = Date.now();
let cleanedCount = 0;
for (const [auditId, audit] of this.auditStorage.entries()) {
if (now - audit.timestamp.getTime() > retentionMs) {
this.auditStorage.delete(auditId);
cleanedCount++;
}
}
if (cleanedCount > 0) {
console.log(`[AUDIT TRAIL] Cleaned up ${cleanedCount} expired audit entries`);
}
}, 60 * 60 * 1000); // Run cleanup every hour
}
getStorageStats(): { totalAudits: number; oldestAudit: string | null; newestAudit: string | null } {
const audits = Array.from(this.auditStorage.values());
if (audits.length === 0) {
return { totalAudits: 0, oldestAudit: null, newestAudit: null };
}
const sorted = audits.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
return {
totalAudits: audits.length,
oldestAudit: sorted[0].timestamp.toISOString(),
newestAudit: sorted[sorted.length - 1].timestamp.toISOString()
};
}
getCurrentAuditId(): string | null {
return this.currentAudit?.auditId || null;
}
isAuditInProgress(): boolean {
return this.currentAudit !== null;
}
}
// Export singleton instance
export const auditTrailService = new ForensicAuditTrailService();
export type { ForensicAuditEntry, QueryClassification, RetrievalAudit, SelectionAudit, BiasAnalysisEntry, MicroTaskAudit, QualityMetrics };

13
src/utils/embeddings.ts
View File

@ -24,6 +24,10 @@ interface EmbeddingsDatabase {
embeddings: EmbeddingData[]; embeddings: EmbeddingData[];
} }
interface EmbeddingSearchResult extends EmbeddingData {
similarity: number;
}
class EmbeddingsService { class EmbeddingsService {
private embeddings: EmbeddingData[] = []; private embeddings: EmbeddingData[] = [];
private isInitialized = false; private isInitialized = false;
@ -211,23 +215,20 @@ class EmbeddingsService {
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
} }
async findSimilar(query: string, maxResults: number = 30, threshold: number = 0.3): Promise<EmbeddingData[]> { async findSimilar(query: string, maxResults: number = 30, threshold: number = 0.3): Promise<EmbeddingSearchResult[]> {
if (!this.enabled || !this.isInitialized || this.embeddings.length === 0) { if (!this.enabled || !this.isInitialized || this.embeddings.length === 0) {
return []; return [];
} }
try { try {
// Generate embedding for query
const queryEmbeddings = await this.generateEmbeddingsBatch([query.toLowerCase()]); const queryEmbeddings = await this.generateEmbeddingsBatch([query.toLowerCase()]);
const queryEmbedding = queryEmbeddings[0]; const queryEmbedding = queryEmbeddings[0];
// Calculate similarities
const similarities = this.embeddings.map(item => ({ const similarities = this.embeddings.map(item => ({
...item, ...item,
similarity: this.cosineSimilarity(queryEmbedding, item.embedding) similarity: this.cosineSimilarity(queryEmbedding, item.embedding)
})); }));
// Filter by threshold and sort by similarity
return similarities return similarities
.filter(item => item.similarity >= threshold) .filter(item => item.similarity >= threshold)
.sort((a, b) => b.similarity - a.similarity) .sort((a, b) => b.similarity - a.similarity)
@ -254,12 +255,10 @@ class EmbeddingsService {
// Global instance
const embeddingsService = new EmbeddingsService(); const embeddingsService = new EmbeddingsService();
export { embeddingsService, type EmbeddingData }; export { embeddingsService, type EmbeddingData, type EmbeddingSearchResult };
// Auto-initialize on import in server environment
if (typeof window === 'undefined' && process.env.NODE_ENV !== 'test') { if (typeof window === 'undefined' && process.env.NODE_ENV !== 'test') {
embeddingsService.initialize().catch(error => { embeddingsService.initialize().catch(error => {
console.error('[EMBEDDINGS] Auto-initialization failed:', error); console.error('[EMBEDDINGS] Auto-initialization failed:', error);

301
src/utils/forensicConfig.ts Normal file
View File

@ -0,0 +1,301 @@
// src/utils/forensicConfig.ts - Centralized Forensic Configuration Management
interface AIModelConfig {
endpoint: string;
apiKey: string;
model: string;
maxContextTokens: number;
maxOutputTokens: number;
temperature: number;
purpose: 'strategic' | 'tactical';
}
interface ForensicThresholds {
// AI Selection Thresholds
maxSelectedItems: number;
embeddingCandidates: number;
similarityThreshold: number;
confidenceThreshold: number;
// Bias Detection Thresholds
biasAlertThreshold: number;
popularityBiasThreshold: number;
embeddingsConfidenceThreshold: number;
selectionConfidenceMinimum: number;
// Performance Thresholds
microTaskTimeoutMs: number;
microTaskDelayMs: number;
rateLimitDelayMs: number;
rateLimitMaxRequests: number;
}
interface ForensicConfig {
// AI Models Configuration
aiModels: {
strategic: AIModelConfig;
tactical: AIModelConfig;
};
// Legacy model (for backward compatibility)
legacyModel: AIModelConfig;
// Audit Trail Settings
auditTrail: {
enabled: boolean;
retentionDays: number;
detailLevel: 'minimal' | 'standard' | 'detailed' | 'comprehensive';
};
// Feature Flags
features: {
confidenceScoring: boolean;
biasDetection: boolean;
performanceMetrics: boolean;
debugMode: boolean;
};
// All configurable thresholds
thresholds: ForensicThresholds;
// Queue and Performance
queue: {
maxSize: number;
cleanupIntervalMs: number;
};
// Embeddings Configuration
embeddings: {
enabled: boolean;
endpoint: string;
apiKey: string;
model: string;
batchSize: number;
batchDelayMs: number;
};
}
class ForensicConfigManager {
private static instance: ForensicConfigManager;
private config: ForensicConfig;
private constructor() {
this.config = this.loadConfig();
this.validateConfig();
}
static getInstance(): ForensicConfigManager {
if (!ForensicConfigManager.instance) {
ForensicConfigManager.instance = new ForensicConfigManager();
}
return ForensicConfigManager.instance;
}
private getEnv(key: string, defaultValue?: string): string {
const value = process.env[key];
if (!value && defaultValue === undefined) {
throw new Error(`Missing required environment variable: ${key}`);
}
return value || defaultValue || '';
}
private getEnvNumber(key: string, defaultValue: number): number {
const value = process.env[key];
if (!value) return defaultValue;
const parsed = parseInt(value, 10);
if (isNaN(parsed)) {
console.warn(`[CONFIG] Invalid number for ${key}: ${value}, using default: ${defaultValue}`);
return defaultValue;
}
return parsed;
}
private getEnvFloat(key: string, defaultValue: number): number {
const value = process.env[key];
if (!value) return defaultValue;
const parsed = parseFloat(value);
if (isNaN(parsed)) {
console.warn(`[CONFIG] Invalid float for ${key}: ${value}, using default: ${defaultValue}`);
return defaultValue;
}
return parsed;
}
private getEnvBoolean(key: string, defaultValue: boolean): boolean {
const value = process.env[key];
if (!value) return defaultValue;
return value.toLowerCase() === 'true';
}
private loadConfig(): ForensicConfig {
// Strategic AI Model Configuration
const strategicModel: AIModelConfig = {
endpoint: this.getEnv('AI_STRATEGIC_ENDPOINT', this.getEnv('AI_ANALYZER_ENDPOINT')),
apiKey: this.getEnv('AI_STRATEGIC_API_KEY', this.getEnv('AI_ANALYZER_API_KEY')),
model: this.getEnv('AI_STRATEGIC_MODEL', this.getEnv('AI_ANALYZER_MODEL')),
maxContextTokens: this.getEnvNumber('AI_STRATEGIC_MAX_CONTEXT_TOKENS', 32000),
maxOutputTokens: this.getEnvNumber('AI_STRATEGIC_MAX_OUTPUT_TOKENS', 1000),
temperature: this.getEnvFloat('AI_STRATEGIC_TEMPERATURE', 0.2),
purpose: 'strategic'
};
// Tactical AI Model Configuration
const tacticalModel: AIModelConfig = {
endpoint: this.getEnv('AI_TACTICAL_ENDPOINT', this.getEnv('AI_ANALYZER_ENDPOINT')),
apiKey: this.getEnv('AI_TACTICAL_API_KEY', this.getEnv('AI_ANALYZER_API_KEY')),
model: this.getEnv('AI_TACTICAL_MODEL', this.getEnv('AI_ANALYZER_MODEL')),
maxContextTokens: this.getEnvNumber('AI_TACTICAL_MAX_CONTEXT_TOKENS', 8000),
maxOutputTokens: this.getEnvNumber('AI_TACTICAL_MAX_OUTPUT_TOKENS', 500),
temperature: this.getEnvFloat('AI_TACTICAL_TEMPERATURE', 0.3),
purpose: 'tactical'
};
// Legacy model (backward compatibility)
const legacyModel: AIModelConfig = {
endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'),
apiKey: this.getEnv('AI_ANALYZER_API_KEY'),
model: this.getEnv('AI_ANALYZER_MODEL'),
maxContextTokens: 8000,
maxOutputTokens: 1000,
temperature: 0.3,
purpose: 'tactical'
};
return {
aiModels: {
strategic: strategicModel,
tactical: tacticalModel
},
legacyModel,
auditTrail: {
enabled: this.getEnvBoolean('FORENSIC_AUDIT_ENABLED', true),
retentionDays: this.getEnvNumber('FORENSIC_AUDIT_RETENTION_DAYS', 90),
detailLevel: (this.getEnv('FORENSIC_AUDIT_DETAIL_LEVEL', 'detailed') as any) || 'detailed'
},
features: {
confidenceScoring: this.getEnvBoolean('FORENSIC_CONFIDENCE_SCORING_ENABLED', true),
biasDetection: this.getEnvBoolean('FORENSIC_BIAS_DETECTION_ENABLED', true),
performanceMetrics: this.getEnvBoolean('AI_PERFORMANCE_METRICS', true),
debugMode: this.getEnvBoolean('AI_MICRO_TASK_DEBUG', false)
},
thresholds: {
maxSelectedItems: this.getEnvNumber('AI_MAX_SELECTED_ITEMS', 60),
embeddingCandidates: this.getEnvNumber('AI_EMBEDDING_CANDIDATES', 60),
similarityThreshold: this.getEnvFloat('AI_SIMILARITY_THRESHOLD', 0.3),
confidenceThreshold: this.getEnvFloat('AI_CONFIDENCE_THRESHOLD', 0.7),
biasAlertThreshold: this.getEnvFloat('AI_BIAS_ALERT_THRESHOLD', 0.8),
popularityBiasThreshold: this.getEnvFloat('TOOL_POPULARITY_BIAS_THRESHOLD', 0.75),
embeddingsConfidenceThreshold: this.getEnvFloat('EMBEDDINGS_CONFIDENCE_THRESHOLD', 0.6),
selectionConfidenceMinimum: this.getEnvFloat('SELECTION_CONFIDENCE_MINIMUM', 0.5),
microTaskTimeoutMs: this.getEnvNumber('AI_MICRO_TASK_TIMEOUT_MS', 25000),
microTaskDelayMs: this.getEnvNumber('AI_MICRO_TASK_DELAY_MS', 500),
rateLimitDelayMs: this.getEnvNumber('AI_RATE_LIMIT_DELAY_MS', 3000),
rateLimitMaxRequests: this.getEnvNumber('AI_RATE_LIMIT_MAX_REQUESTS', 6)
},
queue: {
maxSize: this.getEnvNumber('AI_QUEUE_MAX_SIZE', 50),
cleanupIntervalMs: this.getEnvNumber('AI_QUEUE_CLEANUP_INTERVAL_MS', 300000)
},
embeddings: {
enabled: this.getEnvBoolean('AI_EMBEDDINGS_ENABLED', true),
endpoint: this.getEnv('AI_EMBEDDINGS_ENDPOINT', ''),
apiKey: this.getEnv('AI_EMBEDDINGS_API_KEY', ''),
model: this.getEnv('AI_EMBEDDINGS_MODEL', 'mistral-embed'),
batchSize: this.getEnvNumber('AI_EMBEDDINGS_BATCH_SIZE', 20),
batchDelayMs: this.getEnvNumber('AI_EMBEDDINGS_BATCH_DELAY_MS', 1000)
}
};
}
private validateConfig(): void {
const errors: string[] = [];
// Validate AI models
if (!this.config.aiModels.strategic.endpoint) {
errors.push('Strategic AI endpoint is required');
}
if (!this.config.aiModels.tactical.endpoint) {
errors.push('Tactical AI endpoint is required');
}
// Validate thresholds
const t = this.config.thresholds;
if (t.similarityThreshold < 0 || t.similarityThreshold > 1) {
errors.push('Similarity threshold must be between 0 and 1');
}
if (t.confidenceThreshold < 0 || t.confidenceThreshold > 1) {
errors.push('Confidence threshold must be between 0 and 1');
}
if (errors.length > 0) {
throw new Error(`Configuration validation failed:\n${errors.join('\n')}`);
}
console.log('[FORENSIC CONFIG] Configuration loaded and validated successfully');
console.log(`[FORENSIC CONFIG] Strategic Model: ${this.config.aiModels.strategic.model}`);
console.log(`[FORENSIC CONFIG] Tactical Model: ${this.config.aiModels.tactical.model}`);
console.log(`[FORENSIC CONFIG] Audit Trail: ${this.config.auditTrail.enabled ? 'Enabled' : 'Disabled'}`);
console.log(`[FORENSIC CONFIG] Confidence Scoring: ${this.config.features.confidenceScoring ? 'Enabled' : 'Disabled'}`);
console.log(`[FORENSIC CONFIG] Bias Detection: ${this.config.features.biasDetection ? 'Enabled' : 'Disabled'}`);
}
// Public access methods
getConfig(): ForensicConfig {
return { ...this.config }; // Return a copy to prevent modification
}
getAIModel(purpose: 'strategic' | 'tactical'): AIModelConfig {
return { ...this.config.aiModels[purpose] };
}
getLegacyAIModel(): AIModelConfig {
return { ...this.config.legacyModel };
}
getThresholds(): ForensicThresholds {
return { ...this.config.thresholds };
}
isFeatureEnabled(feature: keyof ForensicConfig['features']): boolean {
return this.config.features[feature];
}
// Utility methods for common configurations
getMaxTokensForTask(taskType: 'analysis' | 'description' | 'selection' | 'evaluation'): number {
switch (taskType) {
case 'analysis':
case 'selection':
return this.config.aiModels.strategic.maxOutputTokens;
case 'description':
case 'evaluation':
return this.config.aiModels.tactical.maxOutputTokens;
default:
return this.config.aiModels.tactical.maxOutputTokens;
}
}
getModelForTask(taskType: 'analysis' | 'description' | 'selection' | 'evaluation'): 'strategic' | 'tactical' {
switch (taskType) {
case 'analysis':
case 'selection':
return 'strategic'; // Use strategic model for complex reasoning
case 'description':
case 'evaluation':
return 'tactical'; // Use tactical model for text generation
default:
return 'tactical';
}
}
}
// Export singleton instance
export const forensicConfig = ForensicConfigManager.getInstance();
export type { ForensicConfig, AIModelConfig, ForensicThresholds };

5
src/utils/rateLimitedQueue.ts
View File

@ -1,10 +1,11 @@
// src/utils/rateLimitedQueue.ts - FIXED: Memory leak and better cleanup // src/utils/rateLimitedQueue.ts
import { forensicConfig } from './forensicConfig.js';
import dotenv from "dotenv"; import dotenv from "dotenv";
dotenv.config(); dotenv.config();
const RATE_LIMIT_DELAY_MS = Number.parseInt(process.env.AI_RATE_LIMIT_DELAY_MS ?? "2000", 10) || 2000; const RATE_LIMIT_DELAY_MS = forensicConfig.getThresholds().rateLimitDelayMs;
export type Task<T = unknown> = () => Promise<T>; export type Task<T = unknown> = () => Promise<T>;