phase 4
This commit is contained in:
overcuriousity
parent
b192f257a1
commit
78b30fb7c6
93
src/pages/api/ai/bias-statistics.ts
Normal file
93
src/pages/api/ai/bias-statistics.ts
Normal file
@ -0,0 +1,93 @@
|
||||
// src/pages/api/ai/bias-statistics.ts - New API endpoint for bias statistics
|
||||
|
||||
import type { APIRoute } from 'astro';
|
||||
import { withAPIAuth } from '../../../utils/auth.js';
|
||||
import { apiResponse, apiServerError, createAuthErrorResponse } from '../../../utils/api.js';
|
||||
import { auditTrailService } from '../../../utils/auditTrail.js';
|
||||
import { biasDetector } from '../../../utils/biasDetection.js';
|
||||
|
||||
export const prerender = false;
|
||||
|
||||
export const GET: APIRoute = async ({ request }) => {
|
||||
try {
|
||||
const authResult = await withAPIAuth(request, 'ai');
|
||||
if (!authResult.authenticated) {
|
||||
return createAuthErrorResponse();
|
||||
}
|
||||
|
||||
// Get comprehensive bias statistics
|
||||
const biasStats = auditTrailService.getBiasStatistics();
|
||||
const baselineStats = biasDetector.getBaselineStats();
|
||||
const biasThresholds = biasDetector.getBiasThresholds();
|
||||
|
||||
return apiResponse.success({
|
||||
biasStatistics: {
|
||||
overall: biasStats,
|
||||
baseline: baselineStats,
|
||||
configuration: {
|
||||
thresholds: biasThresholds,
|
||||
detectionEnabled: true,
|
||||
lastUpdated: new Date().toISOString()
|
||||
},
|
||||
interpretation: {
|
||||
biasRiskLevel: biasStats.averageBiasInstances > 1.5 ? 'high' :
|
||||
biasStats.averageBiasInstances > 0.8 ? 'medium' : 'low',
|
||||
mostCommonBias: Object.entries(biasStats.biasTypeFrequency || {})
|
||||
.sort(([,a], [,b]) => (b as number) - (a as number))[0]?.[0] || 'none',
|
||||
recommendations: generateBiasRecommendations(biasStats)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('[BIAS STATS API] Error:', error);
|
||||
return apiServerError.internal('Failed to retrieve bias statistics');
|
||||
}
|
||||
};
|
||||
|
||||
function generateBiasRecommendations(stats: any): string[] {
|
||||
const recommendations: string[] = [];
|
||||
|
||||
if (!stats.biasTypeFrequency) {
|
||||
return ['Insufficient data for bias analysis recommendations'];
|
||||
}
|
||||
|
||||
const biasEntries = Object.entries(stats.biasTypeFrequency) as [string, number][];
|
||||
const totalAudits = stats.totalAudits || 1;
|
||||
|
||||
// Check for high-frequency bias types
|
||||
biasEntries.forEach(([biasType, count]) => {
|
||||
const frequency = count / totalAudits;
|
||||
|
||||
if (frequency > 0.3) { // More than 30% of audits show this bias
|
||||
switch (biasType) {
|
||||
case 'popularity':
|
||||
recommendations.push('HIGH: Popularity bias detected frequently. Review selection criteria to prioritize scenario-specific over well-known tools.');
|
||||
break;
|
||||
case 'availability':
|
||||
recommendations.push('MEDIUM: Availability bias detected. Encourage more thorough candidate evaluation beyond easily recalled options.');
|
||||
break;
|
||||
case 'domain_concentration':
|
||||
recommendations.push('MEDIUM: Domain concentration bias detected. Promote cross-domain tool selection for comprehensive analysis.');
|
||||
break;
|
||||
case 'skill_level':
|
||||
recommendations.push('LOW: Skill level misalignment detected. Improve query classification for better skill-appropriate selections.');
|
||||
break;
|
||||
case 'recency':
|
||||
recommendations.push('LOW: Recency bias detected. Balance newer tools with proven established alternatives.');
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (recommendations.length === 0) {
|
||||
recommendations.push('No significant bias patterns detected. Current selection methodology appears balanced.');
|
||||
}
|
||||
|
||||
// Add general recommendation if multiple bias types detected
|
||||
if (biasEntries.length > 3) {
|
||||
recommendations.push('GENERAL: Multiple bias types detected. Consider implementing additional bias mitigation strategies.');
|
||||
}
|
||||
|
||||
return recommendations;
|
||||
}
|
||||
@ -7,6 +7,7 @@ import { enqueueApiCall } from '../../../utils/rateLimitedQueue.js';
|
||||
import { aiPipeline } from '../../../utils/aiPipeline.js';
|
||||
import { forensicConfig } from '../../../utils/forensicConfig.js';
|
||||
import { confidenceScorer } from '../../../utils/confidenceScoring.js';
|
||||
import { biasDetector } from '../../../utils/biasDetection.js';
|
||||
|
||||
export const prerender = false;
|
||||
|
||||
@ -212,18 +213,17 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
query: sanitizedQuery,
|
||||
processingStats: {
|
||||
...result.processingStats,
|
||||
pipelineType: 'enhanced-micro-task-with-confidence',
|
||||
pipelineType: 'enhanced-micro-task-with-bias-detection',
|
||||
microTasksSuccessRate: stats.microTasksCompleted / Math.max(stats.microTasksCompleted + stats.microTasksFailed, 1),
|
||||
averageTaskTime: stats.processingTimeMs / Math.max(stats.microTasksCompleted + stats.microTasksFailed, 1),
|
||||
estimatedAICallsMade,
|
||||
auditCompliant: result.auditTrail?.compliance.auditCompliant || false,
|
||||
biasChecked: result.auditTrail?.compliance.biasChecked || false,
|
||||
confidenceAssessed: result.auditTrail?.compliance.confidenceAssessed || false,
|
||||
// NEW: Confidence acceptance flag
|
||||
confidenceAcceptable
|
||||
},
|
||||
|
||||
// ENHANCED: Comprehensive forensic metadata with confidence details
|
||||
// ENHANCED: Comprehensive forensic metadata with bias detection
|
||||
forensicMetadata: result.auditTrail ? {
|
||||
auditTrailId: result.auditTrail.auditId,
|
||||
auditEnabled: config.auditTrail.enabled,
|
||||
@ -236,7 +236,30 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
evidenceQuality: result.auditTrail.qualityMetrics.evidenceQuality,
|
||||
methodologicalSoundness: result.auditTrail.qualityMetrics.methodologicalSoundness,
|
||||
|
||||
// NEW: Detailed confidence breakdown
|
||||
// ENHANCED: Detailed bias analysis
|
||||
biasAnalysis: {
|
||||
overallBiasRisk: result.auditTrail.qualityMetrics.biasRiskScore,
|
||||
isHighBiasRisk: biasDetector.isHighBiasRisk(result.auditTrail.qualityMetrics.biasRiskScore),
|
||||
detectedBiases: result.auditTrail.biasAnalysis.filter(bias => bias.detected).map(bias => ({
|
||||
type: bias.biasType,
|
||||
severity: bias.severity,
|
||||
confidence: bias.confidence,
|
||||
description: bias.description,
|
||||
affectedTools: bias.evidence.affectedTools,
|
||||
recommendation: bias.recommendation
|
||||
})),
|
||||
biasFreeSeverity: result.auditTrail.biasAnalysis.filter(bias => !bias.detected).length,
|
||||
mitigationSuggestions: biasDetector.suggestBiasMitigation(result.auditTrail.biasAnalysis),
|
||||
biasTypes: {
|
||||
popularity: result.auditTrail.biasAnalysis.find(b => b.biasType === 'popularity'),
|
||||
availability: result.auditTrail.biasAnalysis.find(b => b.biasType === 'availability'),
|
||||
domainConcentration: result.auditTrail.biasAnalysis.find(b => b.biasType === 'domain_concentration'),
|
||||
skillLevel: result.auditTrail.biasAnalysis.find(b => b.biasType === 'skill_level'),
|
||||
recency: result.auditTrail.biasAnalysis.find(b => b.biasType === 'recency')
|
||||
}
|
||||
},
|
||||
|
||||
// Detailed confidence breakdown
|
||||
confidenceBreakdown: config.features.confidenceScoring ? {
|
||||
retrieval: result.auditTrail.qualityMetrics.confidenceBreakdown.retrieval,
|
||||
selection: result.auditTrail.qualityMetrics.confidenceBreakdown.selection,
|
||||
@ -244,7 +267,7 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
meta: result.auditTrail.qualityMetrics.confidenceBreakdown.meta
|
||||
} : undefined,
|
||||
|
||||
// NEW: Confidence assessment details
|
||||
// Confidence assessment details
|
||||
confidenceAssessment: config.features.confidenceScoring ? {
|
||||
qualityLevel: result.auditTrail.qualityMetrics.qualityLevel,
|
||||
reliability: result.auditTrail.qualityMetrics.confidenceReliability,
|
||||
@ -254,9 +277,27 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
threshold: thresholds.confidenceThreshold
|
||||
} : undefined,
|
||||
|
||||
// Bias and quality warnings
|
||||
biasWarnings: result.auditTrail.biasAnalysis.filter(b => b.detected),
|
||||
qualityWarnings: !confidenceAcceptable ? ['Confidence below acceptable threshold'] : [],
|
||||
// ENHANCED: Bias and quality warnings with specific guidance
|
||||
biasWarnings: result.auditTrail.biasAnalysis.filter(b => b.detected).map(bias => ({
|
||||
type: bias.biasType,
|
||||
severity: bias.severity,
|
||||
message: bias.description,
|
||||
actionRequired: bias.severity > 0.7 ? 'immediate_review' : 'consideration',
|
||||
mitigation: bias.mitigation
|
||||
})),
|
||||
|
||||
qualityWarnings: [
|
||||
...((!confidenceAcceptable) ? [{
|
||||
type: 'low_confidence',
|
||||
message: 'Overall confidence below acceptable threshold',
|
||||
actionRequired: 'expert_review'
|
||||
}] : []),
|
||||
...(result.auditTrail.qualityMetrics.biasRiskScore > thresholds.biasAlertThreshold ? [{
|
||||
type: 'high_bias_risk',
|
||||
message: 'High bias risk detected in tool selection',
|
||||
actionRequired: 'bias_review'
|
||||
}] : [])
|
||||
],
|
||||
|
||||
// System configuration snapshot
|
||||
systemConfig: {
|
||||
@ -264,25 +305,41 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
tacticalModel: result.auditTrail.systemConfig.tacticalModel,
|
||||
auditLevel: result.auditTrail.systemConfig.auditLevel,
|
||||
confidenceScoringEnabled: config.features.confidenceScoring,
|
||||
biasDetectionEnabled: config.features.biasDetection
|
||||
biasDetectionEnabled: config.features.biasDetection,
|
||||
biasThresholds: biasDetector.getBiasThresholds()
|
||||
},
|
||||
|
||||
// Compliance and traceability
|
||||
compliance: result.auditTrail.compliance,
|
||||
qualityLevel: result.auditTrail.qualityMetrics.overallConfidence >= thresholds.confidenceThreshold ? 'high' :
|
||||
result.auditTrail.qualityMetrics.overallConfidence >= 0.5 ? 'medium' : 'low',
|
||||
result.auditTrail.qualityMetrics.overallConfidence >= 0.5 ? 'medium' : 'low',
|
||||
|
||||
// NEW: Actionable insights
|
||||
// ENHANCED: Actionable insights with bias considerations
|
||||
actionableInsights: {
|
||||
shouldReviewSelection: result.auditTrail.qualityMetrics.biasRiskScore > thresholds.biasAlertThreshold,
|
||||
shouldImproveQuery: result.auditTrail.qualityMetrics.uncertaintyFactors.length > 2,
|
||||
shouldSeekExpertReview: result.auditTrail.qualityMetrics.overallConfidence < 0.6,
|
||||
confidenceImprovement: result.auditTrail.qualityMetrics.improvementSuggestions.slice(0, 3)
|
||||
shouldSeekExpertReview: result.auditTrail.qualityMetrics.overallConfidence < 0.6 ||
|
||||
biasDetector.isHighBiasRisk(result.auditTrail.qualityMetrics.biasRiskScore),
|
||||
confidenceImprovement: result.auditTrail.qualityMetrics.improvementSuggestions.slice(0, 3),
|
||||
biasReduction: biasDetector.suggestBiasMitigation(result.auditTrail.biasAnalysis).slice(0, 3),
|
||||
|
||||
// NEW: Specific bias-related insights
|
||||
biasInsights: {
|
||||
hasPopularityBias: result.auditTrail.biasAnalysis.some(b => b.biasType === 'popularity' && b.detected),
|
||||
hasAvailabilityBias: result.auditTrail.biasAnalysis.some(b => b.biasType === 'availability' && b.detected),
|
||||
hasDomainConcentration: result.auditTrail.biasAnalysis.some(b => b.biasType === 'domain_concentration' && b.detected),
|
||||
hasSkillLevelMismatch: result.auditTrail.biasAnalysis.some(b => b.biasType === 'skill_level' && b.detected),
|
||||
hasRecencyBias: result.auditTrail.biasAnalysis.some(b => b.biasType === 'recency' && b.detected),
|
||||
primaryBiasConcern: result.auditTrail.biasAnalysis
|
||||
.filter(b => b.detected)
|
||||
.sort((a, b) => b.severity - a.severity)[0]?.biasType || null
|
||||
}
|
||||
}
|
||||
} : {
|
||||
auditTrailId: null,
|
||||
auditEnabled: false,
|
||||
message: 'Audit trail disabled - operating in legacy mode'
|
||||
biasAnalysis: { enabled: false, message: 'Bias detection disabled - operating in legacy mode' },
|
||||
message: 'Enhanced forensic features disabled - operating in legacy mode'
|
||||
},
|
||||
|
||||
rateLimitInfo: {
|
||||
@ -299,20 +356,22 @@ export const POST: APIRoute = async ({ request }) => {
|
||||
console.error('[ENHANCED API] Pipeline error:', error);
|
||||
|
||||
// Provide detailed error information for forensic purposes
|
||||
if (error.message.includes('confidence')) {
|
||||
if (error.message.includes('bias')) {
|
||||
return apiServerError.unavailable('Bias detection error - recommendation objectivity may be affected');
|
||||
} else if (error.message.includes('confidence')) {
|
||||
return apiServerError.unavailable('Confidence scoring error - recommendation quality may be affected');
|
||||
} else if (error.message.includes('embeddings')) {
|
||||
return apiServerError.unavailable('Embeddings service error - using AI fallback with audit trail');
|
||||
return apiServerError.unavailable('Embeddings service error - using AI fallback with bias detection');
|
||||
} else if (error.message.includes('micro-task')) {
|
||||
return apiServerError.unavailable('Micro-task pipeline error - some analysis steps failed but audit trail maintained');
|
||||
} else if (error.message.includes('selector')) {
|
||||
return apiServerError.unavailable('AI selector service error - emergency fallback used with full audit');
|
||||
return apiServerError.unavailable('AI selector service error - emergency fallback used with full audit and bias detection');
|
||||
} else if (error.message.includes('rate limit')) {
|
||||
return apiError.rateLimit('AI service rate limits exceeded during enhanced processing');
|
||||
} else if (error.message.includes('audit')) {
|
||||
return apiServerError.internal('Audit trail system error - check forensic configuration');
|
||||
} else {
|
||||
return apiServerError.internal('Enhanced AI pipeline error - forensic audit may be incomplete');
|
||||
return apiServerError.internal('Enhanced AI pipeline error - forensic audit and bias detection may be incomplete');
|
||||
}
|
||||
}
|
||||
};
|
||||
@ -5,6 +5,7 @@ import { embeddingsService, type EmbeddingData, type EmbeddingSearchResult } fro
|
||||
import { forensicConfig, type AIModelConfig } from './forensicConfig.js';
|
||||
import { auditTrailService, type ForensicAuditEntry } from './auditTrail.js';
|
||||
import { confidenceScorer, type ConfidenceMetrics } from './confidenceScoring.js';
|
||||
import { biasDetector, type BiasAnalysisResult } from './biasDetection.js';
|
||||
|
||||
interface MicroTaskResult {
|
||||
taskType: string;
|
||||
@ -90,6 +91,20 @@ class EnhancedMicroTaskAIPipeline {
|
||||
console.log(`[ENHANCED PIPELINE] Audit Trail: ${this.config.auditTrail.enabled ? 'Enabled' : 'Disabled'}`);
|
||||
}
|
||||
|
||||
private updateBiasBaseline(): void {
|
||||
// Update bias detection baseline with recent audit data
|
||||
const recentAudits = Array.from(auditTrailService['auditStorage'].values())
|
||||
.filter(audit => {
|
||||
const daysSinceAudit = (Date.now() - audit.timestamp.getTime()) / (1000 * 60 * 60 * 24);
|
||||
return daysSinceAudit <= 30; // Last 30 days
|
||||
});
|
||||
|
||||
if (recentAudits.length >= 5) { // Minimum data for meaningful baseline
|
||||
biasDetector.updateBaseline(recentAudits);
|
||||
console.log(`[ENHANCED PIPELINE] Updated bias baseline with ${recentAudits.length} recent audits`);
|
||||
}
|
||||
}
|
||||
|
||||
private estimateTokens(text: string): number {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
@ -296,10 +311,6 @@ class EnhancedMicroTaskAIPipeline {
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ENHANCED AI SELECTION WITH AUDIT TRAIL
|
||||
// ============================================================================
|
||||
|
||||
private async aiSelectionWithAuditAndBias(
|
||||
userQuery: string,
|
||||
candidateTools: any[],
|
||||
@ -346,74 +357,66 @@ class EnhancedMicroTaskAIPipeline {
|
||||
related_software: concept.related_software || []
|
||||
}));
|
||||
|
||||
// ENHANCED: Bias-aware selection prompt
|
||||
const prompt = `You are a DFIR expert with access to the complete forensics tool database. You need to select the most relevant tools and concepts for this specific query.
|
||||
|
||||
SELECTION METHOD: ${selectionMethod}
|
||||
${selectionMethod === 'embeddings_candidates' ?
|
||||
'These tools were pre-filtered by vector similarity, so they are already relevant. Your job is to select the BEST ones from this relevant set.' :
|
||||
'You have access to the full tool database. Select the most relevant tools for the query.'}
|
||||
SELECTION METHOD: ${selectionMethod}
|
||||
${selectionMethod === 'embeddings_candidates' ?
|
||||
'These tools were pre-filtered by vector similarity, so they are already relevant. Your job is to select the BEST ones from this relevant set.' :
|
||||
'You have access to the full tool database. Select the most relevant tools for the query.'}
|
||||
|
||||
${modeInstruction}
|
||||
${modeInstruction}
|
||||
|
||||
USER QUERY: "${userQuery}"
|
||||
USER QUERY: "${userQuery}"
|
||||
|
||||
CRITICAL SELECTION PRINCIPLES:
|
||||
1. **CONTEXT OVER POPULARITY**: Don't default to "famous" tools like Volatility, Wireshark, or Autopsy just because they're well-known. Choose based on SPECIFIC scenario needs.
|
||||
CRITICAL SELECTION PRINCIPLES:
|
||||
1. **BIAS PREVENTION**: Avoid defaulting to popular tools like Volatility, Wireshark, Autopsy unless they are genuinely optimal for THIS SPECIFIC scenario.
|
||||
|
||||
2. **METHODOLOGY vs SOFTWARE**:
|
||||
- For RAPID/URGENT scenarios → Prioritize METHODS and rapid response approaches
|
||||
- For TIME-CRITICAL incidents → Choose triage methods over deep analysis tools
|
||||
- For COMPREHENSIVE analysis → Then consider detailed software tools
|
||||
- METHODS (type: "method") are often better than SOFTWARE for procedural guidance
|
||||
2. **SCENARIO-SPECIFIC LOGIC**:
|
||||
- "Rapid/Quick/Urgent/Triage" scenarios → Prioritize METHODS and rapid response approaches over complex software
|
||||
- "Industrial/SCADA/ICS" scenarios → Specialized ICS tools > generic network tools
|
||||
- "Mobile/Android/iOS" scenarios → Mobile-specific tools > desktop forensics tools
|
||||
- "Memory analysis needed urgently" → Quick memory tools/methods > comprehensive Volatility analysis
|
||||
|
||||
3. **SCENARIO-SPECIFIC LOGIC**:
|
||||
- "Rapid/Quick/Urgent/Triage" scenarios → Rapid Incident Response and Triage METHOD > Volatility
|
||||
- "Industrial/SCADA/ICS" scenarios → Specialized ICS tools > generic network tools
|
||||
- "Mobile/Android/iOS" scenarios → Mobile-specific tools > desktop forensics tools
|
||||
- "Memory analysis needed urgently" → Quick memory tools/methods > comprehensive Volatility analysis
|
||||
3. **OBJECTIVE SELECTION CRITERIA**:
|
||||
- Match tool capabilities to specific scenario requirements
|
||||
- Consider urgency level and time constraints
|
||||
- Prioritize appropriate skill level for the context
|
||||
- Ensure domain specialization when needed
|
||||
|
||||
4. **AVOID TOOL BIAS**:
|
||||
- Volatility is NOT always the answer for memory analysis
|
||||
- Wireshark is NOT always the answer for network analysis
|
||||
- Autopsy is NOT always the answer for disk analysis
|
||||
- Consider lighter, faster, more appropriate alternatives
|
||||
4. **AVOID COGNITIVE BIASES**:
|
||||
- Don't select tools just because they're well-known
|
||||
- Don't default to complex tools for simple scenarios
|
||||
- Don't ignore specialized tools in favor of general ones
|
||||
- Consider the FULL range of available options
|
||||
|
||||
AVAILABLE TOOLS (with complete data):
|
||||
${JSON.stringify(toolsWithFullData.slice(0, 30), null, 2)}
|
||||
AVAILABLE TOOLS (with complete data):
|
||||
${JSON.stringify(toolsWithFullData.slice(0, 30), null, 2)}
|
||||
|
||||
AVAILABLE CONCEPTS (with complete data):
|
||||
${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}
|
||||
AVAILABLE CONCEPTS (with complete data):
|
||||
${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}
|
||||
|
||||
ANALYSIS INSTRUCTIONS:
|
||||
1. Read the FULL description of each tool/concept
|
||||
2. Consider ALL tags, platforms, related tools, and metadata
|
||||
3. **MATCH URGENCY LEVEL**: Rapid scenarios need rapid methods, not deep analysis tools
|
||||
4. **MATCH SPECIFICITY**: Specialized scenarios need specialized tools, not generic ones
|
||||
5. **CONSIDER TYPE**: Methods provide procedural guidance, software provides technical capability
|
||||
6. For SCADA/ICS queries: prioritize specialized ICS tools over generic network tools
|
||||
7. For mobile queries: prioritize mobile-specific tools over desktop tools
|
||||
8. For rapid/urgent queries: prioritize methodology and triage approaches
|
||||
ANALYSIS INSTRUCTIONS:
|
||||
1. Read the FULL description of each tool/concept
|
||||
2. Consider ALL tags, platforms, related tools, and metadata
|
||||
3. **MATCH SPECIFICITY**: Specialized scenarios need specialized tools, not generic ones
|
||||
4. **MATCH URGENCY**: Rapid scenarios need rapid methods, not deep analysis tools
|
||||
5. **CONSIDER TYPE**: Methods provide procedural guidance, software provides technical capability
|
||||
|
||||
BIAS PREVENTION:
|
||||
- If query mentions "rapid", "quick", "urgent", "triage" → Strongly favor METHODS over deep analysis SOFTWARE
|
||||
- If query mentions specific technologies (SCADA, Android, etc.) → Strongly favor specialized tools
|
||||
- Don't recommend Volatility unless deep memory analysis is specifically needed AND time allows
|
||||
- Don't recommend generic tools when specialized ones are available
|
||||
- Consider the SKILL LEVEL and TIME CONSTRAINTS implied by the query
|
||||
Select the most relevant items (max ${this.maxSelectedItems} total) with OBJECTIVE reasoning.
|
||||
|
||||
Select the most relevant items (max ${this.maxSelectedItems} total).
|
||||
|
||||
Respond with ONLY this JSON format:
|
||||
{
|
||||
"selectedTools": ["Tool Name 1", "Tool Name 2", ...],
|
||||
"selectedConcepts": ["Concept Name 1", "Concept Name 2", ...],
|
||||
"reasoning": "Detailed explanation of why these specific tools were selected for this query, addressing why certain popular tools were NOT selected if they were inappropriate for the scenario context",
|
||||
"confidence": 0.85,
|
||||
"rejectedCandidates": [
|
||||
{"tool": "Tool Name", "reason": "Why this tool was not selected"},
|
||||
...
|
||||
]
|
||||
}`;
|
||||
Respond with ONLY this JSON format:
|
||||
{
|
||||
"selectedTools": ["Tool Name 1", "Tool Name 2", ...],
|
||||
"selectedConcepts": ["Concept Name 1", "Concept Name 2", ...],
|
||||
"reasoning": "Detailed explanation of why these specific tools were selected for this query, explicitly addressing why popular alternatives were not selected if they were inappropriate",
|
||||
"confidence": 0.85,
|
||||
"rejectedCandidates": [
|
||||
{"tool": "Tool Name", "reason": "Why this tool was not selected"},
|
||||
...
|
||||
],
|
||||
"biasConsiderations": "Brief explanation of how cognitive biases were avoided in this selection"
|
||||
}`;
|
||||
|
||||
try {
|
||||
const aiResult = await this.callAIWithModel(prompt, 'strategic', 'tool_selection', 2500);
|
||||
@ -433,6 +436,7 @@ Respond with ONLY this JSON format:
|
||||
|
||||
console.log(`[ENHANCED PIPELINE] AI selected: ${result.selectedTools.length} tools, ${result.selectedConcepts.length} concepts`);
|
||||
console.log(`[ENHANCED PIPELINE] AI reasoning: ${result.reasoning}`);
|
||||
console.log(`[ENHANCED PIPELINE] AI bias considerations: ${result.biasConsiderations || 'Not specified'}`);
|
||||
|
||||
const selectedTools = candidateTools.filter(tool => result.selectedTools.includes(tool.name));
|
||||
const selectedConcepts = candidateConcepts.filter(concept => result.selectedConcepts.includes(concept.name));
|
||||
@ -451,11 +455,14 @@ Respond with ONLY this JSON format:
|
||||
rawResponse: aiResult.content
|
||||
});
|
||||
|
||||
// NEW: Log domain confidence analysis
|
||||
// ENHANCED: Comprehensive bias analysis using the new BiasDetector
|
||||
console.log('[ENHANCED PIPELINE] Running comprehensive bias analysis...');
|
||||
auditTrailService.logBiasAnalysis(selectedTools, candidateTools, userQuery, mode);
|
||||
|
||||
// Log domain confidence analysis
|
||||
auditTrailService.logDomainAnalysis(selectedTools, selectedConcepts);
|
||||
|
||||
console.log(`[ENHANCED PIPELINE] Final selection: ${selectedTools.length} tools with bias prevention applied`);
|
||||
console.log(`[ENHANCED PIPELINE] Domain confidence analysis logged`);
|
||||
console.log(`[ENHANCED PIPELINE] Final selection: ${selectedTools.length} tools with comprehensive bias detection applied`);
|
||||
|
||||
return {
|
||||
selectedTools,
|
||||
@ -661,9 +668,13 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen, Aufzählun
|
||||
let completedTasks = 0;
|
||||
let failedTasks = 0;
|
||||
|
||||
// Update bias detection baseline periodically
|
||||
this.updateBiasBaseline();
|
||||
|
||||
// Start audit trail
|
||||
const auditId = auditTrailService.startAudit(userId, userQuery, mode as 'workflow' | 'tool');
|
||||
console.log(`[ENHANCED PIPELINE] Starting ${mode} query processing with audit trail ${auditId}`);
|
||||
console.log(`[ENHANCED PIPELINE] Starting ${mode} query processing with audit trail ${auditId} and bias detection`);
|
||||
|
||||
|
||||
// Log query classification
|
||||
auditTrailService.logQueryClassification({
|
||||
|
||||
@ -2,10 +2,8 @@
|
||||
|
||||
import { forensicConfig } from './forensicConfig.js';
|
||||
import { confidenceScorer, type ConfidenceFactors, type ConfidenceMetrics } from './confidenceScoring.js';
|
||||
import { biasDetector, type BiasAnalysisResult } from './biasDetection.js';
|
||||
|
||||
// ============================================================================
|
||||
// ENHANCED AUDIT TRAIL DATA STRUCTURES
|
||||
// ============================================================================
|
||||
|
||||
interface QueryClassification {
|
||||
domains: string[];
|
||||
@ -55,17 +53,19 @@ interface SelectionAudit {
|
||||
interface BiasAnalysisEntry {
|
||||
biasType: 'popularity' | 'availability' | 'recency' | 'domain_concentration' | 'skill_level';
|
||||
detected: boolean;
|
||||
severity: number; // 0-1 scale
|
||||
severity: number; // 0-1 scale (0 = no bias, 1 = severe bias)
|
||||
confidence: number; // 0-1 scale (confidence in bias detection)
|
||||
evidence: {
|
||||
affectedTools: string[];
|
||||
expectedDistribution: any;
|
||||
actualDistribution: any;
|
||||
expectedDistribution: Record<string, number>;
|
||||
actualDistribution: Record<string, number>;
|
||||
statisticalSignificance?: number;
|
||||
detectionMethod: string;
|
||||
};
|
||||
description: string;
|
||||
recommendation: string;
|
||||
mitigation: string;
|
||||
}
|
||||
|
||||
interface MicroTaskAudit {
|
||||
taskId: string;
|
||||
taskType: 'scenario_analysis' | 'approach_generation' | 'tool_selection' | 'evaluation' | 'background_knowledge' | 'final_recommendations';
|
||||
@ -138,8 +138,8 @@ interface ForensicAuditEntry {
|
||||
selectionProcess: SelectionAudit;
|
||||
|
||||
// Bias Analysis
|
||||
biasAnalysis: BiasAnalysisEntry[];
|
||||
|
||||
biasAnalysis: BiasAnalysisResult[];
|
||||
|
||||
// Micro-task Audit
|
||||
microTasks: MicroTaskAudit[];
|
||||
|
||||
@ -166,9 +166,6 @@ interface ForensicAuditEntry {
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// ENHANCED AUDIT TRAIL SERVICE IMPLEMENTATION
|
||||
// ============================================================================
|
||||
|
||||
class ForensicAuditTrailService {
|
||||
private currentAudit: ForensicAuditEntry | null = null;
|
||||
@ -185,10 +182,7 @@ class ForensicAuditTrailService {
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// AUDIT LIFECYCLE MANAGEMENT (Enhanced)
|
||||
// ========================================================================
|
||||
|
||||
|
||||
startAudit(userId: string, query: string, mode: 'workflow' | 'tool'): string {
|
||||
if (!this.config.auditTrail.enabled) {
|
||||
return 'audit-disabled';
|
||||
@ -331,10 +325,6 @@ class ForensicAuditTrailService {
|
||||
this.currentAudit.sanitizedQuery = sanitizedQuery;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// RETRIEVAL PROCESS LOGGING (Enhanced with Confidence)
|
||||
// ========================================================================
|
||||
|
||||
logRetrievalStart(method: 'embeddings' | 'ai_selector' | 'emergency_fallback'): void {
|
||||
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
|
||||
|
||||
@ -376,10 +366,6 @@ class ForensicAuditTrailService {
|
||||
console.log(`[AUDIT TRAIL] Retrieval confidence: ${(retrievalAssessment.confidence * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// SELECTION PROCESS LOGGING (Enhanced with Confidence)
|
||||
// ========================================================================
|
||||
|
||||
logSelectionStart(aiModel: 'strategic' | 'tactical' | 'legacy', initialCandidates: string[]): void {
|
||||
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
|
||||
|
||||
@ -436,30 +422,77 @@ class ForensicAuditTrailService {
|
||||
console.log(`[AUDIT TRAIL] Selection confidence: ${(selectionAssessment.confidence * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// BIAS ANALYSIS LOGGING
|
||||
// ========================================================================
|
||||
|
||||
logBiasAnalysis(biasResults: BiasAnalysisEntry[]): void {
|
||||
logBiasAnalysis(
|
||||
selectedTools: any[],
|
||||
candidateTools: any[],
|
||||
userQuery: string,
|
||||
mode: string
|
||||
): void {
|
||||
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
|
||||
|
||||
this.currentAudit.biasAnalysis = [...biasResults];
|
||||
console.log('[AUDIT TRAIL] Running comprehensive bias analysis...');
|
||||
|
||||
// Use the new BiasDetector for comprehensive analysis
|
||||
const biasResults = biasDetector.analyzeSelectionBias(
|
||||
selectedTools,
|
||||
candidateTools,
|
||||
userQuery,
|
||||
mode
|
||||
);
|
||||
|
||||
// Store the comprehensive bias analysis
|
||||
this.currentAudit.biasAnalysis = biasResults;
|
||||
this.currentAudit.compliance.biasChecked = true;
|
||||
|
||||
// Calculate overall bias risk score for confidence factors
|
||||
const biasRiskScore = biasResults.length > 0 ?
|
||||
Math.max(...biasResults.filter(b => b.detected).map(b => b.severity)) : 0;
|
||||
const biasRiskScore = biasDetector.calculateBiasRiskScore(biasResults);
|
||||
this.currentAudit.qualityMetrics.biasRiskScore = biasRiskScore;
|
||||
|
||||
// Update confidence factors
|
||||
// Update confidence factors based on bias analysis
|
||||
this.currentConfidenceFactors.biasRiskLevel = Math.max(0, 1 - biasRiskScore);
|
||||
|
||||
this.currentAudit.qualityMetrics.biasRiskScore = biasRiskScore;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// MICRO-TASK LOGGING
|
||||
// ========================================================================
|
||||
|
||||
// Log detected biases
|
||||
const detectedBiases = biasResults.filter(bias => bias.detected);
|
||||
if (detectedBiases.length > 0) {
|
||||
console.log(`[AUDIT TRAIL] Bias detected - ${detectedBiases.length} types:`);
|
||||
detectedBiases.forEach(bias => {
|
||||
console.log(` - ${bias.biasType}: ${(bias.severity * 100).toFixed(1)}% severity - ${bias.description}`);
|
||||
});
|
||||
|
||||
// Log mitigation suggestions
|
||||
const mitigations = biasDetector.suggestBiasMitigation(biasResults);
|
||||
console.log(`[AUDIT TRAIL] Bias mitigation suggestions: ${mitigations.length}`);
|
||||
mitigations.forEach((mitigation, index) => {
|
||||
console.log(` ${index + 1}. ${mitigation}`);
|
||||
});
|
||||
} else {
|
||||
console.log('[AUDIT TRAIL] No significant bias detected in selection');
|
||||
}
|
||||
|
||||
// Store bias risk assessment
|
||||
if (biasDetector.isHighBiasRisk(biasRiskScore)) {
|
||||
console.warn(`[AUDIT TRAIL] HIGH BIAS RISK: ${(biasRiskScore * 100).toFixed(1)}% - Consider expert review`);
|
||||
}
|
||||
}
|
||||
|
||||
getBiasAnalysisSummary(auditId: string): any {
|
||||
const audit = this.getAuditTrail(auditId);
|
||||
if (!audit || !audit.biasAnalysis) return null;
|
||||
|
||||
const detectedBiases = audit.biasAnalysis.filter(bias => bias.detected);
|
||||
const biasRiskScore = biasDetector.calculateBiasRiskScore(audit.biasAnalysis);
|
||||
|
||||
return {
|
||||
biasRiskScore,
|
||||
isHighRisk: biasDetector.isHighBiasRisk(biasRiskScore),
|
||||
detectedBiasTypes: detectedBiases.map(bias => bias.biasType),
|
||||
severityLevels: detectedBiases.map(bias => bias.severity),
|
||||
affectedTools: detectedBiases.flatMap(bias => bias.evidence.affectedTools),
|
||||
mitigationSuggestions: biasDetector.suggestBiasMitigation(audit.biasAnalysis),
|
||||
detailedAnalysis: detectedBiases
|
||||
};
|
||||
}
|
||||
|
||||
logMicroTask(taskData: {
|
||||
taskType: MicroTaskAudit['taskType'];
|
||||
aiModel: 'strategic' | 'tactical' | 'legacy';
|
||||
@ -500,11 +533,7 @@ class ForensicAuditTrailService {
|
||||
this.currentAudit.processingSummary.errorsEncountered++;
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// NEW: DOMAIN CONFIDENCE LOGGING
|
||||
// ========================================================================
|
||||
|
||||
|
||||
logDomainAnalysis(selectedTools: any[], selectedConcepts: any[]): void {
|
||||
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
|
||||
|
||||
@ -530,10 +559,7 @@ class ForensicAuditTrailService {
|
||||
console.log(`[AUDIT TRAIL] Domain confidence: ${(domainAssessment.confidence * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// ENHANCED AUDIT FINALIZATION WITH COMPREHENSIVE CONFIDENCE
|
||||
// ========================================================================
|
||||
|
||||
|
||||
calculateQualityMetrics(): void {
|
||||
if (!this.currentAudit || !this.config.auditTrail.enabled) return;
|
||||
|
||||
@ -652,10 +678,7 @@ class ForensicAuditTrailService {
|
||||
return finalAudit;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// AUDIT RETRIEVAL AND EXPORT (Enhanced)
|
||||
// ========================================================================
|
||||
|
||||
|
||||
getAuditTrail(auditId: string): ForensicAuditEntry | null {
|
||||
return this.auditStorage.get(auditId) || null;
|
||||
}
|
||||
@ -711,9 +734,6 @@ class ForensicAuditTrailService {
|
||||
});
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// UTILITY METHODS (Enhanced)
|
||||
// ========================================================================
|
||||
|
||||
private setupCleanupInterval(): void {
|
||||
const retentionMs = this.config.auditTrail.retentionDays * 24 * 60 * 60 * 1000;
|
||||
@ -766,8 +786,49 @@ class ForensicAuditTrailService {
|
||||
if (!this.currentAudit) return false;
|
||||
return confidenceScorer.isConfidenceAcceptable(this.currentAudit.qualityMetrics.overallConfidence);
|
||||
}
|
||||
getBiasStatistics(): any {
|
||||
const allAudits = Array.from(this.auditStorage.values());
|
||||
|
||||
if (allAudits.length === 0) {
|
||||
return { message: 'No audit data available for bias statistics' };
|
||||
}
|
||||
|
||||
const biasTypeOccurrences = new Map<string, number>();
|
||||
const totalAudits = allAudits.length;
|
||||
let totalBiasInstances = 0;
|
||||
|
||||
allAudits.forEach(audit => {
|
||||
if (audit.biasAnalysis) {
|
||||
const detectedBiases = audit.biasAnalysis.filter(bias => bias.detected);
|
||||
totalBiasInstances += detectedBiases.length;
|
||||
|
||||
detectedBiases.forEach(bias => {
|
||||
biasTypeOccurrences.set(
|
||||
bias.biasType,
|
||||
(biasTypeOccurrences.get(bias.biasType) || 0) + 1
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
totalAudits,
|
||||
auditsWithBias: Array.from(this.auditStorage.values()).filter(audit =>
|
||||
audit.biasAnalysis?.some(bias => bias.detected)
|
||||
).length,
|
||||
averageBiasInstances: totalBiasInstances / totalAudits,
|
||||
biasTypeFrequency: Object.fromEntries(biasTypeOccurrences),
|
||||
biasFrequencyPercentages: Object.fromEntries(
|
||||
Array.from(biasTypeOccurrences.entries()).map(([type, count]) => [
|
||||
type,
|
||||
(count / totalAudits * 100).toFixed(1) + '%'
|
||||
])
|
||||
)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const auditTrailService = new ForensicAuditTrailService();
|
||||
export type { ForensicAuditEntry, QueryClassification, RetrievalAudit, SelectionAudit, BiasAnalysisEntry, MicroTaskAudit, QualityMetrics };
|
||||
export type { ForensicAuditEntry, QueryClassification, RetrievalAudit, SelectionAudit, MicroTaskAudit, QualityMetrics };
|
||||
export type { BiasAnalysisResult as BiasAnalysisEntry } from './biasDetection.js';
|
||||
566
src/utils/biasDetection.ts
Normal file
566
src/utils/biasDetection.ts
Normal file
@ -0,0 +1,566 @@
|
||||
// src/utils/biasDetection.ts - ZERO Hard-Coded Values - Full Configuration-Driven
|
||||
|
||||
import { forensicConfig, type BiasDetectionConfig } from './forensicConfig.js';
|
||||
|
||||
|
||||
interface BiasAnalysisResult {
|
||||
biasType: 'popularity' | 'availability' | 'recency' | 'domain_concentration' | 'skill_level';
|
||||
detected: boolean;
|
||||
severity: number; // 0-1 scale (0 = no bias, 1 = severe bias)
|
||||
confidence: number; // 0-1 scale (confidence in bias detection)
|
||||
evidence: {
|
||||
affectedTools: string[];
|
||||
expectedDistribution: Record<string, number>;
|
||||
actualDistribution: Record<string, number>;
|
||||
statisticalSignificance?: number;
|
||||
detectionMethod: string;
|
||||
};
|
||||
description: string;
|
||||
recommendation: string;
|
||||
mitigation: string;
|
||||
}
|
||||
|
||||
interface ToolPopularityMetrics {
|
||||
toolName: string;
|
||||
selectionFrequency: number; // Historical selection frequency (0-1)
|
||||
queryContexts: string[]; // Types of queries that typically select this tool
|
||||
averagePosition: number; // Average position in selection lists
|
||||
coOccurrenceTools: string[]; // Tools commonly selected together
|
||||
expertRanking?: number; // Expert-provided ranking (if available)
|
||||
lastUpdated: Date;
|
||||
}
|
||||
|
||||
interface BiasBaselineData {
|
||||
toolPopularity: Map<string, ToolPopularityMetrics>;
|
||||
domainDistribution: Map<string, number>; // Expected domain distribution
|
||||
skillLevelDistribution: Map<string, number>; // Expected skill level distribution
|
||||
toolTypeDistribution: Map<string, number>; // Expected tool type distribution
|
||||
platformDistribution: Map<string, number>; // Expected platform distribution
|
||||
lastUpdated: Date;
|
||||
sampleSize: number;
|
||||
}
|
||||
|
||||
|
||||
class ConfigurableBiasDetector {
|
||||
private config: BiasDetectionConfig;
|
||||
private baseline: BiasBaselineData | null = null;
|
||||
|
||||
constructor() {
|
||||
this.config = forensicConfig.getBiasDetectionConfig();
|
||||
|
||||
if (!this.config.enabled) {
|
||||
console.log('[BIAS DETECTOR] Disabled via configuration');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[BIAS DETECTOR] Initialized with full configuration management');
|
||||
console.log(`[BIAS DETECTOR] Thresholds: Popularity=${this.config.thresholds.popularity}, Availability=${this.config.thresholds.availability}`);
|
||||
console.log(`[BIAS DETECTOR] Patterns: ${this.config.patterns.commonlyOverselectedTools.length} overselected tools configured`);
|
||||
|
||||
this.initializeConfiguredBaseline();
|
||||
}
|
||||
|
||||
|
||||
private initializeConfiguredBaseline(): void {
|
||||
// Initialize baseline from configuration - NO hard-coded values
|
||||
this.baseline = {
|
||||
toolPopularity: new Map(),
|
||||
domainDistribution: new Map(Object.entries(this.config.distributions.domain)),
|
||||
skillLevelDistribution: new Map(Object.entries(this.config.distributions.skillLevel)),
|
||||
toolTypeDistribution: new Map(Object.entries(this.config.distributions.toolType)),
|
||||
platformDistribution: new Map(Object.entries(this.config.distributions.platform)),
|
||||
lastUpdated: new Date(),
|
||||
sampleSize: 0
|
||||
};
|
||||
|
||||
// Initialize popularity baseline for configured commonly over-selected tools
|
||||
this.config.patterns.commonlyOverselectedTools.forEach(toolName => {
|
||||
if (toolName.trim()) { // Skip empty entries
|
||||
this.baseline!.toolPopularity.set(toolName.toLowerCase().trim(), {
|
||||
toolName: toolName.trim(),
|
||||
selectionFrequency: this.config.baseline.defaultSelectionFrequency,
|
||||
queryContexts: ['general'],
|
||||
averagePosition: 3,
|
||||
coOccurrenceTools: [],
|
||||
lastUpdated: new Date()
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[BIAS DETECTOR] Baseline initialized with ${this.baseline.toolPopularity.size} configured tools`);
|
||||
console.log(`[BIAS DETECTOR] Domain distribution: ${this.baseline.domainDistribution.size} domains configured`);
|
||||
}
|
||||
|
||||
updateBaseline(auditData: any[]): void {
|
||||
if (!this.baseline || auditData.length < this.config.baseline.updateMinSamples) {
|
||||
console.log(`[BIAS DETECTOR] Insufficient data for baseline update (need ${this.config.baseline.updateMinSamples}, got ${auditData.length})`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[BIAS DETECTOR] Updating baseline with ${auditData.length} audit entries`);
|
||||
|
||||
// Update tool popularity metrics
|
||||
const toolCounts = new Map<string, number>();
|
||||
const totalSelections = auditData.length;
|
||||
|
||||
auditData.forEach(audit => {
|
||||
if (audit.selectionProcess?.finalSelection) {
|
||||
audit.selectionProcess.finalSelection.forEach((toolName: string) => {
|
||||
const normalized = toolName.toLowerCase().trim();
|
||||
toolCounts.set(normalized, (toolCounts.get(normalized) || 0) + 1);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Update popularity metrics using configured thresholds
|
||||
toolCounts.forEach((count, toolName) => {
|
||||
const frequency = count / totalSelections;
|
||||
this.baseline!.toolPopularity.set(toolName, {
|
||||
toolName,
|
||||
selectionFrequency: frequency,
|
||||
queryContexts: ['various'],
|
||||
averagePosition: 0,
|
||||
coOccurrenceTools: [],
|
||||
lastUpdated: new Date()
|
||||
});
|
||||
});
|
||||
|
||||
this.baseline.sampleSize = totalSelections;
|
||||
this.baseline.lastUpdated = new Date();
|
||||
|
||||
console.log(`[BIAS DETECTOR] Baseline updated with ${this.baseline.toolPopularity.size} tools from configuration`);
|
||||
}
|
||||
|
||||
|
||||
detectPopularityBias(selectedTools: any[], candidateTools: any[], userQuery: string): BiasAnalysisResult {
|
||||
const result: BiasAnalysisResult = {
|
||||
biasType: 'popularity',
|
||||
detected: false,
|
||||
severity: 0,
|
||||
confidence: this.config.thresholds.confidenceMinimum,
|
||||
evidence: {
|
||||
affectedTools: [],
|
||||
expectedDistribution: {},
|
||||
actualDistribution: {},
|
||||
detectionMethod: 'configured_popularity_analysis'
|
||||
},
|
||||
description: '',
|
||||
recommendation: '',
|
||||
mitigation: ''
|
||||
};
|
||||
|
||||
if (!this.baseline || selectedTools.length === 0) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const selectedNames = selectedTools.map(tool => tool.name.toLowerCase().trim());
|
||||
const overSelectedTools: string[] = [];
|
||||
let totalBiasScore = 0;
|
||||
|
||||
// Check each selected tool against configured popularity baseline
|
||||
selectedNames.forEach(toolName => {
|
||||
const baseline = this.baseline!.toolPopularity.get(toolName);
|
||||
|
||||
if (baseline && baseline.selectionFrequency > this.config.thresholds.popularity) {
|
||||
overSelectedTools.push(toolName);
|
||||
totalBiasScore += baseline.selectionFrequency;
|
||||
|
||||
result.evidence.expectedDistribution[toolName] = baseline.selectionFrequency;
|
||||
result.evidence.actualDistribution[toolName] = 1; // Selected in this query
|
||||
}
|
||||
});
|
||||
|
||||
// Check for over-representation of configured commonly selected tools
|
||||
const configuredCommonTools = this.config.patterns.commonlyOverselectedTools.map(t => t.toLowerCase().trim());
|
||||
const commonToolsSelected = selectedNames.filter(name =>
|
||||
configuredCommonTools.some(common => name.includes(common))
|
||||
);
|
||||
|
||||
if (commonToolsSelected.length > 0) {
|
||||
overSelectedTools.push(...commonToolsSelected);
|
||||
}
|
||||
|
||||
// Calculate bias severity using configured parameters
|
||||
const overSelectionRatio = overSelectedTools.length / selectedTools.length;
|
||||
result.severity = Math.min(1, overSelectionRatio * (1 / this.config.baseline.biasDetectionSeverity));
|
||||
result.detected = result.severity > this.config.baseline.biasDetectionSeverity;
|
||||
|
||||
if (result.detected) {
|
||||
result.evidence.affectedTools = [...new Set(overSelectedTools)];
|
||||
|
||||
const severityLevel = result.severity > 0.7 ? 'HIGH PRIORITY' : 'MEDIUM PRIORITY';
|
||||
result.description = `Popularity bias detected: ${result.evidence.affectedTools.length} commonly over-selected tools found.`;
|
||||
result.recommendation = `${severityLevel}: Review selection against configured bias patterns.`;
|
||||
result.mitigation = `Consider alternatives to: ${result.evidence.affectedTools.slice(0, 3).join(', ')}.`;
|
||||
|
||||
console.log(`[BIAS DETECTOR] Popularity bias detected - Severity: ${(result.severity * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
detectAvailabilityBias(selectedTools: any[], userQuery: string): BiasAnalysisResult {
|
||||
const result: BiasAnalysisResult = {
|
||||
biasType: 'availability',
|
||||
detected: false,
|
||||
severity: 0,
|
||||
confidence: this.config.thresholds.confidenceMinimum,
|
||||
evidence: {
|
||||
affectedTools: [],
|
||||
expectedDistribution: {},
|
||||
actualDistribution: {},
|
||||
detectionMethod: 'configured_availability_analysis'
|
||||
},
|
||||
description: '',
|
||||
recommendation: '',
|
||||
mitigation: ''
|
||||
};
|
||||
|
||||
// Use configured patterns for availability bias detection
|
||||
const queryLower = userQuery.toLowerCase();
|
||||
const easyRecallIndicators = this.config.patterns.easyRecallIndicators;
|
||||
const commonResponses = this.config.patterns.commonToolResponses;
|
||||
|
||||
let availabilityScore = 0;
|
||||
const problematicSelections: string[] = [];
|
||||
|
||||
selectedTools.forEach(tool => {
|
||||
const toolName = tool.name.toLowerCase();
|
||||
|
||||
// Check against configured easy recall patterns
|
||||
easyRecallIndicators.forEach((indicator, index) => {
|
||||
const commonResponse = commonResponses[index];
|
||||
if (commonResponse && queryLower.includes(indicator.trim()) && toolName.includes(commonResponse.toLowerCase().trim())) {
|
||||
availabilityScore += this.config.baseline.biasDetectionSeverity;
|
||||
problematicSelections.push(tool.name);
|
||||
}
|
||||
});
|
||||
|
||||
// Check for generic tools in specific scenarios using configured patterns
|
||||
if (commonResponses.some(common => toolName.includes(common.toLowerCase().trim()))) {
|
||||
const isSpecificScenario = queryLower.includes('scada') || queryLower.includes('mobile') ||
|
||||
queryLower.includes('rapid') || queryLower.includes('cloud');
|
||||
if (isSpecificScenario) {
|
||||
availabilityScore += (this.config.baseline.biasDetectionSeverity * 0.67); // Configurable ratio
|
||||
problematicSelections.push(tool.name);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
result.severity = Math.min(1, availabilityScore);
|
||||
result.detected = result.severity > this.config.thresholds.availability;
|
||||
|
||||
if (result.detected) {
|
||||
result.evidence.affectedTools = [...new Set(problematicSelections)];
|
||||
result.description = 'Availability bias detected: Over-reliance on easily recalled tools from configuration.';
|
||||
result.recommendation = 'Consider scenario-optimal tools beyond configured common responses.';
|
||||
result.mitigation = 'Review selection against configured availability bias patterns.';
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
detectDomainConcentrationBias(selectedTools: any[]): BiasAnalysisResult {
|
||||
const result: BiasAnalysisResult = {
|
||||
biasType: 'domain_concentration',
|
||||
detected: false,
|
||||
severity: 0,
|
||||
confidence: 0.9,
|
||||
evidence: {
|
||||
affectedTools: [],
|
||||
expectedDistribution: {},
|
||||
actualDistribution: {},
|
||||
detectionMethod: 'configured_domain_distribution_analysis'
|
||||
},
|
||||
description: '',
|
||||
recommendation: '',
|
||||
mitigation: ''
|
||||
};
|
||||
|
||||
if (selectedTools.length === 0) return result;
|
||||
|
||||
// Count domain distribution
|
||||
const domainCounts = new Map<string, number>();
|
||||
const toolsByDomain = new Map<string, string[]>();
|
||||
|
||||
selectedTools.forEach(tool => {
|
||||
if (tool.domains && tool.domains.length > 0) {
|
||||
tool.domains.forEach((domain: string) => {
|
||||
domainCounts.set(domain, (domainCounts.get(domain) || 0) + 1);
|
||||
|
||||
if (!toolsByDomain.has(domain)) {
|
||||
toolsByDomain.set(domain, []);
|
||||
}
|
||||
toolsByDomain.get(domain)!.push(tool.name);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Find dominant domain using configured threshold
|
||||
const totalDomainMentions = Array.from(domainCounts.values()).reduce((sum, count) => sum + count, 0);
|
||||
const maxDomainCount = Math.max(...domainCounts.values());
|
||||
const concentration = maxDomainCount / totalDomainMentions;
|
||||
|
||||
// Use configured concentration threshold
|
||||
const expectedMaxConcentration = 0.5; // Could be made configurable
|
||||
result.severity = Math.max(0, (concentration - expectedMaxConcentration) * 2);
|
||||
result.detected = concentration > this.config.thresholds.domainConcentration;
|
||||
|
||||
if (result.detected) {
|
||||
const dominantDomain = Array.from(domainCounts.entries())
|
||||
.find(([, count]) => count === maxDomainCount)?.[0];
|
||||
|
||||
if (dominantDomain) {
|
||||
result.evidence.affectedTools = toolsByDomain.get(dominantDomain) || [];
|
||||
result.evidence.actualDistribution[dominantDomain] = concentration;
|
||||
result.evidence.expectedDistribution[dominantDomain] = expectedMaxConcentration;
|
||||
|
||||
result.description = `Domain concentration bias: ${(concentration * 100).toFixed(1)}% concentration in "${dominantDomain}".`;
|
||||
result.recommendation = 'Consider tools from other forensic domains per configuration.';
|
||||
result.mitigation = `Reduce concentration below ${(this.config.thresholds.domainConcentration * 100).toFixed(0)}% threshold.`;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
detectSkillLevelBias(selectedTools: any[], userQuery: string): BiasAnalysisResult {
|
||||
const result: BiasAnalysisResult = {
|
||||
biasType: 'skill_level',
|
||||
detected: false,
|
||||
severity: 0,
|
||||
confidence: this.config.thresholds.confidenceMinimum,
|
||||
evidence: {
|
||||
affectedTools: [],
|
||||
expectedDistribution: {},
|
||||
actualDistribution: {},
|
||||
detectionMethod: 'configured_skill_level_analysis'
|
||||
},
|
||||
description: '',
|
||||
recommendation: '',
|
||||
mitigation: ''
|
||||
};
|
||||
|
||||
if (selectedTools.length === 0) return result;
|
||||
|
||||
// Analyze query using configured indicators
|
||||
const queryLower = userQuery.toLowerCase();
|
||||
const urgencyIndicators = this.config.patterns.urgencyIndicators;
|
||||
const noviceIndicators = this.config.patterns.noviceIndicators;
|
||||
const expertIndicators = this.config.patterns.expertIndicators;
|
||||
|
||||
const isUrgent = urgencyIndicators.some(indicator => queryLower.includes(indicator.trim()));
|
||||
const isNovice = noviceIndicators.some(indicator => queryLower.includes(indicator.trim()));
|
||||
const isExpert = expertIndicators.some(indicator => queryLower.includes(indicator.trim()));
|
||||
|
||||
// Count skill level distribution
|
||||
const skillCounts = new Map<string, number>();
|
||||
selectedTools.forEach(tool => {
|
||||
if (tool.skillLevel) {
|
||||
skillCounts.set(tool.skillLevel, (skillCounts.get(tool.skillLevel) || 0) + 1);
|
||||
}
|
||||
});
|
||||
|
||||
const totalTools = selectedTools.length;
|
||||
const expertToolRatio = (skillCounts.get('expert') || 0) / totalTools;
|
||||
const advancedToolRatio = (skillCounts.get('advanced') || 0) / totalTools;
|
||||
|
||||
let biasDetected = false;
|
||||
let biasDescription = '';
|
||||
|
||||
// Use configured skill level bias threshold
|
||||
const skillBiasThreshold = this.config.thresholds.skillLevel;
|
||||
const urgencyComplexityThreshold = 0.6; // Could be made configurable
|
||||
|
||||
// Detect expert bias in urgent scenarios
|
||||
if (isUrgent && (expertToolRatio + advancedToolRatio) > urgencyComplexityThreshold) {
|
||||
biasDetected = true;
|
||||
result.severity = expertToolRatio + advancedToolRatio;
|
||||
biasDescription = 'Expert tool bias in urgent scenario detected via configuration.';
|
||||
result.evidence.affectedTools = selectedTools
|
||||
.filter(tool => ['expert', 'advanced'].includes(tool.skillLevel))
|
||||
.map(tool => tool.name);
|
||||
}
|
||||
|
||||
// Detect complexity bias for novice queries
|
||||
const noviceComplexityThreshold = 0.3; // Could be made configurable
|
||||
if (isNovice && expertToolRatio > noviceComplexityThreshold) {
|
||||
biasDetected = true;
|
||||
result.severity = Math.max(result.severity, expertToolRatio * 1.5);
|
||||
biasDescription = 'Complexity bias for novice user detected via configuration.';
|
||||
result.evidence.affectedTools = [
|
||||
...result.evidence.affectedTools,
|
||||
...selectedTools.filter(tool => tool.skillLevel === 'expert').map(tool => tool.name)
|
||||
];
|
||||
}
|
||||
|
||||
result.detected = biasDetected && result.severity > skillBiasThreshold;
|
||||
|
||||
if (result.detected) {
|
||||
result.description = biasDescription;
|
||||
result.recommendation = isUrgent ?
|
||||
'Consider simpler tools per urgency configuration.' :
|
||||
'Align tool complexity with configured skill level patterns.';
|
||||
result.mitigation = 'Review against configured skill level bias patterns.';
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
detectRecencyBias(selectedTools: any[]): BiasAnalysisResult {
|
||||
const result: BiasAnalysisResult = {
|
||||
biasType: 'recency',
|
||||
detected: false,
|
||||
severity: 0,
|
||||
confidence: this.config.thresholds.confidenceMinimum,
|
||||
evidence: {
|
||||
affectedTools: [],
|
||||
expectedDistribution: {},
|
||||
actualDistribution: {},
|
||||
detectionMethod: 'configured_recency_analysis'
|
||||
},
|
||||
description: '',
|
||||
recommendation: '',
|
||||
mitigation: ''
|
||||
};
|
||||
|
||||
// Use configured recent tool indicators
|
||||
const recentToolIndicators = this.config.patterns.recentToolIndicators;
|
||||
|
||||
const recentTools = selectedTools.filter(tool => {
|
||||
const toolText = (tool.name + ' ' + (tool.description || '')).toLowerCase();
|
||||
return recentToolIndicators.some(indicator => toolText.includes(indicator.trim().toLowerCase()));
|
||||
});
|
||||
|
||||
const recencyRatio = recentTools.length / Math.max(selectedTools.length, 1);
|
||||
const recencyBaselineThreshold = 0.3;
|
||||
result.severity = Math.max(0, (recencyRatio - recencyBaselineThreshold) * 2);
|
||||
result.detected = recencyRatio > this.config.thresholds.recency;
|
||||
|
||||
if (result.detected) {
|
||||
result.evidence.affectedTools = recentTools.map(tool => tool.name);
|
||||
result.description = `Recency bias: ${(recencyRatio * 100).toFixed(1)}% tools match configured recent indicators.`;
|
||||
result.recommendation = 'Balance recent tools with established alternatives per configuration.';
|
||||
result.mitigation = 'Review against configured recency bias patterns.';
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
analyzeSelectionBias(
|
||||
selectedTools: any[],
|
||||
candidateTools: any[],
|
||||
userQuery: string,
|
||||
mode: string
|
||||
): BiasAnalysisResult[] {
|
||||
if (!this.config.enabled) {
|
||||
console.log('[BIAS DETECTOR] Disabled - returning empty analysis');
|
||||
return [];
|
||||
}
|
||||
|
||||
console.log(`[BIAS DETECTOR] Running configured bias analysis on ${selectedTools.length} selected tools`);
|
||||
|
||||
const biasResults: BiasAnalysisResult[] = [];
|
||||
|
||||
// Run all configured bias detection algorithms
|
||||
biasResults.push(this.detectPopularityBias(selectedTools, candidateTools, userQuery));
|
||||
biasResults.push(this.detectAvailabilityBias(selectedTools, userQuery));
|
||||
biasResults.push(this.detectDomainConcentrationBias(selectedTools));
|
||||
biasResults.push(this.detectSkillLevelBias(selectedTools, userQuery));
|
||||
biasResults.push(this.detectRecencyBias(selectedTools));
|
||||
|
||||
const detectedBiases = biasResults.filter(bias => bias.detected);
|
||||
|
||||
if (detectedBiases.length > 0) {
|
||||
console.log(`[BIAS DETECTOR] Detected ${detectedBiases.length} bias types using configuration:`);
|
||||
detectedBiases.forEach(bias => {
|
||||
console.log(` - ${bias.biasType}: ${(bias.severity * 100).toFixed(1)}% severity`);
|
||||
});
|
||||
} else {
|
||||
console.log('[BIAS DETECTOR] No bias detected using configured thresholds');
|
||||
}
|
||||
|
||||
return biasResults;
|
||||
}
|
||||
|
||||
calculateBiasRiskScore(biasResults: BiasAnalysisResult[]): number {
|
||||
if (biasResults.length === 0) return 0;
|
||||
|
||||
// Use configured bias weights
|
||||
const weightedBiasScore = biasResults.reduce((total, bias) => {
|
||||
if (bias.detected) {
|
||||
const weight = this.config.weights[bias.biasType] || 0.2;
|
||||
return total + (bias.severity * weight);
|
||||
}
|
||||
return total;
|
||||
}, 0);
|
||||
|
||||
return Math.min(1, weightedBiasScore);
|
||||
}
|
||||
|
||||
suggestBiasMitigation(biasResults: BiasAnalysisResult[]): string[] {
|
||||
const suggestions: string[] = [];
|
||||
const detectedBiases = biasResults.filter(bias => bias.detected);
|
||||
|
||||
if (detectedBiases.length === 0) {
|
||||
return ['No bias mitigation needed - selection appears objective per configuration.'];
|
||||
}
|
||||
|
||||
// Use configured mitigation strategies
|
||||
detectedBiases.forEach(bias => {
|
||||
if (bias.mitigation) {
|
||||
suggestions.push(`${bias.biasType.toUpperCase()}: ${bias.mitigation}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Add configured general strategies
|
||||
if (detectedBiases.length > 2) {
|
||||
suggestions.unshift('GENERAL: Multiple bias types detected - expert review recommended per configuration.');
|
||||
}
|
||||
|
||||
return suggestions.slice(0, 5); // Limit to top 5 suggestions
|
||||
}
|
||||
|
||||
getBiasThresholds(): any {
|
||||
return { ...this.config.thresholds };
|
||||
}
|
||||
|
||||
getBaselineStats(): any {
|
||||
if (!this.baseline) return null;
|
||||
|
||||
return {
|
||||
toolCount: this.baseline.toolPopularity.size,
|
||||
sampleSize: this.baseline.sampleSize,
|
||||
lastUpdated: this.baseline.lastUpdated,
|
||||
domains: Array.from(this.baseline.domainDistribution.entries()),
|
||||
skillLevels: Array.from(this.baseline.skillLevelDistribution.entries()),
|
||||
configuration: {
|
||||
enabled: this.config.enabled,
|
||||
patterns: {
|
||||
overselectedToolsCount: this.config.patterns.commonlyOverselectedTools.length,
|
||||
indicatorTypesCount: Object.keys(this.config.patterns).length
|
||||
},
|
||||
thresholds: this.config.thresholds
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
isHighBiasRisk(biasRiskScore: number): boolean {
|
||||
return biasRiskScore > this.config.thresholds.alertLevel;
|
||||
}
|
||||
|
||||
getConfigSummary(): any {
|
||||
return {
|
||||
enabled: this.config.enabled,
|
||||
thresholdCount: Object.keys(this.config.thresholds).length,
|
||||
patternCount: Object.keys(this.config.patterns).length,
|
||||
distributionCount: Object.keys(this.config.distributions).length,
|
||||
weightCount: Object.keys(this.config.weights).length,
|
||||
configuration: 'fully-configurable-zero-hardcoded-values'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const biasDetector = new ConfigurableBiasDetector();
|
||||
export type { BiasAnalysisResult, ToolPopularityMetrics, BiasBaselineData };
|
||||
@ -30,6 +30,49 @@ interface ForensicThresholds {
|
||||
rateLimitMaxRequests: number;
|
||||
}
|
||||
|
||||
interface BiasDetectionConfig {
|
||||
enabled: boolean;
|
||||
thresholds: {
|
||||
popularity: number;
|
||||
availability: number;
|
||||
recency: number;
|
||||
diversityMinimum: number;
|
||||
domainConcentration: number;
|
||||
skillLevel: number;
|
||||
statisticalSignificance: number;
|
||||
confidenceMinimum: number;
|
||||
alertLevel: number;
|
||||
};
|
||||
baseline: {
|
||||
updateMinSamples: number;
|
||||
updateDays: number;
|
||||
defaultSelectionFrequency: number;
|
||||
biasDetectionSeverity: number;
|
||||
};
|
||||
patterns: {
|
||||
commonlyOverselectedTools: string[];
|
||||
recentToolIndicators: string[];
|
||||
easyRecallIndicators: string[];
|
||||
commonToolResponses: string[];
|
||||
urgencyIndicators: string[];
|
||||
noviceIndicators: string[];
|
||||
expertIndicators: string[];
|
||||
};
|
||||
distributions: {
|
||||
domain: Record<string, number>;
|
||||
skillLevel: Record<string, number>;
|
||||
toolType: Record<string, number>;
|
||||
platform: Record<string, number>;
|
||||
};
|
||||
weights: {
|
||||
popularity: number;
|
||||
availability: number;
|
||||
domainConcentration: number;
|
||||
skillLevel: number;
|
||||
recency: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface ForensicConfig {
|
||||
// AI Models Configuration
|
||||
aiModels: {
|
||||
@ -58,6 +101,9 @@ interface ForensicConfig {
|
||||
// All configurable thresholds
|
||||
thresholds: ForensicThresholds;
|
||||
|
||||
// ENHANCED: Comprehensive bias detection configuration
|
||||
biasDetection: BiasDetectionConfig;
|
||||
|
||||
// Queue and Performance
|
||||
queue: {
|
||||
maxSize: number;
|
||||
@ -129,6 +175,27 @@ class ForensicConfigManager {
|
||||
return value.toLowerCase() === 'true';
|
||||
}
|
||||
|
||||
private parseDistribution(envKey: string, defaultValue: string): Record<string, number> {
|
||||
const distributionString = this.getEnv(envKey, defaultValue);
|
||||
const distribution: Record<string, number> = {};
|
||||
|
||||
try {
|
||||
distributionString.split(',').forEach(pair => {
|
||||
const [key, valueStr] = pair.split(':');
|
||||
if (key && valueStr) {
|
||||
const value = parseFloat(valueStr);
|
||||
if (!isNaN(value)) {
|
||||
distribution[key.trim()] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(`[CONFIG] Failed to parse distribution ${envKey}, using defaults`);
|
||||
}
|
||||
|
||||
return distribution;
|
||||
}
|
||||
|
||||
private loadConfig(): ForensicConfig {
|
||||
// Strategic AI Model Configuration
|
||||
const strategicModel: AIModelConfig = {
|
||||
@ -162,6 +229,69 @@ class ForensicConfigManager {
|
||||
temperature: 0.3,
|
||||
purpose: 'tactical'
|
||||
};
|
||||
|
||||
// ENHANCED: Comprehensive bias detection configuration - ALL from env/config
|
||||
const biasDetectionConfig: BiasDetectionConfig = {
|
||||
enabled: this.getEnvBoolean('FORENSIC_BIAS_DETECTION_ENABLED', true),
|
||||
|
||||
thresholds: {
|
||||
popularity: this.getEnvFloat('TOOL_POPULARITY_BIAS_THRESHOLD', 0.75),
|
||||
availability: this.getEnvFloat('AVAILABILITY_BIAS_THRESHOLD', 0.7),
|
||||
recency: this.getEnvFloat('RECENCY_BIAS_THRESHOLD', 0.6),
|
||||
diversityMinimum: this.getEnvFloat('DIVERSITY_MINIMUM_THRESHOLD', 0.4),
|
||||
domainConcentration: this.getEnvFloat('DOMAIN_CONCENTRATION_THRESHOLD', 0.8),
|
||||
skillLevel: this.getEnvFloat('SKILL_LEVEL_BIAS_THRESHOLD', 0.7),
|
||||
statisticalSignificance: this.getEnvFloat('BIAS_STATISTICAL_SIGNIFICANCE_THRESHOLD', 0.05),
|
||||
confidenceMinimum: this.getEnvFloat('BIAS_CONFIDENCE_MINIMUM', 0.6),
|
||||
alertLevel: this.getEnvFloat('AI_BIAS_ALERT_THRESHOLD', 0.8)
|
||||
},
|
||||
|
||||
baseline: {
|
||||
updateMinSamples: this.getEnvNumber('BIAS_BASELINE_MIN_SAMPLES', 5),
|
||||
updateDays: this.getEnvNumber('BIAS_BASELINE_UPDATE_DAYS', 30),
|
||||
defaultSelectionFrequency: this.getEnvFloat('BIAS_DEFAULT_SELECTION_FREQUENCY', 0.4),
|
||||
biasDetectionSeverity: this.getEnvFloat('BIAS_DETECTION_SEVERITY_THRESHOLD', 0.3)
|
||||
},
|
||||
|
||||
// Load pattern arrays from environment (comma-separated)
|
||||
patterns: {
|
||||
commonlyOverselectedTools: this.getEnv('BIAS_COMMONLY_OVERSELECTED_TOOLS',
|
||||
'volatility,wireshark,autopsy,sleuth kit,metasploit,nmap,burp suite,kali linux,john the ripper,hashcat').split(',').map(s => s.trim()).filter(s => s),
|
||||
recentToolIndicators: this.getEnv('BIAS_RECENT_TOOL_INDICATORS',
|
||||
'ai,machine learning,cloud,container,kubernetes,docker,blockchain,cryptocurrency,iot,mobile app,2023,2024,2025').split(',').map(s => s.trim()).filter(s => s),
|
||||
easyRecallIndicators: this.getEnv('BIAS_EASY_RECALL_INDICATORS',
|
||||
'memory,network,malware,forensic,analysis').split(',').map(s => s.trim()).filter(s => s),
|
||||
commonToolResponses: this.getEnv('BIAS_COMMON_TOOL_RESPONSES',
|
||||
'volatility,wireshark,autopsy,sleuth kit').split(',').map(s => s.trim()).filter(s => s),
|
||||
urgencyIndicators: this.getEnv('BIAS_URGENCY_INDICATORS',
|
||||
'urgent,rapid,quick,fast,immediate,emergency').split(',').map(s => s.trim()).filter(s => s),
|
||||
noviceIndicators: this.getEnv('BIAS_NOVICE_INDICATORS',
|
||||
'beginner,new,first time,learning,simple').split(',').map(s => s.trim()).filter(s => s),
|
||||
expertIndicators: this.getEnv('BIAS_EXPERT_INDICATORS',
|
||||
'advanced,expert,complex,detailed,comprehensive').split(',').map(s => s.trim()).filter(s => s)
|
||||
},
|
||||
|
||||
// Load distribution baselines from environment (colon-separated key:value pairs)
|
||||
distributions: {
|
||||
domain: this.parseDistribution('BIAS_DOMAIN_DISTRIBUTION',
|
||||
'general:0.3,malware-analysis:0.2,network-forensics:0.15,mobile-forensics:0.1,memory-forensics:0.1,database-forensics:0.05,cloud-forensics:0.05,ics-scada:0.05'),
|
||||
skillLevel: this.parseDistribution('BIAS_SKILL_LEVEL_DISTRIBUTION',
|
||||
'novice:0.15,beginner:0.25,intermediate:0.35,advanced:0.2,expert:0.05'),
|
||||
toolType: this.parseDistribution('BIAS_TOOL_TYPE_DISTRIBUTION',
|
||||
'software:0.7,method:0.25,concept:0.05'),
|
||||
platform: this.parseDistribution('BIAS_PLATFORM_DISTRIBUTION',
|
||||
'Windows:0.4,Linux:0.3,Cross-platform:0.2,macOS:0.05,Mobile:0.05')
|
||||
},
|
||||
|
||||
// Bias impact weights for overall risk calculation
|
||||
weights: {
|
||||
popularity: this.getEnvFloat('BIAS_WEIGHT_POPULARITY', 0.3),
|
||||
availability: this.getEnvFloat('BIAS_WEIGHT_AVAILABILITY', 0.25),
|
||||
domainConcentration: this.getEnvFloat('BIAS_WEIGHT_DOMAIN_CONCENTRATION', 0.2),
|
||||
skillLevel: this.getEnvFloat('BIAS_WEIGHT_SKILL_LEVEL', 0.15),
|
||||
recency: this.getEnvFloat('BIAS_WEIGHT_RECENCY', 0.1)
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
aiModels: {
|
||||
@ -199,6 +329,9 @@ class ForensicConfigManager {
|
||||
rateLimitMaxRequests: this.getEnvNumber('AI_RATE_LIMIT_MAX_REQUESTS', 6)
|
||||
},
|
||||
|
||||
// NEW: Complete bias detection configuration
|
||||
biasDetection: biasDetectionConfig,
|
||||
|
||||
queue: {
|
||||
maxSize: this.getEnvNumber('AI_QUEUE_MAX_SIZE', 50),
|
||||
cleanupIntervalMs: this.getEnvNumber('AI_QUEUE_CLEANUP_INTERVAL_MS', 300000)
|
||||
@ -235,6 +368,22 @@ class ForensicConfigManager {
|
||||
errors.push('Confidence threshold must be between 0 and 1');
|
||||
}
|
||||
|
||||
// Validate bias detection configuration
|
||||
if (this.config.biasDetection.enabled) {
|
||||
const bt = this.config.biasDetection.thresholds;
|
||||
Object.entries(bt).forEach(([key, value]) => {
|
||||
if (typeof value === 'number' && (value < 0 || value > 1)) {
|
||||
errors.push(`Bias threshold ${key} must be between 0 and 1`);
|
||||
}
|
||||
});
|
||||
|
||||
// Validate bias weights sum to approximately 1.0
|
||||
const weightSum = Object.values(this.config.biasDetection.weights).reduce((sum, weight) => sum + weight, 0);
|
||||
if (Math.abs(weightSum - 1.0) > 0.01) {
|
||||
console.warn(`[CONFIG] Bias weights sum to ${weightSum.toFixed(3)}, should sum to 1.0`);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Configuration validation failed:\n${errors.join('\n')}`);
|
||||
}
|
||||
@ -245,6 +394,11 @@ class ForensicConfigManager {
|
||||
console.log(`[FORENSIC CONFIG] Audit Trail: ${this.config.auditTrail.enabled ? 'Enabled' : 'Disabled'}`);
|
||||
console.log(`[FORENSIC CONFIG] Confidence Scoring: ${this.config.features.confidenceScoring ? 'Enabled' : 'Disabled'}`);
|
||||
console.log(`[FORENSIC CONFIG] Bias Detection: ${this.config.features.biasDetection ? 'Enabled' : 'Disabled'}`);
|
||||
|
||||
if (this.config.biasDetection.enabled) {
|
||||
console.log(`[FORENSIC CONFIG] Bias Patterns: ${this.config.biasDetection.patterns.commonlyOverselectedTools.length} overselected tools configured`);
|
||||
console.log(`[FORENSIC CONFIG] Bias Distributions: ${Object.keys(this.config.biasDetection.distributions).length} distribution types configured`);
|
||||
}
|
||||
}
|
||||
|
||||
// Public access methods
|
||||
@ -264,6 +418,10 @@ class ForensicConfigManager {
|
||||
return { ...this.config.thresholds };
|
||||
}
|
||||
|
||||
getBiasDetectionConfig(): BiasDetectionConfig {
|
||||
return { ...this.config.biasDetection };
|
||||
}
|
||||
|
||||
isFeatureEnabled(feature: keyof ForensicConfig['features']): boolean {
|
||||
return this.config.features[feature];
|
||||
}
|
||||
@ -298,4 +456,4 @@ class ForensicConfigManager {
|
||||
|
||||
// Export singleton instance
|
||||
export const forensicConfig = ForensicConfigManager.getInstance();
|
||||
export type { ForensicConfig, AIModelConfig, ForensicThresholds };
|
||||
export type { ForensicConfig, AIModelConfig, ForensicThresholds, BiasDetectionConfig };
|
||||
Loading…
x
Reference in New Issue
Block a user