Merge branch 'forensic-ai' of https://git.cc24.dev/mstoeck3/forensic-pathways into forensic-ai

2025-08-05 11:59:49 +02:00 · 2025-08-05 11:59:49 +02:00 · c267681e7d
commit c267681e7d
parent 1eb49315fa f00e2d3cfd
11 changed files with 547 additions and 630 deletions
--- a/.env.example
+++ b/.env.example
@ -42,32 +42,40 @@ AI_EMBEDDINGS_MODEL=mistral-embed
 # How many similar tools/concepts embeddings search returns as candidates
 # 🔍 This is the FIRST filter - vector similarity matching
 # Lower = faster, less comprehensive | Higher = slower, more comprehensive
-AI_EMBEDDING_CANDIDATES=40
+AI_EMBEDDING_CANDIDATES=50

 # Minimum similarity score threshold (0.0-1.0)
 # Lower = more results but less relevant | Higher = fewer but more relevant
 AI_SIMILARITY_THRESHOLD=0.3

+# === AI SELECTION FROM EMBEDDINGS ===
+# When embeddings are enabled, how many top tools to send with full context
+# 🎯 This is the SECOND filter - take best N from embeddings results
+AI_EMBEDDING_SELECTION_LIMIT=30
+AI_EMBEDDING_CONCEPTS_LIMIT=15
+
+# Maximum tools/concepts sent to AI when embeddings are DISABLED
+# Set to 0 for no limit (WARNING: may cause token overflow with large datasets)
+AI_NO_EMBEDDINGS_TOOL_LIMIT=0
+AI_NO_EMBEDDINGS_CONCEPT_LIMIT=0
+
 # === AI SELECTION STAGE ===
 # Maximum tools the AI can select from embedding candidates
 # 🤖 This is the SECOND filter - AI intelligent selection
 # Should be ≤ AI_EMBEDDING_CANDIDATES
 AI_MAX_SELECTED_ITEMS=25

-# Maximum tools sent to AI for detailed analysis (micro-tasks)
-# 📋 This is the FINAL context size sent to AI models
-# Lower = less AI context, faster responses | Higher = more context, slower
-AI_MAX_TOOLS_TO_ANALYZE=20
+# === EMBEDDINGS EFFICIENCY THRESHOLDS ===
+# Minimum tools required for embeddings to be considered useful
+AI_EMBEDDINGS_MIN_TOOLS=8

-# Maximum concepts sent to AI for background knowledge selection
-# 📚 Concepts are smaller than tools, so can be higher
-AI_MAX_CONCEPTS_TO_ANALYZE=10
+# Maximum percentage of total tools that embeddings can return to be considered "filtering"
+AI_EMBEDDINGS_MAX_REDUCTION_RATIO=0.75

 # === CONTEXT FLOW SUMMARY ===
 # 1. Vector Search: 111 total tools → AI_EMBEDDING_CANDIDATES (40) most similar
 # 2. AI Selection: 40 candidates → AI_MAX_SELECTED_ITEMS (25) best matches  
-# 3. AI Analysis: 25 selected → AI_MAX_TOOLS_TO_ANALYZE (20) for micro-tasks
-# 4. Final Output: Recommendations based on analyzed subset
+# 3. Final Output: Recommendations based on analyzed subset

 # ============================================================================
 # 4. AI PERFORMANCE & RATE LIMITING
@ -95,23 +103,21 @@ AI_EMBEDDINGS_BATCH_SIZE=10
 # Delay between embedding batches (milliseconds)
 AI_EMBEDDINGS_BATCH_DELAY_MS=1000

+# Maximum tools sent to AI for detailed analysis (micro-tasks)
+AI_MAX_TOOLS_TO_ANALYZE=20
+AI_MAX_CONCEPTS_TO_ANALYZE=10
+
 # ============================================================================
 # 5. AI CONTEXT & TOKEN MANAGEMENT
 # ============================================================================

 # Maximum context tokens to maintain across micro-tasks
 # Controls how much conversation history is preserved between AI calls
-AI_MAX_CONTEXT_TOKENS=3000
+AI_MAX_CONTEXT_TOKENS=4000

 # Maximum tokens per individual AI prompt
 # Larger = more context per call | Smaller = faster responses
-AI_MAX_PROMPT_TOKENS=1200
-
-# Timeout for individual micro-tasks (milliseconds)
-AI_MICRO_TASK_TIMEOUT_MS=25000
-
-# Maximum size of the processing queue
-AI_QUEUE_MAX_SIZE=50
+AI_MAX_PROMPT_TOKENS=1500

 # ============================================================================
 # 6. AUTHENTICATION & AUTHORIZATION (OPTIONAL)
@ -172,7 +178,7 @@ GIT_API_TOKEN=your-git-api-token
 # ============================================================================

 # Enable detailed audit trail of AI decision-making
-FORENSIC_AUDIT_ENABLED=false
+FORENSIC_AUDIT_ENABLED=true

 # Audit detail level: minimal, standard, verbose
 FORENSIC_AUDIT_DETAIL_LEVEL=standard
@ -183,15 +189,6 @@ FORENSIC_AUDIT_RETENTION_HOURS=24
 # Maximum audit entries per request
 FORENSIC_AUDIT_MAX_ENTRIES=50

-# Enable detailed AI pipeline logging
-AI_PIPELINE_DEBUG=false
-
-# Enable performance metrics collection
-AI_PERFORMANCE_METRICS=false
-
-# Enable detailed micro-task debugging
-AI_MICRO_TASK_DEBUG=false
-
 # ============================================================================
 # 10. QUALITY CONTROL & BIAS DETECTION (ADVANCED)
 # ============================================================================
@ -207,61 +204,20 @@ CONFIDENCE_MINIMUM_THRESHOLD=40
 CONFIDENCE_MEDIUM_THRESHOLD=60
 CONFIDENCE_HIGH_THRESHOLD=80

-# Bias detection settings
-BIAS_DETECTION_ENABLED=false
-BIAS_POPULARITY_THRESHOLD=0.7
-BIAS_DIVERSITY_MINIMUM=0.6
-BIAS_CELEBRITY_TOOLS=""
-
-# Quality control thresholds
-QUALITY_MIN_RESPONSE_LENGTH=50
-QUALITY_MIN_SELECTION_COUNT=1
-QUALITY_MAX_PROCESSING_TIME_MS=30000
-
-# ============================================================================
-# 11. USER INTERFACE DEFAULTS (OPTIONAL)
-# ============================================================================
-
-# Default UI behavior (users can override)
-UI_SHOW_AUDIT_TRAIL_DEFAULT=false
-UI_SHOW_CONFIDENCE_SCORES=true
-UI_SHOW_BIAS_WARNINGS=true
-UI_AUDIT_TRAIL_COLLAPSIBLE=true
-
-# ============================================================================
-# 12. CACHING & PERFORMANCE (OPTIONAL)
-# ============================================================================
-
-# Cache AI responses (milliseconds)
-AI_RESPONSE_CACHE_TTL_MS=3600000
-
-# Queue cleanup interval (milliseconds)
-AI_QUEUE_CLEANUP_INTERVAL_MS=300000
-
 # ============================================================================
 # PERFORMANCE TUNING PRESETS
 # ============================================================================

-# 🚀 FOR FASTER RESPONSES (less comprehensive):
-# AI_EMBEDDING_CANDIDATES=20
-# AI_MAX_SELECTED_ITEMS=15  
-# AI_MAX_TOOLS_TO_ANALYZE=10
-# AI_MICRO_TASK_DELAY_MS=200
-# AI_MAX_CONTEXT_TOKENS=2000
+# 🚀 FOR FASTER RESPONSES (prevent token overflow):
+# AI_NO_EMBEDDINGS_TOOL_LIMIT=25
+# AI_NO_EMBEDDINGS_CONCEPT_LIMIT=10

-# 🎯 FOR BETTER QUALITY (more comprehensive):
-# AI_EMBEDDING_CANDIDATES=60
-# AI_MAX_SELECTED_ITEMS=40
-# AI_MAX_TOOLS_TO_ANALYZE=30
-# AI_MICRO_TASK_DELAY_MS=800
-# AI_MAX_CONTEXT_TOKENS=4000
+# 🎯 FOR FULL DATABASE ACCESS (risk of truncation):
+# AI_NO_EMBEDDINGS_TOOL_LIMIT=0
+# AI_NO_EMBEDDINGS_CONCEPT_LIMIT=0

-# 🔋 FOR LOW-POWER SYSTEMS (minimal resources):
-# AI_EMBEDDING_CANDIDATES=15
-# AI_MAX_SELECTED_ITEMS=10
-# AI_MAX_TOOLS_TO_ANALYZE=8
-# AI_RATE_LIMIT_MAX_REQUESTS=2
-# AI_MICRO_TASK_DELAY_MS=1000
+# 🔋 FOR LOW-POWER SYSTEMS:
+# AI_NO_EMBEDDINGS_TOOL_LIMIT=15

 # ============================================================================
 # FEATURE COMBINATIONS GUIDE
@ -285,7 +241,6 @@ AI_QUEUE_CLEANUP_INTERVAL_MS=300000

 # 🔍 WITH FULL MONITORING:
 # - Enable FORENSIC_AUDIT_ENABLED=true
-# - Enable AI_PIPELINE_DEBUG=true
 # - Configure audit retention and detail level

 # ============================================================================
--- a/src/components/AIQueryInterface.astro
+++ b/src/components/AIQueryInterface.astro
@ -15,7 +15,7 @@ const domainAgnosticSoftware = data['domain-agnostic-software'] || [];
          <path d="M9 11H5a2 2 0 0 0-2 2v7a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2v-7a2 2 0 0 0-2-2h-4"/>
          <path d="M9 11V7a3 3 0 0 1 6 0v4"/>
        </svg>
-        KI-gestützte Workflow-Empfehlungen
+        Forensic AI
      </h2>
      <p id="ai-description" class="text-muted" style="max-width: 700px; margin: 0 auto; line-height: 1.6;">
        Beschreiben Sie Ihr forensisches Szenario und erhalten Sie maßgeschneiderte Workflow-Empfehlungen 
@ -169,16 +169,16 @@ const domainAgnosticSoftware = data['domain-agnostic-software'] || [];
          <!-- Micro-task Progress -->
          <div id="micro-task-progress" class="micro-task-progress hidden">
            <div class="micro-task-header">
-              <span class="micro-task-label">🔬 Micro-Task Analyse</span>
+              <span class="micro-task-label">🔬 micro-Agent-Analysis</span>
              <span id="micro-task-counter" class="micro-task-counter">1/6</span>
            </div>
            <div class="micro-task-steps">
-              <div class="micro-step" data-step="scenario">📋 Szenario</div>
-              <div class="micro-step" data-step="approach">🎯 Ansatz</div>
-              <div class="micro-step" data-step="considerations">⚠️ Kritisches</div>
-              <div class="micro-step" data-step="tools">🔧 Tools</div>
-              <div class="micro-step" data-step="knowledge">📚 Wissen</div>
-              <div class="micro-step" data-step="final">✅ Final</div>
+              <div class="micro-step" data-step="scenario">📋 Problemanalyse</div>
+              <div class="micro-step" data-step="approach">🎯 Ermittlungsansatz</div>
+              <div class="micro-step" data-step="considerations">⚠️ Herausforderungen</div>
+              <div class="micro-step" data-step="tools">🔧 Methoden</div>
+              <div class="micro-step" data-step="knowledge">📚 Evaluation</div>
+              <div class="micro-step" data-step="final">✅ Audit-Trail</div>
            </div>
          </div>
          
@ -292,13 +292,13 @@ class AIQueryInterface {
    return {
      workflow: {
        placeholder: "Beschreiben Sie Ihr forensisches Szenario... z.B. 'Verdacht auf Ransomware-Angriff auf Windows-Domänencontroller'",
-        description: "Beschreiben Sie Ihr forensisches Szenario und erhalten Sie maßgeschneiderte Workflow-Empfehlungen.",
+        description: "Beschreiben Sie Ihre Untersuchungssituation und erhalten Empfehlungen für alle Phasen der Untersuchung.",
        submitText: "Empfehlungen generieren",
        loadingText: "Analysiere Szenario und generiere Empfehlungen..."
      },
      tool: {
        placeholder: "Beschreiben Sie Ihr Problem... z.B. 'Analyse von Android-Backups mit WhatsApp-Nachrichten'",
-        description: "Beschreiben Sie Ihr Problem und erhalten Sie 1-3 gezielt passende Empfehlungen.",
+        description: "Beschreiben Sie Ihre Untersuchungssituation und erhalten Empfehlungen für eine spezifische Aufgabenstellung.",
        submitText: "Empfehlungen finden",
        loadingText: "Analysiere Anforderungen und suche passende Methode..."
      }
@ -706,7 +706,7 @@ class AIQueryInterface {

    const html = `
      <div class="workflow-container">
-        ${this.renderHeader('Empfohlener DFIR-Workflow', originalQuery)}
+        ${this.renderHeader('Untersuchungsansatz', originalQuery)}
        ${this.renderContextualAnalysis(recommendation, 'workflow')}
        ${this.renderBackgroundKnowledge(recommendation.background_knowledge)}
        ${this.renderWorkflowPhases(toolsByPhase, phaseOrder, phaseNames)}
@ -721,7 +721,7 @@ class AIQueryInterface {
  displayToolResults(recommendation, originalQuery) {
    const html = `
      <div class="tool-results-container">
-        ${this.renderHeader('Passende Empfehlungen', originalQuery)}
+        ${this.renderHeader('Handlungsempfehlung', originalQuery)}
        ${this.renderContextualAnalysis(recommendation, 'tool')}
        ${this.renderBackgroundKnowledge(recommendation.background_knowledge)}
        ${this.renderToolRecommendations(recommendation.recommended_tools)}
--- a/src/config/forensic.config.ts
+++ b/src/config/forensic.config.ts
@ -1,126 +0,0 @@
-// src/config/forensic.config.ts
-// Centralized configuration for forensic RAG enhancements
-
-export const FORENSIC_CONFIG = {
-  audit: {
-    enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true',
-    detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as 'minimal' | 'standard' | 'verbose') || 'standard',
-    retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10),
-    maxEntriesPerRequest: parseInt(process.env.FORENSIC_AUDIT_MAX_ENTRIES || '50', 10)
-  },
-  confidence: {
-    embeddingsWeight: parseFloat(process.env.CONFIDENCE_EMBEDDINGS_WEIGHT || '0.3'),
-    consensusWeight: parseFloat(process.env.CONFIDENCE_CONSENSUS_WEIGHT || '0.25'),
-    domainMatchWeight: parseFloat(process.env.CONFIDENCE_DOMAIN_MATCH_WEIGHT || '0.25'),
-    freshnessWeight: parseFloat(process.env.CONFIDENCE_FRESHNESS_WEIGHT || '0.2'),
-    minimumThreshold: parseInt(process.env.CONFIDENCE_MINIMUM_THRESHOLD || '40', 10),
-    highThreshold: parseInt(process.env.CONFIDENCE_HIGH_THRESHOLD || '80', 10),
-    mediumThreshold: parseInt(process.env.CONFIDENCE_MEDIUM_THRESHOLD || '60', 10)
-  },
-  bias: {
-    enabled: process.env.BIAS_DETECTION_ENABLED === 'true',
-    popularityThreshold: parseFloat(process.env.BIAS_POPULARITY_THRESHOLD || '0.7'),
-    diversityMinimum: parseFloat(process.env.BIAS_DIVERSITY_MINIMUM || '0.6'),
-    domainMismatchThreshold: parseFloat(process.env.BIAS_DOMAIN_MISMATCH_THRESHOLD || '0.3'),
-    warningThreshold: parseInt(process.env.BIAS_WARNING_THRESHOLD || '3', 10),
-    celebrityTools: (process.env.BIAS_CELEBRITY_TOOLS || 'Volatility 3,Wireshark,Autopsy,Maltego').split(',').map(t => t.trim())
-  },
-  // Quality thresholds for various metrics
-  quality: {
-    minResponseLength: parseInt(process.env.QUALITY_MIN_RESPONSE_LENGTH || '50', 10),
-    minSelectionCount: parseInt(process.env.QUALITY_MIN_SELECTION_COUNT || '1', 10),
-    maxProcessingTime: parseInt(process.env.QUALITY_MAX_PROCESSING_TIME_MS || '30000', 10)
-  },
-  // Display preferences
-  ui: {
-    showAuditTrailByDefault: process.env.UI_SHOW_AUDIT_TRAIL_DEFAULT === 'true',
-    showConfidenceScores: process.env.UI_SHOW_CONFIDENCE_SCORES !== 'false',
-    showBiasWarnings: process.env.UI_SHOW_BIAS_WARNINGS !== 'false',
-    auditTrailCollapsible: process.env.UI_AUDIT_TRAIL_COLLAPSIBLE !== 'false'
-  }
-};
-
-// Validation function to ensure configuration is valid
-export function validateForensicConfig(): { valid: boolean; errors: string[] } {
-  const errors: string[] = [];
-  
-  // Validate audit configuration
-  if (FORENSIC_CONFIG.audit.retentionHours < 1 || FORENSIC_CONFIG.audit.retentionHours > 168) {
-    errors.push('FORENSIC_AUDIT_RETENTION_HOURS must be between 1 and 168 (1 week)');
-  }
-  
-  if (!['minimal', 'standard', 'verbose'].includes(FORENSIC_CONFIG.audit.detailLevel)) {
-    errors.push('FORENSIC_AUDIT_DETAIL_LEVEL must be one of: minimal, standard, verbose');
-  }
-  
-  // Validate confidence weights sum to approximately 1.0
-  const weightSum = FORENSIC_CONFIG.confidence.embeddingsWeight + 
-                    FORENSIC_CONFIG.confidence.consensusWeight + 
-                    FORENSIC_CONFIG.confidence.domainMatchWeight + 
-                    FORENSIC_CONFIG.confidence.freshnessWeight;
-  
-  if (Math.abs(weightSum - 1.0) > 0.05) {
-    errors.push(`Confidence weights must sum to 1.0 (currently ${weightSum.toFixed(3)})`);
-  }
-  
-  // Validate threshold ranges
-  if (FORENSIC_CONFIG.confidence.minimumThreshold < 0 || FORENSIC_CONFIG.confidence.minimumThreshold > 100) {
-    errors.push('CONFIDENCE_MINIMUM_THRESHOLD must be between 0 and 100');
-  }
-  
-  if (FORENSIC_CONFIG.confidence.highThreshold <= FORENSIC_CONFIG.confidence.mediumThreshold) {
-    errors.push('CONFIDENCE_HIGH_THRESHOLD must be greater than CONFIDENCE_MEDIUM_THRESHOLD');
-  }
-  
-  // Validate bias thresholds
-  if (FORENSIC_CONFIG.bias.popularityThreshold < 0 || FORENSIC_CONFIG.bias.popularityThreshold > 1) {
-    errors.push('BIAS_POPULARITY_THRESHOLD must be between 0 and 1');
-  }
-  
-  if (FORENSIC_CONFIG.bias.diversityMinimum < 0 || FORENSIC_CONFIG.bias.diversityMinimum > 1) {
-    errors.push('BIAS_DIVERSITY_MINIMUM must be between 0 and 1');
-  }
-  
-  return {
-    valid: errors.length === 0,
-    errors
-  };
-}
-
-// Helper functions for configuration access
-export function isAuditEnabled(): boolean {
-  return FORENSIC_CONFIG.audit.enabled;
-}
-
-export function getAuditDetailLevel(): 'minimal' | 'standard' | 'verbose' {
-  return FORENSIC_CONFIG.audit.detailLevel;
-}
-
-export function getConfidenceThresholds() {
-  return {
-    minimum: FORENSIC_CONFIG.confidence.minimumThreshold,
-    medium: FORENSIC_CONFIG.confidence.mediumThreshold,
-    high: FORENSIC_CONFIG.confidence.highThreshold
-  };
-}
-
-export function isBiasDetectionEnabled(): boolean {
-  return FORENSIC_CONFIG.bias.enabled;
-}
-
-// Initialize and validate configuration on module load
-const configValidation = validateForensicConfig();
-if (!configValidation.valid) {
-  console.warn('[FORENSIC CONFIG] Configuration validation failed:', configValidation.errors);
-  // In development, we might want to throw an error
-  if (process.env.NODE_ENV === 'development') {
-    throw new Error(`Forensic configuration invalid: ${configValidation.errors.join(', ')}`);
-  }
-}
-
-console.log('[FORENSIC CONFIG] Configuration loaded:', {
-  auditEnabled: FORENSIC_CONFIG.audit.enabled,
-  confidenceEnabled: true, // Always enabled
-  biasDetectionEnabled: FORENSIC_CONFIG.bias.enabled,
-  detailLevel: FORENSIC_CONFIG.audit.detailLevel
-});
--- a/src/data/tools.yaml
+++ b/src/data/tools.yaml
@ -113,64 +113,6 @@ tools:
    accessType: download
    license: VSL
    knowledgebase: false
-  - name: TheHive 5
-    icon: 🐝
-    type: software
-    description: >-
-      Die zentrale Incident-Response-Plattform orchestriert komplexe 
-      Sicherheitsvorfälle vom ersten Alert bis zum Abschlussbericht. Jeder Case 
-      wird strukturiert durch Observables (IOCs), Tasks und Zeitleisten 
-      abgebildet. Die Cortex-Integration automatisiert Analysen durch Dutzende 
-      Analyzer - von VirusTotal-Checks bis Sandbox-Detonation. 
-      MISP-Synchronisation reichert Cases mit Threat-Intelligence an. Das 
-      ausgeklügelte Rollen- und Rechtesystem ermöglicht sichere Zusammenarbeit 
-      zwischen SOC-Analysten, Forensikern und Management. Templates 
-      standardisieren Response-Prozesse nach Incident-Typ. Die RESTful API 
-      integriert nahtlos mit SIEM, SOAR und Ticketing-Systemen. Metrics und
-      KPIs  messen die Team-Performance. Die Community Edition bleibt kostenlos
-      für  kleinere Teams, während Gold/Platinum-Lizenzen Enterprise-Features
-      bieten.
-    domains:
-      - incident-response
-      - static-investigations
-      - malware-analysis
-      - network-forensics
-      - fraud-investigation
-    phases:
-      - data-collection
-      - examination
-      - analysis
-      - reporting
-    platforms:
-      - Web
-    related_software:
-      - MISP
-      - Cortex
-      - Elasticsearch
-    domain-agnostic-software:
-      - collaboration-general
-    skillLevel: intermediate
-    accessType: server-based
-    url: https://strangebee.com/thehive/
-    projectUrl: ''
-    license: Community Edition (Discontinued) / Commercial
-    knowledgebase: false
-    statusUrl: https://uptime.example.lab/api/badge/1/status
-    tags:
-      - web-interface
-      - case-management
-      - collaboration
-      - api
-      - workflow
-      - multi-user-support
-      - cortex-analyzer
-      - misp-integration
-      - playbooks
-      - metrics
-      - rbac
-      - template-driven
-    related_concepts:
-      - Digital Evidence Chain of Custody
  - name: MISP
    icon: 🌐
    type: software
@ -223,7 +165,6 @@ tools:
    related_concepts:
      - Hash Functions & Digital Signatures
    related_software:
-      - TheHive 5
      - Cortex
      - OpenCTI
  - name: DFIR-IRIS
@ -260,7 +201,6 @@ tools:
    platforms:
      - Web
    related_software:
-      - TheHive 5
      - MISP
      - OpenCTI
    domain-agnostic-software:
@ -3427,6 +3367,244 @@ tools:
    accessType: download
    license: "MPL\_/ AGPL"
    knowledgebase: false
+  - name: ShadowExplorer
+    icon: 🗂️
+    type: software
+    description: >-
+      Das schlanke Windows-Tool macht Volume-Shadow-Copy-Snapshots auch in Home-Editionen sichtbar und erlaubt das komfortable Durchstöbern sowie Wiederherstellen früherer Datei-Versionen. Damit lassen sich versehentlich gelöschte oder überschriebene Dateien in Sekunden zurückholen – geeignet für schnelle Triage und klassische Datenträgerforensik.
+    domains:
+      - static-investigations
+      - incident-response
+    phases:
+      - examination
+      - analysis
+    platforms:
+      - Windows
+    related_software:
+      - OSFMount
+      - PhotoRec
+    domain-agnostic-software: null
+    skillLevel: novice
+    accessType: download
+    url: https://www.shadowexplorer.com/
+    license: Freeware
+    knowledgebase: false
+    tags:
+      - gui
+      - shadow-copy
+      - snapshot-browsing
+      - file-recovery
+      - previous-versions
+      - scenario:file_recovery
+      - point-in-time-restore
+    related_concepts:
+      - Digital Evidence Chain of Custody
+
+
+  - name: Sonic Visualiser
+    icon: 🎵
+    type: software
+    description: >-
+      Die Open-Source-Audio-Analyse-Suite wird in der Forensik eingesetzt,
+      um Wave- und Kompressionsformate bis auf Sample-Ebene zu untersuchen.
+      Spektrogramm-Visualisierung, Zeit-/Frequenz-Annotationen und
+      Transkriptions-Plugins (Vamp) helfen, Manipulationen wie
+      Bandpass-Filter, Time-Stretching oder Insert-Edits nachzuweisen.
+      FFT- und Mel-Spectral-Views decken versteckte Audio-Watermarks oder
+      Steganografie auf. Export-Funktionen in CSV/JSON erlauben die
+      Weiterverarbeitung in Python-Notebooks oder SIEM-Pipelines.
+      Ideal für Voice-Authentication-Checks, Deep-Fake-Erkennung
+      und Beweisaufbereitung vor Gericht.
+    skillLevel: intermediate
+    url: https://www.sonicvisualiser.org/
+    domains:
+      - static-investigations
+      - fraud-investigation
+    phases:
+      - examination
+      - analysis
+      - reporting
+    platforms:
+      - Windows
+      - Linux
+      - macOS
+    accessType: download
+    license: GPL-2.0
+    knowledgebase: false
+    tags:
+      - gui
+      - audio-forensics
+      - spectrogram
+      - plugin-support
+      - annotation
+      - csv-export
+    related_concepts: []
+    related_software:
+      - Audacity
+
+  - name: Dissect
+    icon: 🧩
+    type: software
+    description: >-
+      Fox-ITs Python-Framework abstrahiert Windows- und Linux-Speicherabbilder
+      in virtuelle Objekte (Prozesse, Dateien, Registry, Kernel-Strukturen),
+      ohne zuvor ein Profil definieren zu müssen. Modularer
+      Hypervisor-Layer erlaubt das Mounten und gleichzeitige Analysieren
+      mehrerer Memory-Dumps – perfekt für großflächige Incident-Response.
+      Plugins dekodieren PTEs, handle tables, APC-Queues und liefern
+      YARA-kompatible Scans. Die Zero-Copy-Architektur beschleunigt Queries auf
+      Multi-GB-Images signifikant. Unterstützt Windows 11 24H2-Kernel sowie
+      Linux 6.x-schichten ab Juli 2025.
+    skillLevel: advanced
+    url: https://github.com/fox-it/dissect
+    domains:
+      - incident-response
+      - malware-analysis
+      - static-investigations
+    phases:
+      - examination
+      - analysis
+    platforms:
+      - Windows
+      - Linux
+      - macOS
+    accessType: download
+    license: Apache 2.0
+    knowledgebase: false
+    tags:
+      - command-line
+      - memory-analysis
+      - plugin-support
+      - python-library
+      - zero-copy
+      - profile-less
+    related_concepts:
+      - Regular Expressions (Regex)
+    related_software:
+      - Volatility 3
+      - Rekall
+
+  - name: Docker Explorer
+    icon: 🐳
+    type: software
+    description: >-
+      Googles Forensik-Toolkit zerlegt Offline-Docker-Volumes und
+      Overlay-Dateisysteme ohne laufenden Daemon. Es extrahiert
+      Container-Config, Image-Layer, ENV-Variablen, Mounted-Secrets
+      und schreibt Timeline-fähige Metadata-JSONs. Unterstützt btrfs,
+      overlay2 und zfs Storage-Driver sowie Docker Desktop (macOS/Windows).
+      Perfekt, um bösartige Images nach Supply-Chain-Attacken zu enttarnen
+      oder flüchtige Container nach einem Incident nachträglich zu analysieren.
+    skillLevel: intermediate
+    url: https://github.com/google/docker-explorer
+    domains:
+      - cloud-forensics
+      - incident-response
+      - static-investigations
+    phases:
+      - data-collection
+      - examination
+      - analysis
+    platforms:
+      - Linux
+      - macOS
+      - Windows
+    accessType: download
+    license: Apache 2.0
+    knowledgebase: false
+    tags:
+      - command-line
+      - container-forensics
+      - docker
+      - timeline
+      - json-export
+      - supply-chain
+    related_concepts: []
+    related_software:
+      - Velociraptor
+      - osquery
+
+  - name: Ghiro
+    icon: 🖼️
+    type: software
+    description: >-
+      Die Web-basierte Bildforensik-Plattform automatisiert EXIF-Analyse,
+      Hash-Matching, Error-Level-Evaluation (ELA) und
+      Steganografie-Erkennung für große Dateibatches. Unterstützt
+      Gesichts- und NSFW-Detection sowie GPS-Reverse-Geocoding für
+      Bewegungsprofile. Reports sind gerichtsfest
+      versioniert, REST-API und Celery-Worker skalieren auf
+      Millionen Bilder – ideal für CSAM-Ermittlungen oder Fake-News-Prüfung.
+    skillLevel: intermediate
+    url: https://getghiro.org/
+    domains:
+      - static-investigations
+      - fraud-investigation
+      - mobile-forensics
+    phases:
+      - examination
+      - analysis
+      - reporting
+    platforms:
+      - Web
+      - Linux
+    accessType: server-based
+    license: GPL-2.0
+    knowledgebase: false
+    tags:
+      - web-interface
+      - image-forensics
+      - exif-analysis
+      - steganography
+      - nsfw-detection
+      - batch-processing
+    related_concepts:
+      - Hash Functions & Digital Signatures
+    related_software:
+      - ExifTool
+      - PhotoRec
+
+  - name: Sherloq
+    icon: 🔍
+    type: software
+    description: >-
+      Das Python-GUI-Toolkit für visuelle Datei-Analyse kombiniert
+      klassische Reverse-Steganografie-Techniken (LSB, Palette-Tweaking,
+      DCT-Coefficient-Scanning) mit modernen CV-Algorithmen.
+      Heatmaps und Histogramm-Diffs zeigen Manipulations-Hotspots,
+      während eine „Carve-All-Layers“-Funktion versteckte Daten in PNG,
+      JPEG, BMP, GIF und Audio-Spectra aufspürt. Plugins für zsteg,
+      binwalk und exiftool erweitern die Pipeline.
+      Eine Must-have-Ergänzung zu Ghidra & friends, wenn
+      Malware Dateien als Dead-Drop nutzt.
+    skillLevel: intermediate
+    url: https://github.com/GuidoBartoli/sherloq
+    domains:
+      - malware-analysis
+      - static-investigations
+    phases:
+      - examination
+      - analysis
+    platforms:
+      - Windows
+      - Linux
+      - macOS
+    accessType: download
+    license: MIT
+    knowledgebase: false
+    tags:
+      - gui
+      - image-forensics
+      - steganography
+      - lsb-extraction
+      - histogram-analysis
+      - plugin-support
+    related_concepts:
+      - Regular Expressions (Regex)
+    related_software:
+      - Ghiro
+      - CyberChef
+
  - name: Cortex
    type: software
    description: >-
--- a/src/pages/api/ai/enhance-input.ts
+++ b/src/pages/api/ai/enhance-input.ts
@ -94,18 +94,15 @@ ${input}
  `.trim();
 }

-// Enhanced AI service call function
 async function callAIService(prompt: string): Promise<Response> {
  const endpoint = AI_ENDPOINT;
  const apiKey = AI_ANALYZER_API_KEY;
  const model = AI_ANALYZER_MODEL;
  
-  // Simple headers - add auth only if API key exists
  let headers: Record<string, string> = {
    'Content-Type': 'application/json'
  };
  
-  // Add authentication if API key is provided
  if (apiKey) {
    headers['Authorization'] = `Bearer ${apiKey}`;
    console.log('[ENHANCE API] Using API key authentication');
@ -113,7 +110,6 @@ async function callAIService(prompt: string): Promise<Response> {
    console.log('[ENHANCE API] No API key - making request without authentication');
  }
  
-  // Simple request body
  const requestBody = {
    model,
    messages: [{ role: 'user', content: prompt }],
@ -124,8 +120,6 @@ async function callAIService(prompt: string): Promise<Response> {
    presence_penalty: 0.1
  };
  
-  // FIXED: This function is already being called through enqueueApiCall in the main handler
-  // So we can use direct fetch here since the queuing happens at the caller level
  return fetch(`${endpoint}/v1/chat/completions`, {
    method: 'POST',
    headers,
@ -214,7 +208,7 @@ export const POST: APIRoute = async ({ request }) => {
      success: true,
      questions,
      taskId,
-      inputComplete: questions.length === 0 // Flag to indicate if input seems complete
+      inputComplete: questions.length === 0 
    }), {
      status: 200,
      headers: { 'Content-Type': 'application/json' }
--- a/src/styles/global.css
+++ b/src/styles/global.css
@ -2087,6 +2087,7 @@ input[type="checkbox"] {
  gap: 1rem;
  max-width: 1200px;
  margin: 0 auto;
+  margin-top: 1rem;
 }

 .phase-header {
--- a/src/utils/aiPipeline.ts
+++ b/src/utils/aiPipeline.ts
@ -31,7 +31,6 @@ interface AnalysisResult {
  };
 }

-// NEW: Audit Trail Types
 interface AuditEntry {
  timestamp: number;
  phase: string;           // 'retrieval', 'selection', 'micro-task-N'
@ -40,10 +39,9 @@ interface AuditEntry {
  output: any;             // What came out of this step
  confidence: number;      // 0-100: How confident we are in this step
  processingTimeMs: number;
-  metadata: Record<string, any>; // Additional context
+  metadata: Record<string, any>;
 }

-// Enhanced AnalysisContext with Audit Trail
 interface AnalysisContext {
  userQuery: string;
  mode: string;
@ -62,7 +60,6 @@ interface AnalysisContext {
  
  seenToolNames: Set<string>;
  
-  // NEW: Audit Trail
  auditTrail: AuditEntry[];
 }

@ -78,17 +75,24 @@ class ImprovedMicroTaskAIPipeline {
  private similarityThreshold: number;
  private microTaskDelay: number;
  
+  private embeddingSelectionLimit: number;
+  private embeddingConceptsLimit: number;
+
+  private noEmbeddingsToolLimit: number;
+  private noEmbeddingsConceptLimit: number;
+  
+  private embeddingsMinTools: number;
+  private embeddingsMaxReductionRatio: number;
+  
  private maxContextTokens: number;
  private maxPromptTokens: number;
  
-  // NEW: Audit Configuration
  private auditConfig: {
    enabled: boolean;
    detailLevel: 'minimal' | 'standard' | 'verbose';
    retentionHours: number;
  };
  
-  // NEW: Temporary audit storage for pre-context operations
  private tempAuditEntries: AuditEntry[] = [];

  constructor() {
@ -98,20 +102,35 @@ class ImprovedMicroTaskAIPipeline {
      model: this.getEnv('AI_ANALYZER_MODEL')
    };

-    this.maxSelectedItems = parseInt(process.env.AI_MAX_SELECTED_ITEMS || '60', 10);
-    this.embeddingCandidates = parseInt(process.env.AI_EMBEDDING_CANDIDATES || '60', 10); 
-    this.similarityThreshold = 0.3; 
+    this.maxSelectedItems = parseInt(process.env.AI_MAX_SELECTED_ITEMS || '25', 10);
+    this.embeddingCandidates = parseInt(process.env.AI_EMBEDDING_CANDIDATES || '50', 10); 
+    this.similarityThreshold = parseFloat(process.env.AI_SIMILARITY_THRESHOLD || '0.3');
    this.microTaskDelay = parseInt(process.env.AI_MICRO_TASK_DELAY_MS || '500', 10);
    
+    this.embeddingSelectionLimit = parseInt(process.env.AI_EMBEDDING_SELECTION_LIMIT || '30', 10);
+    this.embeddingConceptsLimit = parseInt(process.env.AI_EMBEDDING_CONCEPTS_LIMIT || '15', 10);
+
+    this.noEmbeddingsToolLimit = parseInt(process.env.AI_NO_EMBEDDINGS_TOOL_LIMIT || '0', 10);
+    this.noEmbeddingsConceptLimit = parseInt(process.env.AI_NO_EMBEDDINGS_CONCEPT_LIMIT || '0', 10);
+    
+    this.embeddingsMinTools = parseInt(process.env.AI_EMBEDDINGS_MIN_TOOLS || '8', 10);
+    this.embeddingsMaxReductionRatio = parseFloat(process.env.AI_EMBEDDINGS_MAX_REDUCTION_RATIO || '0.75');
+    
    this.maxContextTokens = parseInt(process.env.AI_MAX_CONTEXT_TOKENS || '4000', 10);
    this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10);
    
-    // NEW: Initialize Audit Configuration
    this.auditConfig = {
      enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true',
      detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as any) || 'standard',
      retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10)
    };
+    
+    console.log('[AI PIPELINE] Configuration loaded:', {
+      embeddingCandidates: this.embeddingCandidates,
+      embeddingSelection: `${this.embeddingSelectionLimit} tools, ${this.embeddingConceptsLimit} concepts`,
+      noEmbeddingsLimits: `${this.noEmbeddingsToolLimit || 'unlimited'} tools, ${this.noEmbeddingsConceptLimit || 'unlimited'} concepts`,
+      auditEnabled: this.auditConfig.enabled
+    });
  }

  private getEnv(key: string): string {
@ -122,7 +141,6 @@ class ImprovedMicroTaskAIPipeline {
    return value;
  }

-  // NEW: Audit Trail Utility Functions
  private addAuditEntry(
    context: AnalysisContext | null, 
    phase: string, 
@ -149,22 +167,18 @@ class ImprovedMicroTaskAIPipeline {
    if (context) {
      context.auditTrail.push(auditEntry);
    } else {
-      // Store in temporary array for later merging
      this.tempAuditEntries.push(auditEntry);
    }
    
-    // Log for debugging when audit is enabled
    console.log(`[AUDIT] ${phase}/${action}: ${confidence}% confidence, ${Date.now() - startTime}ms`);
  }
  
-  // NEW: Merge temporary audit entries into context
  private mergeTemporaryAuditEntries(context: AnalysisContext): void {
    if (!this.auditConfig.enabled || this.tempAuditEntries.length === 0) return;
    
    const entryCount = this.tempAuditEntries.length;
-    // Add temp entries to the beginning of the context audit trail
    context.auditTrail.unshift(...this.tempAuditEntries);
-    this.tempAuditEntries = []; // Clear temp storage
+    this.tempAuditEntries = []; 
    
    console.log(`[AUDIT] Merged ${entryCount} temporary audit entries into context`);
  }
@ -196,15 +210,12 @@ class ImprovedMicroTaskAIPipeline {
    
    let confidence = 60; // Base confidence
    
-    // Good selection ratio (not too many, not too few)
    if (selectionRatio > 0.05 && selectionRatio < 0.3) confidence += 20;
    else if (selectionRatio <= 0.05) confidence -= 10; // Too few
    else confidence -= 15; // Too many
    
-    // Has detailed reasoning
    if (hasReasoning) confidence += 15;
    
-    // Selected tools have good distribution
    if (result.selectedConcepts && result.selectedConcepts.length > 0) confidence += 5;
    
    return Math.min(95, Math.max(25, confidence));
@ -228,26 +239,106 @@ class ImprovedMicroTaskAIPipeline {

  private safeParseJSON(jsonString: string, fallback: any = null): any {
    try {
-      const cleaned = jsonString
+      let cleaned = jsonString
        .replace(/^```json\s*/i, '')
        .replace(/\s*```\s*$/g, '')
        .trim();
      
+      if (!cleaned.endsWith('}') && !cleaned.endsWith(']')) {
+        console.warn('[AI PIPELINE] JSON appears truncated, attempting recovery...');
+        
+        let lastCompleteStructure = '';
+        let braceCount = 0;
+        let bracketCount = 0;
+        let inString = false;
+        let escaped = false;
+        
+        for (let i = 0; i < cleaned.length; i++) {
+          const char = cleaned[i];
+          
+          if (escaped) {
+            escaped = false;
+            continue;
+          }
+          
+          if (char === '\\') {
+            escaped = true;
+            continue;
+          }
+          
+          if (char === '"' && !escaped) {
+            inString = !inString;
+            continue;
+          }
+          
+          if (!inString) {
+            if (char === '{') braceCount++;
+            if (char === '}') braceCount--;
+            if (char === '[') bracketCount++;
+            if (char === ']') bracketCount--;
+            
+            if (braceCount === 0 && bracketCount === 0 && (char === '}' || char === ']')) {
+              lastCompleteStructure = cleaned.substring(0, i + 1);
+            }
+          }
+        }
+        
+        if (lastCompleteStructure) {
+          console.log('[AI PIPELINE] Attempting to parse recovered JSON structure...');
+          cleaned = lastCompleteStructure;
+        } else {
+          if (braceCount > 0) {
+            cleaned += '}';
+            console.log('[AI PIPELINE] Added closing brace to truncated JSON');
+          }
+          if (bracketCount > 0) {
+            cleaned += ']';
+            console.log('[AI PIPELINE] Added closing bracket to truncated JSON');
+          }
+        }
+      }
+      
      const parsed = JSON.parse(cleaned);
+      
+      if (parsed && typeof parsed === 'object') {
+        if (parsed.selectedTools === undefined) parsed.selectedTools = [];
+        if (parsed.selectedConcepts === undefined) parsed.selectedConcepts = [];
+        
+        if (!Array.isArray(parsed.selectedTools)) parsed.selectedTools = [];
+        if (!Array.isArray(parsed.selectedConcepts)) parsed.selectedConcepts = [];
+      }
+      
      return parsed;
    } catch (error) {
      console.warn('[AI PIPELINE] JSON parsing failed:', error.message);
-      console.warn('[AI PIPELINE] Raw content:', jsonString.slice(0, 200));
+      console.warn('[AI PIPELINE] Raw content (first 300 chars):', jsonString.slice(0, 300));
+      console.warn('[AI PIPELINE] Raw content (last 300 chars):', jsonString.slice(-300));
+      
+      if (jsonString.includes('selectedTools')) {
+        const toolMatches = jsonString.match(/"([^"]+)"/g);
+        if (toolMatches && toolMatches.length > 0) {
+          console.log('[AI PIPELINE] Attempting partial recovery from broken JSON...');
+          const possibleTools = toolMatches
+            .map(match => match.replace(/"/g, ''))
+            .filter(name => name.length > 2 && !['selectedTools', 'selectedConcepts', 'reasoning'].includes(name))
+            .slice(0, 15); // Reasonable limit
+          
+          if (possibleTools.length > 0) {
+            console.log(`[AI PIPELINE] Recovered ${possibleTools.length} possible tool names from broken JSON`);
+            return {
+              selectedTools: possibleTools,
+              selectedConcepts: [],
+              reasoning: 'Recovered from truncated response'
+            };
+          }
+        }
+      }
+      
      return fallback;
    }
  }

  private addToolToSelection(context: AnalysisContext, tool: any, phase: string, priority: string, justification?: string): boolean {   
-    if (context.seenToolNames.has(tool.name)) {
-      console.log(`[AI PIPELINE] Skipping duplicate tool: ${tool.name}`);
-      return false;
-    }
-    
    context.seenToolNames.add(tool.name);
    if (!context.selectedTools) context.selectedTools = [];
    
@ -266,56 +357,62 @@ class ImprovedMicroTaskAIPipeline {
    let candidateConcepts: any[] = [];
    let selectionMethod = 'unknown';
    
+    // WAIT for embeddings initialization if embeddings are enabled
+    if (process.env.AI_EMBEDDINGS_ENABLED === 'true') {
+      try {
+        console.log('[AI PIPELINE] Waiting for embeddings initialization...');
+        await embeddingsService.waitForInitialization();
+        console.log('[AI PIPELINE] Embeddings ready, proceeding with similarity search');
+      } catch (error) {
+        console.error('[AI PIPELINE] Embeddings initialization failed, falling back to full dataset:', error);
+      }
+    }
+    
    if (embeddingsService.isEnabled()) {
      const embeddingsStart = Date.now();
      const similarItems = await embeddingsService.findSimilar(
        userQuery, 
        this.embeddingCandidates, 
        this.similarityThreshold
-      ) as SimilarityResult[]; // Type assertion for similarity property
+      ) as SimilarityResult[];
      
-      console.log(`[IMPROVED PIPELINE] Embeddings found ${similarItems.length} similar items`);
+      console.log(`[AI PIPELINE] Embeddings found ${similarItems.length} similar items`);
      
-      // FIXED: Create lookup maps for O(1) access while preserving original data
      const toolsMap = new Map<string, any>(toolsData.tools.map((tool: any) => [tool.name, tool]));
      const conceptsMap = new Map<string, any>(toolsData.concepts.map((concept: any) => [concept.name, concept]));
      
-      // FIXED: Process in similarity order, preserving the ranking
      const similarTools = similarItems
        .filter((item): item is SimilarityResult => item.type === 'tool')
        .map(item => toolsMap.get(item.name))
-        .filter((tool): tool is any => tool !== undefined); // Proper type guard
+        .filter((tool): tool is any => tool !== undefined);
      
      const similarConcepts = similarItems
        .filter((item): item is SimilarityResult => item.type === 'concept')
        .map(item => conceptsMap.get(item.name))
-        .filter((concept): concept is any => concept !== undefined); // Proper type guard
+        .filter((concept): concept is any => concept !== undefined);
      
-      console.log(`[IMPROVED PIPELINE] Similarity-ordered results: ${similarTools.length} tools, ${similarConcepts.length} concepts`);
+      console.log(`[AI PIPELINE] Similarity-ordered results: ${similarTools.length} tools, ${similarConcepts.length} concepts`);
      
-      // Log the first few tools to verify ordering is preserved
-      if (similarTools.length > 0) {
-        console.log(`[IMPROVED PIPELINE] Top similar tools (in similarity order):`);
-        similarTools.slice(0, 5).forEach((tool, idx) => {
-          const originalSimilarItem = similarItems.find(item => item.name === tool.name);
-          console.log(`  ${idx + 1}. ${tool.name} (similarity: ${originalSimilarItem?.similarity?.toFixed(4) || 'N/A'})`);
-        });
-      }
+      const totalAvailableTools = toolsData.tools.length;
+      const reductionRatio = similarTools.length / totalAvailableTools;
      
-      if (similarTools.length >= 15) {
+      if (similarTools.length >= this.embeddingsMinTools && reductionRatio <= this.embeddingsMaxReductionRatio) {
        candidateTools = similarTools;
        candidateConcepts = similarConcepts;
        selectionMethod = 'embeddings_candidates';
        
-        console.log(`[IMPROVED PIPELINE] Using embeddings candidates in similarity order: ${candidateTools.length} tools`);
+        console.log(`[AI PIPELINE] Using embeddings filtering: ${totalAvailableTools} → ${similarTools.length} tools (${(reductionRatio * 100).toFixed(1)}% reduction)`);
      } else {
-        console.log(`[IMPROVED PIPELINE] Embeddings insufficient (${similarTools.length} < 15), using full dataset`);
+        if (similarTools.length < this.embeddingsMinTools) {
+          console.log(`[AI PIPELINE] Embeddings found too few tools (${similarTools.length} < ${this.embeddingsMinTools}), using full dataset`);
+        } else {
+          console.log(`[AI PIPELINE] Embeddings didn't filter enough (${(reductionRatio * 100).toFixed(1)}% > ${(this.embeddingsMaxReductionRatio * 100).toFixed(1)}%), using full dataset`);
+        }
        candidateTools = toolsData.tools;
        candidateConcepts = toolsData.concepts;
        selectionMethod = 'full_dataset';
      }
      
-      // NEW: Add Audit Entry for Embeddings Search with ordering verification
      if (this.auditConfig.enabled) {
        this.addAuditEntry(null, 'retrieval', 'embeddings-search', 
          { query: userQuery, threshold: this.similarityThreshold, candidates: this.embeddingCandidates }, 
@ -323,21 +420,29 @@ class ImprovedMicroTaskAIPipeline {
            candidatesFound: similarItems.length, 
            toolsInOrder: similarTools.slice(0, 3).map((t: any) => t.name),
            conceptsInOrder: similarConcepts.slice(0, 3).map((c: any) => c.name),
-            orderingPreserved: true
+            reductionRatio: reductionRatio,
+            usingEmbeddings: selectionMethod === 'embeddings_candidates',
+            totalAvailable: totalAvailableTools,
+            filtered: similarTools.length
          },
-          similarTools.length >= 15 ? 85 : 60,
+          selectionMethod === 'embeddings_candidates' ? 85 : 60,
          embeddingsStart,
-          { selectionMethod, embeddingsEnabled: true, orderingFixed: true }
+          { 
+            selectionMethod, 
+            embeddingsEnabled: true, 
+            reductionAchieved: selectionMethod === 'embeddings_candidates',
+            tokenSavingsExpected: selectionMethod === 'embeddings_candidates'
+          }
        );
      }
    } else {
-      console.log(`[IMPROVED PIPELINE] Embeddings disabled, using full dataset`);
+      console.log(`[AI PIPELINE] Embeddings disabled or not ready, using full dataset`);
      candidateTools = toolsData.tools;
      candidateConcepts = toolsData.concepts;
      selectionMethod = 'full_dataset';
    }

-    console.log(`[IMPROVED PIPELINE] AI will analyze ${candidateTools.length} candidate tools (ordering preserved: ${selectionMethod === 'embeddings_candidates'})`);
+    console.log(`[AI PIPELINE] AI will analyze ${candidateTools.length} candidate tools (method: ${selectionMethod})`);
    const finalSelection = await this.aiSelectionWithFullData(userQuery, candidateTools, candidateConcepts, mode, selectionMethod);
    
    return {
@ -387,15 +492,44 @@ class ImprovedMicroTaskAIPipeline {
      related_software: concept.related_software || []
    }));

-    // Generate the German prompt with tool data
+    let toolsToSend: any[];
+    let conceptsToSend: any[];
+    
+    if (selectionMethod === 'embeddings_candidates') {
+      toolsToSend = toolsWithFullData.slice(0, this.embeddingSelectionLimit);
+      conceptsToSend = conceptsWithFullData.slice(0, this.embeddingConceptsLimit);
+      
+      console.log(`[AI PIPELINE] Embeddings enabled: sending top ${toolsToSend.length} similarity-ordered tools`);
+    } else {
+      const maxTools = this.noEmbeddingsToolLimit > 0 ? 
+        Math.min(this.noEmbeddingsToolLimit, candidateTools.length) : 
+        candidateTools.length;
+      
+      const maxConcepts = this.noEmbeddingsConceptLimit > 0 ? 
+        Math.min(this.noEmbeddingsConceptLimit, candidateConcepts.length) : 
+        candidateConcepts.length;
+      
+      toolsToSend = toolsWithFullData.slice(0, maxTools);
+      conceptsToSend = conceptsWithFullData.slice(0, maxConcepts);
+      
+      console.log(`[AI PIPELINE] Embeddings disabled: sending ${toolsToSend.length}/${candidateTools.length} tools (limit: ${this.noEmbeddingsToolLimit || 'none'})`);
+    }
+
    const basePrompt = getPrompt('toolSelection', mode, userQuery, selectionMethod, this.maxSelectedItems);
    const prompt = `${basePrompt}

 VERFÜGBARE TOOLS (mit vollständigen Daten):
-${JSON.stringify(toolsWithFullData.slice(0, 30), null, 2)}
+${JSON.stringify(toolsToSend, null, 2)}

 VERFÜGBARE KONZEPTE (mit vollständigen Daten):
-${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
+${JSON.stringify(conceptsToSend, null, 2)}`;
+
+    const estimatedTokens = this.estimateTokens(prompt);
+    console.log(`[AI PIPELINE] Method: ${selectionMethod}, Tools: ${toolsToSend.length}, Estimated tokens: ~${estimatedTokens}`);
+
+    if (estimatedTokens > 35000) {
+      console.warn(`[AI PIPELINE] WARNING: Prompt tokens (${estimatedTokens}) may exceed model limits`);
+    }

    try {
      const response = await this.callAI(prompt, 2500);
@ -403,16 +537,15 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
      const result = this.safeParseJSON(response, null);
      
      if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) {
-        console.error('[IMPROVED PIPELINE] AI selection returned invalid structure:', response.slice(0, 200));
+        console.error('[AI PIPELINE] AI selection returned invalid structure:', response.slice(0, 200));
        
-        // NEW: Add Audit Entry for Failed Selection
        if (this.auditConfig.enabled) {
          this.addAuditEntry(null, 'selection', 'ai-tool-selection-failed',
            { candidateCount: candidateTools.length, mode, prompt: prompt.slice(0, 200) },
            { error: 'Invalid JSON structure', response: response.slice(0, 200) },
-            10, // Very low confidence
+            10,
            selectionStart,
-            { aiModel: this.config.model, selectionMethod }
+            { aiModel: this.config.model, selectionMethod, tokensSent: estimatedTokens, toolsSent: toolsToSend.length }
          );
        }
        
@ -421,19 +554,15 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;

      const totalSelected = result.selectedTools.length + result.selectedConcepts.length;
      if (totalSelected === 0) {
-        console.error('[IMPROVED PIPELINE] AI selection returned no tools');
+        console.error('[AI PIPELINE] AI selection returned no tools');
        throw new Error('AI selection returned empty selection');
      }
      
-      console.log(`[IMPROVED PIPELINE] AI selected: ${result.selectedTools.length} tools, ${result.selectedConcepts.length} concepts`);
-      console.log(`[IMPROVED PIPELINE] AI reasoning: ${result.reasoning}`);
+      console.log(`[AI PIPELINE] AI selected: ${result.selectedTools.length} tools, ${result.selectedConcepts.length} concepts from ${toolsToSend.length} candidates`);

      const selectedTools = candidateTools.filter(tool => result.selectedTools.includes(tool.name));
      const selectedConcepts = candidateConcepts.filter(concept => result.selectedConcepts.includes(concept.name));
      
-      console.log(`[IMPROVED PIPELINE] Final selection: ${selectedTools.length} tools with bias prevention applied`);
-      
-      // NEW: Add Audit Entry for Successful Selection
      if (this.auditConfig.enabled) {
        const confidence = this.calculateSelectionConfidence(result, candidateTools.length);
        
@ -443,11 +572,12 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
            selectedToolCount: result.selectedTools.length, 
            selectedConceptCount: result.selectedConcepts.length,
            reasoning: result.reasoning?.slice(0, 200) + '...',
-            finalToolNames: selectedTools.map(t => t.name)
+            finalToolNames: selectedTools.map(t => t.name),
+            selectionEfficiency: `${toolsToSend.length} → ${result.selectedTools.length}`
          },
          confidence,
          selectionStart,
-          { aiModel: this.config.model, selectionMethod, promptTokens: this.estimateTokens(prompt) }
+          { aiModel: this.config.model, selectionMethod, promptTokens: estimatedTokens, toolsSent: toolsToSend.length }
        );
      }
      
@ -457,74 +587,26 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
      };

    } catch (error) {
-      console.error('[IMPROVED PIPELINE] AI selection failed:', error);
+      console.error('[AI PIPELINE] AI selection failed:', error);
      
-      // NEW: Add Audit Entry for Selection Error
      if (this.auditConfig.enabled) {
        this.addAuditEntry(null, 'selection', 'ai-tool-selection-error',
          { candidateCount: candidateTools.length, mode },
          { error: error.message },
-          5, // Very low confidence
+          5,
          selectionStart,
-          { aiModel: this.config.model, selectionMethod }
+          { aiModel: this.config.model, selectionMethod, tokensSent: estimatedTokens }
        );
      }
-      
-      console.log('[IMPROVED PIPELINE] Using emergency keyword-based selection');
-      return this.emergencyKeywordSelection(userQuery, candidateTools, candidateConcepts, mode);
+      throw error;
    }
  }

-  private emergencyKeywordSelection(userQuery: string, candidateTools: any[], candidateConcepts: any[], mode: string) {
-    const emergencyStart = Date.now();
-    
-    const queryLower = userQuery.toLowerCase();
-    const keywords = queryLower.split(/\s+/).filter(word => word.length > 3);
-    
-    const scoredTools = candidateTools.map(tool => {
-      const toolText = (
-        tool.name + ' ' + 
-        tool.description + ' ' + 
-        (tool.tags || []).join(' ') + ' ' +
-        (tool.platforms || []).join(' ') + ' ' +
-        (tool.domains || []).join(' ')
-      ).toLowerCase();
-      
-      const score = keywords.reduce((acc, keyword) => {
-        return acc + (toolText.includes(keyword) ? 1 : 0);
-      }, 0);
-      
-      return { tool, score };
-    }).filter(item => item.score > 0)
-      .sort((a, b) => b.score - a.score);
-    
-    const maxTools = mode === 'workflow' ? 20 : 8;
-    const selectedTools = scoredTools.slice(0, maxTools).map(item => item.tool);
-    
-    console.log(`[IMPROVED PIPELINE] Emergency selection: ${selectedTools.length} tools, keywords: ${keywords.slice(0, 5).join(', ')}`);
-    
-    // NEW: Add Audit Entry for Emergency Selection
-    if (this.auditConfig.enabled) {
-      this.addAuditEntry(null, 'selection', 'emergency-keyword-selection',
-        { keywords: keywords.slice(0, 10), candidateCount: candidateTools.length },
-        { selectedCount: selectedTools.length, topScores: scoredTools.slice(0, 5).map(s => ({ name: s.tool.name, score: s.score })) },
-        40, // Moderate confidence for emergency selection
-        emergencyStart,
-        { selectionMethod: 'emergency_keyword' }
-      );
-    }
-    
-    return {
-      selectedTools,
-      selectedConcepts: candidateConcepts.slice(0, 3)
-    };
-  }
-
  private async delay(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

-  private async callMicroTaskAI(prompt: string, context: AnalysisContext, maxTokens: number = 300): Promise<MicroTaskResult> {
+  private async callMicroTaskAI(prompt: string, context: AnalysisContext, maxTokens: number = 500): Promise<MicroTaskResult> {
    const startTime = Date.now();
    
    let contextPrompt = prompt;
@ -549,11 +631,10 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
        success: true
      };
      
-      // NEW: Add Audit Entry for Successful Micro-Task
      this.addAuditEntry(context, 'micro-task', 'ai-analysis',
        { promptLength: contextPrompt.length, maxTokens },
        { responseLength: response.length, contentPreview: response.slice(0, 100) },
-        response.length > 50 ? 80 : 60, // Confidence based on response quality
+        response.length > 50 ? 80 : 60,
        startTime,
        { aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 }
      );
@ -569,11 +650,10 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
        error: error.message
      };
      
-      // NEW: Add Audit Entry for Failed Micro-Task
      this.addAuditEntry(context, 'micro-task', 'ai-analysis-failed',
        { promptLength: contextPrompt.length, maxTokens },
        { error: error.message },
-        5, // Very low confidence
+        5,
        startTime,
        { aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 }
      );
@ -586,7 +666,7 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('scenarioAnalysis', isWorkflow, context.userQuery);

-    const result = await this.callMicroTaskAI(prompt, context, 220);
+    const result = await this.callMicroTaskAI(prompt, context, 400);
    
    if (result.success) {
      if (isWorkflow) {
@ -605,7 +685,7 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('investigationApproach', isWorkflow, context.userQuery);

-    const result = await this.callMicroTaskAI(prompt, context, 220);
+    const result = await this.callMicroTaskAI(prompt, context, 400);
    
    if (result.success) {
      context.investigationApproach = result.content;
@ -619,7 +699,7 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('criticalConsiderations', isWorkflow, context.userQuery);

-    const result = await this.callMicroTaskAI(prompt, context, 180);
+    const result = await this.callMicroTaskAI(prompt, context, 350);
    
    if (result.success) {
      context.criticalConsiderations = result.content;
@ -645,7 +725,7 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;

    const prompt = getPrompt('phaseToolSelection', context.userQuery, phase, phaseTools);

-    const result = await this.callMicroTaskAI(prompt, context, 450);
+    const result = await this.callMicroTaskAI(prompt, context, 800);
    
    if (result.success) {
      const selections = this.safeParseJSON(result.content, []);
@ -662,7 +742,6 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
          }
        });
        
-        // NEW: Add audit entry for tool selection
        this.addAuditEntry(context, 'micro-task', 'phase-tool-selection',
          { phase: phase.id, availableTools: phaseTools.length },
          { validSelections: validSelections.length, selectedTools: validSelections.map(s => s.toolName) },
@ -679,7 +758,7 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
  private async evaluateSpecificTool(context: AnalysisContext, tool: any, rank: number): Promise<MicroTaskResult> {
    const prompt = getPrompt('toolEvaluation', context.userQuery, tool, rank);

-    const result = await this.callMicroTaskAI(prompt, context, 650);
+    const result = await this.callMicroTaskAI(prompt, context, 1200);
    
    if (result.success) {
      const evaluation = this.safeParseJSON(result.content, {
@ -699,7 +778,6 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
        }
      }, 'evaluation', evaluation.suitability_score);
      
-      // NEW: Add audit entry for tool evaluation
      this.addAuditEntry(context, 'micro-task', 'tool-evaluation',
        { toolName: tool.name, rank },
        { suitabilityScore: evaluation.suitability_score, hasExplanation: !!evaluation.detailed_explanation },
@ -727,7 +805,7 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
    const selectedToolNames = context.selectedTools?.map(st => st.tool.name) || [];
    const prompt = getPrompt('backgroundKnowledgeSelection', context.userQuery, context.mode, selectedToolNames, availableConcepts);

-    const result = await this.callMicroTaskAI(prompt, context, 400);
+    const result = await this.callMicroTaskAI(prompt, context, 700);
    
    if (result.success) {
      const selections = this.safeParseJSON(result.content, []);
@ -740,7 +818,6 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
          relevance: sel.relevance
        }));
        
-        // NEW: Add audit entry for background knowledge selection
        this.addAuditEntry(context, 'micro-task', 'background-knowledge-selection',
          { availableConcepts: availableConcepts.length },
          { selectedConcepts: context.backgroundKnowledge?.length || 0 },
@ -758,21 +835,19 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
    const selectedToolNames = context.selectedTools?.map(st => st.tool.name) || [];
    const prompt = getPrompt('finalRecommendations', context.mode === 'workflow', context.userQuery, selectedToolNames);

-    const result = await this.callMicroTaskAI(prompt, context, 180);
+    const result = await this.callMicroTaskAI(prompt, context, 350);
    return result;
  }

-  private async callAI(prompt: string, maxTokens: number = 1000): Promise<string> {
+  private async callAI(prompt: string, maxTokens: number = 1500): Promise<string> {
    const endpoint = this.config.endpoint;
    const apiKey = this.config.apiKey;
    const model = this.config.model;
    
-    // Simple headers - add auth only if API key exists
    let headers: Record<string, string> = {
      'Content-Type': 'application/json'
    };
    
-    // Add authentication if API key is provided
    if (apiKey) {
      headers['Authorization'] = `Bearer ${apiKey}`;
      console.log('[AI PIPELINE] Using API key authentication');
@ -780,7 +855,6 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
      console.log('[AI PIPELINE] No API key - making request without authentication');
    }
    
-    // Simple request body
    const requestBody = {
      model,
      messages: [{ role: 'user', content: prompt }],
@ -789,7 +863,6 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
    };
    
    try {
-      // FIXED: Use direct fetch since entire pipeline is already queued at query.ts level
      const response = await fetch(`${endpoint}/v1/chat/completions`, {
        method: 'POST',
        headers,
@ -823,13 +896,11 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
    let completedTasks = 0;
    let failedTasks = 0;
    
-    // NEW: Clear any previous temporary audit entries
    this.tempAuditEntries = [];
    
-    console.log(`[IMPROVED PIPELINE] Starting ${mode} query processing with context continuity and audit trail`);
+    console.log(`[AI PIPELINE] Starting ${mode} query processing with context continuity and audit trail`);

    try {
-      // Stage 1: Get intelligent candidates (embeddings + AI selection)
      const toolsData = await getCompressedToolsDataForAI();
      const filteredData = await this.getIntelligentCandidates(userQuery, toolsData, mode);
      
@ -841,20 +912,17 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
        maxContextLength: this.maxContextTokens,
        currentContextLength: 0,
        seenToolNames: new Set<string>(),
-        // NEW: Initialize audit trail
        auditTrail: []
      };

-      // NEW: Merge any temporary audit entries from pre-context operations
      this.mergeTemporaryAuditEntries(context);

-      console.log(`[IMPROVED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`);
+      console.log(`[AI PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`);

-      // NEW: Add initial audit entry
      this.addAuditEntry(context, 'initialization', 'pipeline-start',
        { userQuery, mode, toolsDataLoaded: !!toolsData },
        { candidateTools: filteredData.tools.length, candidateConcepts: filteredData.concepts.length },
-        90, // High confidence for initialization
+        90, 
        startTime,
        { auditEnabled: this.auditConfig.enabled }
      );
@ -893,19 +961,15 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
        }
      }

-      // Task 5: Background Knowledge Selection
      const knowledgeResult = await this.selectBackgroundKnowledge(context);
      if (knowledgeResult.success) completedTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

-      // Task 6: Final Recommendations
      const finalResult = await this.generateFinalRecommendations(context);
      if (finalResult.success) completedTasks++; else failedTasks++;

-      // Build final recommendation
      const recommendation = this.buildRecommendation(context, mode, finalResult.content);

-      // NEW: Add final audit entry
      this.addAuditEntry(context, 'completion', 'pipeline-end',
        { completedTasks, failedTasks },
        { finalRecommendation: !!recommendation, auditEntriesGenerated: context.auditTrail.length },
@ -925,23 +989,21 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
        contextContinuityUsed: true
      };

-      console.log(`[IMPROVED PIPELINE] Completed: ${completedTasks} tasks, Failed: ${failedTasks} tasks`);
-      console.log(`[IMPROVED PIPELINE] Unique tools selected: ${context.seenToolNames.size}`);
-      console.log(`[IMPROVED PIPELINE] Audit trail entries: ${context.auditTrail.length}`);
+      console.log(`[AI PIPELINE] Completed: ${completedTasks} tasks, Failed: ${failedTasks} tasks`);
+      console.log(`[AI PIPELINE] Unique tools selected: ${context.seenToolNames.size}`);
+      console.log(`[AI PIPELINE] Audit trail entries: ${context.auditTrail.length}`);

      return {
        recommendation: {
          ...recommendation,
-          // NEW: Include audit trail in response
          auditTrail: this.auditConfig.enabled ? context.auditTrail : undefined
        },
        processingStats
      };

    } catch (error) {
-      console.error('[IMPROVED PIPELINE] Processing failed:', error);
+      console.error('[AI PIPELINE] Processing failed:', error);
      
-      // NEW: Ensure temp audit entries are cleared even on error
      this.tempAuditEntries = [];
      
      throw error;
--- a/src/utils/dataService.ts
+++ b/src/utils/dataService.ts
@ -77,33 +77,8 @@ interface EnhancedCompressedToolsData {
  domains: any[];
  phases: any[];
  'domain-agnostic-software': any[];
-  scenarios?: any[]; // Optional for AI processing
+  scenarios?: any[];
  skill_levels: any;
-  // Enhanced context for micro-tasks
-  domain_relationships: DomainRelationship[];
-  phase_dependencies: PhaseDependency[];
-  tool_compatibility_matrix: CompatibilityMatrix[];
-}
-
-interface DomainRelationship {
-  domain_id: string;
-  tool_count: number;
-  common_tags: string[];
-  skill_distribution: Record<string, number>;
-}
-
-interface PhaseDependency {
-  phase_id: string;
-  order: number;
-  depends_on: string | null;
-  enables: string | null;
-  is_parallel_capable: boolean;
-  typical_duration: string;
-}
-
-interface CompatibilityMatrix {
-  type: string;
-  groups: Record<string, string[]>;
 }

 let cachedData: ToolsData | null = null;
@ -146,104 +121,6 @@ function generateDataVersion(data: any): string {
  return Math.abs(hash).toString(36);
 }

-// Enhanced: Generate domain relationships for better AI understanding
-function generateDomainRelationships(domains: any[], tools: any[]): DomainRelationship[] {
-  const relationships: DomainRelationship[] = [];
-  
-  for (const domain of domains) {
-    const domainTools = tools.filter(tool => 
-      tool.domains && tool.domains.includes(domain.id)
-    );
-    
-    const commonTags = domainTools
-      .flatMap(tool => tool.tags || [])
-      .reduce((acc: any, tag: string) => {
-        acc[tag] = (acc[tag] || 0) + 1;
-        return acc;
-      }, {});
-      
-    const topTags = Object.entries(commonTags)
-      .sort(([,a], [,b]) => (b as number) - (a as number))
-      .slice(0, 5)
-      .map(([tag]) => tag);
-    
-    relationships.push({
-      domain_id: domain.id,
-      tool_count: domainTools.length,
-      common_tags: topTags,
-      skill_distribution: domainTools.reduce((acc: any, tool: any) => {
-        acc[tool.skillLevel] = (acc[tool.skillLevel] || 0) + 1;
-        return acc;
-      }, {})
-    });
-  }
-  
-  return relationships;
-}
-
-// Enhanced: Generate phase dependencies
-function generatePhaseDependencies(phases: any[]): PhaseDependency[] {
-  const dependencies: PhaseDependency[] = [];
-  
-  for (let i = 0; i < phases.length; i++) {
-    const phase = phases[i];
-    const nextPhase = phases[i + 1];
-    const prevPhase = phases[i - 1];
-    
-    dependencies.push({
-      phase_id: phase.id,
-      order: i + 1,
-      depends_on: prevPhase?.id || null,
-      enables: nextPhase?.id || null,
-      is_parallel_capable: ['examination', 'analysis'].includes(phase.id), // Some phases can run in parallel
-      typical_duration: phase.id === 'data-collection' ? 'hours-days' :
-                       phase.id === 'examination' ? 'hours-weeks' :
-                       phase.id === 'analysis' ? 'days-weeks' :
-                       'hours-days'
-    });
-  }
-  
-  return dependencies;
-}
-
-// Enhanced: Generate tool compatibility matrix
-function generateToolCompatibilityMatrix(tools: any[]): CompatibilityMatrix[] {
-  const matrix: CompatibilityMatrix[] = [];
-  
-  // Group tools by common characteristics
-  const platformGroups = tools.reduce((acc: any, tool: any) => {
-    if (tool.platforms) {
-      tool.platforms.forEach((platform: string) => {
-        if (!acc[platform]) acc[platform] = [];
-        acc[platform].push(tool.name);
-      });
-    }
-    return acc;
-  }, {});
-  
-  const phaseGroups = tools.reduce((acc: any, tool: any) => {
-    if (tool.phases) {
-      tool.phases.forEach((phase: string) => {
-        if (!acc[phase]) acc[phase] = [];
-        acc[phase].push(tool.name);
-      });
-    }
-    return acc;
-  }, {});
-  
-  matrix.push({
-    type: 'platform_compatibility',
-    groups: platformGroups
-  });
-  
-  matrix.push({
-    type: 'phase_synergy',
-    groups: phaseGroups
-  });
-  
-  return matrix;
-}
-
 async function loadRawData(): Promise<ToolsData> {
  if (!cachedData) {
    const yamlPath = path.join(process.cwd(), 'src/data/tools.yaml');
@ -253,7 +130,6 @@ async function loadRawData(): Promise<ToolsData> {
    try {
      cachedData = ToolsDataSchema.parse(rawData);
      
-      // Enhanced: Add default skill level descriptions if not provided
      if (!cachedData.skill_levels || Object.keys(cachedData.skill_levels).length === 0) {
        cachedData.skill_levels = {
          novice: "Minimal technical background required, guided interfaces",
@ -301,21 +177,18 @@ export async function getCompressedToolsDataForAI(): Promise<EnhancedCompressedT
  if (!cachedCompressedData) {
    const data = await getToolsData();
    
-    // Enhanced: More detailed tool information for micro-tasks
    const compressedTools = data.tools
      .filter(tool => tool.type !== 'concept') 
      .map(tool => {
        const { projectUrl, statusUrl, ...compressedTool } = tool;
        return {
          ...compressedTool,
-          // Enhanced: Add computed fields for AI
          is_hosted: projectUrl !== undefined && projectUrl !== null && projectUrl !== "" && projectUrl.trim() !== "",
          is_open_source: tool.license && tool.license !== 'Proprietary',
          complexity_score: tool.skillLevel === 'expert' ? 5 :
                           tool.skillLevel === 'advanced' ? 4 :
                           tool.skillLevel === 'intermediate' ? 3 :
                           tool.skillLevel === 'beginner' ? 2 : 1,
-          // Enhanced: Phase-specific suitability hints
          phase_suitability: tool.phases?.map(phase => ({
            phase,
            primary_use: tool.tags?.find(tag => tag.includes(phase)) ? 'primary' : 'secondary'
@ -329,7 +202,6 @@ export async function getCompressedToolsDataForAI(): Promise<EnhancedCompressedT
        const { projectUrl, statusUrl, platforms, accessType, license, ...compressedConcept } = concept;
        return {
          ...compressedConcept,
-          // Enhanced: Learning difficulty indicator
          learning_complexity: concept.skillLevel === 'expert' ? 'very_high' :
                              concept.skillLevel === 'advanced' ? 'high' :
                              concept.skillLevel === 'intermediate' ? 'medium' :
@ -337,27 +209,16 @@ export async function getCompressedToolsDataForAI(): Promise<EnhancedCompressedT
        };
      });
    
-    // Enhanced: Add rich context data
-    const domainRelationships = generateDomainRelationships(data.domains, compressedTools);
-    const phaseDependencies = generatePhaseDependencies(data.phases);
-    const toolCompatibilityMatrix = generateToolCompatibilityMatrix(compressedTools);
-    
    cachedCompressedData = {
      tools: compressedTools,
      concepts: concepts,
      domains: data.domains,
      phases: data.phases,
      'domain-agnostic-software': data['domain-agnostic-software'],
-      scenarios: data.scenarios, // Include scenarios for context
+      scenarios: data.scenarios,
      skill_levels: data.skill_levels || {},
-      // Enhanced context for micro-tasks
-      domain_relationships: domainRelationships,
-      phase_dependencies: phaseDependencies,
-      tool_compatibility_matrix: toolCompatibilityMatrix
    };
    
-    console.log(`[DATA SERVICE] Generated enhanced compressed data: ${compressedTools.length} tools, ${concepts.length} concepts`);
-    console.log(`[DATA SERVICE] Added context: ${domainRelationships.length} domain relationships, ${phaseDependencies.length} phase dependencies`);
  }
  
  return cachedCompressedData;
--- a/src/utils/embeddings.ts
+++ b/src/utils/embeddings.ts
@ -31,6 +31,7 @@ interface SimilarityResult extends EmbeddingData {
 class EmbeddingsService {
  private embeddings: EmbeddingData[] = [];
  private isInitialized = false;
+  private initializationPromise: Promise<void> | null = null; // ADD THIS LINE
  private readonly embeddingsPath = path.join(process.cwd(), 'data', 'embeddings.json');
  private readonly batchSize: number;
  private readonly batchDelay: number;
@ -42,7 +43,25 @@ class EmbeddingsService {
    this.batchDelay = parseInt(process.env.AI_EMBEDDINGS_BATCH_DELAY_MS || '1000', 10);
  }

+  // REPLACE the existing initialize method with this:
  async initialize(): Promise<void> {
+    // If initialization is already in progress, wait for it
+    if (this.initializationPromise) {
+      return this.initializationPromise;
+    }
+
+    // If already initialized, return immediately
+    if (this.isInitialized) {
+      return Promise.resolve();
+    }
+
+    // Start initialization and store the promise
+    this.initializationPromise = this.performInitialization();
+    return this.initializationPromise;
+  }
+
+  // ADD THIS NEW METHOD:
+  private async performInitialization(): Promise<void> {
    if (!this.enabled) {
      console.log('[EMBEDDINGS] Embeddings disabled, skipping initialization');
      return;
@ -74,9 +93,29 @@ class EmbeddingsService {
    } catch (error) {
      console.error('[EMBEDDINGS] Failed to initialize:', error);
      this.isInitialized = false;
+      throw error;
+    } finally {
+      this.initializationPromise = null;
    }
  }

+  async waitForInitialization(): Promise<void> {
+    if (!this.enabled) {
+      return Promise.resolve();
+    }
+
+    if (this.isInitialized) {
+      return Promise.resolve();
+    }
+
+    if (this.initializationPromise) {
+      await this.initializationPromise;
+      return;
+    }
+
+    return this.initialize();
+  }
+
  private hashData(data: any): string {
    return Buffer.from(JSON.stringify(data)).toString('base64').slice(0, 32);
  }
@ -127,7 +166,6 @@ class EmbeddingsService {
      'Content-Type': 'application/json'
    };

-    // API key is optional for Ollama but required for Mistral/OpenAI
    if (apiKey) {
      headers['Authorization'] = `Bearer ${apiKey}`;
    }
@ -148,12 +186,10 @@ class EmbeddingsService {

    const data = await response.json();

-    // Detect Ollama format
    if (Array.isArray(data.embeddings)) {
      return data.embeddings;
    }

-    // Detect OpenAI/Mistral format
    if (Array.isArray(data.data)) {
      return data.data.map((item: any) => item.embedding);
    }
@ -170,7 +206,6 @@ class EmbeddingsService {
    const contents = allItems.map(item => this.createContentString(item));
    this.embeddings = [];

-    // Process in batches to respect rate limits
    for (let i = 0; i < contents.length; i += this.batchSize) {
      const batch = contents.slice(i, i + this.batchSize);
      const batchItems = allItems.slice(i, i + this.batchSize);
@ -198,7 +233,6 @@ class EmbeddingsService {
          });
        });
        
-        // Rate limiting delay between batches
        if (i + this.batchSize < contents.length) {
          await new Promise(resolve => setTimeout(resolve, this.batchDelay));
        }
@ -213,7 +247,6 @@ class EmbeddingsService {
  }

  public async embedText(text: string): Promise<number[]> {
-    // Re‑use the private batch helper to avoid auth duplication
    const [embedding] = await this.generateEmbeddingsBatch([text.toLowerCase()]);
    return embedding;
  }
@ -239,25 +272,21 @@ class EmbeddingsService {
    }

    try {
-      // Generate embedding for query
      const queryEmbeddings = await this.generateEmbeddingsBatch([query.toLowerCase()]);
      const queryEmbedding = queryEmbeddings[0];

      console.log(`[EMBEDDINGS] Computing similarities for ${this.embeddings.length} items`);

-      // Calculate similarities - properly typed
      const similarities: SimilarityResult[] = this.embeddings.map(item => ({
        ...item,
        similarity: this.cosineSimilarity(queryEmbedding, item.embedding)
      }));

-      // Filter by threshold and sort by similarity (descending - highest first)
      const results = similarities
        .filter(item => item.similarity >= threshold)
-        .sort((a, b) => b.similarity - a.similarity) // CRITICAL: Ensure descending order
+        .sort((a, b) => b.similarity - a.similarity) 
        .slice(0, maxResults);

-      // ENHANCED: Verify ordering is correct
      const orderingValid = results.every((item, index) => {
        if (index === 0) return true;
        return item.similarity <= results[index - 1].similarity;
@ -270,15 +299,13 @@ class EmbeddingsService {
        });
      }

-      // ENHANCED: Log top results for debugging
      console.log(`[EMBEDDINGS] Found ${results.length} similar items (threshold: ${threshold})`);
      if (results.length > 0) {
-        console.log('[EMBEDDINGS] Top 5 similarity matches:');
-        results.slice(0, 5).forEach((item, idx) => {
+        console.log('[EMBEDDINGS] Top 10 similarity matches:');
+        results.slice(0, 10).forEach((item, idx) => {
          console.log(`  ${idx + 1}. ${item.name} (${item.type}) = ${item.similarity.toFixed(4)}`);
        });
        
-        // Verify first result is indeed the highest
        const topSimilarity = results[0].similarity;
        const hasHigherSimilarity = results.some(item => item.similarity > topSimilarity);
        if (hasHigherSimilarity) {
--- a/src/utils/rateLimitedQueue.ts
+++ b/src/utils/rateLimitedQueue.ts
@ -157,15 +157,6 @@ class RateLimitedQueue {
    return status;
  }

-  setDelay(ms: number): void {
-    if (!Number.isFinite(ms) || ms < 0) return;
-    this.delayMs = ms;
-  }
-
-  getDelay(): number {
-    return this.delayMs;
-  }
-
  private async processQueue(): Promise<void> {
    if (this.isProcessing) {
      return;
--- a/src/utils/toolHelpers.ts
+++ b/src/utils/toolHelpers.ts
@ -1,8 +1,3 @@
-/**
- * CONSOLIDATED Tool utility functions for consistent tool operations across the app
- * Works in both server (Node.js) and client (browser) environments
- */
-
 export interface Tool {
  name: string;
  type?: 'software' | 'method' | 'concept';
@ -18,10 +13,6 @@ export interface Tool {
  related_concepts?: string[];
 }

-/**
- * Creates a URL-safe slug from a tool name
- * Used for URLs, IDs, and file names consistently across the app
- */
 export function createToolSlug(toolName: string): string {
  if (!toolName || typeof toolName !== 'string') {
    console.warn('[toolHelpers] Invalid toolName provided to createToolSlug:', toolName);
@ -35,9 +26,6 @@ export function createToolSlug(toolName: string): string {
    .replace(/^-|-$/g, '');           // Remove leading/trailing hyphens
 }

-/**
- * Finds a tool by name or slug from tools array
- */
 export function findToolByIdentifier(tools: Tool[], identifier: string): Tool | undefined {
  if (!identifier || !Array.isArray(tools)) return undefined;
  
@ -47,23 +35,9 @@ export function findToolByIdentifier(tools: Tool[], identifier: string): Tool |
  );
 }

-/**
- * Checks if tool has a valid project URL (hosted on CC24 server)
- */
 export function isToolHosted(tool: Tool): boolean {
  return tool.projectUrl !== undefined && 
         tool.projectUrl !== null && 
         tool.projectUrl !== "" && 
         tool.projectUrl.trim() !== "";
 }
-
-/**
- * Determines tool category for styling/logic
- */
-export function getToolCategory(tool: Tool): 'concept' | 'method' | 'hosted' | 'oss' | 'proprietary' {
-  if (tool.type === 'concept') return 'concept';
-  if (tool.type === 'method') return 'method';
-  if (isToolHosted(tool)) return 'hosted';
-  if (tool.license && tool.license !== 'Proprietary') return 'oss';
-  return 'proprietary';
-}