cleanup

2025-08-05 22:09:46 +02:00 · 2025-08-05 22:09:38 +02:00
9 changed files with 22 additions and 2534 deletions
--- a/check-unused-css.js
+++ b/check-unused-css.js
@ -1,102 +0,0 @@
-#!/usr/bin/env node
-// find-unused-css.js
-// Usage: node find-unused-css <cssFile> <sourceRoot> [--verbose]
-
-import fs from 'fs/promises';
-import path from 'path';
-import fg from 'fast-glob';
-import pc from 'picocolors';
-import postcss from 'postcss';
-import safeParser from 'postcss-safe-parser';
-
-const [,, cssPath, srcRoot = '.', ...rest] = process.argv;
-const verbose = rest.includes('--verbose');
-if (!cssPath) {
-  console.error('Usage: node find-unused-css <cssFile> <sourceRoot>');
-  process.exit(1);
-}
-
-/* -------------------------------------------------- */
-/* 1.  Parse the CSS and harvest class/id tokens       */
-/* -------------------------------------------------- */
-const cssRaw = await fs.readFile(cssPath, 'utf8');
-const root   = postcss().process(cssRaw, { parser: safeParser }).root;
-
-const selectorTokens = new Map();          // selector → Set('.foo', '#bar')
-const CLASS = /\.([\w-]+)/g;
-const ID    = /#([\w-]+)/g;
-
-root.walkRules(rule => {
-  rule.selectors.forEach(sel => {
-    const tokens = new Set();
-    sel.replace(CLASS, (_, c) => tokens.add('.'+c));
-    sel.replace(ID,    (_, i) => tokens.add('#'+i));
-    if (tokens.size) selectorTokens.set(sel, tokens);
-  });
-});
-
-/* -------------------------------------------------- */
-/* 2.  Dynamic classes you add via JS (safe keep)      */
-/* -------------------------------------------------- */
-const dynamicAllow = new Set([
-  'hidden', 'active', 'loading', 'open', 'closed'
-]);
-
-/* -------------------------------------------------- */
-/* 3.  Read every source file once                     */
-/* -------------------------------------------------- */
-const files = await fg([
-  `${srcRoot}/**/*.{html,htm,js,jsx,ts,tsx,vue,svelte,astro}`,
-  `!${srcRoot}/**/node_modules/**`
-]);
-const sources = await Promise.all(files.map(f => fs.readFile(f, 'utf8')));
-
-/* -------------------------------------------------- */
-/* 4.  Fast search helpers                             */
-/* -------------------------------------------------- */
-const makeClassRE = cls =>
-  new RegExp(
-    `(class|className)=['"][^'"]*\\b${cls}\\b[^'"]*['"]|['"\`]${cls}['"\`]`,
-    'i'
-  );
-const makeIdRE = id =>
-  new RegExp(`id=['"]${id}['"]|['"\`]${id}['"\`]`, 'i');
-
-const tokenInSources = token => {
-  // dynamic allow-list
-  if (dynamicAllow.has(token)) return true;
-
-  const re = token.startsWith('.')
-    ? makeClassRE(token.slice(1))
-    : makeIdRE(token.slice(1));
-
-  return sources.some(txt => re.test(txt));
-};
-
-/* -------------------------------------------------- */
-/* 5.  Decide used vs unused                           */
-/* -------------------------------------------------- */
-const used   = [];
-const unused = [];
-
-for (const [selector, tokens] of selectorTokens.entries()) {
-  const isUsed = [...tokens].some(tokenInSources); // **ANY** token keeps rule
-  (isUsed ? used : unused).push(selector);
-  if (verbose) {
-    console.log(isUsed ? pc.green('✓ '+selector) : pc.red('✗ '+selector));
-  }
-}
-
-/* -------------------------------------------------- */
-/* 6.  Report & write list                             */
-/* -------------------------------------------------- */
-console.log(
-  `\n${pc.bold(pc.blue('🎯 CSS usage summary'))}\n` +
-  `  selectors total : ${selectorTokens.size}\n` +
-  `  still used       : ${pc.green(used.length)}\n` +
-  `  maybe unused     : ${pc.red(unused.length)}\n`
-);
-
-const outFile = path.resolve('unused-selectors.txt');
-await fs.writeFile(outFile, unused.join('\n'));
-console.log(`📝  Unused selector list → ${pc.yellow(outFile)}\n`);
--- a/context.md
+++ b/context.md
@ -1,337 +0,0 @@
-# ForensicPathways Architecture System Prompt
-
-## Project Overview
-ForensicPathways is a curated directory of Digital Forensics and Incident Response (DFIR) tools, methods, and concepts built with Astro and TypeScript. It serves as an educational and professional resource for forensic investigators, following the NIST SP 800-86 framework (Kent, Chevalier, Grance & Dang).
-
-## Core Technology Stack
- **Framework**: Astro (static site generator with islands architecture)
- **Language**: TypeScript with strict typing
- **Styling**: Vanilla CSS with custom properties (CSS variables)
- **Data Storage**: YAML files for tool catalog
- **Content**: Astro Content Collections for knowledge base articles
- **Authentication**: OIDC (OpenID Connect) with configurable requirements
- **AI Integration**: Mistral API for workflow recommendations
- **File Storage**: Nextcloud with local fallback
- **Version Control**: Git integration for community contributions
-
-## Data Model Architecture
-
-### Core Entity: Tool
-```yaml
-name: string           # Tool identifier
-icon: string?          # Emoji icon
-type: 'software'|'method'|'concept'  # Tool classification
-description: string    # Detailed description
-domains: string[]      # Forensic domains (incident-response, malware-analysis, etc.)
-phases: string[]       # NIST framework phases (data-collection, examination, analysis, reporting)
-platforms: string[]   # Operating systems (for software only)
-skillLevel: string     # novice|beginner|intermediate|advanced|expert
-url: string           # Primary documentation/homepage
-projectUrl: string?   # Hosted instance URL (CC24 server)
-license: string?      # Software license
-knowledgebase: boolean? # Has detailed documentation
-tags: string[]        # Searchable keywords
-related_concepts: string[]? # Links to concept-type tools
-related_software: string[]? #Links to software-type-tools
-```
-
-### Taxonomies
- **Domains**: Forensic specializations (7 main domains)
- **Phases**: NIST investigation phases (4 phases)
- **Domain-Agnostic-Software**: Cross-cutting tools and platforms
- **Skill Levels**: Standardized competency requirements
-
-## Component Architecture
-
-### Layout System
- **BaseLayout.astro**: Global layout with theme system, authentication setup, shared utilities
- **Navigation.astro**: Main navigation with active state management
- **Footer.astro**: Site footer with links and licensing info
-
-### Core Components
- **ToolCard.astro**: Individual tool display with metadata, actions, and type-specific styling
- **ToolMatrix.astro**: Matrix view showing tools by domain/phase intersection
- **ToolFilters.astro**: Search, filtering, and view controls
- **AIQueryInterface.astro**: AI-powered workflow recommendation system
-
-### Utility Components
- **ShareButton.astro**: URL sharing with multiple view options
- **ContributionButton.astro**: Authenticated contribution links
- **ThemeToggle.astro**: Light/dark/auto theme switching
-
-## Feature Systems
-
-### 1. Authentication System (`src/utils/auth.ts`)
- **OIDC Integration**: Uses environment-configured provider
- **Contextual Requirements**: Different auth requirements for contributions vs AI features
- **Session Management**: JWT-based with configurable expiration
- **Client-Side Utilities**: Window-level auth checking functions
-
-### 2. AI Recommendation System
- **Dual Modes**: 
-  - Workflow mode: Multi-phase recommendations for scenarios
-  - Tool mode: Specific tool recommendations for problems
- **Rate Limiting**: Queue-based system with status updates
- **Data Integration**: Uses compressed tool database for AI context
- **Response Validation**: Ensures AI only recommends existing tools
-
-### 3. Contribution System
- **Git Integration**: Automated issue creation via Gitea/GitHub/GitLab APIs
- **Tool Contributions**: Form-based tool submissions
- **Knowledge Base**: Rich text with file upload support
- **Validation**: Client and server-side validation with Zod schemas
-
-### 4. File Upload System
- **Primary**: Nextcloud integration with public link generation
- **Fallback**: Local file storage with public URL generation
- **Validation**: File type and size restrictions
- **Rate Limiting**: Per-user upload quotas
-
-## Styling Architecture
-
-### CSS Custom Properties System
-```css
-/* Core color system */
--color-primary: /* Adaptive based on theme */
--color-accent: /* Secondary brand color */
--color-bg: /* Main background */
--color-text: /* Primary text */
-
-/* Component-specific colors */
--color-hosted: /* CC24 server hosted tools */
--color-oss: /* Open source tools */
--color-method: /* Methodology entries */
--color-concept: /* Knowledge concepts */
-
-/* Theme system */
-[data-theme="light"] { /* Light theme values */ }
-[data-theme="dark"] { /* Dark theme values */ }
-```
-
-### Component Styling Patterns
- **Card System**: Consistent `.card` base with type-specific variants
- **Badge System**: Status and metadata indicators
- **Button System**: Semantic button classes with size variants
- **Grid System**: CSS Grid with responsive breakpoints
-
-## Data Flow Architecture
-
-### 1. Data Loading (`src/utils/dataService.ts`)
-```typescript
-// Daily randomization with seeded shuffle
-getToolsData() → shuffled tools array
-getCompressedToolsDataForAI() → AI-optimized dataset
-clearCache() → cache invalidation
-```
-
-### 2. Client-Side State Management
- **Global Tools Data**: Attached to `window.toolsData`
- **Filter State**: Component-local state with event emission
- **View State**: Grid/Matrix/AI view switching
- **Modal State**: Tool detail overlays with multi-modal support
-
-### 3. Search and Filtering
- **Text Search**: Name, description, tags
- **Faceted Filtering**: Domain, phase, skill level, license type
- **Tag Cloud**: Frequency-weighted tag selection
- **Real-time Updates**: Immediate filtering with event system
-
-## API Architecture
-
-### Endpoint Structure
-```
-/api/auth/         # Authentication endpoints
-  login.ts         # OIDC initiation
-  process.ts       # OIDC callback processing
-  status.ts        # Auth status checking
-
-/api/contribute/   # Contribution endpoints
-  tool.ts          # Tool submissions
-  knowledgebase.ts # KB article submissions
-
-/api/ai/          # AI features
-  query.ts         # Workflow recommendations
-  queue-status.ts  # Queue monitoring
-
-/api/upload/      # File handling
-  media.ts         # File upload processing
-
-/api/health.ts    # System health check
-```
-
-### Response Patterns
-All APIs use consolidated response utilities (`src/utils/api.ts`):
- **Success**: `apiResponse.success()`, `apiResponse.created()`
- **Errors**: `apiError.badRequest()`, `apiError.unauthorized()`, etc.
- **Server Errors**: `apiServerError.internal()`, etc.
-
-## File Organization Patterns
-
-### Page Structure
- **Static Pages**: About, impressum, status
- **Dynamic Pages**: Tool details, knowledge base articles
- **Authenticated Pages**: Contribution forms
- **API Routes**: RESTful endpoints with consistent naming
-
-### Utility Organization
- **Tool Operations**: `toolHelpers.ts` - Core tool manipulation
- **Data Management**: `dataService.ts` - YAML loading and caching
- **Authentication**: `auth.ts` - OIDC flow and session management
- **External APIs**: `gitContributions.ts`, `nextcloud.ts`
- **Rate Limiting**: `rateLimitedQueue.ts` - AI request queuing
-
-## Key Architectural Decisions
-
-### 1. Static-First with Dynamic Islands
- Astro's islands architecture for interactivity
- Static generation for performance
- Selective hydration for complex components
-
-### 2. YAML-Based Data Management
- Human-readable tool catalog
- Git-friendly versioning
- Type-safe loading with Zod validation
-
-### 3. Contextual Authentication
- Optional authentication based on feature
- Graceful degradation for unauthenticated users
- Environment-configurable requirements
-
-### 4. Multi-Modal UI Patterns
- Grid view for browsing
- Matrix view for relationship visualization
- AI interface for guided recommendations
-
-### 5. Progressive Enhancement
- Core functionality works without JavaScript
- Enhanced features require client-side hydration
- Responsive design with mobile-first approach
-
-## Development Patterns
-
-### Component Design
- Props interfaces with TypeScript
- Consistent styling via CSS classes
- Event-driven communication between components
- Server-side rendering with client-side enhancement
-
-### Error Handling
- Comprehensive try-catch in API routes
- User-friendly error messages
- Graceful fallbacks for external services
- Logging for debugging and monitoring
-
-### Performance Optimizations
- Daily data randomization with caching
- Compressed datasets for AI context
- Rate limiting for external API calls
- Efficient DOM updates with targeted selectors
-
-This architecture emphasizes maintainability, user experience, and extensibility while managing the complexity of a multi-feature forensics tool directory.
-
-
-## Absolutely Essential (Core Architecture - 8 files)
-
-**File**: `src/data/tools.yaml.example`
-**Why**: Defines the core data model - what a "tool" is, the schema, domains, phases, etc. Without this, an AI can't understand what the application manages.
-
-**File**: `src/pages/index.astro` 
-**Why**: Main application entry point showing the core functionality (filters, matrix, AI interface, tool grid). Contains the primary application logic flow.
-
-**File**: `src/layouts/BaseLayout.astro`
-**Why**: Global layout with theme system, authentication setup, and shared utility functions. Shows the overall app structure.
-
-**File**: `src/utils/dataService.ts`
-**Why**: Core data loading and processing logic. Essential for understanding how data flows through the application.
-
-**File**: `src/utils/toolHelpers.ts`
-**Why**: Core utility functions used throughout the app (slug creation, tool identification, hosting checks).
-
-**File**: `src/styles/global.css`
-**Why**: Complete styling system - defines visual architecture, component styling, theme system, responsive design.
-
-**File**: `src/env.d.ts`
-**Why**: Global TypeScript definitions and window interface extensions. Shows the global API surface.
-
-**File**: `src/components/ToolCard.astro`
-**Why**: Shows how the core entity (tools) are rendered and structured. Represents the component architecture pattern.
-
-## Secondary Priority (Major Features - 4 files)
-
-**File**: `src/components/ToolMatrix.astro` - Matrix view functionality
-**File**: `src/components/AIQueryInterface.astro` - AI recommendation system  
-**File**: `src/utils/auth.ts` - Authentication system
-**File**: `src/content/config.ts` - Content collection schema
-
-## Strategy for Context Management
-
-1. **Always provide**: The 8 essential files above
-2. **Add selectively**: Include 1-3 secondary files based on the specific development task
-3. **Reference others**: Mention other relevant files by name/purpose without including full content
-
-user01@altiera /v/h/u/P/forensic-pathways (main)> tree src
-src
-├── components
-│   ├── AIQueryInterface.astro
-│   ├── ContributionButton.astro
-│   ├── Footer.astro
-│   ├── Navigation.astro
-│   ├── ShareButton.astro
-│   ├── ThemeToggle.astro
-│   ├── ToolCard.astro
-│   ├── ToolFilters.astro
-│   └── ToolMatrix.astro
-├── content
-│   ├── config.ts
-│   └── knowledgebase
-│       ├── android-logical-imaging.md
-│       ├── kali-linux.md
-│       ├── misp.md
-│       ├── nextcloud.md
-│       ├── regular-expressions-regex.md
-│       └── velociraptor.md
-├── data
-│   ├── tools.yaml
-│   └── tools.yaml.example
-├── env.d.ts
-├── layouts
-│   └── BaseLayout.astro
-├── pages
-│   ├── about.astro
-│   ├── api
-│   │   ├── ai
-│   │   │   ├── query.ts
-│   │   │   └── queue-status.ts
-│   │   ├── auth
-│   │   │   ├── login.ts
-│   │   │   ├── process.ts
-│   │   │   └── status.ts
-│   │   ├── contribute
-│   │   │   ├── knowledgebase.ts
-│   │   │   └── tool.ts
-│   │   ├── health.ts
-│   │   └── upload
-│   │       └── media.ts
-│   ├── auth
-│   │   └── callback.astro
-│   ├── contribute
-│   │   ├── index.astro
-│   │   ├── knowledgebase.astro
-│   │   └── tool.astro
-│   ├── impressum.astro
-│   ├── index.astro
-│   ├── knowledgebase
-│   │   └── [slug].astro
-│   ├── knowledgebase.astro
-│   └── status.astro
-├── styles
-│   └── global.css
-└── utils
-    ├── api.ts
-    ├── auth.ts
-    ├── dataService.ts
-    ├── gitContributions.ts
-    ├── nextcloud.ts
-    ├── rateLimitedQueue.ts
-    └── toolHelpers.ts
-17 directories, 47 files
--- a/dfir_yaml_editor.html
+++ b/dfir_yaml_editor.html
--- a/src/components/AIQueryInterface.astro
+++ b/src/components/AIQueryInterface.astro
@ -702,12 +702,10 @@ class AIQueryInterface {
      toolsByPhase[phase] = [];
    });

-    // DEBUG: Log recommendation structure
    console.log('[AI Results] Recommendation structure:', recommendation);
    console.log('[AI Results] Recommended tools:', recommendation.recommended_tools);

    recommendation.recommended_tools?.forEach(recTool => {
-      // DEBUG: Log each tool's confidence data
      console.log('[AI Results] Tool confidence data:', recTool.name, recTool.confidence);
      
      if (toolsByPhase[recTool.phase]) {
@ -716,7 +714,7 @@ class AIQueryInterface {
          toolsByPhase[recTool.phase].push({
            ...fullTool,
            recommendation: recTool,
-            confidence: recTool.confidence, // Ensure confidence is passed
+            confidence: recTool.confidence, 
            justification: recTool.justification,
            priority: recTool.priority,
            recommendationStrength: recTool.recommendationStrength
@ -836,13 +834,11 @@ class AIQueryInterface {
      return '';
    }
    
-    // Calculate summary statistics
    const totalTime = auditTrail.reduce((sum, entry) => sum + entry.processingTimeMs, 0);
    const avgConfidence = auditTrail.reduce((sum, entry) => sum + entry.confidence, 0) / auditTrail.length;
    const lowConfidenceSteps = auditTrail.filter(entry => entry.confidence < 60).length;
    const highConfidenceSteps = auditTrail.filter(entry => entry.confidence >= 80).length;
    
-    // Group entries by phase for better organization
    const groupedEntries = auditTrail.reduce((groups, entry) => {
      if (!groups[entry.phase]) groups[entry.phase] = [];
      groups[entry.phase].push(entry);
@ -1048,7 +1044,6 @@ class AIQueryInterface {
      second: '2-digit' 
    });
    
-    // Reuse existing grid and text utilities
    return `
      <div class="border-l-2 pl-3 py-2 mb-2" style="border-left-color: ${confidenceColor};">
        <div class="flex justify-between items-center mb-1">
--- a/src/config/prompts.ts
+++ b/src/config/prompts.ts
@ -2,7 +2,6 @@

 export const AI_PROMPTS = {
  
-  // Main tool selection prompt
  toolSelection: (mode: string, userQuery: string, selectionMethod: string, maxSelectedItems: number) => {
    const modeInstruction = mode === 'workflow' 
      ? 'Der Benutzer möchte einen UMFASSENDEN WORKFLOW mit mehreren Tools/Methoden über verschiedene Phasen. Wählen Sie 15-25 Tools aus, die den vollständigen Untersuchungslebenszyklus abdecken.'
@ -51,7 +50,6 @@ Antworten Sie NUR mit diesem JSON-Format:
 }`;
  },

-  // Scenario analysis prompt
  scenarioAnalysis: (isWorkflow: boolean, userQuery: string) => {
    const analysisType = isWorkflow ? 'forensische Szenario' : 'technische Problem';
    const considerations = isWorkflow ? 
@ -74,7 +72,6 @@ ${considerations}
 WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen, Aufzählungen oder Markdown-Formatierung. Maximum 150 Wörter.`;
  },

-  // Investigation approach prompt
  investigationApproach: (isWorkflow: boolean, userQuery: string) => {
    const approachType = isWorkflow ? 'Untersuchungsansatz' : 'Lösungsansatz';
    const considerations = isWorkflow ?
@ -96,7 +93,6 @@ ${considerations}
 WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdown. Maximum 150 Wörter.`;
  },

-  // Critical considerations prompt
  criticalConsiderations: (isWorkflow: boolean, userQuery: string) => {
    const considerationType = isWorkflow ? 'kritische forensische Überlegungen' : 'wichtige methodische Voraussetzungen';
    const aspects = isWorkflow ?
@ -179,7 +175,6 @@ WICHTIG:
 - "pros" soll die Stärken für diese spezifische Aufgabe hervorheben`;
  },

-  // Background knowledge selection prompt
  backgroundKnowledgeSelection: (userQuery: string, mode: string, selectedToolNames: string[], availableConcepts: any[]) => {
    return `Wählen Sie relevante forensische Konzepte für das Verständnis der empfohlenen Methodik.

@ -200,7 +195,6 @@ Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format:
 ]`;
  },

-  // Final recommendations prompt
  finalRecommendations: (isWorkflow: boolean, userQuery: string, selectedToolNames: string[]) => {
    const prompt = isWorkflow ? 
      `Erstellen Sie eine Workflow-Empfehlung basierend auf DFIR-Prinzipien.
@ -225,7 +219,6 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
  }
 } as const;

-// Type-safe prompt function with proper overloads
 export function getPrompt(key: 'toolSelection', mode: string, userQuery: string, selectionMethod: string, maxSelectedItems: number): string;
 export function getPrompt(key: 'scenarioAnalysis', isWorkflow: boolean, userQuery: string): string;
 export function getPrompt(key: 'investigationApproach', isWorkflow: boolean, userQuery: string): string;
@ -238,7 +231,6 @@ export function getPrompt(promptKey: keyof typeof AI_PROMPTS, ...args: any[]): s
  try {
    const promptFunction = AI_PROMPTS[promptKey];
    if (typeof promptFunction === 'function') {
-      // Use type assertion since we've validated the function exists
      return (promptFunction as (...args: any[]) => string)(...args);
    } else {
      console.error(`[PROMPTS] Invalid prompt key: ${promptKey}`);
--- a/src/pages/index.astro
+++ b/src/pages/index.astro
@ -265,20 +265,15 @@ const phases = data.phases;
      return;
    }
    
-    // AI Button click handler
    if (aiQueryBtn) {
      aiQueryBtn.addEventListener('click', () => {
-        // Visual feedback
        aiQueryBtn.classList.add('activated');
        setTimeout(() => aiQueryBtn.classList.remove('activated'), 400);
        
-        // Switch to AI view
        switchToView('ai');
        
-        // Trigger existing view change system
        window.dispatchEvent(new CustomEvent('viewChanged', { detail: 'ai' }));
        
-        // Scroll to AI interface
        if (window.scrollToElementById) {
          window.scrollToElementById('ai-interface');
        } else {
@ -294,14 +289,12 @@ const phases = data.phases;
    const filtersSection = document.getElementById('filters-section');
    const noResults = document.getElementById('no-results');
    
-    // Hide all views first
    if (toolsGrid) toolsGrid.style.display = 'none';
    if (matrixContainer) matrixContainer.style.display = 'none';
    if (aiInterface) aiInterface.style.display = 'none';
    if (filtersSection) filtersSection.style.display = 'none';
    if (noResults) noResults.style.display = 'none';
    
-    // Show selected view
    switch (view) {
      case 'grid':
        if (toolsGrid) toolsGrid.style.display = 'block';
--- a/src/utils/aiPipeline.ts
+++ b/src/utils/aiPipeline.ts
@ -63,17 +63,15 @@ interface AnalysisContext {
  
  auditTrail: AuditEntry[];
  
-  // Store actual similarity data from embeddings
  embeddingsSimilarities: Map<string, number>;
 }

 interface ConfidenceMetrics {
-  overall: number;                    // 0-100: Combined confidence score
-  semanticRelevance: number;          // How well tool description matches query (from embeddings)
-  taskSuitability: number;           // AI-determined fitness for this specific task  
-  methodologicalConsistency: number; // How well different analysis steps agree
-  uncertaintyFactors: string[];      // Specific reasons why this might not work
-  strengthIndicators: string[];      // Specific reasons why this is a good choice
+  overall: number;                    
+  semanticRelevance: number;         
+  taskSuitability: number;          
+  uncertaintyFactors: string[];   
+  strengthIndicators: string[]; 
 }

 class ImprovedMicroTaskAIPipeline {
@ -102,10 +100,10 @@ class ImprovedMicroTaskAIPipeline {
  };

  private confidenceConfig: {
-    semanticWeight: number;        // Weight for embeddings similarity
-    suitabilityWeight: number;     // Weight for AI task fit evaluation
-    consistencyWeight: number;     // Weight for cross-validation agreement
-    reliabilityWeight: number;     // Weight for tool quality indicators
+    semanticWeight: number;       
+    suitabilityWeight: number;  
+    consistencyWeight: number;    
+    reliabilityWeight: number;    
    minimumThreshold: number;
    mediumThreshold: number;
    highThreshold: number;
@ -143,10 +141,9 @@ class ImprovedMicroTaskAIPipeline {
      retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10)
    };
    
-    // Updated confidence weights - more focused on AI evaluation
    this.confidenceConfig = {
-      semanticWeight: parseFloat(process.env.CONFIDENCE_SEMANTIC_WEIGHT || '0.3'),     // Embeddings similarity
-      suitabilityWeight: parseFloat(process.env.CONFIDENCE_SUITABILITY_WEIGHT || '0.7'), // AI task fit evaluation  
+      semanticWeight: parseFloat(process.env.CONFIDENCE_SEMANTIC_WEIGHT || '0.3'),   
+      suitabilityWeight: parseFloat(process.env.CONFIDENCE_SUITABILITY_WEIGHT || '0.7'), 
      consistencyWeight: 0,    
      reliabilityWeight: 0,
      minimumThreshold: parseInt(process.env.CONFIDENCE_MINIMUM_THRESHOLD || '40', 10),
@ -235,7 +232,7 @@ class ImprovedMicroTaskAIPipeline {
    const selectionRatio = result.selectedTools.length / candidateCount;
    const hasReasoning = result.reasoning && result.reasoning.length > 50;
    
-    let confidence = 60; // Base confidence
+    let confidence = 60; 
    
    if (selectionRatio > 0.05 && selectionRatio < 0.3) confidence += 20;
    else if (selectionRatio <= 0.05) confidence -= 10; 
@ -386,7 +383,6 @@ class ImprovedMicroTaskAIPipeline {
    let candidateConcepts: any[] = [];
    let selectionMethod = 'unknown';
    
-    // Initialize embeddings similarities storage
    context.embeddingsSimilarities = new Map<string, number>();
    
    if (process.env.AI_EMBEDDINGS_ENABLED === 'true') {
@ -409,7 +405,6 @@ class ImprovedMicroTaskAIPipeline {
      
      console.log(`[AI PIPELINE] Embeddings found ${similarItems.length} similar items`);
      
-      // Store actual similarity scores for confidence calculation
      similarItems.forEach(item => {
        context.embeddingsSimilarities.set(item.name, item.similarity);
      });
@ -707,18 +702,14 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
    limitations: string[] = []
  ): ConfidenceMetrics {
    
-    // 1. Semantic Relevance: Real embeddings similarity score
    const rawSemanticRelevance = context.embeddingsSimilarities.has(tool.name) ? 
      context.embeddingsSimilarities.get(tool.name)! * 100 : 50;
    
-    // 2. Task Suitability: Enhanced with phase-awareness for workflow mode
    let enhancedTaskSuitability = taskRelevance;
    
    if (context.mode === 'workflow') {
-      // In workflow mode, boost score if tool is well-matched to its assigned phase
      const toolSelection = context.selectedTools?.find(st => st.tool.name === tool.name);
      if (toolSelection && tool.phases && tool.phases.includes(toolSelection.phase)) {
-        // Boost for phase alignment (but cap at 100)
        const phaseBonus = Math.min(15, 100 - taskRelevance);
        enhancedTaskSuitability = Math.min(100, taskRelevance + phaseBonus);
        
@ -726,7 +717,6 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
      }
    }
    
-    // Simple weighted combination - no artificial scaling
    const overall = (
      rawSemanticRelevance * this.confidenceConfig.semanticWeight +
      enhancedTaskSuitability * this.confidenceConfig.suitabilityWeight
@ -747,7 +737,6 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
      overall: Math.round(overall),
      semanticRelevance: Math.round(rawSemanticRelevance),
      taskSuitability: Math.round(enhancedTaskSuitability), 
-      methodologicalConsistency: 0,
      uncertaintyFactors,
      strengthIndicators
    };
@ -756,18 +745,15 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
  private identifySpecificUncertaintyFactors(tool: any, context: AnalysisContext, limitations: string[], confidence: number): string[] {
    const factors: string[] = [];
    
-    // Add AI-identified limitations first (most specific)
    if (limitations && limitations.length > 0) {
-      factors.push(...limitations.slice(0, 2)); // Limit to top 2 to leave room for others
+      factors.push(...limitations.slice(0, 2));
    }
    
-    // Low semantic similarity
    const similarity = context.embeddingsSimilarities.get(tool.name) || 0.5;
    if (similarity < 0.7) {
      factors.push('Geringe semantische Ähnlichkeit zur Anfrage - Tool-Beschreibung passt möglicherweise nicht optimal');
    }
    
-    // Skill level vs scenario complexity mismatch
    if (tool.skillLevel === 'expert' && /schnell|rapid|triage|urgent|sofort/i.test(context.userQuery)) {
      factors.push('Experten-Tool für zeitkritisches Szenario - Setup und Einarbeitung könnten zu lange dauern');
    }
@ -776,35 +762,29 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
      factors.push('Einsteiger-Tool für komplexe Analyse - könnte funktionale Limitierungen haben');
    }
    
-    // Access and deployment concerns
    if (tool.type === 'software' && !isToolHosted(tool) && tool.accessType === 'download') {
      factors.push('Installation und Setup erforderlich');
    }
    
-    // License restrictions
    if (tool.license === 'Proprietary') {
      factors.push('Kommerzielle Software - Lizenzkosten und rechtliche Beschränkungen zu beachten');
    }
    
-    // Low overall confidence warning
    if (confidence < 60) {
      factors.push('Moderate Gesamtbewertung - alternative Ansätze sollten ebenfalls betrachtet werden');
    }
    
-    return factors.slice(0, 4); // Limit to 4 most relevant factors
+    return factors.slice(0, 4);
  }

-  // NEW: Identify specific strength indicators
  private identifySpecificStrengthIndicators(tool: any, context: AnalysisContext, confidence: number): string[] {
    const indicators: string[] = [];
    
-    // High semantic similarity
    const similarity = context.embeddingsSimilarities.get(tool.name) || 0.5;
    if (similarity >= 0.7) {
      indicators.push('Sehr gute semantische Übereinstimmung mit Ihrer Anfrage');
    }
    
-    // Quality indicators
    if (tool.knowledgebase === true) {
      indicators.push('Umfassende Dokumentation und Wissensbasis verfügbar');
    }
@ -813,17 +793,15 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
      indicators.push('Sofort verfügbar über gehostete Lösung - kein Setup erforderlich');
    }
    
-    // Skill level match
    if (tool.skillLevel === 'intermediate' || tool.skillLevel === 'advanced') {
      indicators.push('Ausgewogenes Verhältnis zwischen Funktionalität und Benutzerfreundlichkeit');
    }
    
-    // Method alignment
    if (tool.type === 'method' && /methodik|vorgehen|prozess|ansatz/i.test(context.userQuery)) {
      indicators.push('Methodischer Ansatz passt zu Ihrer prozeduralen Anfrage');
    }
    
-    return indicators.slice(0, 4); // Limit to 4 most important indicators
+    return indicators.slice(0, 4); 
  }

  private async analyzeScenario(context: AnalysisContext): Promise<MicroTaskResult> {
@ -902,11 +880,9 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
        validSelections.forEach((sel: any) => {
          const tool = phaseTools.find((t: any) => t.name === sel.toolName);
          if (tool) {
-            // Ensure taskRelevance is a number
            const taskRelevance = typeof sel.taskRelevance === 'number' ? 
              sel.taskRelevance : parseInt(String(sel.taskRelevance)) || 70;
            
-            // Derive priority automatically from score
            const priority = this.derivePriorityFromScore(taskRelevance);
            
            this.addToolToSelection(context, tool, phase.id, priority, sel.justification, taskRelevance, sel.limitations);
@ -967,7 +943,7 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
          hasExplanation: !!evaluation.detailed_explanation,
          hasImplementationApproach: !!evaluation.implementation_approach,
          prosCount: evaluation.pros?.length || 0,
-          limitationsCount: evaluation.limitations?.length || 0, // ← Updated field name
+          limitationsCount: evaluation.limitations?.length || 0,
          hasLimitations: Array.isArray(evaluation.limitations) && evaluation.limitations.length > 0
        },
        70,
@ -1101,7 +1077,7 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
      const context: AnalysisContext = {
        userQuery,
        mode,
-        filteredData: {}, // Will be populated by getIntelligentCandidates
+        filteredData: {},
        contextHistory: [],
        maxContextLength: this.maxContextTokens,
        currentContextLength: 0,
@ -1124,9 +1100,7 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
        startTime,
        { auditEnabled: this.auditConfig.enabled, confidenceScoringEnabled: true }
      );
-
-      // MICRO-TASK SEQUENCE WITH ENHANCED CONFIDENCE TRACKING
-      
+   
      const analysisResult = await this.analyzeScenario(context);
      if (analysisResult.success) completeTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);
@ -1234,7 +1208,6 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
            components: {
              semantic: confidence.semanticRelevance,
              suitability: confidence.taskSuitability,
-              consistency: confidence.methodologicalConsistency
            }
          },
          confidence.overall,
@ -1286,7 +1259,7 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
          detailed_explanation: st.tool.evaluation?.detailed_explanation || '',
          implementation_approach: st.tool.evaluation?.implementation_approach || '',
          pros: st.tool.evaluation?.pros || [],
-          cons: st.tool.evaluation?.limitations || [], // ← FIXED: Use limitations as cons for display
+          cons: st.tool.evaluation?.limitations || [],
          alternatives: st.tool.evaluation?.alternatives || '',
          confidence: confidence,
          recommendationStrength: confidence.overall >= this.confidenceConfig.highThreshold ? 'strong' : 
--- a/src/utils/embeddings.ts
+++ b/src/utils/embeddings.ts
@ -31,7 +31,7 @@ interface SimilarityResult extends EmbeddingData {
 class EmbeddingsService {
  private embeddings: EmbeddingData[] = [];
  private isInitialized = false;
-  private initializationPromise: Promise<void> | null = null; // ADD THIS LINE
+  private initializationPromise: Promise<void> | null = null;
  private readonly embeddingsPath = path.join(process.cwd(), 'data', 'embeddings.json');
  private readonly batchSize: number;
  private readonly batchDelay: number;
@ -43,24 +43,19 @@ class EmbeddingsService {
    this.batchDelay = parseInt(process.env.AI_EMBEDDINGS_BATCH_DELAY_MS || '1000', 10);
  }

-  // REPLACE the existing initialize method with this:
  async initialize(): Promise<void> {
-    // If initialization is already in progress, wait for it
    if (this.initializationPromise) {
      return this.initializationPromise;
    }

-    // If already initialized, return immediately
    if (this.isInitialized) {
      return Promise.resolve();
    }

-    // Start initialization and store the promise
    this.initializationPromise = this.performInitialization();
    return this.initializationPromise;
  }

-  // ADD THIS NEW METHOD:
  private async performInitialization(): Promise<void> {
    if (!this.enabled) {
      console.log('[EMBEDDINGS] Embeddings disabled, skipping initialization');
@ -70,13 +65,11 @@ class EmbeddingsService {
    try {
      console.log('[EMBEDDINGS] Initializing embeddings system...');
      
-      // Create data directory if it doesn't exist
      await fs.mkdir(path.dirname(this.embeddingsPath), { recursive: true });
      
      const toolsData = await getCompressedToolsDataForAI();
      const currentDataHash = this.hashData(toolsData);
      
-      // Try to load existing embeddings
      const existingEmbeddings = await this.loadEmbeddings();
      
      if (existingEmbeddings && existingEmbeddings.version === currentDataHash) {
@ -336,12 +329,10 @@ class EmbeddingsService {



-// Global instance
 const embeddingsService = new EmbeddingsService();

 export { embeddingsService, type EmbeddingData, type SimilarityResult };

-// Auto-initialize on import in server environment
 if (typeof window === 'undefined' && process.env.NODE_ENV !== 'test') {
  embeddingsService.initialize().catch(error => {
    console.error('[EMBEDDINGS] Auto-initialization failed:', error);
--- a/src/utils/rateLimitedQueue.ts
+++ b/src/utils/rateLimitedQueue.ts
@ -96,7 +96,6 @@ class RateLimitedQueue {

      this.tasks.push(queuedTask);

-      // Kick the processor soon.
      setTimeout(() => {
        this.processQueue();
      }, 100);
@ -170,7 +169,7 @@ class RateLimitedQueue {
          .filter((t) => t.status === "queued")
          .sort((a, b) => a.addedAt - b.addedAt)[0];

-        if (!nextTask) break; // No more work
+        if (!nextTask) break;

        nextTask.status = "processing";
        nextTask.startedAt = Date.now();
Author	SHA1	Message	Date
overcuriousity	b515a45e1e	cleanup	2025-08-05 22:09:46 +02:00
overcuriousity	1c0025796a	cleanup	2025-08-05 22:09:38 +02:00