// src/utils/aiPipeline.ts - Enhanced with Proper Confidence Scoring

import { getCompressedToolsDataForAI } from './dataService.js';
import { embeddingsService, type EmbeddingData, type SimilarityResult } from './embeddings.js';
import { AI_PROMPTS, getPrompt } from '../config/prompts.js';
import { isToolHosted } from './toolHelpers.js';

interface AIConfig {
  endpoint: string;
  apiKey: string;
  model: string;
}

interface MicroTaskResult {
  taskType: string;
  content: string;
  processingTimeMs: number;
  success: boolean;
  error?: string;
}

interface AnalysisResult {
  recommendation: any;
  processingStats: {
    embeddingsUsed: boolean;
    candidatesFromEmbeddings: number;
    finalSelectedItems: number;
    processingTimeMs: number;
    microTasksCompleted: number;
    microTasksFailed: number;
    contextContinuityUsed: boolean;
  };
}

interface AuditEntry {
  timestamp: number;
  phase: string;           
  action: string;          
  input: any;              
  output: any;             
  confidence: number;      
  processingTimeMs: number;
  metadata: Record<string, any>;
}

interface AnalysisContext {
  userQuery: string;
  mode: string;
  filteredData: any;
  contextHistory: string[];
  
  maxContextLength: number;
  currentContextLength: number;
  
  scenarioAnalysis?: string;
  problemAnalysis?: string;
  investigationApproach?: string;
  criticalConsiderations?: string;
  selectedTools?: Array<{tool: any, phase: string, priority: string, justification?: string, taskRelevance?: number, limitations?: string[]}>;
  backgroundKnowledge?: Array<{concept: any, relevance: string}>;
  
  seenToolNames: Set<string>;
  
  auditTrail: AuditEntry[];
  
  embeddingsSimilarities: Map<string, number>;
}

interface ConfidenceMetrics {
  overall: number;                    
  semanticRelevance: number;         
  taskSuitability: number;          
  uncertaintyFactors: string[];   
  strengthIndicators: string[]; 
}

class ImprovedMicroTaskAIPipeline {
  private config: AIConfig;
  private maxSelectedItems: number;
  private embeddingCandidates: number;
  private similarityThreshold: number;
  private microTaskDelay: number;
  
  private embeddingSelectionLimit: number;
  private embeddingConceptsLimit: number;

  private noEmbeddingsToolLimit: number;
  private noEmbeddingsConceptLimit: number;
  
  private embeddingsMinTools: number;
  private embeddingsMaxReductionRatio: number;
  
  private maxContextTokens: number;
  private maxPromptTokens: number;
  
  private auditConfig: {
    enabled: boolean;
    detailLevel: 'minimal' | 'standard' | 'verbose';
    retentionHours: number;
  };

  private confidenceConfig: {
    semanticWeight: number;       
    suitabilityWeight: number;  
    consistencyWeight: number;    
    reliabilityWeight: number;    
    minimumThreshold: number;
    mediumThreshold: number;
    highThreshold: number;
  };
  
  private tempAuditEntries: AuditEntry[] = [];

  constructor() {
    this.config = {
      endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'),
      apiKey: this.getEnv('AI_ANALYZER_API_KEY'), 
      model: this.getEnv('AI_ANALYZER_MODEL')
    };

    this.maxSelectedItems = parseInt(process.env.AI_MAX_SELECTED_ITEMS || '25', 10);
    this.embeddingCandidates = parseInt(process.env.AI_EMBEDDING_CANDIDATES || '50', 10); 
    this.similarityThreshold = parseFloat(process.env.AI_SIMILARITY_THRESHOLD || '0.3');
    this.microTaskDelay = parseInt(process.env.AI_MICRO_TASK_DELAY_MS || '500', 10);
    
    this.embeddingSelectionLimit = parseInt(process.env.AI_EMBEDDING_SELECTION_LIMIT || '30', 10);
    this.embeddingConceptsLimit = parseInt(process.env.AI_EMBEDDING_CONCEPTS_LIMIT || '15', 10);

    this.noEmbeddingsToolLimit = parseInt(process.env.AI_NO_EMBEDDINGS_TOOL_LIMIT || '0', 10);
    this.noEmbeddingsConceptLimit = parseInt(process.env.AI_NO_EMBEDDINGS_CONCEPT_LIMIT || '0', 10);
    
    this.embeddingsMinTools = parseInt(process.env.AI_EMBEDDINGS_MIN_TOOLS || '8', 10);
    this.embeddingsMaxReductionRatio = parseFloat(process.env.AI_EMBEDDINGS_MAX_REDUCTION_RATIO || '0.75');
    
    this.maxContextTokens = parseInt(process.env.AI_MAX_CONTEXT_TOKENS || '4000', 10);
    this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10);
    
    this.auditConfig = {
      enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true',
      detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as any) || 'standard',
      retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10)
    };
    
    this.confidenceConfig = {
      semanticWeight: parseFloat(process.env.CONFIDENCE_SEMANTIC_WEIGHT || '0.3'),   
      suitabilityWeight: parseFloat(process.env.CONFIDENCE_SUITABILITY_WEIGHT || '0.7'), 
      consistencyWeight: 0,    
      reliabilityWeight: 0,
      minimumThreshold: parseInt(process.env.CONFIDENCE_MINIMUM_THRESHOLD || '40', 10),
      mediumThreshold: parseInt(process.env.CONFIDENCE_MEDIUM_THRESHOLD || '60', 10),
      highThreshold: parseInt(process.env.CONFIDENCE_HIGH_THRESHOLD || '80', 10)
    };

    console.log('[AI PIPELINE] Simplified confidence scoring enabled:', {
      weights: `Semantic:${this.confidenceConfig.semanticWeight} Suitability:${this.confidenceConfig.suitabilityWeight}`,
      thresholds: `${this.confidenceConfig.minimumThreshold}/${this.confidenceConfig.mediumThreshold}/${this.confidenceConfig.highThreshold}`
    });
  }

  private getEnv(key: string): string {
    const value = process.env[key];
    if (!value) {
      throw new Error(`Missing environment variable: ${key}`);
    }
    return value;
  }

  private addAuditEntry(
    context: AnalysisContext | null, 
    phase: string, 
    action: string, 
    input: any, 
    output: any, 
    confidence: number, 
    startTime: number, 
    metadata: Record<string, any> = {}
  ): void {
    if (!this.auditConfig.enabled) return;
    
    const auditEntry: AuditEntry = {
      timestamp: Date.now(),
      phase,
      action, 
      input: this.auditConfig.detailLevel === 'verbose' ? input : this.summarizeForAudit(input),
      output: this.auditConfig.detailLevel === 'verbose' ? output : this.summarizeForAudit(output),
      confidence,
      processingTimeMs: Date.now() - startTime,
      metadata
    };
    
    if (context) {
      context.auditTrail.push(auditEntry);
    } else {
      this.tempAuditEntries.push(auditEntry);
    }
    
    console.log(`[AUDIT] ${phase}/${action}: ${confidence}% confidence, ${Date.now() - startTime}ms`);
  }
  
  private mergeTemporaryAuditEntries(context: AnalysisContext): void {
    if (!this.auditConfig.enabled || this.tempAuditEntries.length === 0) return;
    
    const entryCount = this.tempAuditEntries.length;
    context.auditTrail.unshift(...this.tempAuditEntries);
    this.tempAuditEntries = []; 
    
    console.log(`[AUDIT] Merged ${entryCount} temporary audit entries into context`);
  }

  private summarizeForAudit(data: any): any {
    if (this.auditConfig.detailLevel === 'minimal') {
      if (typeof data === 'string' && data.length > 100) {
        return data.slice(0, 100) + '...[truncated]';
      }
      if (Array.isArray(data) && data.length > 3) {
        return [...data.slice(0, 3), `...[${data.length - 3} more items]`];
      }
    } else if (this.auditConfig.detailLevel === 'standard') {
      if (typeof data === 'string' && data.length > 500) {
        return data.slice(0, 500) + '...[truncated]';
      }
      if (Array.isArray(data) && data.length > 10) {
        return [...data.slice(0, 10), `...[${data.length - 10} more items]`];
      }
    }
    return data;
  }

  private calculateSelectionConfidence(result: any, candidateCount: number): number {
    if (!result || !result.selectedTools) return 30;
    
    const selectionRatio = result.selectedTools.length / candidateCount;
    const hasReasoning = result.reasoning && result.reasoning.length > 50;
    
    let confidence = 60; 
    
    if (selectionRatio > 0.05 && selectionRatio < 0.3) confidence += 20;
    else if (selectionRatio <= 0.05) confidence -= 10; 
    else confidence -= 15; 
    
    if (hasReasoning) confidence += 15;
    
    if (result.selectedConcepts && result.selectedConcepts.length > 0) confidence += 5;
    
    return Math.min(95, Math.max(25, confidence));
  }

  private estimateTokens(text: string): number {
    return Math.ceil(text.length / 4); 
  }

  private addToContextHistory(context: AnalysisContext, newEntry: string): void {
    const entryTokens = this.estimateTokens(newEntry);
    
    context.contextHistory.push(newEntry);
    context.currentContextLength += entryTokens;
    
    while (context.currentContextLength > this.maxContextTokens && context.contextHistory.length > 1) {
      const removed = context.contextHistory.shift()!;
      context.currentContextLength -= this.estimateTokens(removed);
    }
  }

  private safeParseJSON(jsonString: string, fallback: any = null): any {
    try {
      let cleaned = jsonString
        .replace(/^```json\s*/i, '')
        .replace(/\s*```\s*$/g, '')
        .trim();
      
      if (!cleaned.endsWith('}') && !cleaned.endsWith(']')) {
        console.warn('[AI PIPELINE] JSON appears truncated, attempting recovery...');
        
        let lastCompleteStructure = '';
        let braceCount = 0;
        let bracketCount = 0;
        let inString = false;
        let escaped = false;
        
        for (let i = 0; i < cleaned.length; i++) {
          const char = cleaned[i];
          
          if (escaped) {
            escaped = false;
            continue;
          }
          
          if (char === '\\') {
            escaped = true;
            continue;
          }
          
          if (char === '"' && !escaped) {
            inString = !inString;
            continue;
          }
          
          if (!inString) {
            if (char === '{') braceCount++;
            if (char === '}') braceCount--;
            if (char === '[') bracketCount++;
            if (char === ']') bracketCount--;
            
            if (braceCount === 0 && bracketCount === 0 && (char === '}' || char === ']')) {
              lastCompleteStructure = cleaned.substring(0, i + 1);
            }
          }
        }
        
        if (lastCompleteStructure) {
          console.log('[AI PIPELINE] Attempting to parse recovered JSON structure...');
          cleaned = lastCompleteStructure;
        } else {
          if (braceCount > 0) {
            cleaned += '}';
            console.log('[AI PIPELINE] Added closing brace to truncated JSON');
          }
          if (bracketCount > 0) {
            cleaned += ']';
            console.log('[AI PIPELINE] Added closing bracket to truncated JSON');
          }
        }
      }
      
      const parsed = JSON.parse(cleaned);
      
      if (parsed && typeof parsed === 'object') {
        if (parsed.selectedTools === undefined) parsed.selectedTools = [];
        if (parsed.selectedConcepts === undefined) parsed.selectedConcepts = [];
        
        if (!Array.isArray(parsed.selectedTools)) parsed.selectedTools = [];
        if (!Array.isArray(parsed.selectedConcepts)) parsed.selectedConcepts = [];
      }
      
      return parsed;
    } catch (error) {
      console.warn('[AI PIPELINE] JSON parsing failed:', error.message);
      console.warn('[AI PIPELINE] Raw content (first 300 chars):', jsonString.slice(0, 300));
      console.warn('[AI PIPELINE] Raw content (last 300 chars):', jsonString.slice(-300));
      
      if (jsonString.includes('selectedTools')) {
        const toolMatches = jsonString.match(/"([^"]+)"/g);
        if (toolMatches && toolMatches.length > 0) {
          console.log('[AI PIPELINE] Attempting partial recovery from broken JSON...');
          const possibleTools = toolMatches
            .map(match => match.replace(/"/g, ''))
            .filter(name => name.length > 2 && !['selectedTools', 'selectedConcepts', 'reasoning'].includes(name))
            .slice(0, 15); 
          
          if (possibleTools.length > 0) {
            console.log(`[AI PIPELINE] Recovered ${possibleTools.length} possible tool names from broken JSON`);
            return {
              selectedTools: possibleTools,
              selectedConcepts: [],
              reasoning: 'Recovered from truncated response'
            };
          }
        }
      }
      
      return fallback;
    }
  }

  private addToolToSelection(context: AnalysisContext, tool: any, phase: string, priority: string, justification?: string, taskRelevance?: number, limitations?: string[]): boolean {   
    context.seenToolNames.add(tool.name);
    if (!context.selectedTools) context.selectedTools = [];
    
    context.selectedTools.push({
      tool,
      phase,
      priority,
      justification,
      taskRelevance,
      limitations
    });
    
    return true;
  }

  private async getIntelligentCandidates(userQuery: string, toolsData: any, mode: string, context: AnalysisContext) {
    let candidateTools: any[] = [];
    let candidateConcepts: any[] = [];
    let selectionMethod = 'unknown';
    
    context.embeddingsSimilarities = new Map<string, number>();
    
    if (process.env.AI_EMBEDDINGS_ENABLED === 'true') {
      try {
        console.log('[AI PIPELINE] Waiting for embeddings initialization...');
        await embeddingsService.waitForInitialization();
        console.log('[AI PIPELINE] Embeddings ready, proceeding with similarity search');
      } catch (error) {
        console.error('[AI PIPELINE] Embeddings initialization failed, falling back to full dataset:', error);
      }
    }
    
    if (embeddingsService.isEnabled()) {
      const embeddingsStart = Date.now();
      const similarItems = await embeddingsService.findSimilar(
        userQuery, 
        this.embeddingCandidates, 
        this.similarityThreshold
      ) as SimilarityResult[];
      
      console.log(`[AI PIPELINE] Embeddings found ${similarItems.length} similar items`);
      
      similarItems.forEach(item => {
        context.embeddingsSimilarities.set(item.name, item.similarity);
      });
      
      const toolsMap = new Map<string, any>(toolsData.tools.map((tool: any) => [tool.name, tool]));
      const conceptsMap = new Map<string, any>(toolsData.concepts.map((concept: any) => [concept.name, concept]));
      
      const similarTools = similarItems
        .filter((item): item is SimilarityResult => item.type === 'tool')
        .map(item => toolsMap.get(item.name))
        .filter((tool): tool is any => tool !== undefined);
      
      const similarConcepts = similarItems
        .filter((item): item is SimilarityResult => item.type === 'concept')
        .map(item => conceptsMap.get(item.name))
        .filter((concept): concept is any => concept !== undefined);
      
      console.log(`[AI PIPELINE] Similarity-ordered results: ${similarTools.length} tools, ${similarConcepts.length} concepts`);
      
      const totalAvailableTools = toolsData.tools.length;
      const reductionRatio = similarTools.length / totalAvailableTools;
      
      if (similarTools.length >= this.embeddingsMinTools && reductionRatio <= this.embeddingsMaxReductionRatio) {
        candidateTools = similarTools;
        candidateConcepts = similarConcepts;
        selectionMethod = 'embeddings_candidates';
        
        console.log(`[AI PIPELINE] Using embeddings filtering: ${totalAvailableTools} → ${similarTools.length} tools (${(reductionRatio * 100).toFixed(1)}% reduction)`);
      } else {
        if (similarTools.length < this.embeddingsMinTools) {
          console.log(`[AI PIPELINE] Embeddings found too few tools (${similarTools.length} < ${this.embeddingsMinTools}), using full dataset`);
        } else {
          console.log(`[AI PIPELINE] Embeddings didn't filter enough (${(reductionRatio * 100).toFixed(1)}% > ${(this.embeddingsMaxReductionRatio * 100).toFixed(1)}%), using full dataset`);
        }
        candidateTools = toolsData.tools;
        candidateConcepts = toolsData.concepts;
        selectionMethod = 'full_dataset';
      }
      
      if (this.auditConfig.enabled) {
        this.addAuditEntry(context, 'retrieval', 'embeddings-search', 
          { query: userQuery, threshold: this.similarityThreshold, candidates: this.embeddingCandidates }, 
          { 
            candidatesFound: similarItems.length, 
            toolsInOrder: similarTools.slice(0, 3).map((t: any) => t.name),
            conceptsInOrder: similarConcepts.slice(0, 3).map((c: any) => c.name),
            reductionRatio: reductionRatio,
            usingEmbeddings: selectionMethod === 'embeddings_candidates',
            totalAvailable: totalAvailableTools,
            filtered: similarTools.length,
            avgSimilarity: similarItems.length > 0 ? similarItems.reduce((sum, item) => sum + item.similarity, 0) / similarItems.length : 0
          },
          selectionMethod === 'embeddings_candidates' ? 85 : 60,
          embeddingsStart,
          { 
            selectionMethod, 
            embeddingsEnabled: true, 
            reductionAchieved: selectionMethod === 'embeddings_candidates',
            tokenSavingsExpected: selectionMethod === 'embeddings_candidates'
          }
        );
      }
    } else {
      console.log(`[AI PIPELINE] Embeddings disabled or not ready, using full dataset`);
      candidateTools = toolsData.tools;
      candidateConcepts = toolsData.concepts;
      selectionMethod = 'full_dataset';
    }

    console.log(`[AI PIPELINE] AI will analyze ${candidateTools.length} candidate tools (method: ${selectionMethod})`);
    const finalSelection = await this.aiSelectionWithFullData(userQuery, candidateTools, candidateConcepts, mode, selectionMethod, context);
    
    return {
      tools: finalSelection.selectedTools,
      concepts: finalSelection.selectedConcepts,
      domains: toolsData.domains,
      phases: toolsData.phases,
      'domain-agnostic-software': toolsData['domain-agnostic-software']
    };
  }

  private async aiSelectionWithFullData(
    userQuery: string, 
    candidateTools: any[], 
    candidateConcepts: any[], 
    mode: string,
    selectionMethod: string,
    context: AnalysisContext
  ) {
    const selectionStart = Date.now();
    
    const toolsWithFullData = candidateTools.map((tool: any) => ({
      name: tool.name,
      type: tool.type,
      description: tool.description,
      domains: tool.domains,
      phases: tool.phases,
      platforms: tool.platforms || [],
      tags: tool.tags || [],
      skillLevel: tool.skillLevel,
      license: tool.license,
      accessType: tool.accessType,
      projectUrl: tool.projectUrl,
      knowledgebase: tool.knowledgebase,
      related_concepts: tool.related_concepts || [],
      related_software: tool.related_software || []
    }));

    const conceptsWithFullData = candidateConcepts.map((concept: any) => ({
      name: concept.name,
      type: 'concept',
      description: concept.description,
      domains: concept.domains,
      phases: concept.phases,
      tags: concept.tags || [],
      skillLevel: concept.skillLevel,
      related_concepts: concept.related_concepts || [],
      related_software: concept.related_software || []
    }));

    let toolsToSend: any[];
    let conceptsToSend: any[];
    
    if (selectionMethod === 'embeddings_candidates') {
      toolsToSend = toolsWithFullData.slice(0, this.embeddingSelectionLimit);
      conceptsToSend = conceptsWithFullData.slice(0, this.embeddingConceptsLimit);
      
      console.log(`[AI PIPELINE] Embeddings enabled: sending top ${toolsToSend.length} similarity-ordered tools`);
    } else {
      const maxTools = this.noEmbeddingsToolLimit > 0 ? 
        Math.min(this.noEmbeddingsToolLimit, candidateTools.length) : 
        candidateTools.length;
      
      const maxConcepts = this.noEmbeddingsConceptLimit > 0 ? 
        Math.min(this.noEmbeddingsConceptLimit, candidateConcepts.length) : 
        candidateConcepts.length;
      
      toolsToSend = toolsWithFullData.slice(0, maxTools);
      conceptsToSend = conceptsWithFullData.slice(0, maxConcepts);
      
      console.log(`[AI PIPELINE] Embeddings disabled: sending ${toolsToSend.length}/${candidateTools.length} tools (limit: ${this.noEmbeddingsToolLimit || 'none'})`);
    }

    const basePrompt = getPrompt('toolSelection', mode, userQuery, selectionMethod, this.maxSelectedItems);
    const prompt = `${basePrompt}

VERFÜGBARE TOOLS (mit vollständigen Daten):
${JSON.stringify(toolsToSend, null, 2)}

VERFÜGBARE KONZEPTE (mit vollständigen Daten):
${JSON.stringify(conceptsToSend, null, 2)}`;

    const estimatedTokens = this.estimateTokens(prompt);
    console.log(`[AI PIPELINE] Method: ${selectionMethod}, Tools: ${toolsToSend.length}, Estimated tokens: ~${estimatedTokens}`);

    if (estimatedTokens > 35000) {
      console.warn(`[AI PIPELINE] WARNING: Prompt tokens (${estimatedTokens}) may exceed model limits`);
    }

    try {
      const response = await this.callAI(prompt, 2500);
      
      const result = this.safeParseJSON(response, null);
      
      if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) {
        console.error('[AI PIPELINE] AI selection returned invalid structure:', response.slice(0, 200));
        
        if (this.auditConfig.enabled) {
          this.addAuditEntry(context, 'selection', 'ai-tool-selection-failed',
            { candidateCount: candidateTools.length, mode, prompt: prompt.slice(0, 200) },
            { error: 'Invalid JSON structure', response: response.slice(0, 200) },
            10,
            selectionStart,
            { aiModel: this.config.model, selectionMethod, tokensSent: estimatedTokens, toolsSent: toolsToSend.length }
          );
        }
        
        throw new Error('AI selection failed to return valid tool selection');
      }

      const totalSelected = result.selectedTools.length + result.selectedConcepts.length;
      if (totalSelected === 0) {
        console.error('[AI PIPELINE] AI selection returned no tools');
        throw new Error('AI selection returned empty selection');
      }
      
      console.log(`[AI PIPELINE] AI selected: ${result.selectedTools.length} tools, ${result.selectedConcepts.length} concepts from ${toolsToSend.length} candidates`);

      const selectedTools = candidateTools.filter(tool => result.selectedTools.includes(tool.name));
      const selectedConcepts = candidateConcepts.filter(concept => result.selectedConcepts.includes(concept.name));
      
      if (this.auditConfig.enabled) {
        const confidence = this.calculateSelectionConfidence(result, candidateTools.length);
        
        this.addAuditEntry(context, 'selection', 'ai-tool-selection',
          { candidateCount: candidateTools.length, mode, promptLength: prompt.length },
          { 
            selectedToolCount: result.selectedTools.length, 
            selectedConceptCount: result.selectedConcepts.length,
            reasoning: result.reasoning?.slice(0, 200) + '...',
            finalToolNames: selectedTools.map(t => t.name),
            selectionEfficiency: `${toolsToSend.length} → ${result.selectedTools.length}`
          },
          confidence,
          selectionStart,
          { aiModel: this.config.model, selectionMethod, promptTokens: estimatedTokens, toolsSent: toolsToSend.length }
        );
      }
      
      return {
        selectedTools,
        selectedConcepts
      };

    } catch (error) {
      console.error('[AI PIPELINE] AI selection failed:', error);
      
      if (this.auditConfig.enabled) {
        this.addAuditEntry(context, 'selection', 'ai-tool-selection-error',
          { candidateCount: candidateTools.length, mode },
          { error: error.message },
          5,
          selectionStart,
          { aiModel: this.config.model, selectionMethod, tokensSent: estimatedTokens }
        );
      }
      throw error;
    }
  }

  private async delay(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  private async callMicroTaskAI(prompt: string, context: AnalysisContext, maxTokens: number = 500): Promise<MicroTaskResult> {
    const startTime = Date.now();
    
    let contextPrompt = prompt;
    if (context.contextHistory.length > 0) {
      const contextSection = `BISHERIGE ANALYSE:\n${context.contextHistory.join('\n\n')}\n\nAKTUELLE AUFGABE:\n`;
      const combinedPrompt = contextSection + prompt;
      
      if (this.estimateTokens(combinedPrompt) <= this.maxPromptTokens) {
        contextPrompt = combinedPrompt;
      } else {
        console.warn('[AI PIPELINE] Context too long, using prompt only');
      }
    }
    
    try {
      const response = await this.callAI(contextPrompt, maxTokens);
      
      const result = {
        taskType: 'micro-task',
        content: response.trim(),
        processingTimeMs: Date.now() - startTime,
        success: true
      };
      
      this.addAuditEntry(context, 'micro-task', 'ai-analysis',
        { promptLength: contextPrompt.length, maxTokens },
        { responseLength: response.length, contentPreview: response.slice(0, 100) },
        response.length > 50 ? 80 : 60,
        startTime,
        { aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 }
      );
      
      return result;

    } catch (error) {
      const result = {
        taskType: 'micro-task',
        content: '',
        processingTimeMs: Date.now() - startTime,
        success: false,
        error: error.message
      };
      
      this.addAuditEntry(context, 'micro-task', 'ai-analysis-failed',
        { promptLength: contextPrompt.length, maxTokens },
        { error: error.message },
        5,
        startTime,
        { aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 }
      );
      
      return result;
    }
  }

  private calculateRecommendationConfidence(
    tool: any, 
    context: AnalysisContext,
    taskRelevance: number = 70,
    limitations: string[] = []
  ): ConfidenceMetrics {
    
    const rawSemanticRelevance = context.embeddingsSimilarities.has(tool.name) ? 
      context.embeddingsSimilarities.get(tool.name)! * 100 : 50;
    
    let enhancedTaskSuitability = taskRelevance;
    
    if (context.mode === 'workflow') {
      const toolSelection = context.selectedTools?.find(st => st.tool.name === tool.name);
      if (toolSelection && tool.phases && tool.phases.includes(toolSelection.phase)) {
        const phaseBonus = Math.min(15, 100 - taskRelevance);
        enhancedTaskSuitability = Math.min(100, taskRelevance + phaseBonus);
        
        console.log(`[CONFIDENCE] Phase bonus for ${tool.name}: ${taskRelevance} -> ${enhancedTaskSuitability} (phase: ${toolSelection.phase})`);
      }
    }
    
    const overall = (
      rawSemanticRelevance * this.confidenceConfig.semanticWeight +
      enhancedTaskSuitability * this.confidenceConfig.suitabilityWeight
    );

    const uncertaintyFactors = this.identifySpecificUncertaintyFactors(tool, context, limitations, overall);
    const strengthIndicators = this.identifySpecificStrengthIndicators(tool, context, overall);

    console.log(`[CONFIDENCE DEBUG] ${tool.name}:`, {
      rawSemantic: Math.round(rawSemanticRelevance),
      rawTaskSuitability: taskRelevance,
      enhancedTaskSuitability: Math.round(enhancedTaskSuitability),
      overall: Math.round(overall),
      mode: context.mode
    });

    return {
      overall: Math.round(overall),
      semanticRelevance: Math.round(rawSemanticRelevance),
      taskSuitability: Math.round(enhancedTaskSuitability), 
      uncertaintyFactors,
      strengthIndicators
    };
  }

  private identifySpecificUncertaintyFactors(tool: any, context: AnalysisContext, limitations: string[], confidence: number): string[] {
    const factors: string[] = [];
    
    if (limitations && limitations.length > 0) {
      factors.push(...limitations.slice(0, 2));
    }
    
    const similarity = context.embeddingsSimilarities.get(tool.name) || 0.5;
    if (similarity < 0.7) {
      factors.push('Geringe semantische Ähnlichkeit zur Anfrage - Tool-Beschreibung passt möglicherweise nicht optimal');
    }
    
    if (tool.skillLevel === 'expert' && /schnell|rapid|triage|urgent|sofort/i.test(context.userQuery)) {
      factors.push('Experten-Tool für zeitkritisches Szenario - Setup und Einarbeitung könnten zu lange dauern');
    }
    
    if (tool.skillLevel === 'novice' && /komplex|erweitert|tiefgehend|advanced|forensisch/i.test(context.userQuery)) {
      factors.push('Einsteiger-Tool für komplexe Analyse - könnte funktionale Limitierungen haben');
    }
    
    if (tool.type === 'software' && !isToolHosted(tool) && tool.accessType === 'download') {
      factors.push('Installation und Setup erforderlich');
    }
    
    if (tool.license === 'Proprietary') {
      factors.push('Kommerzielle Software - Lizenzkosten und rechtliche Beschränkungen zu beachten');
    }
    
    if (confidence < 60) {
      factors.push('Moderate Gesamtbewertung - alternative Ansätze sollten ebenfalls betrachtet werden');
    }
    
    return factors.slice(0, 4);
  }

  private identifySpecificStrengthIndicators(tool: any, context: AnalysisContext, confidence: number): string[] {
    const indicators: string[] = [];
    
    const similarity = context.embeddingsSimilarities.get(tool.name) || 0.5;
    if (similarity >= 0.7) {
      indicators.push('Sehr gute semantische Übereinstimmung mit Ihrer Anfrage');
    }
    
    if (tool.knowledgebase === true) {
      indicators.push('Umfassende Dokumentation und Wissensbasis verfügbar');
    }
    
    if (isToolHosted(tool)) {
      indicators.push('Sofort verfügbar über gehostete Lösung - kein Setup erforderlich');
    }
    
    if (tool.skillLevel === 'intermediate' || tool.skillLevel === 'advanced') {
      indicators.push('Ausgewogenes Verhältnis zwischen Funktionalität und Benutzerfreundlichkeit');
    }
    
    if (tool.type === 'method' && /methodik|vorgehen|prozess|ansatz/i.test(context.userQuery)) {
      indicators.push('Methodischer Ansatz passt zu Ihrer prozeduralen Anfrage');
    }
    
    return indicators.slice(0, 4); 
  }

  private async analyzeScenario(context: AnalysisContext): Promise<MicroTaskResult> {
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('scenarioAnalysis', isWorkflow, context.userQuery);

    const result = await this.callMicroTaskAI(prompt, context, 400);
    
    if (result.success) {
      if (isWorkflow) {
        context.scenarioAnalysis = result.content;
      } else {
        context.problemAnalysis = result.content;
      }
      
      this.addToContextHistory(context, `${isWorkflow ? 'Szenario' : 'Problem'}-Analyse: ${result.content.slice(0, 200)}...`);
    }
    
    return result;
  }

  private async generateApproach(context: AnalysisContext): Promise<MicroTaskResult> {
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('investigationApproach', isWorkflow, context.userQuery);

    const result = await this.callMicroTaskAI(prompt, context, 400);
    
    if (result.success) {
      context.investigationApproach = result.content;
      this.addToContextHistory(context, `${isWorkflow ? 'Untersuchungs' : 'Lösungs'}ansatz: ${result.content.slice(0, 200)}...`);
    }
    
    return result;
  }

  private async generateCriticalConsiderations(context: AnalysisContext): Promise<MicroTaskResult> {
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('criticalConsiderations', isWorkflow, context.userQuery);

    const result = await this.callMicroTaskAI(prompt, context, 350);
    
    if (result.success) {
      context.criticalConsiderations = result.content;
      this.addToContextHistory(context, `Kritische Überlegungen: ${result.content.slice(0, 200)}...`);
    }
    
    return result;
  }

  private async selectToolsForPhase(context: AnalysisContext, phase: any): Promise<MicroTaskResult> {
    const phaseTools = context.filteredData.tools.filter((tool: any) => 
      tool.phases && tool.phases.includes(phase.id)
    );

    if (phaseTools.length === 0) {
      return {
        taskType: 'tool-selection',
        content: JSON.stringify([]),
        processingTimeMs: 0,
        success: true
      };
    }

    const prompt = getPrompt('phaseToolSelection', context.userQuery, phase, phaseTools);

    const result = await this.callMicroTaskAI(prompt, context, 1000);
    
    if (result.success) {
      const selections = this.safeParseJSON(result.content, []);
      
      if (Array.isArray(selections)) {
        const validSelections = selections.filter((sel: any) => 
          sel.toolName && phaseTools.some((tool: any) => tool.name === sel.toolName)
        );
        
        validSelections.forEach((sel: any) => {
          const tool = phaseTools.find((t: any) => t.name === sel.toolName);
          if (tool) {
            const taskRelevance = typeof sel.taskRelevance === 'number' ? 
              sel.taskRelevance : parseInt(String(sel.taskRelevance)) || 70;
            
            const priority = this.derivePriorityFromScore(taskRelevance);
            
            this.addToolToSelection(context, tool, phase.id, priority, sel.justification, taskRelevance, sel.limitations);
          }
        });
        
        this.addAuditEntry(context, 'micro-task', 'phase-tool-selection',
          { phase: phase.id, availableTools: phaseTools.length },
          { 
            validSelections: validSelections.length, 
            selectedTools: validSelections.map(s => ({ 
              name: s.toolName, 
              taskRelevance: s.taskRelevance, 
              derivedPriority: this.derivePriorityFromScore(s.taskRelevance) 
            }))
          },
          validSelections.length > 0 ? 75 : 30,
          Date.now() - result.processingTimeMs,
          { phaseName: phase.name, comparativeEvaluation: true, priorityDerived: true }
        );
      }
    }
    
    return result;
  }

  private async evaluateSpecificTool(context: AnalysisContext, tool: any, rank: number): Promise<MicroTaskResult> {
    const existingSelection = context.selectedTools?.find(st => st.tool.name === tool.name);
    const taskRelevance = existingSelection?.taskRelevance || 70;
    const priority = this.derivePriorityFromScore(taskRelevance);
    
    const prompt = getPrompt('toolEvaluation', context.userQuery, tool, rank, taskRelevance);

    const result = await this.callMicroTaskAI(prompt, context, 1000);
    
    if (result.success) {
      const evaluation = this.safeParseJSON(result.content, {
        detailed_explanation: 'Evaluation failed',
        implementation_approach: '',
        pros: [],
        limitations: [], 
        alternatives: ''
      });
      
      this.addToolToSelection(context, {
        ...tool,
        evaluation: {
          ...evaluation,
          rank,
          task_relevance: taskRelevance
        }
      }, 'evaluation', priority, evaluation.detailed_explanation, 
      taskRelevance, evaluation.limitations); 
      
      this.addAuditEntry(context, 'micro-task', 'tool-evaluation',
        { toolName: tool.name, rank, existingTaskRelevance: taskRelevance, derivedPriority: priority },
        { 
          hasExplanation: !!evaluation.detailed_explanation,
          hasImplementationApproach: !!evaluation.implementation_approach,
          prosCount: evaluation.pros?.length || 0,
          limitationsCount: evaluation.limitations?.length || 0,
          hasLimitations: Array.isArray(evaluation.limitations) && evaluation.limitations.length > 0
        },
        70,
        Date.now() - result.processingTimeMs,
        { toolType: tool.type, explanationOnly: true, priorityDerived: true, limitationsExtracted: true }
      );
    }
    
    return result;
  }

  private async selectBackgroundKnowledge(context: AnalysisContext): Promise<MicroTaskResult> {
    const availableConcepts = context.filteredData.concepts;
    
    if (availableConcepts.length === 0) {
      return {
        taskType: 'background-knowledge',
        content: JSON.stringify([]),
        processingTimeMs: 0,
        success: true
      };
    }

    const selectedToolNames = context.selectedTools?.map(st => st.tool.name) || [];
    const prompt = getPrompt('backgroundKnowledgeSelection', context.userQuery, context.mode, selectedToolNames, availableConcepts);

    const result = await this.callMicroTaskAI(prompt, context, 700);
    
    if (result.success) {
      const selections = this.safeParseJSON(result.content, []);
      
      if (Array.isArray(selections)) {
        context.backgroundKnowledge = selections.filter((sel: any) => 
          sel.conceptName && availableConcepts.some((concept: any) => concept.name === sel.conceptName)
        ).map((sel: any) => ({
          concept: availableConcepts.find((c: any) => c.name === sel.conceptName),
          relevance: sel.relevance
        }));
        
        this.addAuditEntry(context, 'micro-task', 'background-knowledge-selection',
          { availableConcepts: availableConcepts.length },
          { selectedConcepts: context.backgroundKnowledge?.length || 0 },
          context.backgroundKnowledge && context.backgroundKnowledge.length > 0 ? 75 : 40,
          Date.now() - result.processingTimeMs,
          {}
        );
      }
    }
    
    return result;
  }

  private async generateFinalRecommendations(context: AnalysisContext): Promise<MicroTaskResult> {
    const selectedToolNames = context.selectedTools?.map(st => st.tool.name) || [];
    const prompt = getPrompt('finalRecommendations', context.mode === 'workflow', context.userQuery, selectedToolNames);

    const result = await this.callMicroTaskAI(prompt, context, 350);
    return result;
  }

  private async callAI(prompt: string, maxTokens: number = 1500): Promise<string> {
    const endpoint = this.config.endpoint;
    const apiKey = this.config.apiKey;
    const model = this.config.model;
    
    let headers: Record<string, string> = {
      'Content-Type': 'application/json'
    };
    
    if (apiKey) {
      headers['Authorization'] = `Bearer ${apiKey}`;
      console.log('[AI PIPELINE] Using API key authentication');
    } else {
      console.log('[AI PIPELINE] No API key - making request without authentication');
    }
    
    const requestBody = {
      model,
      messages: [{ role: 'user', content: prompt }],
      max_tokens: maxTokens,
      temperature: 0.3
    };
    
    try {
      const response = await fetch(`${endpoint}/v1/chat/completions`, {
        method: 'POST',
        headers,
        body: JSON.stringify(requestBody)
      });

      if (!response.ok) {
        const errorText = await response.text();
        console.error(`[AI PIPELINE] AI API Error ${response.status}:`, errorText);
        throw new Error(`AI API error: ${response.status} - ${errorText}`);
      }

      const data = await response.json();
      const content = data.choices?.[0]?.message?.content;
      
      if (!content) {
        console.error('[AI PIPELINE] No response content:', data);
        throw new Error('No response from AI model');
      }

      return content;
      
    } catch (error) {
      console.error('[AI PIPELINE] AI service call failed:', error.message);
      throw error;
    }
  }

  private derivePriorityFromScore(taskRelevance: number): string {
    if (taskRelevance >= 80) return 'high';
    if (taskRelevance >= 60) return 'medium';
    return 'low';
  }

  async processQuery(userQuery: string, mode: string): Promise<AnalysisResult> {
    const startTime = Date.now();
    let completeTasks = 0;
    let failedTasks = 0;
    
    this.tempAuditEntries = [];
    
    console.log(`[AI PIPELINE] Starting ${mode} query processing with enhanced confidence scoring`);

    try {
      const toolsData = await getCompressedToolsDataForAI();
      
      const context: AnalysisContext = {
        userQuery,
        mode,
        filteredData: {},
        contextHistory: [],
        maxContextLength: this.maxContextTokens,
        currentContextLength: 0,
        seenToolNames: new Set<string>(),
        auditTrail: [],
        embeddingsSimilarities: new Map<string, number>()
      };

      const filteredData = await this.getIntelligentCandidates(userQuery, toolsData, mode, context);
      context.filteredData = filteredData;
      
      this.mergeTemporaryAuditEntries(context);

      console.log(`[AI PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`);

      this.addAuditEntry(context, 'initialization', 'pipeline-start',
        { userQuery, mode, toolsDataLoaded: !!toolsData },
        { candidateTools: filteredData.tools.length, candidateConcepts: filteredData.concepts.length },
        90, 
        startTime,
        { auditEnabled: this.auditConfig.enabled, confidenceScoringEnabled: true }
      );
   
      const analysisResult = await this.analyzeScenario(context);
      if (analysisResult.success) completeTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

      const approachResult = await this.generateApproach(context);
      if (approachResult.success) completeTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

      const considerationsResult = await this.generateCriticalConsiderations(context);
      if (considerationsResult.success) completeTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

      if (mode === 'workflow') {
        const phases = toolsData.phases || [];
        for (const phase of phases) {
          const toolSelectionResult = await this.selectToolsForPhase(context, phase);
          if (toolSelectionResult.success) completeTasks++; else failedTasks++;
          await this.delay(this.microTaskDelay);
        }
      } else {
        const topTools = filteredData.tools.slice(0, 3);
        for (let i = 0; i < topTools.length; i++) {
          const evaluationResult = await this.evaluateSpecificTool(context, topTools[i], i + 1);
          if (evaluationResult.success) completeTasks++; else failedTasks++;
          await this.delay(this.microTaskDelay);
        }
      }

      const knowledgeResult = await this.selectBackgroundKnowledge(context);
      if (knowledgeResult.success) completeTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

      const finalResult = await this.generateFinalRecommendations(context);
      if (finalResult.success) completeTasks++; else failedTasks++;

      const recommendation = this.buildRecommendation(context, mode, finalResult.content);

      this.addAuditEntry(context, 'completion', 'pipeline-end',
        { completedTasks: completeTasks, failedTasks },
        { finalRecommendation: !!recommendation, auditEntriesGenerated: context.auditTrail.length },
        completeTasks > failedTasks ? 85 : 60,
        startTime,
        { totalProcessingTimeMs: Date.now() - startTime, confidenceScoresGenerated: context.selectedTools?.length || 0 }
      );

      const processingStats = {
        embeddingsUsed: embeddingsService.isEnabled(),
        candidatesFromEmbeddings: filteredData.tools.length,
        finalSelectedItems: (context.selectedTools?.length || 0) + 
                           (context.backgroundKnowledge?.length || 0),
        processingTimeMs: Date.now() - startTime,
        microTasksCompleted: completeTasks,
        microTasksFailed: failedTasks,
        contextContinuityUsed: true
      };

      console.log(`[AI PIPELINE] Completed: ${completeTasks} tasks, Failed: ${failedTasks} tasks`);
      console.log(`[AI PIPELINE] Enhanced confidence scores generated: ${context.selectedTools?.length || 0}`);
      console.log(`[AI PIPELINE] Audit trail entries: ${context.auditTrail.length}`);

      return {
        recommendation: {
          ...recommendation,
          auditTrail: this.auditConfig.enabled ? context.auditTrail : undefined
        },
        processingStats
      };

    } catch (error) {
      console.error('[AI PIPELINE] Processing failed:', error);
      
      this.tempAuditEntries = [];
      
      throw error;
    }
  }

  private buildRecommendation(context: AnalysisContext, mode: string, finalContent: string): any {
    const isWorkflow = mode === 'workflow';
    
    const base = {
      [isWorkflow ? 'scenario_analysis' : 'problem_analysis']: 
        isWorkflow ? context.scenarioAnalysis : context.problemAnalysis,
      investigation_approach: context.investigationApproach,
      critical_considerations: context.criticalConsiderations,
      background_knowledge: context.backgroundKnowledge?.map(bk => ({
        concept_name: bk.concept.name,
        relevance: bk.relevance
      })) || []
    };

    if (isWorkflow) {
      const recommendedToolsWithConfidence = context.selectedTools?.map(st => {
        const confidence = this.calculateRecommendationConfidence(
          st.tool,
          context,
          st.taskRelevance || 70,
          st.limitations || []
        );
        
        this.addAuditEntry(context, 'validation', 'confidence-scoring',
          { toolName: st.tool.name, phase: st.phase },
          { 
            overall: confidence.overall,
            components: {
              semantic: confidence.semanticRelevance,
              suitability: confidence.taskSuitability,
            }
          },
          confidence.overall,
          Date.now(),
          { uncertaintyCount: confidence.uncertaintyFactors.length, strengthCount: confidence.strengthIndicators.length }
        );

        return {
          name: st.tool.name,
          phase: st.phase,
          priority: st.priority,
          justification: st.justification || `Empfohlen für ${st.phase}`,
          confidence: confidence,
          recommendationStrength: confidence.overall >= this.confidenceConfig.highThreshold ? 'strong' : 
                                confidence.overall >= this.confidenceConfig.mediumThreshold ? 'moderate' : 'weak'
        };
      }) || [];

      return {
        ...base,
        recommended_tools: recommendedToolsWithConfidence,
        workflow_suggestion: finalContent
      };
    } else {
      const recommendedToolsWithConfidence = context.selectedTools?.map(st => {
        const confidence = this.calculateRecommendationConfidence(
          st.tool,
          context,
          st.taskRelevance || 70,
          st.limitations || [] 
        );
        
        this.addAuditEntry(context, 'validation', 'confidence-scoring',
          { toolName: st.tool.name, rank: st.tool.evaluation?.rank || 1 },
          { 
            overall: confidence.overall,
            suitabilityAlignment: st.priority === 'high' && confidence.overall >= this.confidenceConfig.highThreshold,
            limitationsUsed: st.limitations?.length || 0
          },
          confidence.overall,
          Date.now(),
          { strengthCount: confidence.strengthIndicators.length, limitationsCount: confidence.uncertaintyFactors.length }
        );

        return {
          name: st.tool.name,
          rank: st.tool.evaluation?.rank || 1,
          suitability_score: st.priority,
          detailed_explanation: st.tool.evaluation?.detailed_explanation || '',
          implementation_approach: st.tool.evaluation?.implementation_approach || '',
          pros: st.tool.evaluation?.pros || [],
          cons: st.tool.evaluation?.limitations || [],
          alternatives: st.tool.evaluation?.alternatives || '',
          confidence: confidence,
          recommendationStrength: confidence.overall >= this.confidenceConfig.highThreshold ? 'strong' : 
                                confidence.overall >= this.confidenceConfig.mediumThreshold ? 'moderate' : 'weak'
        };
      }) || [];

      return {
        ...base,
        recommended_tools: recommendedToolsWithConfidence,
        additional_considerations: finalContent
      };
    }
  }
}

const aiPipeline = new ImprovedMicroTaskAIPipeline();

export { aiPipeline, type AnalysisResult };