// src/utils/aiPipeline.ts - Enhanced with Audit Trail System

import { getCompressedToolsDataForAI } from './dataService.js';
import { embeddingsService, type EmbeddingData } from './embeddings.js';
import { AI_PROMPTS, getPrompt } from '../config/prompts.js';

interface AIConfig {
  endpoint: string;
  apiKey: string;
  model: string;
}

interface MicroTaskResult {
  taskType: string;
  content: string;
  processingTimeMs: number;
  success: boolean;
  error?: string;
}

interface AnalysisResult {
  recommendation: any;
  processingStats: {
    embeddingsUsed: boolean;
    candidatesFromEmbeddings: number;
    finalSelectedItems: number;
    processingTimeMs: number;
    microTasksCompleted: number;
    microTasksFailed: number;
    contextContinuityUsed: boolean;
  };
}

// NEW: Audit Trail Types
interface AuditEntry {
  timestamp: number;
  phase: string;           // 'retrieval', 'selection', 'micro-task-N'
  action: string;          // 'embeddings-search', 'ai-selection', 'tool-evaluation'
  input: any;              // What went into this step
  output: any;             // What came out of this step
  confidence: number;      // 0-100: How confident we are in this step
  processingTimeMs: number;
  metadata: Record<string, any>; // Additional context
}

// Enhanced AnalysisContext with Audit Trail
interface AnalysisContext {
  userQuery: string;
  mode: string;
  filteredData: any;
  contextHistory: string[];
  
  maxContextLength: number;
  currentContextLength: number;
  
  scenarioAnalysis?: string;
  problemAnalysis?: string;
  investigationApproach?: string;
  criticalConsiderations?: string;
  selectedTools?: Array<{tool: any, phase: string, priority: string, justification?: string}>;
  backgroundKnowledge?: Array<{concept: any, relevance: string}>;
  
  seenToolNames: Set<string>;
  
  // NEW: Audit Trail
  auditTrail: AuditEntry[];
}

interface SimilarityResult extends EmbeddingData {
  similarity: number;
}


class ImprovedMicroTaskAIPipeline {
  private config: AIConfig;
  private maxSelectedItems: number;
  private embeddingCandidates: number;
  private similarityThreshold: number;
  private microTaskDelay: number;
  
  private maxContextTokens: number;
  private maxPromptTokens: number;
  
  // NEW: Audit Configuration
  private auditConfig: {
    enabled: boolean;
    detailLevel: 'minimal' | 'standard' | 'verbose';
    retentionHours: number;
  };
  
  // NEW: Temporary audit storage for pre-context operations
  private tempAuditEntries: AuditEntry[] = [];

  constructor() {
    this.config = {
      endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'),
      apiKey: this.getEnv('AI_ANALYZER_API_KEY'), 
      model: this.getEnv('AI_ANALYZER_MODEL')
    };

    this.maxSelectedItems = parseInt(process.env.AI_MAX_SELECTED_ITEMS || '60', 10);
    this.embeddingCandidates = parseInt(process.env.AI_EMBEDDING_CANDIDATES || '60', 10); 
    this.similarityThreshold = 0.3; 
    this.microTaskDelay = parseInt(process.env.AI_MICRO_TASK_DELAY_MS || '500', 10);
    
    this.maxContextTokens = parseInt(process.env.AI_MAX_CONTEXT_TOKENS || '4000', 10);
    this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10);
    
    // NEW: Initialize Audit Configuration
    this.auditConfig = {
      enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true',
      detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as any) || 'standard',
      retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10)
    };
  }

  private getEnv(key: string): string {
    const value = process.env[key];
    if (!value) {
      throw new Error(`Missing environment variable: ${key}`);
    }
    return value;
  }

  // NEW: Audit Trail Utility Functions
  private addAuditEntry(
    context: AnalysisContext | null, 
    phase: string, 
    action: string, 
    input: any, 
    output: any, 
    confidence: number, 
    startTime: number, 
    metadata: Record<string, any> = {}
  ): void {
    if (!this.auditConfig.enabled) return;
    
    const auditEntry: AuditEntry = {
      timestamp: Date.now(),
      phase,
      action, 
      input: this.auditConfig.detailLevel === 'verbose' ? input : this.summarizeForAudit(input),
      output: this.auditConfig.detailLevel === 'verbose' ? output : this.summarizeForAudit(output),
      confidence,
      processingTimeMs: Date.now() - startTime,
      metadata
    };
    
    if (context) {
      context.auditTrail.push(auditEntry);
    } else {
      // Store in temporary array for later merging
      this.tempAuditEntries.push(auditEntry);
    }
    
    // Log for debugging when audit is enabled
    console.log(`[AUDIT] ${phase}/${action}: ${confidence}% confidence, ${Date.now() - startTime}ms`);
  }
  
  // NEW: Merge temporary audit entries into context
  private mergeTemporaryAuditEntries(context: AnalysisContext): void {
    if (!this.auditConfig.enabled || this.tempAuditEntries.length === 0) return;
    
    const entryCount = this.tempAuditEntries.length;
    // Add temp entries to the beginning of the context audit trail
    context.auditTrail.unshift(...this.tempAuditEntries);
    this.tempAuditEntries = []; // Clear temp storage
    
    console.log(`[AUDIT] Merged ${entryCount} temporary audit entries into context`);
  }

  private summarizeForAudit(data: any): any {
    if (this.auditConfig.detailLevel === 'minimal') {
      if (typeof data === 'string' && data.length > 100) {
        return data.slice(0, 100) + '...[truncated]';
      }
      if (Array.isArray(data) && data.length > 3) {
        return [...data.slice(0, 3), `...[${data.length - 3} more items]`];
      }
    } else if (this.auditConfig.detailLevel === 'standard') {
      if (typeof data === 'string' && data.length > 500) {
        return data.slice(0, 500) + '...[truncated]';
      }
      if (Array.isArray(data) && data.length > 10) {
        return [...data.slice(0, 10), `...[${data.length - 10} more items]`];
      }
    }
    return data;
  }

  private calculateSelectionConfidence(result: any, candidateCount: number): number {
    if (!result || !result.selectedTools) return 30;
    
    const selectionRatio = result.selectedTools.length / candidateCount;
    const hasReasoning = result.reasoning && result.reasoning.length > 50;
    
    let confidence = 60; // Base confidence
    
    // Good selection ratio (not too many, not too few)
    if (selectionRatio > 0.05 && selectionRatio < 0.3) confidence += 20;
    else if (selectionRatio <= 0.05) confidence -= 10; // Too few
    else confidence -= 15; // Too many
    
    // Has detailed reasoning
    if (hasReasoning) confidence += 15;
    
    // Selected tools have good distribution
    if (result.selectedConcepts && result.selectedConcepts.length > 0) confidence += 5;
    
    return Math.min(95, Math.max(25, confidence));
  }

  private estimateTokens(text: string): number {
    return Math.ceil(text.length / 4); 
  }

  private addToContextHistory(context: AnalysisContext, newEntry: string): void {
    const entryTokens = this.estimateTokens(newEntry);
    
    context.contextHistory.push(newEntry);
    context.currentContextLength += entryTokens;
    
    while (context.currentContextLength > this.maxContextTokens && context.contextHistory.length > 1) {
      const removed = context.contextHistory.shift()!;
      context.currentContextLength -= this.estimateTokens(removed);
    }
  }

  private safeParseJSON(jsonString: string, fallback: any = null): any {
    try {
      const cleaned = jsonString
        .replace(/^```json\s*/i, '')
        .replace(/\s*```\s*$/g, '')
        .trim();
      
      const parsed = JSON.parse(cleaned);
      return parsed;
    } catch (error) {
      console.warn('[AI PIPELINE] JSON parsing failed:', error.message);
      console.warn('[AI PIPELINE] Raw content:', jsonString.slice(0, 200));
      return fallback;
    }
  }

  private addToolToSelection(context: AnalysisContext, tool: any, phase: string, priority: string, justification?: string): boolean {
    if (context.seenToolNames.has(tool.name)) {
      console.log(`[AI PIPELINE] Skipping duplicate tool: ${tool.name}`);
      return false;
    }
    
    context.seenToolNames.add(tool.name);
    if (!context.selectedTools) context.selectedTools = [];
    
    context.selectedTools.push({
      tool,
      phase,
      priority,
      justification
    });
    
    return true;
  }

  private async getIntelligentCandidates(userQuery: string, toolsData: any, mode: string) {
    let candidateTools: any[] = [];
    let candidateConcepts: any[] = [];
    let selectionMethod = 'unknown';
    
    if (embeddingsService.isEnabled()) {
      const embeddingsStart = Date.now();
      const similarItems = await embeddingsService.findSimilar(
        userQuery, 
        this.embeddingCandidates, 
        this.similarityThreshold
      ) as SimilarityResult[]; // Type assertion for similarity property
      
      console.log(`[IMPROVED PIPELINE] Embeddings found ${similarItems.length} similar items`);
      
      // FIXED: Create lookup maps for O(1) access while preserving original data
      const toolsMap = new Map<string, any>(toolsData.tools.map((tool: any) => [tool.name, tool]));
      const conceptsMap = new Map<string, any>(toolsData.concepts.map((concept: any) => [concept.name, concept]));
      
      // FIXED: Process in similarity order, preserving the ranking
      const similarTools = similarItems
        .filter((item): item is SimilarityResult => item.type === 'tool')
        .map(item => toolsMap.get(item.name))
        .filter((tool): tool is any => tool !== undefined); // Proper type guard
      
      const similarConcepts = similarItems
        .filter((item): item is SimilarityResult => item.type === 'concept')
        .map(item => conceptsMap.get(item.name))
        .filter((concept): concept is any => concept !== undefined); // Proper type guard
      
      console.log(`[IMPROVED PIPELINE] Similarity-ordered results: ${similarTools.length} tools, ${similarConcepts.length} concepts`);
      
      // Log the first few tools to verify ordering is preserved
      if (similarTools.length > 0) {
        console.log(`[IMPROVED PIPELINE] Top similar tools (in similarity order):`);
        similarTools.slice(0, 5).forEach((tool, idx) => {
          const originalSimilarItem = similarItems.find(item => item.name === tool.name);
          console.log(`  ${idx + 1}. ${tool.name} (similarity: ${originalSimilarItem?.similarity?.toFixed(4) || 'N/A'})`);
        });
      }
      
      if (similarTools.length >= 15) {
        candidateTools = similarTools;
        candidateConcepts = similarConcepts;
        selectionMethod = 'embeddings_candidates';
        
        console.log(`[IMPROVED PIPELINE] Using embeddings candidates in similarity order: ${candidateTools.length} tools`);
      } else {
        console.log(`[IMPROVED PIPELINE] Embeddings insufficient (${similarTools.length} < 15), using full dataset`);
        candidateTools = toolsData.tools;
        candidateConcepts = toolsData.concepts;
        selectionMethod = 'full_dataset';
      }
      
      // NEW: Add Audit Entry for Embeddings Search with ordering verification
      if (this.auditConfig.enabled) {
        this.addAuditEntry(null, 'retrieval', 'embeddings-search', 
          { query: userQuery, threshold: this.similarityThreshold, candidates: this.embeddingCandidates }, 
          { 
            candidatesFound: similarItems.length, 
            toolsInOrder: similarTools.slice(0, 3).map((t: any) => t.name),
            conceptsInOrder: similarConcepts.slice(0, 3).map((c: any) => c.name),
            orderingPreserved: true
          },
          similarTools.length >= 15 ? 85 : 60,
          embeddingsStart,
          { selectionMethod, embeddingsEnabled: true, orderingFixed: true }
        );
      }
    } else {
      console.log(`[IMPROVED PIPELINE] Embeddings disabled, using full dataset`);
      candidateTools = toolsData.tools;
      candidateConcepts = toolsData.concepts;
      selectionMethod = 'full_dataset';
    }

    console.log(`[IMPROVED PIPELINE] AI will analyze ${candidateTools.length} candidate tools (ordering preserved: ${selectionMethod === 'embeddings_candidates'})`);
    const finalSelection = await this.aiSelectionWithFullData(userQuery, candidateTools, candidateConcepts, mode, selectionMethod);
    
    return {
      tools: finalSelection.selectedTools,
      concepts: finalSelection.selectedConcepts,
      domains: toolsData.domains,
      phases: toolsData.phases,
      'domain-agnostic-software': toolsData['domain-agnostic-software']
    };
  }

  private async aiSelectionWithFullData(
    userQuery: string, 
    candidateTools: any[], 
    candidateConcepts: any[], 
    mode: string,
    selectionMethod: string
  ) {
    const selectionStart = Date.now();
    
    const toolsWithFullData = candidateTools.map((tool: any) => ({
      name: tool.name,
      type: tool.type,
      description: tool.description,
      domains: tool.domains,
      phases: tool.phases,
      platforms: tool.platforms || [],
      tags: tool.tags || [],
      skillLevel: tool.skillLevel,
      license: tool.license,
      accessType: tool.accessType,
      projectUrl: tool.projectUrl,
      knowledgebase: tool.knowledgebase,
      related_concepts: tool.related_concepts || [],
      related_software: tool.related_software || []
    }));

    const conceptsWithFullData = candidateConcepts.map((concept: any) => ({
      name: concept.name,
      type: 'concept',
      description: concept.description,
      domains: concept.domains,
      phases: concept.phases,
      tags: concept.tags || [],
      skillLevel: concept.skillLevel,
      related_concepts: concept.related_concepts || [],
      related_software: concept.related_software || []
    }));

    // Generate the German prompt with tool data
    const basePrompt = getPrompt('toolSelection', mode, userQuery, selectionMethod, this.maxSelectedItems);
    const prompt = `${basePrompt}

VERFÜGBARE TOOLS (mit vollständigen Daten):
${JSON.stringify(toolsWithFullData.slice(0, 30), null, 2)}

VERFÜGBARE KONZEPTE (mit vollständigen Daten):
${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;

    try {
      const response = await this.callAI(prompt, 2500);
      
      const result = this.safeParseJSON(response, null);
      
      if (!result || !Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) {
        console.error('[IMPROVED PIPELINE] AI selection returned invalid structure:', response.slice(0, 200));
        
        // NEW: Add Audit Entry for Failed Selection
        if (this.auditConfig.enabled) {
          this.addAuditEntry(null, 'selection', 'ai-tool-selection-failed',
            { candidateCount: candidateTools.length, mode, prompt: prompt.slice(0, 200) },
            { error: 'Invalid JSON structure', response: response.slice(0, 200) },
            10, // Very low confidence
            selectionStart,
            { aiModel: this.config.model, selectionMethod }
          );
        }
        
        throw new Error('AI selection failed to return valid tool selection');
      }

      const totalSelected = result.selectedTools.length + result.selectedConcepts.length;
      if (totalSelected === 0) {
        console.error('[IMPROVED PIPELINE] AI selection returned no tools');
        throw new Error('AI selection returned empty selection');
      }
      
      console.log(`[IMPROVED PIPELINE] AI selected: ${result.selectedTools.length} tools, ${result.selectedConcepts.length} concepts`);
      console.log(`[IMPROVED PIPELINE] AI reasoning: ${result.reasoning}`);

      const selectedTools = candidateTools.filter(tool => result.selectedTools.includes(tool.name));
      const selectedConcepts = candidateConcepts.filter(concept => result.selectedConcepts.includes(concept.name));
      
      console.log(`[IMPROVED PIPELINE] Final selection: ${selectedTools.length} tools with bias prevention applied`);
      
      // NEW: Add Audit Entry for Successful Selection
      if (this.auditConfig.enabled) {
        const confidence = this.calculateSelectionConfidence(result, candidateTools.length);
        
        this.addAuditEntry(null, 'selection', 'ai-tool-selection',
          { candidateCount: candidateTools.length, mode, promptLength: prompt.length },
          { 
            selectedToolCount: result.selectedTools.length, 
            selectedConceptCount: result.selectedConcepts.length,
            reasoning: result.reasoning?.slice(0, 200) + '...',
            finalToolNames: selectedTools.map(t => t.name)
          },
          confidence,
          selectionStart,
          { aiModel: this.config.model, selectionMethod, promptTokens: this.estimateTokens(prompt) }
        );
      }
      
      return {
        selectedTools,
        selectedConcepts
      };

    } catch (error) {
      console.error('[IMPROVED PIPELINE] AI selection failed:', error);
      
      // NEW: Add Audit Entry for Selection Error
      if (this.auditConfig.enabled) {
        this.addAuditEntry(null, 'selection', 'ai-tool-selection-error',
          { candidateCount: candidateTools.length, mode },
          { error: error.message },
          5, // Very low confidence
          selectionStart,
          { aiModel: this.config.model, selectionMethod }
        );
      }
      
      console.log('[IMPROVED PIPELINE] Using emergency keyword-based selection');
      return this.emergencyKeywordSelection(userQuery, candidateTools, candidateConcepts, mode);
    }
  }

  private emergencyKeywordSelection(userQuery: string, candidateTools: any[], candidateConcepts: any[], mode: string) {
    const emergencyStart = Date.now();
    
    const queryLower = userQuery.toLowerCase();
    const keywords = queryLower.split(/\s+/).filter(word => word.length > 3);
    
    const scoredTools = candidateTools.map(tool => {
      const toolText = (
        tool.name + ' ' + 
        tool.description + ' ' + 
        (tool.tags || []).join(' ') + ' ' +
        (tool.platforms || []).join(' ') + ' ' +
        (tool.domains || []).join(' ')
      ).toLowerCase();
      
      const score = keywords.reduce((acc, keyword) => {
        return acc + (toolText.includes(keyword) ? 1 : 0);
      }, 0);
      
      return { tool, score };
    }).filter(item => item.score > 0)
      .sort((a, b) => b.score - a.score);
    
    const maxTools = mode === 'workflow' ? 20 : 8;
    const selectedTools = scoredTools.slice(0, maxTools).map(item => item.tool);
    
    console.log(`[IMPROVED PIPELINE] Emergency selection: ${selectedTools.length} tools, keywords: ${keywords.slice(0, 5).join(', ')}`);
    
    // NEW: Add Audit Entry for Emergency Selection
    if (this.auditConfig.enabled) {
      this.addAuditEntry(null, 'selection', 'emergency-keyword-selection',
        { keywords: keywords.slice(0, 10), candidateCount: candidateTools.length },
        { selectedCount: selectedTools.length, topScores: scoredTools.slice(0, 5).map(s => ({ name: s.tool.name, score: s.score })) },
        40, // Moderate confidence for emergency selection
        emergencyStart,
        { selectionMethod: 'emergency_keyword' }
      );
    }
    
    return {
      selectedTools,
      selectedConcepts: candidateConcepts.slice(0, 3)
    };
  }

  private async delay(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  private async callMicroTaskAI(prompt: string, context: AnalysisContext, maxTokens: number = 300): Promise<MicroTaskResult> {
    const startTime = Date.now();
    
    let contextPrompt = prompt;
    if (context.contextHistory.length > 0) {
      const contextSection = `BISHERIGE ANALYSE:\n${context.contextHistory.join('\n\n')}\n\nAKTUELLE AUFGABE:\n`;
      const combinedPrompt = contextSection + prompt;
      
      if (this.estimateTokens(combinedPrompt) <= this.maxPromptTokens) {
        contextPrompt = combinedPrompt;
      } else {
        console.warn('[AI PIPELINE] Context too long, using prompt only');
      }
    }
    
    try {
      const response = await this.callAI(contextPrompt, maxTokens);
      
      const result = {
        taskType: 'micro-task',
        content: response.trim(),
        processingTimeMs: Date.now() - startTime,
        success: true
      };
      
      // NEW: Add Audit Entry for Successful Micro-Task
      this.addAuditEntry(context, 'micro-task', 'ai-analysis',
        { promptLength: contextPrompt.length, maxTokens },
        { responseLength: response.length, contentPreview: response.slice(0, 100) },
        response.length > 50 ? 80 : 60, // Confidence based on response quality
        startTime,
        { aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 }
      );
      
      return result;

    } catch (error) {
      const result = {
        taskType: 'micro-task',
        content: '',
        processingTimeMs: Date.now() - startTime,
        success: false,
        error: error.message
      };
      
      // NEW: Add Audit Entry for Failed Micro-Task
      this.addAuditEntry(context, 'micro-task', 'ai-analysis-failed',
        { promptLength: contextPrompt.length, maxTokens },
        { error: error.message },
        5, // Very low confidence
        startTime,
        { aiModel: this.config.model, contextUsed: context.contextHistory.length > 0 }
      );
      
      return result;
    }
  }

  private async analyzeScenario(context: AnalysisContext): Promise<MicroTaskResult> {
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('scenarioAnalysis', isWorkflow, context.userQuery);

    const result = await this.callMicroTaskAI(prompt, context, 220);
    
    if (result.success) {
      if (isWorkflow) {
        context.scenarioAnalysis = result.content;
      } else {
        context.problemAnalysis = result.content;
      }
      
      this.addToContextHistory(context, `${isWorkflow ? 'Szenario' : 'Problem'}-Analyse: ${result.content.slice(0, 200)}...`);
    }
    
    return result;
  }

  private async generateApproach(context: AnalysisContext): Promise<MicroTaskResult> {
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('investigationApproach', isWorkflow, context.userQuery);

    const result = await this.callMicroTaskAI(prompt, context, 220);
    
    if (result.success) {
      context.investigationApproach = result.content;
      this.addToContextHistory(context, `${isWorkflow ? 'Untersuchungs' : 'Lösungs'}ansatz: ${result.content.slice(0, 200)}...`);
    }
    
    return result;
  }

  private async generateCriticalConsiderations(context: AnalysisContext): Promise<MicroTaskResult> {
    const isWorkflow = context.mode === 'workflow';
    const prompt = getPrompt('criticalConsiderations', isWorkflow, context.userQuery);

    const result = await this.callMicroTaskAI(prompt, context, 180);
    
    if (result.success) {
      context.criticalConsiderations = result.content;
      this.addToContextHistory(context, `Kritische Überlegungen: ${result.content.slice(0, 200)}...`);
    }
    
    return result;
  }

  private async selectToolsForPhase(context: AnalysisContext, phase: any): Promise<MicroTaskResult> {
    const phaseTools = context.filteredData.tools.filter((tool: any) => 
      tool.phases && tool.phases.includes(phase.id)
    );

    if (phaseTools.length === 0) {
      return {
        taskType: 'tool-selection',
        content: JSON.stringify([]),
        processingTimeMs: 0,
        success: true
      };
    }

    const prompt = getPrompt('phaseToolSelection', context.userQuery, phase, phaseTools);

    const result = await this.callMicroTaskAI(prompt, context, 450);
    
    if (result.success) {
      const selections = this.safeParseJSON(result.content, []);
      
      if (Array.isArray(selections)) {
        const validSelections = selections.filter((sel: any) => 
          sel.toolName && phaseTools.some((tool: any) => tool.name === sel.toolName)
        );
        
        validSelections.forEach((sel: any) => {
          const tool = phaseTools.find((t: any) => t.name === sel.toolName);
          if (tool) {
            this.addToolToSelection(context, tool, phase.id, sel.priority, sel.justification);
          }
        });
        
        // NEW: Add audit entry for tool selection
        this.addAuditEntry(context, 'micro-task', 'phase-tool-selection',
          { phase: phase.id, availableTools: phaseTools.length },
          { validSelections: validSelections.length, selectedTools: validSelections.map(s => s.toolName) },
          validSelections.length > 0 ? 75 : 30,
          Date.now() - result.processingTimeMs,
          { phaseName: phase.name }
        );
      }
    }
    
    return result;
  }

  private async evaluateSpecificTool(context: AnalysisContext, tool: any, rank: number): Promise<MicroTaskResult> {
    const prompt = getPrompt('toolEvaluation', context.userQuery, tool, rank);

    const result = await this.callMicroTaskAI(prompt, context, 650);
    
    if (result.success) {
      const evaluation = this.safeParseJSON(result.content, {
        suitability_score: 'medium',
        detailed_explanation: 'Evaluation failed',
        implementation_approach: '',
        pros: [],
        cons: [],
        alternatives: ''
      });
      
      this.addToolToSelection(context, {
        ...tool,
        evaluation: {
          ...evaluation,
          rank
        }
      }, 'evaluation', evaluation.suitability_score);
      
      // NEW: Add audit entry for tool evaluation
      this.addAuditEntry(context, 'micro-task', 'tool-evaluation',
        { toolName: tool.name, rank },
        { suitabilityScore: evaluation.suitability_score, hasExplanation: !!evaluation.detailed_explanation },
        evaluation.suitability_score === 'high' ? 85 : evaluation.suitability_score === 'medium' ? 70 : 50,
        Date.now() - result.processingTimeMs,
        { toolType: tool.type }
      );
    }
    
    return result;
  }

  private async selectBackgroundKnowledge(context: AnalysisContext): Promise<MicroTaskResult> {
    const availableConcepts = context.filteredData.concepts;
    
    if (availableConcepts.length === 0) {
      return {
        taskType: 'background-knowledge',
        content: JSON.stringify([]),
        processingTimeMs: 0,
        success: true
      };
    }

    const selectedToolNames = context.selectedTools?.map(st => st.tool.name) || [];
    const prompt = getPrompt('backgroundKnowledgeSelection', context.userQuery, context.mode, selectedToolNames, availableConcepts);

    const result = await this.callMicroTaskAI(prompt, context, 400);
    
    if (result.success) {
      const selections = this.safeParseJSON(result.content, []);
      
      if (Array.isArray(selections)) {
        context.backgroundKnowledge = selections.filter((sel: any) => 
          sel.conceptName && availableConcepts.some((concept: any) => concept.name === sel.conceptName)
        ).map((sel: any) => ({
          concept: availableConcepts.find((c: any) => c.name === sel.conceptName),
          relevance: sel.relevance
        }));
        
        // NEW: Add audit entry for background knowledge selection
        this.addAuditEntry(context, 'micro-task', 'background-knowledge-selection',
          { availableConcepts: availableConcepts.length },
          { selectedConcepts: context.backgroundKnowledge?.length || 0 },
          context.backgroundKnowledge && context.backgroundKnowledge.length > 0 ? 75 : 40,
          Date.now() - result.processingTimeMs,
          {}
        );
      }
    }
    
    return result;
  }

  private async generateFinalRecommendations(context: AnalysisContext): Promise<MicroTaskResult> {
    const selectedToolNames = context.selectedTools?.map(st => st.tool.name) || [];
    const prompt = getPrompt('finalRecommendations', context.mode === 'workflow', context.userQuery, selectedToolNames);

    const result = await this.callMicroTaskAI(prompt, context, 180);
    return result;
  }

  private async callAI(prompt: string, maxTokens: number = 1000): Promise<string> {
    const endpoint = this.config.endpoint;
    const apiKey = this.config.apiKey;
    const model = this.config.model;
    
    // Simple headers - add auth only if API key exists
    let headers: Record<string, string> = {
      'Content-Type': 'application/json'
    };
    
    // Add authentication if API key is provided
    if (apiKey) {
      headers['Authorization'] = `Bearer ${apiKey}`;
      console.log('[AI PIPELINE] Using API key authentication');
    } else {
      console.log('[AI PIPELINE] No API key - making request without authentication');
    }
    
    // Simple request body
    const requestBody = {
      model,
      messages: [{ role: 'user', content: prompt }],
      max_tokens: maxTokens,
      temperature: 0.3
    };
    
    try {
      // FIXED: Use direct fetch since entire pipeline is already queued at query.ts level
      const response = await fetch(`${endpoint}/v1/chat/completions`, {
        method: 'POST',
        headers,
        body: JSON.stringify(requestBody)
      });

      if (!response.ok) {
        const errorText = await response.text();
        console.error(`[AI PIPELINE] AI API Error ${response.status}:`, errorText);
        throw new Error(`AI API error: ${response.status} - ${errorText}`);
      }

      const data = await response.json();
      const content = data.choices?.[0]?.message?.content;
      
      if (!content) {
        console.error('[AI PIPELINE] No response content:', data);
        throw new Error('No response from AI model');
      }

      return content;
      
    } catch (error) {
      console.error('[AI PIPELINE] AI service call failed:', error.message);
      throw error;
    }
  }

  async processQuery(userQuery: string, mode: string): Promise<AnalysisResult> {
    const startTime = Date.now();
    let completedTasks = 0;
    let failedTasks = 0;
    
    // NEW: Clear any previous temporary audit entries
    this.tempAuditEntries = [];
    
    console.log(`[IMPROVED PIPELINE] Starting ${mode} query processing with context continuity and audit trail`);

    try {
      // Stage 1: Get intelligent candidates (embeddings + AI selection)
      const toolsData = await getCompressedToolsDataForAI();
      const filteredData = await this.getIntelligentCandidates(userQuery, toolsData, mode);
      
      const context: AnalysisContext = {
        userQuery,
        mode,
        filteredData,
        contextHistory: [],
        maxContextLength: this.maxContextTokens,
        currentContextLength: 0,
        seenToolNames: new Set<string>(),
        // NEW: Initialize audit trail
        auditTrail: []
      };

      // NEW: Merge any temporary audit entries from pre-context operations
      this.mergeTemporaryAuditEntries(context);

      console.log(`[IMPROVED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`);

      // NEW: Add initial audit entry
      this.addAuditEntry(context, 'initialization', 'pipeline-start',
        { userQuery, mode, toolsDataLoaded: !!toolsData },
        { candidateTools: filteredData.tools.length, candidateConcepts: filteredData.concepts.length },
        90, // High confidence for initialization
        startTime,
        { auditEnabled: this.auditConfig.enabled }
      );

      // MICRO-TASK SEQUENCE
      
      // Task 1: Scenario/Problem Analysis
      const analysisResult = await this.analyzeScenario(context);
      if (analysisResult.success) completedTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

      // Task 2: Investigation/Solution Approach
      const approachResult = await this.generateApproach(context);
      if (approachResult.success) completedTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

      // Task 3: Critical Considerations
      const considerationsResult = await this.generateCriticalConsiderations(context);
      if (considerationsResult.success) completedTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

      // Task 4: Tool Selection/Evaluation (mode-dependent)
      if (mode === 'workflow') {
        const phases = toolsData.phases || [];
        for (const phase of phases) {
          const toolSelectionResult = await this.selectToolsForPhase(context, phase);
          if (toolSelectionResult.success) completedTasks++; else failedTasks++;
          await this.delay(this.microTaskDelay);
        }
      } else {
        const topTools = filteredData.tools.slice(0, 3);
        for (let i = 0; i < topTools.length; i++) {
          const evaluationResult = await this.evaluateSpecificTool(context, topTools[i], i + 1);
          if (evaluationResult.success) completedTasks++; else failedTasks++;
          await this.delay(this.microTaskDelay);
        }
      }

      // Task 5: Background Knowledge Selection
      const knowledgeResult = await this.selectBackgroundKnowledge(context);
      if (knowledgeResult.success) completedTasks++; else failedTasks++;
      await this.delay(this.microTaskDelay);

      // Task 6: Final Recommendations
      const finalResult = await this.generateFinalRecommendations(context);
      if (finalResult.success) completedTasks++; else failedTasks++;

      // Build final recommendation
      const recommendation = this.buildRecommendation(context, mode, finalResult.content);

      // NEW: Add final audit entry
      this.addAuditEntry(context, 'completion', 'pipeline-end',
        { completedTasks, failedTasks },
        { finalRecommendation: !!recommendation, auditEntriesGenerated: context.auditTrail.length },
        completedTasks > failedTasks ? 85 : 60,
        startTime,
        { totalProcessingTimeMs: Date.now() - startTime }
      );

      const processingStats = {
        embeddingsUsed: embeddingsService.isEnabled(),
        candidatesFromEmbeddings: filteredData.tools.length,
        finalSelectedItems: (context.selectedTools?.length || 0) + 
                           (context.backgroundKnowledge?.length || 0),
        processingTimeMs: Date.now() - startTime,
        microTasksCompleted: completedTasks,
        microTasksFailed: failedTasks,
        contextContinuityUsed: true
      };

      console.log(`[IMPROVED PIPELINE] Completed: ${completedTasks} tasks, Failed: ${failedTasks} tasks`);
      console.log(`[IMPROVED PIPELINE] Unique tools selected: ${context.seenToolNames.size}`);
      console.log(`[IMPROVED PIPELINE] Audit trail entries: ${context.auditTrail.length}`);

      return {
        recommendation: {
          ...recommendation,
          // NEW: Include audit trail in response
          auditTrail: this.auditConfig.enabled ? context.auditTrail : undefined
        },
        processingStats
      };

    } catch (error) {
      console.error('[IMPROVED PIPELINE] Processing failed:', error);
      
      // NEW: Ensure temp audit entries are cleared even on error
      this.tempAuditEntries = [];
      
      throw error;
    }
  }

  private buildRecommendation(context: AnalysisContext, mode: string, finalContent: string): any {
    const isWorkflow = mode === 'workflow';
    
    const base = {
      [isWorkflow ? 'scenario_analysis' : 'problem_analysis']: 
        isWorkflow ? context.scenarioAnalysis : context.problemAnalysis,
      investigation_approach: context.investigationApproach,
      critical_considerations: context.criticalConsiderations,
      background_knowledge: context.backgroundKnowledge?.map(bk => ({
        concept_name: bk.concept.name,
        relevance: bk.relevance
      })) || []
    };

    if (isWorkflow) {
      return {
        ...base,
        recommended_tools: context.selectedTools?.map(st => ({
          name: st.tool.name,
          phase: st.phase,
          priority: st.priority,
          justification: st.justification || `Empfohlen für ${st.phase}`
        })) || [],
        workflow_suggestion: finalContent
      };
    } else {
      return {
        ...base,
        recommended_tools: context.selectedTools?.map(st => ({
          name: st.tool.name,
          rank: st.tool.evaluation?.rank || 1,
          suitability_score: st.priority,
          detailed_explanation: st.tool.evaluation?.detailed_explanation || '',
          implementation_approach: st.tool.evaluation?.implementation_approach || '',
          pros: st.tool.evaluation?.pros || [],
          cons: st.tool.evaluation?.cons || [],
          alternatives: st.tool.evaluation?.alternatives || ''
        })) || [],
        additional_considerations: finalContent
      };
    }
  }
}

const aiPipeline = new ImprovedMicroTaskAIPipeline();

export { aiPipeline, type AnalysisResult };