semantic search finalization

src/utils/aiPipeline.ts

@@ -4,6 +4,7 @@ import { getCompressedToolsDataForAI } from './dataService.js';
 import { embeddingsService, type EmbeddingData, type SimilarityResult } from './embeddings.js';
 import { AI_PROMPTS, getPrompt } from '../config/prompts.js';
 import { isToolHosted } from './toolHelpers.js';
+import { auditService } from './auditService.js';  // Add this import
 
 interface AIConfig {
   endpoint: string;
@@ -95,9 +96,9 @@ class ImprovedMicroTaskAIPipeline {
   
   private auditConfig: {
     enabled: boolean;
-    detailLevel: 'minimal' | 'standard' | 'verbose';
-    retentionHours: number;
+    detailLevel: string;
   };
+  private tempAuditEntries: AuditEntry[] = [];
 
   private confidenceConfig: {
     semanticWeight: number;       
@@ -109,8 +110,6 @@ class ImprovedMicroTaskAIPipeline {
     highThreshold: number;
   };
   
-  private tempAuditEntries: AuditEntry[] = [];
-
   constructor() {
     this.config = {
       endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'),
@@ -136,11 +135,12 @@ class ImprovedMicroTaskAIPipeline {
     this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10);
     
     this.auditConfig = {
-      enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true',
-      detailLevel: (process.env.FORENSIC_AUDIT_DETAIL_LEVEL as any) || 'standard',
-      retentionHours: parseInt(process.env.FORENSIC_AUDIT_RETENTION_HOURS || '72', 10)
+      enabled: process.env.FORENSIC_AUDIT_ENABLED === 'true' || process.env.NODE_ENV === 'development',
+      detailLevel: process.env.FORENSIC_AUDIT_DETAIL_LEVEL || 'standard'
     };
     
+    console.log('[AI PIPELINE] Audit trail enabled:', this.auditConfig.enabled);
+    
     this.confidenceConfig = {
       semanticWeight: parseFloat(process.env.CONFIDENCE_SEMANTIC_WEIGHT || '0.3'),   
       suitabilityWeight: parseFloat(process.env.CONFIDENCE_SUITABILITY_WEIGHT || '0.7'), 
@@ -166,47 +166,50 @@ class ImprovedMicroTaskAIPipeline {
   }
 
   private addAuditEntry(
-    context: AnalysisContext | null, 
-    phase: string, 
-    action: string, 
-    input: any, 
-    output: any, 
-    confidence: number, 
-    startTime: number, 
+    context: AnalysisContext,
+    phase: string,
+    action: string,
+    input: any,
+    output: any,
+    confidence: number,
+    startTime: number,
     metadata: Record<string, any> = {}
   ): void {
     if (!this.auditConfig.enabled) return;
-    
-    const auditEntry: AuditEntry = {
+
+    const entry: AuditEntry = {
       timestamp: Date.now(),
       phase,
-      action, 
-      input: this.auditConfig.detailLevel === 'verbose' ? input : this.summarizeForAudit(input),
-      output: this.auditConfig.detailLevel === 'verbose' ? output : this.summarizeForAudit(output),
-      confidence,
+      action,
+      input,
+      output,
+      confidence: Math.round(confidence),
       processingTimeMs: Date.now() - startTime,
       metadata
     };
-    
-    if (context) {
-      context.auditTrail.push(auditEntry);
-    } else {
-      this.tempAuditEntries.push(auditEntry);
+
+    // Add to context audit trail instead of temp storage
+    if (!context.auditTrail) {
+      context.auditTrail = [];
     }
+    context.auditTrail.push(entry);
     
-    console.log(`[AUDIT] ${phase}/${action}: ${confidence}% confidence, ${Date.now() - startTime}ms`);
+    console.log(`[AUDIT] ${phase}/${action}: ${confidence}% confidence, ${entry.processingTimeMs}ms`);
   }
   
   private mergeTemporaryAuditEntries(context: AnalysisContext): void {
     if (!this.auditConfig.enabled || this.tempAuditEntries.length === 0) return;
-    
-    const entryCount = this.tempAuditEntries.length;
-    context.auditTrail.unshift(...this.tempAuditEntries);
-    this.tempAuditEntries = []; 
-    
-    console.log(`[AUDIT] Merged ${entryCount} temporary audit entries into context`);
-  }
 
+    if (!context.auditTrail) {
+      context.auditTrail = [];
+    }
+    
+    context.auditTrail.unshift(...this.tempAuditEntries);
+    this.tempAuditEntries = [];
+    
+    console.log('[AUDIT] Merged temporary entries into context');
+  }
+/**
   private summarizeForAudit(data: any): any {
     if (this.auditConfig.detailLevel === 'minimal') {
       if (typeof data === 'string' && data.length > 100) {
@@ -224,7 +227,7 @@ class ImprovedMicroTaskAIPipeline {
       }
     }
     return data;
-  }
+  }**/
 
   private calculateSelectionConfidence(result: any, candidateCount: number): number {
     if (!result || !result.selectedTools) return 30;
@@ -1191,6 +1194,11 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
       })) || []
     };
 
+    // Process audit trail before returning
+    const processedAuditTrail = this.auditConfig.enabled && context.auditTrail 
+      ? context.auditTrail 
+      : [];
+
     if (isWorkflow) {
       const recommendedToolsWithConfidence = context.selectedTools?.map(st => {
         const confidence = this.calculateRecommendationConfidence(
@@ -1228,7 +1236,8 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
       return {
         ...base,
         recommended_tools: recommendedToolsWithConfidence,
-        workflow_suggestion: finalContent
+        workflow_suggestion: finalContent,
+        auditTrail: processedAuditTrail  // Always include audit trail array
       };
     } else {
       const recommendedToolsWithConfidence = context.selectedTools?.map(st => {
@@ -1269,7 +1278,8 @@ ${JSON.stringify(conceptsToSend, null, 2)}`;
       return {
         ...base,
         recommended_tools: recommendedToolsWithConfidence,
-        additional_considerations: finalContent
+        additional_considerations: finalContent,
+        auditTrail: processedAuditTrail  // Always include audit trail array
       };
     }
   }