mstoeck3/forensic-pathways
2
0
Fork 0
Code Issues 17 Pull Requests Projects Releases Activity

remove mindless dev comments

Browse Source
This commit is contained in:
overcuriousity
2025-08-02 12:13:16 +02:00
parent 3973479ae4
commit 57c507915f
12 changed files with 13 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/utils/aiPipeline.ts
View File

@@ -1,4 +1,4 @@
// src/utils/aiPipeline.ts - FIXED: Critical error corrections
// src/utils/aiPipeline.ts
import { getCompressedToolsDataForAI } from './dataService.js';
import { embeddingsService, type EmbeddingData } from './embeddings.js';
@@ -36,7 +36,6 @@ interface AnalysisContext {
filteredData: any;
contextHistory: string[];
// FIXED: Add max context length tracking
maxContextLength: number;
currentContextLength: number;
@@ -47,7 +46,6 @@ interface AnalysisContext {
selectedTools?: Array<{tool: any, phase: string, priority: string, justification?: string}>;
backgroundKnowledge?: Array<{concept: any, relevance: string}>;
// FIXED: Add seen tools tracking to prevent duplicates
seenToolNames: Set<string>;
}
@@ -58,7 +56,6 @@ class ImprovedMicroTaskAIPipeline {
private similarityThreshold: number;
private microTaskDelay: number;
// FIXED: Add proper token management
private maxContextTokens: number;
private maxPromptTokens: number;
@@ -74,7 +71,6 @@ class ImprovedMicroTaskAIPipeline {
this.similarityThreshold = 0.3;
this.microTaskDelay = parseInt(process.env.AI_MICRO_TASK_DELAY_MS || '500', 10);
// FIXED: Token management
this.maxContextTokens = parseInt(process.env.AI_MAX_CONTEXT_TOKENS || '4000', 10);
this.maxPromptTokens = parseInt(process.env.AI_MAX_PROMPT_TOKENS || '1500', 10);
}
@@ -87,27 +83,22 @@ class ImprovedMicroTaskAIPipeline {
return value;
}
// FIXED: Estimate token count (rough approximation)
private estimateTokens(text: string): number {
return Math.ceil(text.length / 4); // Rough estimate: 4 chars per token
return Math.ceil(text.length / 4);
}
// FIXED: Manage context history with token limits
private addToContextHistory(context: AnalysisContext, newEntry: string): void {
const entryTokens = this.estimateTokens(newEntry);
// Add new entry
context.contextHistory.push(newEntry);
context.currentContextLength += entryTokens;
// Prune old entries if exceeding limits
while (context.currentContextLength > this.maxContextTokens && context.contextHistory.length > 1) {
const removed = context.contextHistory.shift()!;
context.currentContextLength -= this.estimateTokens(removed);
}
}
// FIXED: Safe JSON parsing with validation
private safeParseJSON(jsonString: string, fallback: any = null): any {
try {
const cleaned = jsonString
@@ -124,7 +115,6 @@ class ImprovedMicroTaskAIPipeline {
}
}
// FIXED: Add tool deduplication
private addToolToSelection(context: AnalysisContext, tool: any, phase: string, priority: string, justification?: string): boolean {
if (context.seenToolNames.has(tool.name)) {
console.log(`[AI PIPELINE] Skipping duplicate tool: ${tool.name}`);
@@ -166,8 +156,7 @@ class ImprovedMicroTaskAIPipeline {
console.log(`[IMPROVED PIPELINE] Embeddings found: ${toolNames.size} tools, ${conceptNames.size} concepts`);
// FIXED: Use your expected flow - get full data of embeddings results
if (toolNames.size >= 15) { // Reasonable threshold for quality
if (toolNames.size >= 15) {
candidateTools = toolsData.tools.filter((tool: any) => toolNames.has(tool.name));
candidateConcepts = toolsData.concepts.filter((concept: any) => conceptNames.has(concept.name));
selectionMethod = 'embeddings_candidates';
@@ -186,7 +175,6 @@ class ImprovedMicroTaskAIPipeline {
selectionMethod = 'full_dataset';
}
// FIXED: NOW AI ANALYZES FULL DATA of the candidates
console.log(`[IMPROVED PIPELINE] AI will analyze FULL DATA of ${candidateTools.length} candidate tools`);
const finalSelection = await this.aiSelectionWithFullData(userQuery, candidateTools, candidateConcepts, mode, selectionMethod);
@@ -199,8 +187,6 @@ class ImprovedMicroTaskAIPipeline {
};
}
// src/utils/aiPipeline.ts - FIXED: De-biased AI selection prompt
private async aiSelectionWithFullData(
userQuery: string,
candidateTools: any[],
@@ -212,7 +198,6 @@ class ImprovedMicroTaskAIPipeline {
? 'The user wants a COMPREHENSIVE WORKFLOW with multiple tools/methods across different phases. Select 15-25 tools that cover the full investigation lifecycle.'
: 'The user wants SPECIFIC TOOLS/METHODS that directly solve their particular problem. Select 3-8 tools that are most relevant and effective.';
// FIXED: Give AI the COMPLETE tool data, not truncated
const toolsWithFullData = candidateTools.map((tool: any) => ({
name: tool.name,
type: tool.type,
@@ -307,7 +292,7 @@ Respond with ONLY this JSON format:
}`;
try {
const response = await this.callAI(prompt, 2500); // More tokens for bias prevention logic
const response = await this.callAI(prompt, 2500);
const result = this.safeParseJSON(response, null);
@@ -325,7 +310,6 @@ Respond with ONLY this JSON format:
console.log(`[IMPROVED PIPELINE] AI selected: ${result.selectedTools.length} tools, ${result.selectedConcepts.length} concepts`);
console.log(`[IMPROVED PIPELINE] AI reasoning: ${result.reasoning}`);
// Return the actual tool/concept objects
const selectedTools = candidateTools.filter(tool => result.selectedTools.includes(tool.name));
const selectedConcepts = candidateConcepts.filter(concept => result.selectedConcepts.includes(concept.name));
@@ -339,7 +323,6 @@ Respond with ONLY this JSON format:
} catch (error) {
console.error('[IMPROVED PIPELINE] AI selection failed:', error);
// Emergency fallback with bias awareness
console.log('[IMPROVED PIPELINE] Using emergency keyword-based selection');
return this.emergencyKeywordSelection(userQuery, candidateTools, candidateConcepts, mode);
}
@@ -349,7 +332,6 @@ Respond with ONLY this JSON format:
const queryLower = userQuery.toLowerCase();
const keywords = queryLower.split(/\s+/).filter(word => word.length > 3);
// Score tools based on keyword matches in full data
const scoredTools = candidateTools.map(tool => {
const toolText = (
tool.name + ' ' +
@@ -385,18 +367,15 @@ Respond with ONLY this JSON format:
private async callMicroTaskAI(prompt: string, context: AnalysisContext, maxTokens: number = 300): Promise<MicroTaskResult> {
const startTime = Date.now();
// FIXED: Build context prompt with token management
let contextPrompt = prompt;
if (context.contextHistory.length > 0) {
const contextSection = `BISHERIGE ANALYSE:\n${context.contextHistory.join('\n\n')}\n\nAKTUELLE AUFGABE:\n`;
const combinedPrompt = contextSection + prompt;
// Check if combined prompt exceeds limits
if (this.estimateTokens(combinedPrompt) <= this.maxPromptTokens) {
contextPrompt = combinedPrompt;
} else {
console.warn('[AI PIPELINE] Context too long, using prompt only');
// Could implement smarter context truncation here
}
}
@@ -451,7 +430,6 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen, Aufzählun
context.problemAnalysis = result.content;
}
// FIXED: Use new context management
this.addToContextHistory(context, `${isWorkflow ? 'Szenario' : 'Problem'}-Analyse: ${result.content.slice(0, 200)}...`);
}
@@ -559,7 +537,6 @@ Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format (kein zusätzlicher Text):
const result = await this.callMicroTaskAI(prompt, context, 450);
if (result.success) {
// FIXED: Safe JSON parsing with validation
const selections = this.safeParseJSON(result.content, []);
if (Array.isArray(selections)) {
@@ -570,7 +547,6 @@ Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format (kein zusätzlicher Text):
validSelections.forEach((sel: any) => {
const tool = phaseTools.find((t: any) => t.name === sel.toolName);
if (tool) {
// FIXED: Use deduplication helper
this.addToolToSelection(context, tool, phase.id, sel.priority, sel.justification);
}
});
@@ -603,7 +579,6 @@ Bewerten Sie nach forensischen Standards und antworten Sie AUSSCHLIESSLICH mit d
const result = await this.callMicroTaskAI(prompt, context, 650);
if (result.success) {
// FIXED: Safe JSON parsing
const evaluation = this.safeParseJSON(result.content, {
suitability_score: 'medium',
detailed_explanation: 'Evaluation failed',
@@ -613,7 +588,6 @@ Bewerten Sie nach forensischen Standards und antworten Sie AUSSCHLIESSLICH mit d
alternatives: ''
});
// FIXED: Use deduplication helper
this.addToolToSelection(context, {
...tool,
evaluation: {
@@ -661,7 +635,6 @@ Antworten Sie AUSSCHLIESSLICH mit diesem JSON-Format:
const result = await this.callMicroTaskAI(prompt, context, 400);
if (result.success) {
// FIXED: Safe JSON parsing
const selections = this.safeParseJSON(result.content, []);
if (Array.isArray(selections)) {
@@ -745,7 +718,6 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
const toolsData = await getCompressedToolsDataForAI();
const filteredData = await this.getIntelligentCandidates(userQuery, toolsData, mode);
// FIXED: Initialize context with proper state management
const context: AnalysisContext = {
userQuery,
mode,
@@ -753,7 +725,7 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
contextHistory: [],
maxContextLength: this.maxContextTokens,
currentContextLength: 0,
seenToolNames: new Set<string>() // FIXED: Add deduplication tracking
seenToolNames: new Set<string>()
};
console.log(`[IMPROVED PIPELINE] Starting micro-tasks with ${filteredData.tools.length} tools visible`);
@@ -777,7 +749,6 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
// Task 4: Tool Selection/Evaluation (mode-dependent)
if (mode === 'workflow') {
// Select tools for each phase
const phases = toolsData.phases || [];
for (const phase of phases) {
const toolSelectionResult = await this.selectToolsForPhase(context, phase);
@@ -785,7 +756,6 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
await this.delay(this.microTaskDelay);
}
} else {
// Evaluate top 3 tools for specific problem
const topTools = filteredData.tools.slice(0, 3);
for (let i = 0; i < topTools.length; i++) {
const evaluationResult = await this.evaluateSpecificTool(context, topTools[i], i + 1);
@@ -831,7 +801,6 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
}
}
// Build recommendation (same structure but using fixed context)
private buildRecommendation(context: AnalysisContext, mode: string, finalContent: string): any {
const isWorkflow = mode === 'workflow';
@@ -876,7 +845,6 @@ WICHTIG: Antworten Sie NUR in fließendem deutschen Text ohne Listen oder Markdo
}
}
// Global instance
const aiPipeline = new ImprovedMicroTaskAIPipeline();
export { aiPipeline, type AnalysisResult };