progress
This commit is contained in:
overcuriousity
521
src/utils/aiPipeline.ts
Normal file
521
src/utils/aiPipeline.ts
Normal file
@@ -0,0 +1,521 @@
|
||||
// src/utils/aiPipeline.ts
|
||||
import { getCompressedToolsDataForAI } from './dataService.js';
|
||||
import { embeddingsService, type EmbeddingData } from './embeddings.js';
|
||||
|
||||
interface AIConfig {
|
||||
endpoint: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
interface SelectionResult {
|
||||
selectedTools: string[];
|
||||
selectedConcepts: string[];
|
||||
reasoning: string;
|
||||
}
|
||||
|
||||
interface AnalysisResult {
|
||||
recommendation: any;
|
||||
processingStats: {
|
||||
embeddingsUsed: boolean;
|
||||
candidatesFromEmbeddings: number;
|
||||
finalSelectedItems: number;
|
||||
processingTimeMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
class AIProcessingPipeline {
|
||||
private selectorConfig: AIConfig;
|
||||
private analyzerConfig: AIConfig;
|
||||
private maxSelectedItems: number;
|
||||
private embeddingCandidates: number;
|
||||
private similarityThreshold: number;
|
||||
|
||||
constructor() {
|
||||
this.selectorConfig = {
|
||||
endpoint: this.getEnv('AI_SELECTOR_ENDPOINT'),
|
||||
apiKey: this.getEnv('AI_SELECTOR_API_KEY'),
|
||||
model: this.getEnv('AI_SELECTOR_MODEL')
|
||||
};
|
||||
|
||||
this.analyzerConfig = {
|
||||
endpoint: this.getEnv('AI_ANALYZER_ENDPOINT'),
|
||||
apiKey: this.getEnv('AI_ANALYZER_API_KEY'),
|
||||
model: this.getEnv('AI_ANALYZER_MODEL')
|
||||
};
|
||||
|
||||
this.maxSelectedItems = parseInt(process.env.AI_MAX_SELECTED_ITEMS || '15', 10);
|
||||
this.embeddingCandidates = parseInt(process.env.AI_EMBEDDING_CANDIDATES || '30', 10);
|
||||
this.similarityThreshold = parseFloat(process.env.AI_SIMILARITY_THRESHOLD || '0.3');
|
||||
}
|
||||
|
||||
private getEnv(key: string): string {
|
||||
const value = process.env[key];
|
||||
if (!value) {
|
||||
throw new Error(`Missing environment variable: ${key}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private async callAI(config: AIConfig, messages: any[], maxTokens: number = 1000): Promise<string> {
|
||||
const response = await fetch(`${config.endpoint}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${config.apiKey}`
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: config.model,
|
||||
messages,
|
||||
max_tokens: maxTokens,
|
||||
temperature: 0.3
|
||||
})
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(`AI API error (${config.model}): ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
|
||||
if (!content) {
|
||||
throw new Error(`No response from AI model: ${config.model}`);
|
||||
}
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
private createSelectorPrompt(toolsData: any, userQuery: string, mode: string): string {
|
||||
const toolsList = toolsData.tools.map((tool: any) => ({
|
||||
name: tool.name,
|
||||
type: tool.type,
|
||||
description: tool.description.slice(0, 200) + '...',
|
||||
domains: tool.domains,
|
||||
phases: tool.phases,
|
||||
tags: tool.tags?.slice(0, 5) || [],
|
||||
skillLevel: tool.skillLevel
|
||||
}));
|
||||
|
||||
const conceptsList = toolsData.concepts.map((concept: any) => ({
|
||||
name: concept.name,
|
||||
type: 'concept',
|
||||
description: concept.description.slice(0, 200) + '...',
|
||||
domains: concept.domains,
|
||||
phases: concept.phases,
|
||||
tags: concept.tags?.slice(0, 5) || []
|
||||
}));
|
||||
|
||||
const modeInstruction = mode === 'workflow'
|
||||
? 'The user wants a COMPREHENSIVE WORKFLOW with multiple tools/methods across different phases.'
|
||||
: 'The user wants SPECIFIC TOOLS/METHODS that directly solve their particular problem.';
|
||||
|
||||
return `You are a DFIR expert tasked with selecting the most relevant tools and concepts for a user query.
|
||||
|
||||
${modeInstruction}
|
||||
|
||||
AVAILABLE TOOLS:
|
||||
${JSON.stringify(toolsList, null, 2)}
|
||||
|
||||
AVAILABLE CONCEPTS:
|
||||
${JSON.stringify(conceptsList, null, 2)}
|
||||
|
||||
USER QUERY: "${userQuery}"
|
||||
|
||||
Select the most relevant items (max ${this.maxSelectedItems} total). For workflow mode, prioritize breadth across phases. For tool mode, prioritize specificity and direct relevance.
|
||||
|
||||
Respond with ONLY this JSON format:
|
||||
{
|
||||
"selectedTools": ["Tool Name 1", "Tool Name 2", ...],
|
||||
"selectedConcepts": ["Concept Name 1", "Concept Name 2", ...],
|
||||
"reasoning": "Brief explanation of selection criteria and approach"
|
||||
}`;
|
||||
}
|
||||
|
||||
private async selectRelevantItems(toolsData: any, userQuery: string, mode: string): Promise<SelectionResult> {
|
||||
const prompt = this.createSelectorPrompt(toolsData, userQuery, mode);
|
||||
|
||||
const messages = [
|
||||
{ role: 'user', content: prompt }
|
||||
];
|
||||
|
||||
const response = await this.callAI(this.selectorConfig, messages, 1500);
|
||||
|
||||
try {
|
||||
const cleaned = response.replace(/^```json\s*/i, '').replace(/\s*```\s*$/g, '').trim();
|
||||
const result = JSON.parse(cleaned);
|
||||
|
||||
// Validate the structure
|
||||
if (!Array.isArray(result.selectedTools) || !Array.isArray(result.selectedConcepts)) {
|
||||
throw new Error('Invalid selection result structure');
|
||||
}
|
||||
|
||||
// Limit selections
|
||||
const totalSelected = result.selectedTools.length + result.selectedConcepts.length;
|
||||
if (totalSelected > this.maxSelectedItems) {
|
||||
console.warn(`[AI PIPELINE] Selection exceeded limit (${totalSelected}), truncating`);
|
||||
result.selectedTools = result.selectedTools.slice(0, Math.floor(this.maxSelectedItems * 0.8));
|
||||
result.selectedConcepts = result.selectedConcepts.slice(0, Math.ceil(this.maxSelectedItems * 0.2));
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error('[AI PIPELINE] Failed to parse selector response:', response);
|
||||
throw new Error('Invalid JSON response from selector AI');
|
||||
}
|
||||
}
|
||||
|
||||
private filterDataBySelection(toolsData: any, selection: SelectionResult): any {
|
||||
const selectedToolNames = new Set(selection.selectedTools);
|
||||
const selectedConceptNames = new Set(selection.selectedConcepts);
|
||||
|
||||
return {
|
||||
tools: toolsData.tools.filter((tool: any) => selectedToolNames.has(tool.name)),
|
||||
concepts: toolsData.concepts.filter((concept: any) => selectedConceptNames.has(concept.name)),
|
||||
domains: toolsData.domains,
|
||||
phases: toolsData.phases,
|
||||
'domain-agnostic-software': toolsData['domain-agnostic-software']
|
||||
};
|
||||
}
|
||||
|
||||
private async processWithEmbeddings(userQuery: string, toolsData: any, mode: string): Promise<{ filteredData: any; stats: any }> {
|
||||
console.log('[AI PIPELINE] Using embeddings for initial filtering');
|
||||
|
||||
const similarItems = await embeddingsService.findSimilar(
|
||||
userQuery,
|
||||
this.embeddingCandidates,
|
||||
this.similarityThreshold
|
||||
);
|
||||
|
||||
if (similarItems.length === 0) {
|
||||
console.log('[AI PIPELINE] No similar items found with embeddings, using full dataset');
|
||||
return {
|
||||
filteredData: toolsData,
|
||||
stats: { embeddingsUsed: true, candidatesFromEmbeddings: 0, fallbackToFull: true }
|
||||
};
|
||||
}
|
||||
|
||||
// Create filtered dataset from embedding results
|
||||
const similarToolNames = new Set();
|
||||
const similarConceptNames = new Set();
|
||||
|
||||
similarItems.forEach(item => {
|
||||
if (item.type === 'tool') {
|
||||
similarToolNames.add(item.name);
|
||||
} else if (item.type === 'concept') {
|
||||
similarConceptNames.add(item.name);
|
||||
}
|
||||
});
|
||||
|
||||
const embeddingFilteredData = {
|
||||
tools: toolsData.tools.filter((tool: any) => similarToolNames.has(tool.name)),
|
||||
concepts: toolsData.concepts.filter((concept: any) => similarConceptNames.has(concept.name)),
|
||||
domains: toolsData.domains,
|
||||
phases: toolsData.phases,
|
||||
'domain-agnostic-software': toolsData['domain-agnostic-software']
|
||||
};
|
||||
|
||||
console.log(`[AI PIPELINE] Embeddings filtered to ${embeddingFilteredData.tools.length} tools, ${embeddingFilteredData.concepts.length} concepts`);
|
||||
|
||||
return {
|
||||
filteredData: embeddingFilteredData,
|
||||
stats: { embeddingsUsed: true, candidatesFromEmbeddings: similarItems.length }
|
||||
};
|
||||
}
|
||||
|
||||
private async processWithoutEmbeddings(userQuery: string, toolsData: any, mode: string): Promise<{ filteredData: any; stats: any }> {
|
||||
console.log('[AI PIPELINE] Processing without embeddings - using selector AI');
|
||||
|
||||
const selection = await this.selectRelevantItems(toolsData, userQuery, mode);
|
||||
const filteredData = this.filterDataBySelection(toolsData, selection);
|
||||
|
||||
console.log(`[AI PIPELINE] Selector chose ${selection.selectedTools.length} tools, ${selection.selectedConcepts.length} concepts`);
|
||||
console.log(`[AI PIPELINE] Selection reasoning: ${selection.reasoning}`);
|
||||
|
||||
return {
|
||||
filteredData,
|
||||
stats: { embeddingsUsed: false, candidatesFromEmbeddings: 0, selectorReasoning: selection.reasoning }
|
||||
};
|
||||
}
|
||||
|
||||
private createAnalyzerPrompt(filteredData: any, userQuery: string, mode: string): string {
|
||||
// Use existing prompt creation logic but with filtered data
|
||||
if (mode === 'workflow') {
|
||||
return this.createWorkflowAnalyzerPrompt(filteredData, userQuery);
|
||||
} else {
|
||||
return this.createToolAnalyzerPrompt(filteredData, userQuery);
|
||||
}
|
||||
}
|
||||
|
||||
private createWorkflowAnalyzerPrompt(toolsData: any, userQuery: string): string {
|
||||
const toolsList = toolsData.tools.map((tool: any) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
domains: tool.domains,
|
||||
phases: tool.phases,
|
||||
domainAgnostic: tool['domain-agnostic-software'],
|
||||
platforms: tool.platforms,
|
||||
skillLevel: tool.skillLevel,
|
||||
license: tool.license,
|
||||
tags: tool.tags,
|
||||
related_concepts: tool.related_concepts || []
|
||||
}));
|
||||
|
||||
const conceptsList = toolsData.concepts.map((concept: any) => ({
|
||||
name: concept.name,
|
||||
description: concept.description,
|
||||
domains: concept.domains,
|
||||
phases: concept.phases,
|
||||
skillLevel: concept.skillLevel,
|
||||
tags: concept.tags
|
||||
}));
|
||||
|
||||
const regularPhases = toolsData.phases || [];
|
||||
const domainAgnosticSoftware = toolsData['domain-agnostic-software'] || [];
|
||||
const allPhaseItems = [...regularPhases, ...domainAgnosticSoftware];
|
||||
|
||||
const phasesDescription = allPhaseItems.map((phase: any) =>
|
||||
`- ${phase.id}: ${phase.name}`
|
||||
).join('\n');
|
||||
|
||||
const domainsDescription = toolsData.domains.map((domain: any) =>
|
||||
`- ${domain.id}: ${domain.name}`
|
||||
).join('\n');
|
||||
|
||||
const validPhases = [...regularPhases.map((p: any) => p.id), ...domainAgnosticSoftware.map((s: any) => s.id)].join('|');
|
||||
|
||||
return `Du bist ein DFIR (Digital Forensics and Incident Response) Experte. Du erhältst eine vorgefilterte Auswahl relevanter Tools und Konzepte und sollst daraus eine optimale Empfehlung erstellen.
|
||||
|
||||
VERFÜGBARE TOOLS/METHODEN (VORGEFILTERT):
|
||||
${JSON.stringify(toolsList, null, 2)}
|
||||
|
||||
VERFÜGBARE KONZEPTE (VORGEFILTERT):
|
||||
${JSON.stringify(conceptsList, null, 2)}
|
||||
|
||||
UNTERSUCHUNGSPHASEN:
|
||||
${phasesDescription}
|
||||
|
||||
FORENSISCHE DOMÄNEN:
|
||||
${domainsDescription}
|
||||
|
||||
WICHTIGE REGELN:
|
||||
1. Pro Phase 2-3 Tools/Methoden empfehlen (immer mindestens 2 wenn verfügbar)
|
||||
2. Tools/Methoden können in MEHREREN Phasen empfohlen werden wenn sinnvoll
|
||||
3. Für Reporting-Phase: Visualisierungs- und Dokumentationssoftware einschließen
|
||||
4. Gib stets dem spezieller für den Fall geeigneten Werkzeug den Vorzug
|
||||
5. Deutsche Antworten für deutsche Anfragen, English for English queries
|
||||
6. Methoden haben, sofern für das SZENARIO passend, IMMER Vorrang vor Software
|
||||
7. Bevorzuge alles, was nicht proprietär ist (license != "Proprietary"), aber erkenne an, wenn proprietäre Software besser geeignet ist
|
||||
8. WICHTIG: Erwähne relevante Hintergrundwissen-Konzepte wenn Tools verwendet werden, die related_concepts haben
|
||||
9. Konzepte sind NICHT Tools - empfehle sie nicht als actionable Schritte, sondern als Wissensbasis
|
||||
|
||||
ENHANCED CONTEXTUAL ANALYSIS:
|
||||
10. Analysiere das Szenario detailliert und identifiziere Schlüsselelemente, Bedrohungen und forensische Herausforderungen
|
||||
11. Entwickle einen strategischen Untersuchungsansatz basierend auf dem spezifischen Szenario
|
||||
12. Identifiziere zeitkritische oder besonders wichtige Faktoren für diesen Fall
|
||||
|
||||
USER QUERY: "${userQuery}"
|
||||
|
||||
ANTWORT-FORMAT (strict JSON):
|
||||
{
|
||||
"scenario_analysis": "Detaillierte Analyse des Szenarios: Erkannte Schlüsselelemente, Art des Vorfalls, betroffene Systeme, potentielle Bedrohungen und forensische Herausforderungen",
|
||||
"investigation_approach": "Strategischer Untersuchungsansatz für dieses spezifische Szenario: Prioritäten, Reihenfolge der Phasen, besondere Überlegungen",
|
||||
"critical_considerations": "Zeitkritische Faktoren, wichtige Sicherheitsaspekte oder besondere Vorsichtsmaßnahmen für diesen Fall",
|
||||
"recommended_tools": [
|
||||
{
|
||||
"name": "EXAKTER Name aus der Tools-Database",
|
||||
"priority": "high|medium|low",
|
||||
"phase": "${validPhases}",
|
||||
"justification": "Warum diese Methode für diese Phase und dieses spezifische Szenario geeignet ist - mit Bezug zu den erkannten Schlüsselelementen"
|
||||
}
|
||||
],
|
||||
"workflow_suggestion": "Vorgeschlagener Untersuchungsablauf mit konkreten Schritten für dieses Szenario",
|
||||
"background_knowledge": [
|
||||
{
|
||||
"concept_name": "EXAKTER Name aus der Konzepte-Database",
|
||||
"relevance": "Warum dieses Konzept für das Szenario relevant ist, und bei welchen der empfohlenen Methoden/Tools."
|
||||
}
|
||||
],
|
||||
"additional_notes": "Wichtige Überlegungen und Hinweise"
|
||||
}
|
||||
|
||||
Antworte NUR mit validen JSON. Keine zusätzlichen Erklärungen außerhalb des JSON.`;
|
||||
}
|
||||
|
||||
private createToolAnalyzerPrompt(toolsData: any, userQuery: string): string {
|
||||
const toolsList = toolsData.tools.map((tool: any) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
domains: tool.domains,
|
||||
phases: tool.phases,
|
||||
platforms: tool.platforms,
|
||||
skillLevel: tool.skillLevel,
|
||||
license: tool.license,
|
||||
tags: tool.tags,
|
||||
url: tool.url,
|
||||
projectUrl: tool.projectUrl,
|
||||
related_concepts: tool.related_concepts || []
|
||||
}));
|
||||
|
||||
const conceptsList = toolsData.concepts.map((concept: any) => ({
|
||||
name: concept.name,
|
||||
description: concept.description,
|
||||
domains: concept.domains,
|
||||
phases: concept.phases,
|
||||
skillLevel: concept.skillLevel,
|
||||
tags: concept.tags
|
||||
}));
|
||||
|
||||
return `Du bist ein DFIR (Digital Forensics and Incident Response) Experte. Du erhältst eine vorgefilterte Auswahl relevanter Tools und Konzepte und sollst daraus 1-3 optimale Empfehlungen für ein spezifisches Problem erstellen.
|
||||
|
||||
VERFÜGBARE TOOLS/METHODEN (VORGEFILTERT):
|
||||
${JSON.stringify(toolsList, null, 2)}
|
||||
|
||||
VERFÜGBARE KONZEPTE (VORGEFILTERT):
|
||||
${JSON.stringify(conceptsList, null, 2)}
|
||||
|
||||
WICHTIGE REGELN:
|
||||
1. Analysiere das spezifische Problem/die Anforderung sorgfältig
|
||||
2. Empfehle 1-3 Methoden/Tools, sortiert nach Eignung (beste Empfehlung zuerst)
|
||||
3. Gib detaillierte Erklärungen, WARUM und WIE jede Methode/Tool das Problem löst
|
||||
4. Berücksichtige praktische Aspekte: Skill Level, Plattformen, Verfügbarkeit
|
||||
5. Deutsche Antworten für deutsche Anfragen, English for English queries
|
||||
6. Gib konkrete Anwendungshinweise, nicht nur allgemeine Beschreibungen
|
||||
7. Methoden haben, sofern für das SZENARIO passend, IMMER Vorrang vor Software
|
||||
8. Erwähne sowohl Stärken als auch Schwächen/Limitationen
|
||||
9. Schlage alternative Ansätze vor, wenn sinnvoll
|
||||
10. Gib grundsätzliche Hinweise, WIE die Methode/Tool konkret eingesetzt wird
|
||||
11. WICHTIG: Erwähne relevante Hintergrundwissen-Konzepte wenn Tools verwendet werden, die related_concepts haben
|
||||
12. Konzepte sind NICHT Tools - empfehle sie nicht als actionable Schritte, sondern als Wissensbasis
|
||||
|
||||
ENHANCED CONTEXTUAL ANALYSIS:
|
||||
13. Analysiere das Problem detailliert und identifiziere technische Anforderungen, Herausforderungen und Erfolgsfaktoren
|
||||
14. Entwickle einen strategischen Lösungsansatz basierend auf dem spezifischen Problem
|
||||
15. Identifiziere wichtige Voraussetzungen oder Warnungen für die Anwendung
|
||||
|
||||
USER QUERY: "${userQuery}"
|
||||
|
||||
ANTWORT-FORMAT (strict JSON):
|
||||
{
|
||||
"problem_analysis": "Detaillierte Analyse des Problems: Erkannte technische Anforderungen, Herausforderungen, benötigte Fähigkeiten und Erfolgsfaktoren",
|
||||
"investigation_approach": "Strategischer Lösungsansatz für dieses spezifische Problem: Herangehensweise, Prioritäten, optimale Anwendungsreihenfolge",
|
||||
"critical_considerations": "Wichtige Voraussetzungen, potentielle Fallstricke oder Warnungen für die Anwendung der empfohlenen Lösungen",
|
||||
"recommended_tools": [
|
||||
{
|
||||
"name": "EXAKTER Name aus der Tools-Database",
|
||||
"rank": 1,
|
||||
"suitability_score": "high|medium|low",
|
||||
"detailed_explanation": "Detaillierte Erklärung, warum dieses Tool/diese Methode das spezifische Problem löst - mit Bezug zu den erkannten Anforderungen",
|
||||
"implementation_approach": "Konkrete Schritte/Ansatz zur Anwendung für dieses spezifische Problem",
|
||||
"pros": ["Spezifische Vorteile für diesen Anwendungsfall", "Weitere Vorteile"],
|
||||
"cons": ["Potentielle Nachteile oder Limitationen", "Weitere Einschränkungen"],
|
||||
"alternatives": "Alternative Ansätze oder ergänzende Tools/Methoden, falls relevant"
|
||||
}
|
||||
],
|
||||
"background_knowledge": [
|
||||
{
|
||||
"concept_name": "EXAKTER Name aus der Konzepte-Database",
|
||||
"relevance": "Warum dieses Konzept für die empfohlenen Tools/das Problem relevant ist, und für welche der empfohlenen Methoden/Tools."
|
||||
}
|
||||
],
|
||||
"additional_considerations": "Wichtige Überlegungen, Voraussetzungen oder Warnungen"
|
||||
}
|
||||
|
||||
Antworte NUR mit validen JSON. Keine zusätzlichen Erklärungen außerhalb des JSON.`;
|
||||
}
|
||||
|
||||
async processQuery(userQuery: string, mode: string): Promise<AnalysisResult> {
|
||||
const startTime = Date.now();
|
||||
console.log(`[AI PIPELINE] Starting ${mode} query processing`);
|
||||
|
||||
try {
|
||||
// Load full dataset
|
||||
const toolsData = await getCompressedToolsDataForAI();
|
||||
|
||||
let filteredData: any;
|
||||
let processingStats: any = {
|
||||
embeddingsUsed: false,
|
||||
candidatesFromEmbeddings: 0,
|
||||
finalSelectedItems: 0,
|
||||
processingTimeMs: 0
|
||||
};
|
||||
|
||||
// Stage 1: Filter candidates (embeddings or selector AI)
|
||||
if (embeddingsService.isEnabled()) {
|
||||
const result = await this.processWithEmbeddings(userQuery, toolsData, mode);
|
||||
filteredData = result.filteredData;
|
||||
processingStats = { ...processingStats, ...result.stats };
|
||||
} else {
|
||||
const result = await this.processWithoutEmbeddings(userQuery, toolsData, mode);
|
||||
filteredData = result.filteredData;
|
||||
processingStats = { ...processingStats, ...result.stats };
|
||||
}
|
||||
|
||||
// Stage 2: Generate detailed analysis with analyzer AI
|
||||
console.log('[AI PIPELINE] Stage 2: Generating detailed analysis');
|
||||
const analyzerPrompt = this.createAnalyzerPrompt(filteredData, userQuery, mode);
|
||||
|
||||
const messages = [
|
||||
{ role: 'user', content: analyzerPrompt }
|
||||
];
|
||||
|
||||
const analysisResponse = await this.callAI(this.analyzerConfig, messages, 3500);
|
||||
|
||||
// Parse the response
|
||||
let recommendation;
|
||||
try {
|
||||
const cleanedContent = analysisResponse.replace(/^```json\s*/i, '').replace(/\s*```\s*$/g, '').trim();
|
||||
recommendation = JSON.parse(cleanedContent);
|
||||
} catch (error) {
|
||||
console.error('[AI PIPELINE] Failed to parse analysis response:', analysisResponse);
|
||||
throw new Error('Invalid JSON response from analyzer AI');
|
||||
}
|
||||
|
||||
// Validate tool/concept names exist in filtered data
|
||||
const validToolNames = new Set(filteredData.tools.map((t: any) => t.name));
|
||||
const validConceptNames = new Set(filteredData.concepts.map((c: any) => c.name));
|
||||
|
||||
if (recommendation.recommended_tools) {
|
||||
recommendation.recommended_tools = recommendation.recommended_tools.filter((tool: any) => {
|
||||
if (!validToolNames.has(tool.name)) {
|
||||
console.warn(`[AI PIPELINE] Analyzer recommended unknown tool: ${tool.name}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
if (recommendation.background_knowledge) {
|
||||
recommendation.background_knowledge = recommendation.background_knowledge.filter((concept: any) => {
|
||||
if (!validConceptNames.has(concept.concept_name)) {
|
||||
console.warn(`[AI PIPELINE] Analyzer referenced unknown concept: ${concept.concept_name}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
processingStats.finalSelectedItems = (recommendation.recommended_tools?.length || 0) +
|
||||
(recommendation.background_knowledge?.length || 0);
|
||||
processingStats.processingTimeMs = Date.now() - startTime;
|
||||
|
||||
console.log(`[AI PIPELINE] Completed in ${processingStats.processingTimeMs}ms`);
|
||||
console.log(`[AI PIPELINE] Final recommendations: ${recommendation.recommended_tools?.length || 0} tools, ${recommendation.background_knowledge?.length || 0} concepts`);
|
||||
|
||||
return {
|
||||
recommendation,
|
||||
processingStats
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('[AI PIPELINE] Processing failed:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Global instance
|
||||
const aiPipeline = new AIProcessingPipeline();
|
||||
|
||||
export { aiPipeline, type AnalysisResult };
|
||||
Reference in New Issue
Block a user