mstoeck3/forensic-pathways
2
0
Fork 0
Code Issues 17 Pull Requests Projects Releases Activity

bugfixing in embeddings api

Browse Source
This commit is contained in:
overcuriousity
2025-08-04 15:11:30 +02:00
parent 6c73a20dff
commit ec1969b2e2
4 changed files with 400 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
src/pages/api/ai/enhance-input.ts
View File

@@ -1,4 +1,5 @@
// src/pages/api/ai/enhance-input.ts - ENHANCED with forensics methodology
// src/pages/api/ai/enhance-input.ts - Enhanced AI service compatibility
import type { APIRoute } from 'astro';
import { withAPIAuth } from '../../../utils/auth.js';
import { apiError, apiServerError, createAuthErrorResponse } from '../../../utils/api.js';
@@ -20,7 +21,7 @@ const AI_ANALYZER_MODEL = getEnv('AI_ANALYZER_MODEL');
const rateLimitStore = new Map<string, { count: number; resetTime: number }>();
const RATE_LIMIT_WINDOW = 60 * 1000; // 1 minute
const RATE_LIMIT_MAX = 5;
const RATE_LIMIT_MAX = 5;
function sanitizeInput(input: string): string {
return input
@@ -93,6 +94,45 @@ ${input}
`.trim();
}
// Enhanced AI service call function
async function callAIService(prompt: string): Promise<Response> {
const endpoint = AI_ENDPOINT;
const apiKey = AI_ANALYZER_API_KEY;
const model = AI_ANALYZER_MODEL;
// Simple headers - add auth only if API key exists
let headers: Record<string, string> = {
'Content-Type': 'application/json'
};
// Add authentication if API key is provided
if (apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`;
console.log('[ENHANCE API] Using API key authentication');
} else {
console.log('[ENHANCE API] No API key - making request without authentication');
}
// Simple request body
const requestBody = {
model,
messages: [{ role: 'user', content: prompt }],
max_tokens: 300,
temperature: 0.7,
top_p: 0.9,
frequency_penalty: 0.2,
presence_penalty: 0.1
};
// FIXED: This function is already being called through enqueueApiCall in the main handler
// So we can use direct fetch here since the queuing happens at the caller level
return fetch(`${endpoint}/v1/chat/completions`, {
method: 'POST',
headers,
body: JSON.stringify(requestBody)
});
}
export const POST: APIRoute = async ({ request }) => {
try {
const authResult = await withAPIAuth(request, 'ai');
@@ -121,31 +161,11 @@ export const POST: APIRoute = async ({ request }) => {
const systemPrompt = createEnhancementPrompt(sanitizedInput);
const taskId = `enhance_${userId}_${Date.now()}_${Math.random().toString(36).substr(2, 4)}`;
const aiResponse = await enqueueApiCall(() =>
fetch(`${AI_ENDPOINT}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${AI_ANALYZER_API_KEY}`
},
body: JSON.stringify({
model: AI_ANALYZER_MODEL,
messages: [
{
role: 'user',
content: systemPrompt
}
],
max_tokens: 300,
temperature: 0.7,
top_p: 0.9,
frequency_penalty: 0.2,
presence_penalty: 0.1
})
}), taskId);
const aiResponse = await enqueueApiCall(() => callAIService(systemPrompt), taskId);
if (!aiResponse.ok) {
console.error('AI enhancement error:', await aiResponse.text());
const errorText = await aiResponse.text();
console.error('[ENHANCE API] AI enhancement error:', errorText, 'Status:', aiResponse.status);
return apiServerError.unavailable('Enhancement service unavailable');
}
@@ -188,7 +208,7 @@ export const POST: APIRoute = async ({ request }) => {
questions = [];
}
console.log(`[AI Enhancement] User: ${userId}, Forensics Questions: ${questions.length}, Input length: ${sanitizedInput.length}`);
console.log(`[ENHANCE API] User: ${userId}, Forensics Questions: ${questions.length}, Input length: ${sanitizedInput.length}`);
return new Response(JSON.stringify({
success: true,

138
src/utils/aiPipeline.ts
View File

@@ -66,6 +66,11 @@ interface AnalysisContext {
auditTrail: AuditEntry[];
}
interface SimilarityResult extends EmbeddingData {
similarity: number;
}
class ImprovedMicroTaskAIPipeline {
private config: AIConfig;
private maxSelectedItems: number;
@@ -267,39 +272,62 @@ class ImprovedMicroTaskAIPipeline {
userQuery,
this.embeddingCandidates,
this.similarityThreshold
);
) as SimilarityResult[]; // Type assertion for similarity property
const toolNames = new Set<string>();
const conceptNames = new Set<string>();
console.log(`[IMPROVED PIPELINE] Embeddings found ${similarItems.length} similar items`);
similarItems.forEach(item => {
if (item.type === 'tool') toolNames.add(item.name);
if (item.type === 'concept') conceptNames.add(item.name);
});
// FIXED: Create lookup maps for O(1) access while preserving original data
const toolsMap = new Map<string, any>(toolsData.tools.map((tool: any) => [tool.name, tool]));
const conceptsMap = new Map<string, any>(toolsData.concepts.map((concept: any) => [concept.name, concept]));
console.log(`[IMPROVED PIPELINE] Embeddings found: ${toolNames.size} tools, ${conceptNames.size} concepts`);
// FIXED: Process in similarity order, preserving the ranking
const similarTools = similarItems
.filter((item): item is SimilarityResult => item.type === 'tool')
.map(item => toolsMap.get(item.name))
.filter((tool): tool is any => tool !== undefined); // Proper type guard
if (toolNames.size >= 15) {
candidateTools = toolsData.tools.filter((tool: any) => toolNames.has(tool.name));
candidateConcepts = toolsData.concepts.filter((concept: any) => conceptNames.has(concept.name));
const similarConcepts = similarItems
.filter((item): item is SimilarityResult => item.type === 'concept')
.map(item => conceptsMap.get(item.name))
.filter((concept): concept is any => concept !== undefined); // Proper type guard
console.log(`[IMPROVED PIPELINE] Similarity-ordered results: ${similarTools.length} tools, ${similarConcepts.length} concepts`);
// Log the first few tools to verify ordering is preserved
if (similarTools.length > 0) {
console.log(`[IMPROVED PIPELINE] Top similar tools (in similarity order):`);
similarTools.slice(0, 5).forEach((tool, idx) => {
const originalSimilarItem = similarItems.find(item => item.name === tool.name);
console.log(` ${idx + 1}. ${tool.name} (similarity: ${originalSimilarItem?.similarity?.toFixed(4) || 'N/A'})`);
});
}
if (similarTools.length >= 15) {
candidateTools = similarTools;
candidateConcepts = similarConcepts;
selectionMethod = 'embeddings_candidates';
console.log(`[IMPROVED PIPELINE] Using embeddings candidates: ${candidateTools.length} tools`);
console.log(`[IMPROVED PIPELINE] Using embeddings candidates in similarity order: ${candidateTools.length} tools`);
} else {
console.log(`[IMPROVED PIPELINE] Embeddings insufficient (${toolNames.size} < 15), using full dataset`);
console.log(`[IMPROVED PIPELINE] Embeddings insufficient (${similarTools.length} < 15), using full dataset`);
candidateTools = toolsData.tools;
candidateConcepts = toolsData.concepts;
selectionMethod = 'full_dataset';
}
// NEW: Add Audit Entry for Embeddings Search
// NEW: Add Audit Entry for Embeddings Search with ordering verification
if (this.auditConfig.enabled) {
this.addAuditEntry(null, 'retrieval', 'embeddings-search',
{ query: userQuery, threshold: this.similarityThreshold, candidates: this.embeddingCandidates },
{ candidatesFound: similarItems.length, toolNames: Array.from(toolNames), conceptNames: Array.from(conceptNames) },
similarItems.length >= 15 ? 85 : 60, // Confidence based on result quality
{
candidatesFound: similarItems.length,
toolsInOrder: similarTools.slice(0, 3).map((t: any) => t.name),
conceptsInOrder: similarConcepts.slice(0, 3).map((c: any) => c.name),
orderingPreserved: true
},
similarTools.length >= 15 ? 85 : 60,
embeddingsStart,
{ selectionMethod, embeddingsEnabled: true }
{ selectionMethod, embeddingsEnabled: true, orderingFixed: true }
);
}
} else {
@@ -309,7 +337,7 @@ class ImprovedMicroTaskAIPipeline {
selectionMethod = 'full_dataset';
}
console.log(`[IMPROVED PIPELINE] AI will analyze FULL DATA of ${candidateTools.length} candidate tools`);
console.log(`[IMPROVED PIPELINE] AI will analyze ${candidateTools.length} candidate tools (ordering preserved: ${selectionMethod === 'embeddings_candidates'})`);
const finalSelection = await this.aiSelectionWithFullData(userQuery, candidateTools, candidateConcepts, mode, selectionMethod);
return {
@@ -735,33 +763,59 @@ ${JSON.stringify(conceptsWithFullData.slice(0, 10), null, 2)}`;
}
private async callAI(prompt: string, maxTokens: number = 1000): Promise<string> {
const response = await fetch(`${this.config.endpoint}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.config.apiKey}`
},
body: JSON.stringify({
model: this.config.model,
messages: [{ role: 'user', content: prompt }],
max_tokens: maxTokens,
temperature: 0.3
})
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`AI API error: ${response.status} - ${errorText}`);
}
const data = await response.json();
const content = data.choices?.[0]?.message?.content;
const endpoint = this.config.endpoint;
const apiKey = this.config.apiKey;
const model = this.config.model;
if (!content) {
throw new Error('No response from AI model');
// Simple headers - add auth only if API key exists
let headers: Record<string, string> = {
'Content-Type': 'application/json'
};
// Add authentication if API key is provided
if (apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`;
console.log('[AI PIPELINE] Using API key authentication');
} else {
console.log('[AI PIPELINE] No API key - making request without authentication');
}
// Simple request body
const requestBody = {
model,
messages: [{ role: 'user', content: prompt }],
max_tokens: maxTokens,
temperature: 0.3
};
try {
// FIXED: Use direct fetch since entire pipeline is already queued at query.ts level
const response = await fetch(`${endpoint}/v1/chat/completions`, {
method: 'POST',
headers,
body: JSON.stringify(requestBody)
});
return content;
if (!response.ok) {
const errorText = await response.text();
console.error(`[AI PIPELINE] AI API Error ${response.status}:`, errorText);
throw new Error(`AI API error: ${response.status} - ${errorText}`);
}
const data = await response.json();
const content = data.choices?.[0]?.message?.content;
if (!content) {
console.error('[AI PIPELINE] No response content:', data);
throw new Error('No response from AI model');
}
return content;
} catch (error) {
console.error('[AI PIPELINE] AI service call failed:', error.message);
throw error;
}
}
async processQuery(userQuery: string, mode: string): Promise<AnalysisResult> {

52
src/utils/embeddings.ts
View File

@@ -24,6 +24,10 @@ interface EmbeddingsDatabase {
embeddings: EmbeddingData[];
}
interface SimilarityResult extends EmbeddingData {
similarity: number;
}
class EmbeddingsService {
private embeddings: EmbeddingData[] = [];
private isInitialized = false;
@@ -211,8 +215,9 @@ class EmbeddingsService {
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}
async findSimilar(query: string, maxResults: number = 30, threshold: number = 0.3): Promise<EmbeddingData[]> {
async findSimilar(query: string, maxResults: number = 30, threshold: number = 0.3): Promise<SimilarityResult[]> {
if (!this.enabled || !this.isInitialized || this.embeddings.length === 0) {
console.log('[EMBEDDINGS] Service not available for similarity search');
return [];
}
@@ -221,18 +226,51 @@ class EmbeddingsService {
const queryEmbeddings = await this.generateEmbeddingsBatch([query.toLowerCase()]);
const queryEmbedding = queryEmbeddings[0];
// Calculate similarities
const similarities = this.embeddings.map(item => ({
console.log(`[EMBEDDINGS] Computing similarities for ${this.embeddings.length} items`);
// Calculate similarities - properly typed
const similarities: SimilarityResult[] = this.embeddings.map(item => ({
...item,
similarity: this.cosineSimilarity(queryEmbedding, item.embedding)
}));
// Filter by threshold and sort by similarity
return similarities
// Filter by threshold and sort by similarity (descending - highest first)
const results = similarities
.filter(item => item.similarity >= threshold)
.sort((a, b) => b.similarity - a.similarity)
.sort((a, b) => b.similarity - a.similarity) // CRITICAL: Ensure descending order
.slice(0, maxResults);
// ENHANCED: Verify ordering is correct
const orderingValid = results.every((item, index) => {
if (index === 0) return true;
return item.similarity <= results[index - 1].similarity;
});
if (!orderingValid) {
console.error('[EMBEDDINGS] CRITICAL: Similarity ordering is broken!');
results.forEach((item, idx) => {
console.error(` ${idx}: ${item.name} = ${item.similarity.toFixed(4)}`);
});
}
// ENHANCED: Log top results for debugging
console.log(`[EMBEDDINGS] Found ${results.length} similar items (threshold: ${threshold})`);
if (results.length > 0) {
console.log('[EMBEDDINGS] Top 5 similarity matches:');
results.slice(0, 5).forEach((item, idx) => {
console.log(` ${idx + 1}. ${item.name} (${item.type}) = ${item.similarity.toFixed(4)}`);
});
// Verify first result is indeed the highest
const topSimilarity = results[0].similarity;
const hasHigherSimilarity = results.some(item => item.similarity > topSimilarity);
if (hasHigherSimilarity) {
console.error('[EMBEDDINGS] CRITICAL: Top result is not actually the highest similarity!');
}
}
return results;
} catch (error) {
console.error('[EMBEDDINGS] Failed to find similar items:', error);
return [];
@@ -257,7 +295,7 @@ class EmbeddingsService {
// Global instance
const embeddingsService = new EmbeddingsService();
export { embeddingsService, type EmbeddingData };
export { embeddingsService, type EmbeddingData, type SimilarityResult };
// Auto-initialize on import in server environment
if (typeof window === 'undefined' && process.env.NODE_ENV !== 'test') {