semanticsearch

src/pages/api/ai/embeddings-status.ts

@@ -0,0 +1,37 @@
+// src/pages/api/ai/embeddings-status.ts
+import type { APIRoute } from 'astro';
+
+export const prerender = false;
+
+export const GET: APIRoute = async () => {
+  try {
+    const { embeddingsService } = await import('../../../utils/embeddings.js');
+    await embeddingsService.waitForInitialization();
+    
+    const stats = embeddingsService.getStats();
+    const status = stats.enabled && stats.initialized ? 'ready' : 
+                  stats.enabled && !stats.initialized ? 'initializing' : 'disabled';
+    
+    return new Response(JSON.stringify({
+      success: true,
+      embeddings: stats,
+      timestamp: new Date().toISOString(),
+      status: status
+    }), {
+      status: 200,
+      headers: { 'Content-Type': 'application/json' }
+    });
+    
+  } catch (error) {
+    return new Response(JSON.stringify({
+      success: false,
+      embeddings: { enabled: false, initialized: false, count: 0 },
+      timestamp: new Date().toISOString(),
+      status: 'disabled',
+      error: error.message
+    }), {
+      status: 200,
+      headers: { 'Content-Type': 'application/json' }
+    });
+  }
+};

src/pages/api/ai/embeddings.status.ts

@@ -1,22 +0,0 @@
-// src/pages/api/ai/embeddings-status.ts
-import type { APIRoute } from 'astro';
-import { embeddingsService } from '../../../utils/embeddings.js';
-import { apiResponse, apiServerError } from '../../../utils/api.js';
-
-export const prerender = false;
-
-export const GET: APIRoute = async () => {
-  try {
-    const stats = embeddingsService.getStats();
-    
-    return apiResponse.success({
-      embeddings: stats,
-      timestamp: new Date().toISOString(),
-      status: stats.enabled && stats.initialized ? 'ready' : 
-             stats.enabled && !stats.initialized ? 'initializing' : 'disabled'
-    });
-  } catch (error) {
-    console.error('Embeddings status error:', error);
-    return apiServerError.internal('Failed to get embeddings status');
-  }
-};

src/pages/api/debug-env.ts

@@ -0,0 +1,42 @@
+// src/pages/api/debug-env.ts
+import type { APIRoute } from 'astro';
+import { debugEmbeddings } from '../../utils/embeddings.js';
+
+export const GET: APIRoute = async () => {
+  const embeddingVars = Object.keys(process.env)
+    .filter(key => key.includes('EMBEDDINGS'))
+    .reduce((obj: Record<string, string>, key) => {
+      obj[key] = process.env[key] || 'undefined';
+      return obj;
+    }, {});
+
+  const aiVars = Object.keys(process.env)
+    .filter(key => key.includes('AI_'))
+    .reduce((obj: Record<string, string>, key) => {
+      // Mask sensitive values
+      const value = process.env[key] || 'undefined';
+      obj[key] = key.includes('KEY') || key.includes('SECRET') ? 
+        (value.length > 10 ? `${value.slice(0, 6)}...${value.slice(-4)}` : value) : 
+        value;
+      return obj;
+    }, {});
+
+  // Force recheck embeddings environment
+  await debugEmbeddings.recheckEnvironment();
+  const embeddingsStatus = debugEmbeddings.getStatus();
+
+  return new Response(JSON.stringify({
+    timestamp: new Date().toISOString(),
+    embeddingVars,
+    allAiVars: aiVars,
+    totalEnvVars: Object.keys(process.env).length,
+    embeddingsStatus,
+    nodeEnv: process.env.NODE_ENV,
+    platform: process.platform
+  }, null, 2), {
+    status: 200,
+    headers: {
+      'Content-Type': 'application/json'
+    }
+  });
+};

src/pages/api/search/semantic.ts

@@ -0,0 +1,83 @@
+// src/pages/api/search/semantic.ts
+import type { APIRoute } from 'astro';
+import { getToolsData } from '../../../utils/dataService.js';
+
+export const prerender = false;
+
+export const POST: APIRoute = async ({ request }) => {
+  try {
+    const { query, maxResults = 50, threshold = 0.15 } = await request.json();
+    
+    if (!query || typeof query !== 'string') {
+      return new Response(JSON.stringify({
+        success: false,
+        error: 'Query is required'
+      }), {
+        status: 400,
+        headers: { 'Content-Type': 'application/json' }
+      });
+    }
+
+    // Import embeddings service dynamically
+    const { embeddingsService } = await import('../../../utils/embeddings.js');
+
+    // Check if embeddings are available
+    if (!embeddingsService.isEnabled()) {
+      return new Response(JSON.stringify({
+        success: false,
+        error: 'Semantic search not available'
+      }), {
+        status: 400,
+        headers: { 'Content-Type': 'application/json' }
+      });
+    }
+
+    // Wait for embeddings initialization if needed
+    await embeddingsService.waitForInitialization();
+
+    // Get similar items using embeddings
+    const similarItems = await embeddingsService.findSimilar(
+      query.trim(), 
+      maxResults, 
+      threshold
+    );
+    
+    // Get current tools data
+    const toolsData = await getToolsData();
+    
+    // Map similarity results back to full tool objects, preserving similarity ranking
+    const rankedTools = similarItems
+      .map(similarItem => {
+        const tool = toolsData.tools.find(t => t.name === similarItem.name);
+        return tool ? { 
+          ...tool, 
+          _semanticSimilarity: similarItem.similarity,
+          _semanticRank: similarItems.indexOf(similarItem) + 1
+        } : null;
+      })
+      .filter(Boolean);
+
+    return new Response(JSON.stringify({
+      success: true,
+      query: query.trim(),
+      results: rankedTools,
+      totalFound: rankedTools.length,
+      semanticSearch: true,
+      threshold,
+      maxSimilarity: rankedTools.length > 0 ? rankedTools[0]._semanticSimilarity : 0
+    }), {
+      status: 200,
+      headers: { 'Content-Type': 'application/json' }
+    });
+
+  } catch (error) {
+    console.error('Semantic search error:', error);
+    return new Response(JSON.stringify({
+      success: false,
+      error: 'Semantic search failed'
+    }), {
+      status: 500,
+      headers: { 'Content-Type': 'application/json' }
+    });
+  }
+};

src/pages/index.astro

@@ -198,12 +198,15 @@ const phases = data.phases;
 <script define:vars={{ toolsData: data.tools, phases: data.phases }}>
   window.toolsData = toolsData;
   
+  // CONSOLIDATED: Approach selection - Pure navigation aid
   window.selectApproach = function(approach) {
     console.log(`Selected approach: ${approach}`);
     
+    // Clear any existing AI results
     const aiResults = document.getElementById('ai-results');
     if (aiResults) aiResults.style.display = 'none';
     
+    // Update visual selection state
     document.querySelectorAll('.approach-card').forEach(card => {
       card.classList.remove('selected');
     });
@@ -211,14 +214,20 @@ const phases = data.phases;
     const selectedCard = document.querySelector(`.approach-card.${approach}`);
     if (selectedCard) selectedCard.classList.add('selected');
     
+    // Hide all approach sections first (ensures mutual exclusivity)
+    const methodologySection = document.getElementById('methodology-section');
+    const targetedSection = document.getElementById('targeted-section');
+    
+    if (methodologySection) methodologySection.classList.remove('active');
+    if (targetedSection) targetedSection.classList.remove('active');
+    
+    // Show the selected approach section (navigation aid only)
     if (approach === 'methodology') {
-      const methodologySection = document.getElementById('methodology-section');
       if (methodologySection) {
         methodologySection.classList.add('active');
         window.scrollToElementById('methodology-section');
       }
     } else if (approach === 'targeted') {
-      const targetedSection = document.getElementById('targeted-section');
       if (targetedSection) {
         targetedSection.classList.add('active');
         window.scrollToElementById('targeted-section');
@@ -226,9 +235,11 @@ const phases = data.phases;
     }
   };
 
+  // CONSOLIDATED: Phase selection - Sets unified filter dropdown
   window.selectPhase = function(phase) {
     console.log(`Selected NIST phase: ${phase}`);
     
+    // Update visual selection of phase cards
     document.querySelectorAll('.phase-card').forEach(card => {
       card.classList.remove('active');
     });
@@ -238,17 +249,26 @@ const phases = data.phases;
       selectedCard.classList.add('active');
     }
     
-    const existingPhaseButton = document.querySelector(`[data-phase="${phase}"]`);
-    if (existingPhaseButton && !existingPhaseButton.classList.contains('active')) {
-      existingPhaseButton.click();
+    // CONSOLIDATED: Set the unified phase-select dropdown
+    const phaseSelect = document.getElementById('phase-select');
+    if (phaseSelect) {
+      phaseSelect.value = phase;
+      
+      // Trigger the change event to activate unified filtering
+      const changeEvent = new Event('change', { bubbles: true });
+      phaseSelect.dispatchEvent(changeEvent);
     }
     
+    // Switch to grid view to show filtered results
     const gridToggle = document.querySelector('.view-toggle[data-view="grid"]');
     if (gridToggle && !gridToggle.classList.contains('active')) {
       gridToggle.click();
     }
     
-    window.scrollToElementById('tools-grid');
+    // Scroll to filtered results
+    setTimeout(() => {
+      window.scrollToElementById('tools-grid');
+    }, 200);
   };
 
   document.addEventListener('DOMContentLoaded', () => {
@@ -282,61 +302,68 @@ const phases = data.phases;
       });
     }
 
-  function switchToView(view) {
-    const toolsGrid = document.getElementById('tools-grid');
-    const matrixContainer = document.getElementById('matrix-container');
-    const aiInterface = document.getElementById('ai-interface');
-    const filtersSection = document.getElementById('filters-section');
-    const noResults = document.getElementById('no-results');
-    
-    if (toolsGrid) toolsGrid.style.display = 'none';
-    if (matrixContainer) matrixContainer.style.display = 'none';
-    if (aiInterface) aiInterface.style.display = 'none';
-    if (filtersSection) filtersSection.style.display = 'none';
-    if (noResults) noResults.style.display = 'none';
-    
-    switch (view) {
-      case 'grid':
-        if (toolsGrid) toolsGrid.style.display = 'block';
-        if (filtersSection) filtersSection.style.display = 'block';
-        break;
-      case 'matrix':
-        if (matrixContainer) matrixContainer.style.display = 'block';
-        if (filtersSection) filtersSection.style.display = 'block';
-        break;
-      case 'ai':
-        if (aiInterface) aiInterface.style.display = 'block';
-        break;
-    }
-  }
-
-    function hideFilterControls() {
-      const filterSections = document.querySelectorAll('.filter-section');
-      filterSections.forEach((section, index) => {
-        if (index < filterSections.length - 1) {
-          section.style.display = 'none';
-        }
-      });
-    }
-
-    function showFilterControls() {
-      const filterSections = document.querySelectorAll('.filter-section');
-      const searchInput = document.getElementById('search-input');
-      const tagCloud = document.querySelector('.tag-cloud');
-      const tagControls = document.querySelector('.tag-controls');
-      const checkboxWrappers = document.querySelectorAll('.checkbox-wrapper');
-      const allInputs = filtersSection.querySelectorAll('input, select, textarea');
-      
-      filterSections.forEach(section => section.style.display = 'block');
-      
-      if (searchInput) searchInput.style.display = 'block';
-      if (tagCloud) tagCloud.style.display = 'flex';
-      if (tagControls) tagControls.style.display = 'flex';
-      
-      allInputs.forEach(input => input.style.display = 'block');
-      checkboxWrappers.forEach(wrapper => wrapper.style.display = 'flex');
+    function switchToView(view) {
+      const toolsGrid = document.getElementById('tools-grid');
+      const matrixContainer = document.getElementById('matrix-container');
+      const aiInterface = document.getElementById('ai-interface');
+      const filtersSection = document.getElementById('filters-section');
+      const noResults = document.getElementById('no-results');
+      
+      // FIXED: Hide approach sections when switching to ANY view mode
+      const methodologySection = document.getElementById('methodology-section');
+      const targetedSection = document.getElementById('targeted-section');
+      
+      // Hide all main content areas
+      if (toolsGrid) toolsGrid.style.display = 'none';
+      if (matrixContainer) matrixContainer.style.display = 'none';
+      if (aiInterface) aiInterface.style.display = 'none';
+      if (noResults) noResults.style.display = 'none';
+      
+      // FIXED: Hide approach sections when switching to view modes
+      if (methodologySection) methodologySection.classList.remove('active');
+      if (targetedSection) targetedSection.classList.remove('active');
+      
+      switch (view) {
+        case 'grid':
+          if (toolsGrid) toolsGrid.style.display = 'block';
+          if (filtersSection) filtersSection.style.display = 'block';
+          break;
+        case 'matrix':
+          if (matrixContainer) matrixContainer.style.display = 'block';
+          if (filtersSection) filtersSection.style.display = 'block';
+          break;
+        case 'ai':
+          if (aiInterface) aiInterface.style.display = 'block';
+          
+          // FIXED: Show filters but hide everything except view controls
+          if (filtersSection) {
+            filtersSection.style.display = 'block';
+            
+            // Hide all filter sections except the last one (view controls)
+            const filterSections = filtersSection.querySelectorAll('.filter-section');
+            filterSections.forEach((section, index) => {
+              if (index === filterSections.length - 1) {
+                // Keep view controls visible
+                section.style.display = 'block';
+              } else {
+                // Hide other filter sections
+                section.style.display = 'none';
+              }
+            });
+          }
+          break;
+      }
+      
+      // FIXED: Reset filter sections visibility when not in AI view
+      if (view !== 'ai' && filtersSection) {
+        const filterSections = filtersSection.querySelectorAll('.filter-section');
+        filterSections.forEach(section => {
+          section.style.display = 'block';
+        });
+      }
     }
 
+    // Navigation functions for AI recommendations (unchanged)
     window.navigateToGrid = function(toolName) {
       console.log('Navigating to grid for tool:', toolName);
       
@@ -458,21 +485,101 @@ const phases = data.phases;
       }, 100);
     }
 
-    window.addEventListener('toolsFiltered', (event) => {
-      const filtered = event.detail;
-      const currentView = document.querySelector('.view-toggle.active')?.getAttribute('data-view');
+  // REPLACE the existing toolsFiltered event listener in index.astro with this enhanced version:
+
+  window.addEventListener('toolsFiltered', (event) => {
+    const filtered = event.detail;
+    const semanticSearch = event.semanticSearch || false;
+    const currentView = document.querySelector('.view-toggle.active')?.getAttribute('data-view');
+    
+    if (currentView === 'matrix' || currentView === 'ai') {
+      return;
+    }
+    
+    const allToolCards = document.querySelectorAll('.tool-card');
+    const filteredNames = new Set(filtered.map(tool => tool.name.toLowerCase()));
+    const toolsContainer = document.getElementById('tools-container');
+    
+    let visibleCount = 0;
+    
+    if (semanticSearch && filtered.length > 0) {
+      console.log('[SEMANTIC] Reordering tools by semantic similarity');
       
-      if (currentView === 'matrix' || currentView === 'ai') {
-        return;
-      }
+      // FIXED: Create ordered array of cards based on semantic similarity
+      const orderedCards = [];
+      const remainingCards = [];
       
-      const allToolCards = document.querySelectorAll('.tool-card');
-      const filteredNames = new Set(filtered.map(tool => tool.name.toLowerCase()));
-      
-      let visibleCount = 0;
+      // First pass: collect cards in semantic order
+      filtered.forEach(tool => {
+        const toolName = tool.name.toLowerCase();
+        const matchingCard = Array.from(allToolCards).find(card => 
+          card.getAttribute('data-tool-name') === toolName
+        );
+        
+        if (matchingCard) {
+          matchingCard.style.display = 'block';
+          orderedCards.push(matchingCard);
+          visibleCount++;
+          
+          // Add semantic indicators if available
+          if (tool._semanticSimilarity) {
+            matchingCard.setAttribute('data-semantic-similarity', tool._semanticSimilarity.toFixed(3));
+            matchingCard.setAttribute('data-semantic-rank', tool._semanticRank || '');
+            
+            // Visual indication of semantic ranking (subtle)
+            const header = matchingCard.querySelector('.tool-card-header h3');
+            if (header && tool._semanticRank <= 3) {
+              const existingIndicator = header.querySelector('.semantic-rank-indicator');
+              if (existingIndicator) {
+                existingIndicator.remove();
+              }
+              
+              const indicator = document.createElement('span');
+              indicator.className = 'semantic-rank-indicator';
+              indicator.style.cssText = `
+                display: inline-block;
+                width: 6px;
+                height: 6px;
+                background-color: var(--color-accent);
+                border-radius: 50%;
+                margin-left: 0.5rem;
+                opacity: ${1 - (tool._semanticRank - 1) * 0.3};
+              `;
+              indicator.title = `Semantische Relevanz: ${tool._semanticSimilarity.toFixed(3)}`;
+              header.appendChild(indicator);
+            }
+          }
+        }
+      });
       
+      // Second pass: hide non-matching cards and collect them
       allToolCards.forEach(card => {
         const toolName = card.getAttribute('data-tool-name');
+        if (!filteredNames.has(toolName)) {
+          card.style.display = 'none';
+          remainingCards.push(card);
+        }
+      });
+      
+      // Reorder DOM: semantic results first, then hidden cards
+      const allCards = [...orderedCards, ...remainingCards];
+      allCards.forEach(card => {
+        toolsContainer.appendChild(card);
+      });
+      
+    } else {
+      // FIXED: Standard filtering without semantic ordering
+      allToolCards.forEach(card => {
+        const toolName = card.getAttribute('data-tool-name');
+        
+        // Clean up any semantic indicators
+        card.removeAttribute('data-semantic-similarity');
+        card.removeAttribute('data-semantic-rank');
+        const semanticIndicator = card.querySelector('.semantic-rank-indicator');
+        if (semanticIndicator) {
+          semanticIndicator.remove();
+        }
+        
         if (filteredNames.has(toolName)) {
           card.style.display = 'block';
           visibleCount++;
@@ -481,12 +588,33 @@ const phases = data.phases;
         }
       });
       
-      if (visibleCount === 0) {
-        noResults.style.display = 'block';
-      } else {
-        noResults.style.display = 'none';
+      // Restore original order when not using semantic search
+      if (!semanticSearch) {
+        const originalOrder = Array.from(allToolCards).sort((a, b) => {
+          // Get original indices from data attributes or DOM order
+          const aIndex = Array.from(allToolCards).indexOf(a);
+          const bIndex = Array.from(allToolCards).indexOf(b);
+          return aIndex - bIndex;
+        });
+        
+        originalOrder.forEach(card => {
+          toolsContainer.appendChild(card);
+        });
       }
-    });
+    }
+    
+    // Show/hide no results message
+    if (visibleCount === 0) {
+      noResults.style.display = 'block';
+    } else {
+      noResults.style.display = 'none';
+    }
+    
+    // Log semantic search info
+    if (semanticSearch) {
+      console.log(`[SEMANTIC] Displayed ${visibleCount} tools in semantic order`);
+    }
+  });
     
     window.addEventListener('viewChanged', (event) => {
       const view = event.detail;
@@ -497,4 +625,5 @@ const phases = data.phases;
 
     handleSharedURL();
   });
-</script>
+</script>
+</BaseLayout>