core/graph_manager.py

@@ -56,15 +56,22 @@ class GraphManager:
         self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
 
     def process_correlations_for_node(self, node_id: str):
-        """Process correlations for a given node based on its attributes."""
+        """
+        UPDATED: Process correlations for a given node with enhanced tracking.
+        Now properly tracks which attribute/provider created each correlation.
+        """
         if not self.graph.has_node(node_id):
             return
 
         node_attributes = self.graph.nodes[node_id].get('attributes', [])
+        
+        # Process each attribute for potential correlations
         for attr in node_attributes:
             attr_name = attr.get('name')
             attr_value = attr.get('value')
+            attr_provider = attr.get('provider', 'unknown')
 
+            # Skip excluded attributes and invalid values
             if attr_name in self.EXCLUDED_KEYS or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None:
                 continue
 
@@ -74,25 +81,91 @@ class GraphManager:
             if isinstance(attr_value, str) and (len(attr_value) < 4 or self.date_pattern.match(attr_value)):
                 continue
 
+            # Initialize correlation tracking for this value
             if attr_value not in self.correlation_index:
-                self.correlation_index[attr_value] = set()
+                self.correlation_index[attr_value] = {
+                    'nodes': set(),
+                    'sources': []  # Track which provider/attribute combinations contributed
+                }
 
-            self.correlation_index[attr_value].add(node_id)
+            # Add this node and source information
+            self.correlation_index[attr_value]['nodes'].add(node_id)
             
-            if len(self.correlation_index[attr_value]) > 1:
+            # Track the source of this correlation value
-                self._create_correlation_node_and_edges(attr_value, self.correlation_index[attr_value])
+            source_info = {
+                'node_id': node_id,
+                'provider': attr_provider,
+                'attribute': attr_name,
+                'path': f"{attr_provider}_{attr_name}"
+            }
             
-    def _create_correlation_node_and_edges(self, value, nodes):
+            # Add source if not already present (avoid duplicates)
-        """Create a correlation node and edges to the correlated nodes."""
+            existing_sources = [s for s in self.correlation_index[attr_value]['sources'] 
-        correlation_node_id = f"corr_{value}"
+                            if s['node_id'] == node_id and s['path'] == source_info['path']]
+            if not existing_sources:
+                self.correlation_index[attr_value]['sources'].append(source_info)
+
+            # Create correlation node if we have multiple nodes with this value
+            if len(self.correlation_index[attr_value]['nodes']) > 1:
+                self._create_enhanced_correlation_node_and_edges(attr_value, self.correlation_index[attr_value])
+
+    def _create_enhanced_correlation_node_and_edges(self, value, correlation_data):
+        """
+        UPDATED: Create correlation node and edges with detailed provider tracking.
+        """
+        correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}"
+        nodes = correlation_data['nodes']
+        sources = correlation_data['sources']
+        
+        # Create or update correlation node
         if not self.graph.has_node(correlation_node_id):
-            self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT,
+            # Determine the most common provider/attribute combination
-                        metadata={'value': value, 'correlated_nodes': list(nodes)})
+            provider_counts = {}
+            for source in sources:
+                key = f"{source['provider']}_{source['attribute']}"
+                provider_counts[key] = provider_counts.get(key, 0) + 1
+            
+            # Use the most common provider/attribute as the primary label
+            primary_source = max(provider_counts.items(), key=lambda x: x[1])[0] if provider_counts else "unknown_correlation"
+            
+            metadata = {
+                'value': value,
+                'correlated_nodes': list(nodes),
+                'sources': sources,
+                'primary_source': primary_source,
+                'correlation_count': len(nodes)
+            }
+            
+            self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, metadata=metadata)
+            print(f"Created correlation node {correlation_node_id} for value '{value}' with {len(nodes)} nodes")
+
+        # Create edges from each node to the correlation node
+        for source in sources:
+            node_id = source['node_id']
+            provider = source['provider']
+            attribute = source['attribute']
             
-        for node_id in nodes:
             if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id):
-                self.add_edge(node_id, correlation_node_id, "correlation", confidence_score=0.9)
+                # Format relationship label as "provider: attribute"
+                display_provider = provider
+                display_attribute = attribute.replace('_', ' ').replace('cert ', '').strip()
                 
+                relationship_label = f"{display_provider}: {display_attribute}"
+                
+                self.add_edge(
+                    source_id=node_id,
+                    target_id=correlation_node_id,
+                    relationship_type=relationship_label,
+                    confidence_score=0.9,
+                    source_provider=provider,
+                    raw_data={
+                        'correlation_value': value,
+                        'original_attribute': attribute,
+                        'correlation_type': 'attribute_matching'
+                    }
+                )
+                
+                print(f"Added correlation edge: {node_id} -> {correlation_node_id} ({relationship_label})")
 
     def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None,
                 description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
@@ -335,7 +408,7 @@ class GraphManager:
     def get_graph_data(self) -> Dict[str, Any]:
         """
         Export graph data formatted for frontend visualization.
-        Compatible with unified data model - preserves all attribute information for frontend display.
+        UPDATED: Fixed certificate validity styling logic for unified data model.
         """
         nodes = []
         for node_id, attrs in self.graph.nodes(data=True):
@@ -345,16 +418,46 @@ class GraphManager:
                         'metadata': attrs.get('metadata', {}),
                         'added_timestamp': attrs.get('added_timestamp')}
             
-            # Customize node appearance based on type and attributes
+            # UPDATED: Fixed certificate validity styling logic
             node_type = node_data['type']
             attributes_list = node_data['attributes']
             
-            # CORRECTED LOGIC: Handle certificate validity styling
             if node_type == 'domain' and isinstance(attributes_list, list):
-                # Find the certificates attribute in the list
+                # Check for certificate-related attributes
-                cert_attr = next((attr for attr in attributes_list if attr.get('name') == 'certificates'), None)
+                has_certificates = False
-                if cert_attr and cert_attr.get('value', {}).get('has_valid_cert') is False:
+                has_valid_certificates = False
-                    node_data['color'] = {'background': '#c7c7c7', 'border': '#999'} # Gray for invalid cert
+                has_expired_certificates = False
+                
+                for attr in attributes_list:
+                    attr_name = attr.get('name', '').lower()
+                    attr_provider = attr.get('provider', '').lower()
+                    attr_value = attr.get('value')
+                    
+                    # Look for certificate attributes from crt.sh provider
+                    if attr_provider == 'crtsh' or 'cert' in attr_name:
+                        has_certificates = True
+                        
+                        # Check certificate validity
+                        if attr_name == 'cert_is_currently_valid':
+                            if attr_value is True:
+                                has_valid_certificates = True
+                            elif attr_value is False:
+                                has_expired_certificates = True
+                        
+                        # Also check for certificate expiry indicators
+                        elif 'expires_soon' in attr_name and attr_value is True:
+                            has_expired_certificates = True
+                        elif 'expired' in attr_name and attr_value is True:
+                            has_expired_certificates = True
+                
+                # Apply styling based on certificate status
+                if has_expired_certificates and not has_valid_certificates:
+                    # Red for expired/invalid certificates
+                    node_data['color'] = {'background': '#ff6b6b', 'border': '#cc5555'}
+                elif not has_certificates:
+                    # Grey for domains with no certificates
+                    node_data['color'] = {'background': '#c7c7c7', 'border': '#999999'}
+                # Default green styling is handled by the frontend for domains with valid certificates
             
             # Add incoming and outgoing edges to node data
             if self.graph.has_node(node_id):

static/js/main.js

@@ -822,8 +822,8 @@ class DNSReconApp {
     }
 
     /**
-     * UPDATED: Enhanced node details HTML generation for unified data model
+     * Node details HTML generation for unified data model
-     * Now processes StandardAttribute objects instead of simple key-value pairs
+     * processes StandardAttribute objects 
      */
     generateNodeDetailsHtml(node) {
         if (!node) return '<div class="detail-row"><span class="detail-value">Details not available.</span></div>';
@@ -871,7 +871,7 @@ class DNSReconApp {
     }
 
     /**
-     * UPDATED: Generate details for standard nodes using unified data model
+     * Generate details for standard nodes using unified data model
      */
     generateStandardNodeDetails(node) {
         let html = '';
@@ -879,9 +879,9 @@ class DNSReconApp {
         // Relationships sections
         html += this.generateRelationshipsSection(node);
         
-        // UPDATED: Simplified attributes section for the flat model
+        // Attributes section with grouping
         if (node.attributes && Array.isArray(node.attributes) && node.attributes.length > 0) {
-            html += this.generateAttributesSection(node.attributes);
+            html += this.generateEnhancedAttributesSection(node.attributes, node.type);
         }
         
         // Description section
@@ -893,6 +893,242 @@ class DNSReconApp {
         return html;
     }
 
+    generateEnhancedAttributesSection(attributes, nodeType) {
+        if (!Array.isArray(attributes) || attributes.length === 0) {
+            return '';
+        }
+
+        // Group attributes by provider and type for better organization
+        const groupedAttributes = this.groupAttributesIntelligently(attributes, nodeType);
+        
+        let html = '';
+        
+        for (const [groupName, groupData] of Object.entries(groupedAttributes)) {
+            const isDefaultOpen = groupData.priority === 'high';
+            
+            html += `
+                <div class="modal-section">
+                    <details ${isDefaultOpen ? 'open' : ''}>
+                        <summary>
+                            <span>${groupData.icon} ${groupName}</span>
+                            <span class="count-badge">${groupData.attributes.length}</span>
+                        </summary>
+                        <div class="modal-section-content">
+                            <div class="attribute-list">
+            `;
+
+            groupData.attributes.forEach(attr => {
+                html += `
+                    <div class="attribute-item-compact">
+                        <span class="attribute-key-compact">${this.formatAttributeLabel(attr.name)}</span>
+                        <span class="attribute-value-compact">${this.formatAttributeValueEnhanced(attr)}</span>
+                    </div>
+                `;
+            });
+
+            html += '</div></div></details></div>';
+        }
+
+        return html;
+    }
+
+    formatAttributeValueEnhanced(attr) {
+        const value = attr.value;
+        
+        if (value === null || value === undefined) {
+            return '<em>None</em>';
+        }
+        
+        if (Array.isArray(value)) {
+            if (value.length === 0) return '<em>None</em>';
+            if (value.length === 1) return this.escapeHtml(String(value[0]));
+            
+            // Complex array - make it collapsible
+            const previewItems = value.slice(0, 2);
+            const hasMore = value.length > 2;
+            
+            let html = '<div class="expandable-array">';
+            html += `<div class="array-preview">`;
+            
+            previewItems.forEach(item => {
+                html += `<div class="array-item-preview">${this.escapeHtml(String(item))}</div>`;
+            });
+            
+            if (hasMore) {
+                html += `
+                    <button class="expand-array-btn" onclick="this.parentElement.style.display='none'; this.parentElement.nextElementSibling.style.display='block';">
+                        +${value.length - 2} more...
+                    </button>
+                `;
+            }
+            
+            html += '</div>';
+            
+            if (hasMore) {
+                html += `<div class="array-full" style="display: none;">`;
+                value.forEach(item => {
+                    html += `<div class="array-item-full">${this.escapeHtml(String(item))}</div>`;
+                });
+                html += `
+                    <button class="collapse-array-btn" onclick="this.parentElement.style.display='none'; this.parentElement.previousElementSibling.style.display='block';">
+                        Show less
+                    </button>
+                `;
+                html += '</div>';
+            }
+            
+            html += '</div>';
+            return html;
+        }
+        
+        if (typeof value === 'object' && value !== null) {
+            return this.formatObjectExpandable(value);
+        }
+        
+        return this.escapeHtml(String(value));
+    }
+
+    /**
+     * NEW: Format objects as expandable content
+     */
+    formatObjectExpandable(obj) {
+        if (!obj || typeof obj !== 'object') return '';
+        
+        const entries = Object.entries(obj);
+        if (entries.length === 0) return '<em>Empty</em>';
+        
+        if (entries.length <= 3) {
+            // Simple inline display for small objects
+            let html = '<div class="simple-object">';
+            entries.forEach(([key, value]) => {
+                html += `<div><strong>${key}:</strong> ${this.escapeHtml(String(value))}</div>`;
+            });
+            html += '</div>';
+            return html;
+        }
+        
+        // Expandable display for complex objects
+        let html = '<div class="expandable-object">';
+        html += `
+            <div class="object-preview">
+                <strong>${entries[0][0]}:</strong> ${this.escapeHtml(String(entries[0][1]))}
+                <button class="expand-object-btn" onclick="this.parentElement.style.display='none'; this.parentElement.nextElementSibling.style.display='block';">
+                    +${entries.length - 1} more properties...
+                </button>
+            </div>
+            <div class="object-full" style="display: none;">
+        `;
+        
+        entries.forEach(([key, value]) => {
+            html += `<div class="object-property"><strong>${key}:</strong> ${this.escapeHtml(String(value))}</div>`;
+        });
+        
+        html += `
+                <button class="collapse-object-btn" onclick="this.parentElement.style.display='none'; this.parentElement.previousElementSibling.style.display='block';">
+                    Show less
+                </button>
+            </div>
+        </div>
+        `;
+        
+        return html;
+    }
+
+    formatAttributeLabel(name) {
+        // Handle provider prefixed attributes
+        if (name.includes('_')) {
+            const parts = name.split('_');
+            if (parts.length >= 2) {
+                const provider = parts[0];
+                const attribute = parts.slice(1).join('_');
+                return `${this.provider}: ${this.formatLabel(attribute)}`;
+            }
+        }
+        
+        return this.formatLabel(name);
+    }
+
+    groupAttributesIntelligently(attributes, nodeType) {
+        const groups = {};
+
+        // Define group configurations
+        const groupConfigs = {
+            'DNS Records': {
+                icon: '🔍',
+                priority: 'high',
+                keywords: ['dns', 'record', 'a_record', 'cname', 'mx', 'ns', 'txt', 'ptr'],
+                providers: ['dns']
+            },
+            'Certificate Information': {
+                icon: '🔒',
+                priority: 'high', 
+                keywords: ['cert', 'certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'],
+                providers: ['crtsh']
+            },
+            'Network Information': {
+                icon: '🌐',
+                priority: 'high',
+                keywords: ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city'],
+                providers: ['shodan']
+            },
+            'Correlation Data': {
+                icon: '🔗',
+                priority: 'medium',
+                keywords: ['correlation', 'shared', 'common'],
+                providers: []
+            }
+        };
+
+        // Initialize groups
+        Object.entries(groupConfigs).forEach(([name, config]) => {
+            groups[name] = {
+                ...config,
+                attributes: []
+            };
+        });
+
+        // Add a catch-all group
+        groups['Other Information'] = {
+            icon: '📋',
+            priority: 'low',
+            attributes: []
+        };
+
+        // Classify attributes into groups
+        attributes.forEach(attr => {
+            let assigned = false;
+            
+            // Try to assign to a specific group based on provider or keywords
+            for (const [groupName, config] of Object.entries(groupConfigs)) {
+                const matchesProvider = config.providers.includes(attr.provider);
+                const matchesKeyword = config.keywords.some(keyword => 
+                    attr.name.toLowerCase().includes(keyword) || 
+                    attr.type.toLowerCase().includes(keyword)
+                );
+                
+                if (matchesProvider || matchesKeyword) {
+                    groups[groupName].attributes.push(attr);
+                    assigned = true;
+                    break;
+                }
+            }
+            
+            // If not assigned to any specific group, put in "Other"
+            if (!assigned) {
+                groups['Other Information'].attributes.push(attr);
+            }
+        });
+
+        // Remove empty groups
+        Object.keys(groups).forEach(groupName => {
+            if (groups[groupName].attributes.length === 0) {
+                delete groups[groupName];
+            }
+        });
+
+        return groups;
+    }
+
     /**
      * UPDATED: Generate large entity details using unified data model
      */