diff --git a/core/graph_manager.py b/core/graph_manager.py index edd31c3..5054505 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -41,7 +41,7 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days'] + self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days','crtsh_cert_source'] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" @@ -112,7 +112,7 @@ class GraphManager: def _create_enhanced_correlation_node_and_edges(self, value, correlation_data): """ - UPDATED: Create correlation node and edges with detailed provider tracking. + UPDATED: Create correlation node and edges with raw provider data (no formatting). """ correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}" nodes = correlation_data['nodes'] @@ -120,13 +120,14 @@ class GraphManager: # Create or update correlation node if not self.graph.has_node(correlation_node_id): - # Determine the most common provider/attribute combination + # Use raw provider/attribute data - no formatting provider_counts = {} for source in sources: + # Keep original provider and attribute names key = f"{source['provider']}_{source['attribute']}" provider_counts[key] = provider_counts.get(key, 0) + 1 - # Use the most common provider/attribute as the primary label + # Use the most common provider/attribute as the primary label (raw) primary_source = max(provider_counts.items(), key=lambda x: x[1])[0] if provider_counts else "unknown_correlation" metadata = { @@ -303,18 +304,18 @@ class GraphManager: return is_new_node def add_edge(self, source_id: str, target_id: str, relationship_type: str, - confidence_score: float = 0.5, source_provider: str = "unknown", - raw_data: Optional[Dict[str, Any]] = None) -> bool: - """Add or update an edge between two nodes, ensuring nodes exist.""" + confidence_score: float = 0.5, source_provider: str = "unknown", + raw_data: Optional[Dict[str, Any]] = None) -> bool: + """ + UPDATED: Add or update an edge between two nodes with raw relationship labels. + """ if not self.graph.has_node(source_id) or not self.graph.has_node(target_id): return False new_confidence = confidence_score - if relationship_type.startswith("c_"): - edge_label = relationship_type - else: - edge_label = f"{source_provider}_{relationship_type}" + # UPDATED: Use raw relationship type - no formatting + edge_label = relationship_type if self.graph.has_edge(source_id, target_id): # If edge exists, update confidence if the new score is higher. @@ -324,7 +325,7 @@ class GraphManager: self.graph.edges[source_id, target_id]['updated_by'] = source_provider return False - # Add a new edge with all attributes. + # Add a new edge with raw attributes self.graph.add_edge(source_id, target_id, relationship_type=edge_label, confidence_score=new_confidence, @@ -333,7 +334,7 @@ class GraphManager: raw_data=raw_data or {}) self.last_modified = datetime.now(timezone.utc).isoformat() return True - + def extract_node_from_large_entity(self, large_entity_id: str, node_id_to_extract: str) -> bool: """ Removes a node from a large entity's internal lists and updates its count. @@ -417,73 +418,45 @@ class GraphManager: def get_graph_data(self) -> Dict[str, Any]: """ Export graph data formatted for frontend visualization. - UPDATED: Fixed certificate validity styling logic for unified data model. + SIMPLIFIED: No certificate styling - frontend handles all visual styling. """ nodes = [] for node_id, attrs in self.graph.nodes(data=True): - node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'), - 'attributes': attrs.get('attributes', []), # Ensure attributes is a list - 'description': attrs.get('description', ''), - 'metadata': attrs.get('metadata', {}), - 'added_timestamp': attrs.get('added_timestamp')} - - # UPDATED: Fixed certificate validity styling logic - node_type = node_data['type'] - attributes_list = node_data['attributes'] - - if node_type == 'domain' and isinstance(attributes_list, list): - # Check for certificate-related attributes - has_certificates = False - has_valid_certificates = False - has_expired_certificates = False - - for attr in attributes_list: - attr_name = attr.get('name', '').lower() - attr_provider = attr.get('provider', '').lower() - attr_value = attr.get('value') - - # Look for certificate attributes from crt.sh provider - if attr_provider == 'crtsh' or 'cert' in attr_name: - has_certificates = True - - # Check certificate validity - if attr_name == 'cert_is_currently_valid': - if attr_value is True: - has_valid_certificates = True - elif attr_value is False: - has_expired_certificates = True - - # Also check for certificate expiry indicators - elif 'expires_soon' in attr_name and attr_value is True: - has_expired_certificates = True - elif 'expired' in attr_name and attr_value is True: - has_expired_certificates = True - - # Apply styling based on certificate status - if has_expired_certificates and not has_valid_certificates: - # Red for expired/invalid certificates - node_data['color'] = {'background': '#ff6b6b', 'border': '#cc5555'} - elif not has_certificates: - # Grey for domains with no certificates - node_data['color'] = {'background': '#c7c7c7', 'border': '#999999'} - # Default green styling is handled by the frontend for domains with valid certificates + node_data = { + 'id': node_id, + 'label': node_id, + 'type': attrs.get('type', 'unknown'), + 'attributes': attrs.get('attributes', []), # Raw attributes list + 'description': attrs.get('description', ''), + 'metadata': attrs.get('metadata', {}), + 'added_timestamp': attrs.get('added_timestamp') + } # Add incoming and outgoing edges to node data if self.graph.has_node(node_id): - node_data['incoming_edges'] = [{'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)] - node_data['outgoing_edges'] = [{'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)] + node_data['incoming_edges'] = [ + {'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True) + ] + node_data['outgoing_edges'] = [ + {'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True) + ] nodes.append(node_data) edges = [] for source, target, attrs in self.graph.edges(data=True): - edges.append({'from': source, 'to': target, - 'label': attrs.get('relationship_type', ''), - 'confidence_score': attrs.get('confidence_score', 0), - 'source_provider': attrs.get('source_provider', ''), - 'discovery_timestamp': attrs.get('discovery_timestamp')}) + edges.append({ + 'from': source, + 'to': target, + 'label': attrs.get('relationship_type', ''), + 'confidence_score': attrs.get('confidence_score', 0), + 'source_provider': attrs.get('source_provider', ''), + 'discovery_timestamp': attrs.get('discovery_timestamp') + }) + return { - 'nodes': nodes, 'edges': edges, + 'nodes': nodes, + 'edges': edges, 'statistics': self.get_statistics()['basic_metrics'] } diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index d8bdfba..6adde15 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -355,7 +355,7 @@ class CrtShProvider(BaseProvider): 'not_before': cert_data.get('not_before'), 'not_after': cert_data.get('not_after'), 'entry_timestamp': cert_data.get('entry_timestamp'), - 'source': 'crt.sh' + 'source': 'crtsh' } try: @@ -367,8 +367,9 @@ class CrtShProvider(BaseProvider): metadata['is_currently_valid'] = self._is_cert_valid(cert_data) metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30 - metadata['not_before'] = not_before.strftime('%Y-%m-%d %H:%M:%S UTC') - metadata['not_after'] = not_after.strftime('%Y-%m-%d %H:%M:%S UTC') + # UPDATED: Keep raw date format or convert to standard format + metadata['not_before'] = not_before.isoformat() + metadata['not_after'] = not_after.isoformat() except Exception as e: self.logger.logger.debug(f"Error computing certificate metadata: {e}") diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 5d972d1..7abaf58 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -155,12 +155,7 @@ class DNSProvider(BaseProvider): def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None: """ - Query a specific type of DNS record for the domain and add results to ProviderResult. - - Args: - domain: Domain to query - record_type: DNS record type (A, AAAA, CNAME, etc.) - result: ProviderResult to populate + UPDATED: Query DNS records with minimal formatting - keep raw values. """ try: self.total_requests += 1 @@ -180,13 +175,14 @@ class DNSProvider(BaseProvider): elif record_type == 'SOA': target = str(record.mname).rstrip('.') elif record_type in ['TXT']: - # TXT records are treated as attributes, not relationships + # UPDATED: Keep raw TXT record value txt_value = str(record).strip('"') dns_records.append(f"TXT: {txt_value}") continue elif record_type == 'SRV': target = str(record.target).rstrip('.') elif record_type == 'CAA': + # UPDATED: Keep raw CAA record format caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\"" dns_records.append(f"CAA: {caa_value}") continue @@ -200,8 +196,8 @@ class DNSProvider(BaseProvider): 'value': target, 'ttl': response.ttl } - relationship_type = f"{record_type.lower()}_record" - confidence = 0.8 # Standard confidence for DNS records + relationship_type = f"{record_type.lower()}_record" # Raw relationship type + confidence = 0.8 # Add relationship result.add_relationship( @@ -213,7 +209,7 @@ class DNSProvider(BaseProvider): raw_data=raw_data ) - # Add DNS record as attribute to the source domain + # UPDATED: Keep raw DNS record format dns_records.append(f"{record_type}: {target}") # Log relationship discovery @@ -226,7 +222,7 @@ class DNSProvider(BaseProvider): discovery_method=f"dns_{record_type.lower()}_record" ) - # Add DNS records as a consolidated attribute + # Add DNS records as a consolidated attribute (raw format) if dns_records: result.add_attribute( target_node=domain, @@ -241,5 +237,5 @@ class DNSProvider(BaseProvider): except Exception as e: self.failed_requests += 1 self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}") - # Re-raise the exception so the scanner can handle it - raise e \ No newline at end of file + raise e + diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 6f3a044..00d0cea 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -211,14 +211,7 @@ class ShodanProvider(BaseProvider): def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult: """ - Process Shodan data to extract relationships and attributes. - - Args: - ip: IP address queried - data: Raw Shodan response data - - Returns: - ProviderResult with relationships and attributes + UPDATED: Process Shodan data with raw attribute names and values. """ result = ProviderResult() @@ -271,9 +264,10 @@ class ShodanProvider(BaseProvider): confidence=0.9 ) elif isinstance(value, (str, int, float, bool)) and value is not None: + # UPDATED: Keep raw Shodan field names (no "shodan_" prefix) result.add_attribute( target_node=ip, - name=f"shodan_{key}", + name=key, # Raw field name from Shodan API value=value, attr_type='shodan_info', provider=self.name, diff --git a/static/js/graph.js b/static/js/graph.js index dd5bc13..9391150 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -382,7 +382,6 @@ class GraphManager { graphData.nodes.forEach(node => { if (node.type === 'large_entity' && node.attributes) { - // UPDATED: Handle unified data model - look for 'nodes' attribute in the attributes list const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes'); if (nodesAttribute && Array.isArray(nodesAttribute.value)) { nodesAttribute.value.forEach(nodeId => { @@ -394,15 +393,30 @@ class GraphManager { }); const filteredNodes = graphData.nodes.filter(node => { - // Only include nodes that are NOT members of large entities, but always include the container itself return !this.largeEntityMembers.has(node.id) || node.type === 'large_entity'; }); console.log(`Filtered ${graphData.nodes.length - filteredNodes.length} large entity member nodes from visualization`); - // Process only the filtered nodes + // FIXED: Process nodes with proper certificate coloring const processedNodes = filteredNodes.map(node => { - return this.processNode(node); + const processed = this.processNode(node); + + // FIXED: Apply certificate-based coloring here in frontend + if (node.type === 'domain' && Array.isArray(node.attributes)) { + const certInfo = this.analyzeCertificateInfo(node.attributes); + + if (certInfo.hasExpiredOnly) { + // Red for domains with only expired/invalid certificates + processed.color = { background: '#ff6b6b', border: '#cc5555' }; + } else if (!certInfo.hasCertificates) { + // Grey for domains with no certificates + processed.color = { background: '#c7c7c7', border: '#999999' }; + } + // Valid certificates use default green (handled by processNode) + } + + return processed; }); const mergedEdges = {}; @@ -439,24 +453,19 @@ class GraphManager { const existingNodeIds = this.nodes.getIds(); const existingEdgeIds = this.edges.getIds(); - // Add new nodes with fade-in animation const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id)); const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id)); - // Update existing data this.nodes.update(processedNodes); this.edges.update(processedEdges); - // After data is loaded, apply filters this.updateFilterControls(); this.applyAllFilters(); - // Highlight new additions briefly if (newNodes.length > 0 || newEdges.length > 0) { setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100); } - // Auto-fit view for small graphs or first update if (processedNodes.length <= 10 || existingNodeIds.length === 0) { setTimeout(() => this.fitView(), 800); } @@ -470,6 +479,46 @@ class GraphManager { } } + analyzeCertificateInfo(attributes) { + let hasCertificates = false; + let hasValidCertificates = false; + let hasExpiredCertificates = false; + + for (const attr of attributes) { + const attrName = (attr.name || '').toLowerCase(); + const attrProvider = (attr.provider || '').toLowerCase(); + const attrValue = attr.value; + + // Look for certificate attributes from crtsh provider + if (attrProvider === 'crtsh' || attrName.startsWith('cert_')) { + hasCertificates = true; + + // Check certificate validity using raw attribute names + if (attrName === 'cert_is_currently_valid') { + if (attrValue === true) { + hasValidCertificates = true; + } else if (attrValue === false) { + hasExpiredCertificates = true; + } + } + // Check for expiry indicators + else if (attrName === 'cert_expires_soon' && attrValue === true) { + hasExpiredCertificates = true; + } + else if (attrName.includes('expired') && attrValue === true) { + hasExpiredCertificates = true; + } + } + } + + return { + hasCertificates, + hasValidCertificates, + hasExpiredCertificates, + hasExpiredOnly: hasExpiredCertificates && !hasValidCertificates + }; + } + /** * UPDATED: Helper method to find an attribute by name in the standardized attributes list * @param {Array} attributes - List of StandardAttribute objects @@ -496,7 +545,7 @@ class GraphManager { size: this.getNodeSize(node.type), borderColor: this.getNodeBorderColor(node.type), shape: this.getNodeShape(node.type), - attributes: node.attributes || [], // Keep as standardized attributes list + attributes: node.attributes || [], description: node.description || '', metadata: node.metadata || {}, type: node.type, @@ -509,19 +558,33 @@ class GraphManager { processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); } - // Handle merged correlation objects (similar to large entities) + // FIXED: Certificate-based domain coloring + if (node.type === 'domain' && Array.isArray(node.attributes)) { + const certInfo = this.analyzeCertificateInfo(node.attributes); + + if (certInfo.hasExpiredOnly) { + // Red for domains with only expired/invalid certificates + processedNode.color = '#ff6b6b'; + processedNode.borderColor = '#cc5555'; + } else if (!certInfo.hasCertificates) { + // Grey for domains with no certificates + processedNode.color = '#c7c7c7'; + processedNode.borderColor = '#999999'; + } + // Green for valid certificates (default color) + } + + // Handle merged correlation objects if (node.type === 'correlation_object') { const metadata = node.metadata || {}; const values = metadata.values || []; const mergeCount = metadata.merge_count || 1; if (mergeCount > 1) { - // Display as merged correlation container processedNode.label = `Correlations (${mergeCount})`; processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`; - processedNode.borderWidth = 3; // Thicker border for merged nodes + processedNode.borderWidth = 3; } else { - // Single correlation value const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown'); const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value; processedNode.label = `${displayValue}`; @@ -532,6 +595,7 @@ class GraphManager { return processedNode; } + /** * Process edge data with styling and metadata * @param {Object} edge - Raw edge data diff --git a/static/js/main.js b/static/js/main.js index 27be70b..72f5fdf 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -868,20 +868,14 @@ class DNSReconApp { return html; } - /** - * NEW: Organized attributes section with provider/semantic grouping (no formatting) - */ generateOrganizedAttributesSection(attributes, nodeType) { if (!Array.isArray(attributes) || attributes.length === 0) { return ''; } - // Group attributes intelligently const groups = this.groupAttributesByProviderAndType(attributes, nodeType); - let html = ''; - // Sort groups by priority const sortedGroups = Object.entries(groups).sort((a, b) => { const priorityOrder = { 'high': 0, 'medium': 1, 'low': 2 }; return priorityOrder[a[1].priority] - priorityOrder[b[1].priority]; @@ -904,22 +898,10 @@ class DNSReconApp { `; groupData.attributes.forEach(attr => { - // Format the value appropriately - let displayValue = ''; - if (attr.value === null || attr.value === undefined) { - displayValue = 'N/A'; - } else if (Array.isArray(attr.value)) { - displayValue = attr.value.length > 0 ? `Array (${attr.value.length} items)` : 'Empty Array'; - } else if (typeof attr.value === 'object') { - displayValue = 'Object'; - } else { - displayValue = String(attr.value); - } - html += `
${this.escapeHtml(String(value))}