diff --git a/core/graph_manager.py b/core/graph_manager.py index 69dbda8..b7c2159 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -40,6 +40,7 @@ class GraphManager: self.correlation_index = {} # Compile regex for date filtering for efficiency self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') + self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type'] def __getstate__(self): """Prepare GraphManager for pickling, excluding compiled regex.""" @@ -54,145 +55,44 @@ class GraphManager: self.__dict__.update(state) self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') - def _update_correlation_index(self, node_id: str, data: Any, path: List[str] = [], parent_attr: str = ""): - """Recursively traverse metadata and add hashable values to the index with better path tracking.""" - if path is None: - path = [] - - if isinstance(data, dict): - for key, value in data.items(): - self._update_correlation_index(node_id, value, path + [key], key) - elif isinstance(data, list): - for i, item in enumerate(data): - # Instead of just using [i], include the parent attribute context - list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]" - self._update_correlation_index(node_id, item, path + [list_path_component], parent_attr) - else: - self._add_to_correlation_index(node_id, data, ".".join(path), parent_attr) - - def _add_to_correlation_index(self, node_id: str, value: Any, path_str: str, parent_attr: str = ""): - """Add a hashable value to the correlation index, filtering out noise.""" - if not isinstance(value, (str, int, float, bool)) or value is None: + def process_correlations_for_node(self, node_id: str): + """Process correlations for a given node based on its attributes.""" + if not self.graph.has_node(node_id): return - # Ignore certain paths that contain noisy, non-unique identifiers - if any(keyword in path_str.lower() for keyword in ['count', 'total', 'timestamp', 'date']): - return + node_attributes = self.graph.nodes[node_id].get('attributes', []) + for attr in node_attributes: + attr_name = attr.get('name') + attr_value = attr.get('value') - # Filter out common low-entropy values and date-like strings - if isinstance(value, str): - # FIXED: Prevent correlation on date/time strings. - if self.date_pattern.match(value): - return - if len(value) < 4 or value.lower() in ['true', 'false', 'unknown', 'none', 'crt.sh']: - return - elif isinstance(value, int) and (abs(value) < 1024 or abs(value) > 65535): - return # Ignore small integers and common port numbers - elif isinstance(value, bool): - return # Ignore boolean values + if attr_name in self.EXCLUDED_KEYS or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None: + continue - # Add the valuable correlation data to the index - if value not in self.correlation_index: - self.correlation_index[value] = {} - if node_id not in self.correlation_index[value]: - self.correlation_index[value][node_id] = [] - - # Store both the full path and the parent attribute for better edge labeling - correlation_entry = { - 'path': path_str, - 'parent_attr': parent_attr, - 'meaningful_attr': self._extract_meaningful_attribute(path_str, parent_attr) - } - - if correlation_entry not in self.correlation_index[value][node_id]: - self.correlation_index[value][node_id].append(correlation_entry) - - def _extract_meaningful_attribute(self, path_str: str, parent_attr: str = "") -> str: - """Extract the most meaningful attribute name from a path string.""" - if not path_str: - return "unknown" - - path_parts = path_str.split('.') - - # Look for the last non-array-index part - for part in reversed(path_parts): - # Skip array indices like [0], [1], etc. - if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()): - # Clean up compound names like "hostnames[0]" to just "hostnames" - clean_part = re.sub(r'\[\d+\]$', '', part) - if clean_part: - return clean_part - - # Fallback to parent attribute if available - if parent_attr: - return parent_attr - - # Last resort - use the first meaningful part - for part in path_parts: - if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()): - clean_part = re.sub(r'\[\d+\]$', '', part) - if clean_part: - return clean_part - - return "correlation" - - def _check_for_correlations(self, new_node_id: str, data: Any, path: List[str] = [], parent_attr: str = "") -> List[Dict]: - """Recursively traverse metadata to find correlations with existing data.""" - if path is None: - path = [] - - all_correlations = [] - if isinstance(data, dict): - for key, value in data.items(): - if key == 'source': # Avoid correlating on the provider name - continue - all_correlations.extend(self._check_for_correlations(new_node_id, value, path + [key], key)) - elif isinstance(data, list): - for i, item in enumerate(data): - list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]" - all_correlations.extend(self._check_for_correlations(new_node_id, item, path + [list_path_component], parent_attr)) - else: - value = data - if value in self.correlation_index: - existing_nodes_with_paths = self.correlation_index[value] - unique_nodes = set(existing_nodes_with_paths.keys()) - unique_nodes.add(new_node_id) - - if len(unique_nodes) < 2: - return all_correlations # Correlation must involve at least two distinct nodes - - new_source = { - 'node_id': new_node_id, - 'path': ".".join(path), - 'parent_attr': parent_attr, - 'meaningful_attr': self._extract_meaningful_attribute(".".join(path), parent_attr) - } - all_sources = [new_source] + if isinstance(attr_value, bool): + continue - for node_id, path_entries in existing_nodes_with_paths.items(): - for entry in path_entries: - if isinstance(entry, dict): - all_sources.append({ - 'node_id': node_id, - 'path': entry['path'], - 'parent_attr': entry.get('parent_attr', ''), - 'meaningful_attr': entry.get('meaningful_attr', self._extract_meaningful_attribute(entry['path'], entry.get('parent_attr', ''))) - }) - else: - # Handle legacy string-only entries - all_sources.append({ - 'node_id': node_id, - 'path': str(entry), - 'parent_attr': '', - 'meaningful_attr': self._extract_meaningful_attribute(str(entry)) - }) + if isinstance(attr_value, str) and (len(attr_value) < 4 or self.date_pattern.match(attr_value)): + continue + + if attr_value not in self.correlation_index: + self.correlation_index[attr_value] = set() + + self.correlation_index[attr_value].add(node_id) + + if len(self.correlation_index[attr_value]) > 1: + self._create_correlation_node_and_edges(attr_value, self.correlation_index[attr_value]) + + def _create_correlation_node_and_edges(self, value, nodes): + """Create a correlation node and edges to the correlated nodes.""" + correlation_node_id = f"corr_{value}" + if not self.graph.has_node(correlation_node_id): + self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, + metadata={'value': value, 'correlated_nodes': list(nodes)}) + + for node_id in nodes: + if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id): + self.add_edge(node_id, correlation_node_id, "correlation", confidence_score=0.9) - all_correlations.append({ - 'value': value, - 'sources': all_sources, - 'nodes': list(unique_nodes) - }) - return all_correlations def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None, description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool: @@ -232,78 +132,9 @@ class GraphManager: existing_metadata.update(metadata) self.graph.nodes[node_id]['metadata'] = existing_metadata - if attributes and node_type != NodeType.CORRELATION_OBJECT: - correlations = self._check_for_correlations(node_id, attributes) - for corr in correlations: - value = corr['value'] - - # STEP 1: Substring check against all existing nodes - if self._correlation_value_matches_existing_node(value): - # Skip creating correlation node - would be redundant - continue - - eligible_nodes = set(corr['nodes']) - - if len(eligible_nodes) < 2: - # Need at least 2 nodes to create a correlation - continue - - # STEP 3: Check for existing correlation node with same connection pattern - correlation_nodes_with_pattern = self._find_correlation_nodes_with_same_pattern(eligible_nodes) - - if correlation_nodes_with_pattern: - # STEP 4: Merge with existing correlation node - target_correlation_node = correlation_nodes_with_pattern[0] - self._merge_correlation_values(target_correlation_node, value, corr) - else: - # STEP 5: Create new correlation node for eligible nodes only - correlation_node_id = f"corr_{abs(hash(str(sorted(eligible_nodes))))}" - self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, - metadata={'values': [value], 'sources': corr['sources'], - 'correlated_nodes': list(eligible_nodes)}) - - # Create edges from eligible nodes to this correlation node with better labeling - for c_node_id in eligible_nodes: - if self.graph.has_node(c_node_id): - # Find the best attribute name for this node - meaningful_attr = self._find_best_attribute_name_for_node(c_node_id, corr['sources']) - relationship_type = f"c_{meaningful_attr}" - self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9) - - self._update_correlation_index(node_id, attributes) - self.last_modified = datetime.now(timezone.utc).isoformat() return is_new_node - def _find_best_attribute_name_for_node(self, node_id: str, sources: List[Dict]) -> str: - """Find the best attribute name for a correlation edge by looking at the sources.""" - node_sources = [s for s in sources if s['node_id'] == node_id] - - if not node_sources: - return "correlation" - - # Use the meaningful_attr if available - for source in node_sources: - meaningful_attr = source.get('meaningful_attr') - if meaningful_attr and meaningful_attr != "unknown": - return meaningful_attr - - # Fallback to parent_attr - for source in node_sources: - parent_attr = source.get('parent_attr') - if parent_attr: - return parent_attr - - # Last resort - extract from path - for source in node_sources: - path = source.get('path', '') - if path: - extracted = self._extract_meaningful_attribute(path) - if extracted != "unknown": - return extracted - - return "correlation" - def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool: """ Check if there's a direct edge between two nodes in either direction. diff --git a/core/scanner.py b/core/scanner.py index 11b5493..6b234b6 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -506,6 +506,7 @@ class Scanner: large_entity_members.update(discovered) else: new_targets.update(discovered) + self.graph.process_correlations_for_node(target) else: print(f"Stop requested after processing results from {provider.get_name()}") except Exception as e: diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index 4ccce97..d8bdfba 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -134,7 +134,7 @@ class CrtShProvider(BaseProvider): self.logger.logger.info(f"Refreshed and merged cache for {domain}") else: # "not_found" # Create new result from processed certs - result = self._process_certificates_to_result(domain, current_processed_certs) + result = self._process_certificates_to_result(domain, raw_certificates) self.logger.logger.info(f"Created fresh result for {domain} ({result.get_relationship_count()} relationships)") # Save the result to cache @@ -272,109 +272,73 @@ class CrtShProvider(BaseProvider): Process certificates to create ProviderResult with relationships and attributes. """ result = ProviderResult() - + if self._stop_event and self._stop_event.is_set(): print(f"CrtSh processing cancelled before processing for domain: {domain}") return result - # Aggregate certificate data by domain - domain_certificates = {} all_discovered_domains = set() - - # Process certificates with cancellation checking + for i, cert_data in enumerate(certificates): if i % 5 == 0 and self._stop_event and self._stop_event.is_set(): print(f"CrtSh processing cancelled at certificate {i} for domain: {domain}") break - - cert_metadata = self._extract_certificate_metadata(cert_data) + cert_domains = self._extract_domains_from_certificate(cert_data) - all_discovered_domains.update(cert_domains) + for cert_domain in cert_domains: if not _is_valid_domain(cert_domain): continue - - if cert_domain not in domain_certificates: - domain_certificates[cert_domain] = [] - - domain_certificates[cert_domain].append(cert_metadata) - + + for key, value in self._extract_certificate_metadata(cert_data).items(): + if value is not None: + result.add_attribute( + target_node=cert_domain, + name=f"cert_{key}", + value=value, + attr_type='certificate_data', + provider=self.name, + confidence=0.9 + ) + if self._stop_event and self._stop_event.is_set(): print(f"CrtSh query cancelled before relationship creation for domain: {domain}") return result - # Create relationships from query domain to ALL discovered domains for i, discovered_domain in enumerate(all_discovered_domains): if discovered_domain == domain: - continue # Skip self-relationships - + continue + if i % 10 == 0 and self._stop_event and self._stop_event.is_set(): print(f"CrtSh relationship creation cancelled for domain: {domain}") break if not _is_valid_domain(discovered_domain): continue - - # Get certificates for both domains - query_domain_certs = domain_certificates.get(domain, []) - discovered_domain_certs = domain_certificates.get(discovered_domain, []) - - # Find shared certificates - shared_certificates = self._find_shared_certificates(query_domain_certs, discovered_domain_certs) - - # Calculate confidence + confidence = self._calculate_domain_relationship_confidence( - domain, discovered_domain, shared_certificates, all_discovered_domains + domain, discovered_domain, [], all_discovered_domains ) - - # Create comprehensive raw data for the relationship - relationship_raw_data = { - 'relationship_type': 'certificate_discovery', - 'shared_certificates': shared_certificates, - 'total_shared_certs': len(shared_certificates), - 'discovery_context': self._determine_relationship_context(discovered_domain, domain), - 'domain_certificates': { - domain: self._summarize_certificates(query_domain_certs), - discovered_domain: self._summarize_certificates(discovered_domain_certs) - } - } - - # Add relationship + result.add_relationship( source_node=domain, target_node=discovered_domain, relationship_type='san_certificate', provider=self.name, confidence=confidence, - raw_data=relationship_raw_data + raw_data={'relationship_type': 'certificate_discovery'} ) - - # Log the relationship discovery + self.log_relationship_discovery( source_node=domain, target_node=discovered_domain, relationship_type='san_certificate', confidence_score=confidence, - raw_data=relationship_raw_data, + raw_data={'relationship_type': 'certificate_discovery'}, discovery_method="certificate_transparency_analysis" ) - # Add certificate summary as attributes for all domains that have certificates - for cert_domain, cert_list in domain_certificates.items(): - if cert_list: - cert_summary = self._summarize_certificates(cert_list) - - result.add_attribute( - target_node=cert_domain, - name='certificates', - value=cert_summary, - attr_type='certificate_data', - provider=self.name, - confidence=0.9, - metadata={'total_certificates': len(cert_list)} - ) - return result def _extract_certificate_metadata(self, cert_data: Dict[str, Any]) -> Dict[str, Any]: diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 21b530c..6f3a044 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -222,110 +222,62 @@ class ShodanProvider(BaseProvider): """ result = ProviderResult() - # Extract hostname relationships - hostnames = data.get('hostnames', []) - for hostname in hostnames: - if _is_valid_domain(hostname): + for key, value in data.items(): + if key == 'hostnames': + for hostname in value: + if _is_valid_domain(hostname): + result.add_relationship( + source_node=ip, + target_node=hostname, + relationship_type='a_record', + provider=self.name, + confidence=0.8, + raw_data=data + ) + self.log_relationship_discovery( + source_node=ip, + target_node=hostname, + relationship_type='a_record', + confidence_score=0.8, + raw_data=data, + discovery_method="shodan_host_lookup" + ) + elif key == 'asn': + asn_name = f"AS{value[2:]}" if isinstance(value, str) and value.startswith('AS') else f"AS{value}" result.add_relationship( source_node=ip, - target_node=hostname, - relationship_type='a_record', + target_node=asn_name, + relationship_type='asn_membership', provider=self.name, - confidence=0.8, + confidence=0.7, raw_data=data ) - self.log_relationship_discovery( source_node=ip, - target_node=hostname, - relationship_type='a_record', - confidence_score=0.8, + target_node=asn_name, + relationship_type='asn_membership', + confidence_score=0.7, raw_data=data, - discovery_method="shodan_host_lookup" + discovery_method="shodan_asn_lookup" + ) + elif key == 'ports': + for port in value: + result.add_attribute( + target_node=ip, + name='open_port', + value=port, + attr_type='network_info', + provider=self.name, + confidence=0.9 + ) + elif isinstance(value, (str, int, float, bool)) and value is not None: + result.add_attribute( + target_node=ip, + name=f"shodan_{key}", + value=value, + attr_type='shodan_info', + provider=self.name, + confidence=0.9 ) - # Extract ASN relationship - asn = data.get('asn') - if asn: - asn_name = f"AS{asn[2:]}" if isinstance(asn, str) and asn.startswith('AS') else f"AS{asn}" - result.add_relationship( - source_node=ip, - target_node=asn_name, - relationship_type='asn_membership', - provider=self.name, - confidence=0.7, - raw_data=data - ) - - self.log_relationship_discovery( - source_node=ip, - target_node=asn_name, - relationship_type='asn_membership', - confidence_score=0.7, - raw_data=data, - discovery_method="shodan_asn_lookup" - ) - - # Add comprehensive Shodan host information as attributes - if 'ports' in data: - result.add_attribute( - target_node=ip, - name='ports', - value=data['ports'], - attr_type='network_info', - provider=self.name, - confidence=0.9 - ) - - if 'os' in data and data['os']: - result.add_attribute( - target_node=ip, - name='operating_system', - value=data['os'], - attr_type='system_info', - provider=self.name, - confidence=0.8 - ) - - if 'org' in data: - result.add_attribute( - target_node=ip, - name='organization', - value=data['org'], - attr_type='network_info', - provider=self.name, - confidence=0.8 - ) - - if 'country_name' in data: - result.add_attribute( - target_node=ip, - name='country', - value=data['country_name'], - attr_type='location_info', - provider=self.name, - confidence=0.9 - ) - - if 'city' in data: - result.add_attribute( - target_node=ip, - name='city', - value=data['city'], - attr_type='location_info', - provider=self.name, - confidence=0.8 - ) - - # Store complete Shodan data as a comprehensive attribute - result.add_attribute( - target_node=ip, - name='shodan_host_info', - value=data, # Complete Shodan response for full forensic detail - attr_type='comprehensive_data', - provider=self.name, - confidence=0.9, - metadata={'data_source': 'shodan_api', 'query_type': 'host_lookup'} - ) - return result \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index 6f8d6f3..dd5bc13 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -1,7 +1,7 @@ /** * Graph visualization module for DNSRecon * Handles network graph rendering using vis.js with proper large entity node hiding - * UPDATED: Now compatible with unified data model (StandardAttribute objects) + * UPDATED: Now compatible with a strictly flat, unified data model for attributes. */ const contextMenuCSS = ` .graph-context-menu { @@ -484,7 +484,7 @@ class GraphManager { } /** - * UPDATED: Process node data with styling and metadata for unified data model + * UPDATED: Process node data with styling and metadata for the flat data model * @param {Object} node - Raw node data with standardized attributes * @returns {Object} Processed node data */ @@ -508,14 +508,6 @@ class GraphManager { if (node.confidence) { processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); } - - // UPDATED: Style based on certificate validity using unified data model - if (node.type === 'domain') { - const certificatesAttr = this.findAttributeByName(node.attributes, 'certificates'); - if (certificatesAttr && certificatesAttr.value && certificatesAttr.value.has_valid_cert === false) { - processedNode.color = { background: '#888888', border: '#666666' }; - } - } // Handle merged correlation objects (similar to large entities) if (node.type === 'correlation_object') { diff --git a/static/js/main.js b/static/js/main.js index 5ce2477..d6dccf7 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -1,7 +1,7 @@ /** * Main application logic for DNSRecon web interface * Handles UI interactions, API communication, and data flow - * UPDATED: Now compatible with unified data model (StandardAttribute objects) + * UPDATED: Now compatible with a strictly flat, unified data model for attributes. */ class DNSReconApp { @@ -879,21 +879,9 @@ class DNSReconApp { // Relationships sections html += this.generateRelationshipsSection(node); - // UPDATED: Enhanced attributes section with special certificate handling for unified model + // UPDATED: Simplified attributes section for the flat model if (node.attributes && Array.isArray(node.attributes) && node.attributes.length > 0) { - // Find certificate attribute separately - const certificatesAttr = this.findAttributeByName(node.attributes, 'certificates'); - - // Handle certificates separately with enhanced display - if (certificatesAttr) { - html += this.generateCertificateSection(certificatesAttr); - } - - // Handle other attributes normally (excluding certificates to avoid duplication) - const otherAttributes = node.attributes.filter(attr => attr.name !== 'certificates'); - if (otherAttributes.length > 0) { - html += this.generateAttributesSection(otherAttributes); - } + html += this.generateAttributesSection(node.attributes); } // Description section @@ -905,213 +893,6 @@ class DNSReconApp { return html; } - /** - * UPDATED: Enhanced certificate section generation for unified data model - */ - generateCertificateSection(certificatesAttr) { - const certificates = certificatesAttr.value; - if (!certificates || typeof certificates !== 'object') { - return ''; - } - - let html = ` -
-
- 🔒 SSL/TLS Certificates -
- `; - - // Certificate summary using existing grid pattern - html += this.generateCertificateSummary(certificates); - - // Latest certificate info using existing attribute display - if (certificates.latest_certificate) { - html += this.generateLatestCertificateInfo(certificates.latest_certificate); - } - - // Detailed certificate list if available - if (certificates.certificate_details && Array.isArray(certificates.certificate_details)) { - html += this.generateCertificateList(certificates.certificate_details); - } - - html += '
'; - return html; - } - - /** - * Generate latest certificate info using existing attribute list - */ - generateLatestCertificateInfo(latest) { - const isValid = latest.is_currently_valid; - const statusText = isValid ? 'Valid' : 'Invalid/Expired'; - const statusColor = isValid ? '#00ff41' : '#ff6b6b'; - - let html = ` -
-
Most Recent Certificate
-
-
- Status: - ${statusText} -
-
- Issued: - ${latest.not_before || 'Unknown'} -
-
- Expires: - ${latest.not_after || 'Unknown'} -
-
- Issuer: - ${this.escapeHtml(latest.issuer_name || 'Unknown')} -
- ${latest.certificate_id ? ` -
- Certificate: - - - View on crt.sh ↗ - - -
- ` : ''} -
-
- `; - - return html; - } - - /** - * Generate certificate list using existing collapsible structure - */ - generateCertificateList(certificateDetails) { - if (!certificateDetails || certificateDetails.length === 0) { - return ''; - } - - // Limit display to prevent overwhelming the UI - const maxDisplay = 8; - const certificates = certificateDetails.slice(0, maxDisplay); - const remaining = certificateDetails.length - maxDisplay; - - let html = ` -
- 📋 Certificate Details (${certificates.length}${remaining > 0 ? ` of ${certificateDetails.length}` : ''}) -
- `; - - certificates.forEach((cert, index) => { - const isValid = cert.is_currently_valid; - let statusText = isValid ? '✅ Valid' : '❌ Invalid/Expired'; - let statusColor = isValid ? '#00ff41' : '#ff6b6b'; - - if (cert.expires_soon && isValid) { - statusText = '⚠️ Valid (Expiring Soon)'; - statusColor = '#ff9900'; - } - - html += ` -
-
- #${index + 1} - ${statusText} - ${cert.certificate_id ? ` - crt.sh ↗ - ` : ''} -
-
-
- Common Name: - ${this.escapeHtml(cert.common_name || 'N/A')} -
-
- Issuer: - ${this.escapeHtml(cert.issuer_name || 'Unknown')} -
-
- Valid From: - ${cert.not_before || 'Unknown'} -
-
- Valid Until: - ${cert.not_after || 'Unknown'} -
- ${cert.validity_period_days ? ` -
- Period: - ${cert.validity_period_days} days -
- ` : ''} -
-
- `; - }); - - if (remaining > 0) { - html += ` -
- 📋 ${remaining} additional certificate${remaining > 1 ? 's' : ''} not shown.
- Use the export function to see all certificates. -
- `; - } - - html += '
'; - return html; - } - - /** - * Generate certificate summary using minimal new CSS - */ - generateCertificateSummary(certificates) { - const total = certificates.total_certificates || 0; - const valid = certificates.valid_certificates || 0; - const expired = certificates.expired_certificates || 0; - const expiringSoon = certificates.expires_soon_count || 0; - const issuers = certificates.unique_issuers || []; - - let html = ` -
-
-
${total}
-
Total
-
-
-
${valid}
-
Valid
-
-
-
${expired}
-
Expired
-
-
-
${expiringSoon}
-
Expiring Soon
-
-
- `; - - // Certificate authorities using existing array display - if (issuers.length > 0) { - html += ` -
- Certificate Authorities: - -
- `; - - issuers.forEach(issuer => { - html += `
${this.escapeHtml(issuer)}
`; - }); - - html += '
'; - } - - return html; - } - /** * UPDATED: Generate large entity details using unified data model */ @@ -1356,71 +1137,32 @@ class DNSReconApp { } /** - * UPDATED: Generate attributes section for unified data model - * Now processes StandardAttribute objects instead of key-value pairs + * UPDATED: Generate attributes section for the new flat data model */ generateAttributesSection(attributes) { if (!Array.isArray(attributes) || attributes.length === 0) { return ''; } - const categorized = this.categorizeStandardAttributes(attributes); - let html = ''; - - Object.entries(categorized).forEach(([category, attrs]) => { - if (attrs.length === 0) return; - - html += ` -
-
- 📊 ${category} -
- `; - - html += '
'; - attrs.forEach(attr => { - html += ` -
- ${this.formatLabel(attr.name)} - ${this.formatStandardAttributeValue(attr)} -
- `; - }); - html += '
'; - - html += '
'; - }); - - return html; - } + let html = ` +
+
+ 📊 Attributes (${attributes.length}) +
+
+ `; - /** - * UPDATED: Categorize StandardAttribute objects by type and content - */ - categorizeStandardAttributes(attributes) { - const categories = { - 'DNS Records': [], - 'Network Info': [], - 'Provider Data': [], - 'Other': [] - }; - attributes.forEach(attr => { - const lowerName = attr.name.toLowerCase(); - const attrType = attr.type ? attr.type.toLowerCase() : ''; - - if (lowerName.includes('dns') || lowerName.includes('record') || attrType.includes('dns')) { - categories['DNS Records'].push(attr); - } else if (lowerName.includes('ip') || lowerName.includes('asn') || lowerName.includes('network') || attrType.includes('network')) { - categories['Network Info'].push(attr); - } else if (lowerName.includes('shodan') || lowerName.includes('crtsh') || lowerName.includes('provider') || attrType.includes('provider')) { - categories['Provider Data'].push(attr); - } else { - categories['Other'].push(attr); - } + html += ` +
+ ${this.formatLabel(attr.name)} + ${this.formatStandardAttributeValue(attr)} +
+ `; }); - - return categories; + + html += '
'; + return html; } /**