From 297431227896c2320ce47d3f17d02e62b1edc6dc Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Sat, 13 Sep 2025 21:10:27 +0200 Subject: [PATCH] data model refinement --- core/graph_manager.py | 40 ++++++---- core/scanner.py | 59 +++++++-------- static/css/main.css | 167 +++++++++++++++++++++++++----------------- static/js/graph.js | 30 ++++---- static/js/main.js | 165 +++++++++++++++-------------------------- 5 files changed, 231 insertions(+), 230 deletions(-) diff --git a/core/graph_manager.py b/core/graph_manager.py index 267cc31..03d769d 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -152,21 +152,31 @@ class GraphManager: }) return all_correlations - def add_node(self, node_id: str, node_type: NodeType, metadata: Optional[Dict[str, Any]] = None) -> bool: - """Add a node to the graph, update metadata, and process correlations.""" + def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None, + description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool: + """Add a node to the graph, update attributes, and process correlations.""" is_new_node = not self.graph.has_node(node_id) if is_new_node: self.graph.add_node(node_id, type=node_type.value, added_timestamp=datetime.now(timezone.utc).isoformat(), + attributes=attributes or {}, + description=description, metadata=metadata or {}) - elif metadata: - # Safely merge new metadata into existing metadata - existing_metadata = self.graph.nodes[node_id].get('metadata', {}) - existing_metadata.update(metadata) - self.graph.nodes[node_id]['metadata'] = existing_metadata + else: + # Safely merge new attributes into existing attributes + if attributes: + existing_attributes = self.graph.nodes[node_id].get('attributes', {}) + existing_attributes.update(attributes) + self.graph.nodes[node_id]['attributes'] = existing_attributes + if description: + self.graph.nodes[node_id]['description'] = description + if metadata: + existing_metadata = self.graph.nodes[node_id].get('metadata', {}) + existing_metadata.update(metadata) + self.graph.nodes[node_id]['metadata'] = existing_metadata - if metadata and node_type != NodeType.CORRELATION_OBJECT: - correlations = self._check_for_correlations(node_id, metadata) + if attributes and node_type != NodeType.CORRELATION_OBJECT: + correlations = self._check_for_correlations(node_id, attributes) for corr in correlations: value = corr['value'] @@ -186,7 +196,7 @@ class GraphManager: continue # Skip creating a redundant correlation node # Proceed to create a new correlation node if no major node was found. - correlation_node_id = f"corr_{hash(value) & 0x7FFFFFFF}" + correlation_node_id = f"{value}" if not self.graph.has_node(correlation_node_id): self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT, metadata={'value': value, 'sources': corr['sources'], @@ -203,7 +213,7 @@ class GraphManager: for c_node_id in set(corr['nodes']): self.add_edge(c_node_id, correlation_node_id, RelationshipType.CORRELATED_TO) - self._update_correlation_index(node_id, metadata) + self._update_correlation_index(node_id, attributes) self.last_modified = datetime.now(timezone.utc).isoformat() return is_new_node @@ -263,12 +273,14 @@ class GraphManager: nodes = [] for node_id, attrs in self.graph.nodes(data=True): node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'), + 'attributes': attrs.get('attributes', {}), + 'description': attrs.get('description', ''), 'metadata': attrs.get('metadata', {}), 'added_timestamp': attrs.get('added_timestamp')} - # Customize node appearance based on type and metadata + # Customize node appearance based on type and attributes node_type = node_data['type'] - metadata = node_data['metadata'] - if node_type == 'domain' and metadata.get('certificate_data', {}).get('has_valid_cert') is False: + attributes = node_data['attributes'] + if node_type == 'domain' and attributes.get('certificates', {}).get('has_valid_cert') is False: node_data['color'] = {'background': '#c7c7c7', 'border': '#999'} # Gray for invalid cert nodes.append(node_data) diff --git a/core/scanner.py b/core/scanner.py index e53f0c1..b619525 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -344,7 +344,7 @@ class Scanner: new_targets = set() large_entity_members = set() - target_metadata = defaultdict(lambda: defaultdict(list)) + node_attributes = defaultdict(lambda: defaultdict(list)) eligible_providers = self._get_eligible_providers(target, is_ip, dns_only) @@ -361,7 +361,7 @@ class Scanner: provider_results = self._query_single_provider_forensic(provider, target, is_ip, depth) if provider_results and not self._is_stop_requested(): discovered, is_large_entity = self._process_provider_results_forensic( - target, provider, provider_results, target_metadata, depth + target, provider, provider_results, node_attributes, depth ) if is_large_entity: large_entity_members.update(discovered) @@ -370,11 +370,11 @@ class Scanner: except Exception as e: self._log_provider_error(target, provider.get_name(), str(e)) - for node_id, metadata_dict in target_metadata.items(): + for node_id, attributes in node_attributes.items(): if self.graph.graph.has_node(node_id): node_is_ip = _is_valid_ip(node_id) node_type_to_add = NodeType.IP if node_is_ip else NodeType.DOMAIN - self.graph.add_node(node_id, node_type_to_add, metadata=metadata_dict) + self.graph.add_node(node_id, node_type_to_add, attributes=attributes) return new_targets, large_entity_members @@ -485,7 +485,7 @@ class Scanner: self.logger.logger.info(f"Provider state updated: {target} -> {provider_name} -> {status} ({results_count} results)") def _process_provider_results_forensic(self, target: str, provider, results: List, - target_metadata: Dict, current_depth: int) -> Tuple[Set[str], bool]: + node_attributes: Dict, current_depth: int) -> Tuple[Set[str], bool]: """Process provider results, returns (discovered_targets, is_large_entity).""" provider_name = provider.get_name() discovered_targets = set() @@ -514,7 +514,7 @@ class Scanner: discovery_method=f"{provider_name}_query_depth_{current_depth}" ) - self._collect_node_metadata_forensic(source, provider_name, rel_type, rel_target, raw_data, target_metadata[source]) + self._collect_node_attributes(source, provider_name, rel_type, rel_target, raw_data, node_attributes[source]) if _is_valid_ip(rel_target): self.graph.add_node(rel_target, NodeType.IP) @@ -532,10 +532,10 @@ class Scanner: if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data): print(f"Added domain relationship: {source} -> {rel_target} ({rel_type.relationship_name})") discovered_targets.add(rel_target) - self._collect_node_metadata_forensic(rel_target, provider_name, rel_type, source, raw_data, target_metadata[rel_target]) + self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target]) else: - self._collect_node_metadata_forensic(source, provider_name, rel_type, rel_target, raw_data, target_metadata[source]) + self._collect_node_attributes(source, provider_name, rel_type, rel_target, raw_data, node_attributes[source]) return discovered_targets, False @@ -555,17 +555,17 @@ class Scanner: for target in targets: self.graph.add_node(target, NodeType.DOMAIN if node_type == 'domain' else NodeType.IP) - metadata = { + attributes = { 'count': len(targets), 'nodes': targets, 'node_type': node_type, 'source_provider': provider_name, 'discovery_depth': current_depth, 'threshold_exceeded': self.config.large_entity_threshold, - 'forensic_note': f'Large entity created due to {len(targets)} results from {provider_name}' } + description = f'Large entity created due to {len(targets)} results from {provider_name}' - self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, metadata=metadata) + self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes, description=description) if results: rel_type = results[0][2] @@ -577,49 +577,50 @@ class Scanner: return set(targets) - def _collect_node_metadata_forensic(self, node_id: str, provider_name: str, rel_type: RelationshipType, - target: str, raw_data: Dict[str, Any], metadata: Dict[str, Any]) -> None: - """Collect and organize metadata for forensic tracking with enhanced logging.""" - self.logger.logger.debug(f"Collecting metadata for {node_id} from {provider_name}: {rel_type.relationship_name}") + def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: RelationshipType, + target: str, raw_data: Dict[str, Any], attributes: Dict[str, Any]) -> None: + """Collect and organize attributes for a node.""" + self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type.relationship_name}") if provider_name == 'dns': record_type = raw_data.get('query_type', 'UNKNOWN') value = raw_data.get('value', target) dns_entry = f"{record_type}: {value}" - if dns_entry not in metadata.get('dns_records', []): - metadata.setdefault('dns_records', []).append(dns_entry) + if dns_entry not in attributes.get('dns_records', []): + attributes.setdefault('dns_records', []).append(dns_entry) elif provider_name == 'crtsh': if rel_type == RelationshipType.SAN_CERTIFICATE: domain_certs = raw_data.get('domain_certificates', {}) if node_id in domain_certs: cert_summary = domain_certs[node_id] - metadata['certificate_data'] = cert_summary - metadata['has_valid_cert'] = cert_summary.get('has_valid_cert', False) - if target not in metadata.get('related_domains_san', []): - metadata.setdefault('related_domains_san', []).append(target) + attributes['certificates'] = cert_summary + if target not in attributes.get('related_domains_san', []): + attributes.setdefault('related_domains_san', []).append(target) elif provider_name == 'shodan': + shodan_attributes = attributes.setdefault('shodan', {}) for key, value in raw_data.items(): - if key not in metadata.get('shodan', {}) or not metadata.get('shodan', {}).get(key): - metadata.setdefault('shodan', {})[key] = value + if key not in shodan_attributes or not shodan_attributes.get(key): + shodan_attributes[key] = value if rel_type == RelationshipType.ASN_MEMBERSHIP: - metadata['asn_data'] = { - 'asn': target, + attributes['asn'] = { + 'id': target, 'description': raw_data.get('org', ''), 'isp': raw_data.get('isp', ''), 'country': raw_data.get('country', '') } record_type_name = rel_type.relationship_name - if record_type_name not in metadata: - metadata[record_type_name] = [] + if record_type_name not in attributes: + attributes[record_type_name] = [] if isinstance(target, list): - metadata[record_type_name].extend(target) + attributes[record_type_name].extend(target) else: - metadata[record_type_name].append(target) + if target not in attributes[record_type_name]: + attributes[record_type_name].append(target) def _log_target_processing_error(self, target: str, error: str) -> None: diff --git a/static/css/main.css b/static/css/main.css index 483d8b8..95d0529 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -581,30 +581,6 @@ input[type="text"]:focus, select:focus { color: #555; } -/* Modal */ -.modal { - display: none; - position: fixed; - z-index: 1000; - left: 0; - top: 0; - width: 100%; - height: 100%; - background-color: rgba(0, 0, 0, 0.8); - animation: fadeIn 0.3s ease-out; -} - -.modal-content { - background-color: #2a2a2a; - border: 1px solid #444; - margin: 5% auto; - width: 80%; - max-width: 600px; - max-height: 80vh; - overflow-y: auto; - animation: slideInDown 0.3s ease-out; -} - @keyframes slideInDown { from { opacity: 0; @@ -616,43 +592,6 @@ input[type="text"]:focus, select:focus { } } -.modal-header { - background-color: #1a1a1a; - padding: 1rem; - border-bottom: 1px solid #444; - display: flex; - justify-content: space-between; - align-items: center; -} - -.modal-header h3 { - color: #00ff41; - font-size: 1.1rem; -} - -.modal-close { - background: transparent; - border: none; - color: #c7c7c7; - font-size: 1.2rem; - cursor: pointer; - font-family: 'Roboto Mono', monospace; -} - -.modal-close:hover { - color: #ff9900; -} - -.modal-body { - padding: 1.5rem; -} - -.modal-description { - color: #999; - margin-bottom: 1.5rem; - line-height: 1.6; -} - .detail-row { display: flex; justify-content: space-between; @@ -801,12 +740,6 @@ input[type="text"]:focus, select:focus { color: #00ff41 !important; } -/* Animations */ -@keyframes fadeIn { - from { opacity: 0; transform: translateY(10px); } - to { opacity: 1; transform: translateY(0); } -} - .fade-in { animation: fadeIn 0.3s ease-out; } @@ -970,4 +903,104 @@ input[type="text"]:focus, select:focus { .large-entity-node-details .detail-section-header { margin-left: 1rem; margin-right: 1rem; +} + +/* dnsrecon/static/css/main.css */ + +/* --- Add these styles for the modal --- */ + +.modal { + display: none; /* Hidden by default */ + position: fixed; /* Stay in place */ + z-index: 1000; /* Sit on top */ + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: auto; /* Enable scroll if needed */ + background-color: rgba(0,0,0,0.6); /* Black w/ opacity */ + backdrop-filter: blur(5px); +} + +.modal-content { + background-color: #1e1e1e; + margin: 10% auto; + padding: 20px; + border: 1px solid #444; + width: 60%; + max-width: 800px; + border-radius: 5px; + box-shadow: 0 5px 15px rgba(0,0,0,0.5); + animation: fadeIn 0.3s; +} + +.modal-header { + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #444; + padding-bottom: 10px; + margin-bottom: 20px; +} + +.modal-header h3 { + margin: 0; + font-family: 'Special Elite', monospace; + color: #00ff41; +} + +.modal-close { + background: none; + border: none; + color: #c7c7c7; + font-size: 24px; + cursor: pointer; + padding: 0 10px; +} +.modal-close:hover { + color: #ff6b6b; +} + +.modal-body { + max-height: 60vh; + overflow-y: auto; +} + +/* Styles for the new data model display */ +.modal-details-grid { + display: grid; + grid-template-columns: 1fr; + gap: 20px; +} + +.modal-section h4 { + font-family: 'Special Elite', monospace; + color: #ff9900; + border-bottom: 1px dashed #555; + padding-bottom: 5px; + margin-top: 0; +} + +.modal-section ul { + list-style-type: none; + padding-left: 15px; +} + +.modal-section li { + margin-bottom: 8px; +} + +.modal-section li > ul { + padding-left: 20px; + margin-top: 5px; +} + +.description-text, .no-data { + color: #aaa; + font-style: italic; +} + +@keyframes fadeIn { + from {opacity: 0; transform: scale(0.95);} + to {opacity: 1; transform: scale(1);} } \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index 6ec5e2d..b522a9d 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -213,12 +213,12 @@ class GraphManager { } }); - // TODO Context menu (right-click) + // FIX: Comment out the problematic context menu handler this.network.on('oncontext', (params) => { params.event.preventDefault(); - if (params.nodes.length > 0) { - this.showNodeContextMenu(params.pointer.DOM, params.nodes[0]); - } + // if (params.nodes.length > 0) { + // this.showNodeContextMenu(params.pointer.DOM, params.nodes[0]); + // } }); // Stabilization events with progress @@ -256,8 +256,8 @@ class GraphManager { const largeEntityMap = new Map(); graphData.nodes.forEach(node => { - if (node.type === 'large_entity' && node.metadata && Array.isArray(node.metadata.nodes)) { - node.metadata.nodes.forEach(nodeId => { + if (node.type === 'large_entity' && node.attributes && Array.isArray(node.attributes.nodes)) { + node.attributes.nodes.forEach(nodeId => { largeEntityMap.set(nodeId, node.id); }); } @@ -274,12 +274,14 @@ class GraphManager { const mergedEdges = {}; graphData.edges.forEach(edge => { const fromNode = largeEntityMap.has(edge.from) ? largeEntityMap.get(edge.from) : edge.from; - const mergeKey = `${fromNode}-${edge.to}-${edge.label}`; + const toNode = largeEntityMap.has(edge.to) ? largeEntityMap.get(edge.to) : edge.to; + const mergeKey = `${fromNode}-${toNode}-${edge.label}`; if (!mergedEdges[mergeKey]) { mergedEdges[mergeKey] = { ...edge, from: fromNode, + to: toNode, count: 0, confidence_score: 0 }; @@ -341,6 +343,8 @@ class GraphManager { size: this.getNodeSize(node.type), borderColor: this.getNodeBorderColor(node.type), shape: this.getNodeShape(node.type), + attributes: node.attributes || {}, + description: node.description || '', metadata: node.metadata || {}, type: node.type }; @@ -352,12 +356,8 @@ class GraphManager { // Style based on certificate validity if (node.type === 'domain') { - if (node.metadata && node.metadata.certificate_data && node.metadata.certificate_data.has_valid_cert === true) { - processedNode.color = '#00ff41'; // Bright green for valid cert - processedNode.borderColor = '#00aa2e'; - } else if (node.metadata && node.metadata.certificate_data && node.metadata.certificate_data.has_valid_cert === false) { - processedNode.color = '#888888'; // Muted grey color - processedNode.borderColor = '#666666'; // Darker grey border + if (node.attributes && node.attributes.certificates && node.attributes.certificates.has_valid_cert === false) { + processedNode.color = { background: '#888888', border: '#666666' }; } } @@ -404,7 +404,7 @@ class GraphManager { * @returns {string} Formatted label */ formatNodeLabel(nodeId, nodeType) { - // Truncate long domain names + if (typeof nodeId !== 'string') return ''; if (nodeId.length > 20) { return nodeId.substring(0, 17) + '...'; } @@ -564,7 +564,7 @@ class GraphManager { // Trigger custom event for main application to handle const event = new CustomEvent('nodeSelected', { - detail: { nodeId, node } + detail: { node } }); document.dispatchEvent(event); } diff --git a/static/js/main.js b/static/js/main.js index b7391c2..a498df4 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -193,9 +193,9 @@ class DNSReconApp { this.elements.resetApiKeys.addEventListener('click', () => this.resetApiKeys()); } - // Custom events + // ** FIX: Listen for the custom event from the graph ** document.addEventListener('nodeSelected', (e) => { - this.showNodeModal(e.detail.nodeId, e.detail.node); + this.showNodeModal(e.detail.node); }); // Keyboard shortcuts @@ -793,129 +793,86 @@ class DNSReconApp { } /** - * Generates the HTML for the node details view. + * **FIX**: Generates the HTML for the node details view using the new data model. * @param {Object} node - The node object. * @returns {string} The HTML string for the node details. */ generateNodeDetailsHtml(node) { if(!node) return '
Details not available.
'; - let detailsHtml = ''; - const createDetailRow = (label, value, statusIcon = '') => { - const baseId = `detail-${node.id.replace(/[^a-zA-Z0-9]/g, '-')}-${label.replace(/[^a-zA-Z0-9]/g, '-')}`; + + let detailsHtml = ''; return detailsHtml; } + + /** + * Recursively formats a JavaScript object into an HTML unordered list. + * @param {Object} obj - The object to format. + * @returns {string} - An HTML string representing the object. + */ + formatObjectToHtml(obj) { + if (!obj || Object.keys(obj).length === 0) { + return '

No data available.

'; + } + + let html = ''; + return html; + } /** * Show node details modal - * @param {string} nodeId - Node identifier * @param {Object} node - Node data */ - showNodeModal(nodeId, node) { - if (!this.elements.nodeModal) return; + showNodeModal(node) { + if (!this.elements.nodeModal || !node) return; if (this.elements.modalTitle) { - this.elements.modalTitle.textContent = `Node Details`; + this.elements.modalTitle.textContent = `${this.formatStatus(node.type)} Node: ${node.id}`; } + let detailsHtml = ''; - if (node.type === 'large_entity') { - const metadata = node.metadata || {}; - const nodes = metadata.nodes || []; - const node_type = metadata.node_type || 'nodes'; - detailsHtml += `
Contains ${metadata.count} ${node_type}s
`; + const attributes = node.attributes || {}; + const nodes = attributes.nodes || []; + const node_type = attributes.node_type || 'nodes'; + detailsHtml += `
Contains ${attributes.count} ${node_type}s
`; detailsHtml += '
'; for(const innerNodeId of nodes) { @@ -926,12 +883,10 @@ class DNSReconApp { detailsHtml += ``; } detailsHtml += '
'; - } else { detailsHtml = this.generateNodeDetailsHtml(node); } - if (this.elements.modalDetails) { this.elements.modalDetails.innerHTML = detailsHtml; }