diff --git a/README.md b/README.md index 35826d0..e07c595 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,6 @@ For power users who require more in-depth information, DNScope can be configured * **In-Memory Graph Analysis**: Uses NetworkX for efficient relationship mapping. * **Real-Time Visualization**: The graph updates dynamically as the scan progresses. * **Forensic Logging**: A complete audit trail of all reconnaissance activities is maintained. - * **Confidence Scoring**: Relationships are weighted based on the reliability of the data source. * **Session Management**: Supports concurrent user sessions with isolated scanner instances. * **Extensible Provider Architecture**: Easily add new data sources to expand the tool's capabilities. * **Web-Based UI**: An intuitive and interactive web interface for managing scans and visualizing results. diff --git a/app.py b/app.py index f5a2a15..c9c475e 100644 --- a/app.py +++ b/app.py @@ -332,7 +332,6 @@ def revert_graph_action(): scanner.graph.add_edge( source_id=edge['from'], target_id=edge['to'], relationship_type=edge['metadata']['relationship_type'], - confidence_score=edge['metadata']['confidence_score'], source_provider=edge['metadata']['source_provider'], raw_data=edge.get('raw_data', {}) ) diff --git a/core/graph_manager.py b/core/graph_manager.py index cf7d1fa..57b066a 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -2,7 +2,7 @@ """ Graph data model for DNScope using NetworkX. -Manages in-memory graph storage with confidence scoring and forensic metadata. +Manages in-memory graph storage with forensic metadata. Now fully compatible with the unified ProviderResult data model. UPDATED: Fixed correlation exclusion keys to match actual attribute names. UPDATED: Removed export_json() method - now handled by ExportManager. @@ -31,7 +31,7 @@ class NodeType(Enum): class GraphManager: """ Thread-safe graph manager for DNScope infrastructure mapping. - Uses NetworkX for in-memory graph storage with confidence scoring. + Uses NetworkX for in-memory graph storage. Compatible with unified ProviderResult data model. """ @@ -83,7 +83,7 @@ class GraphManager: return is_new_node def add_edge(self, source_id: str, target_id: str, relationship_type: str, - confidence_score: float = 0.5, source_provider: str = "unknown", + source_provider: str = "unknown", raw_data: Optional[Dict[str, Any]] = None) -> bool: """ UPDATED: Add or update an edge between two nodes with raw relationship labels. @@ -91,23 +91,13 @@ class GraphManager: if not self.graph.has_node(source_id) or not self.graph.has_node(target_id): return False - new_confidence = confidence_score # UPDATED: Use raw relationship type - no formatting edge_label = relationship_type - - if self.graph.has_edge(source_id, target_id): - # If edge exists, update confidence if the new score is higher. - if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0): - self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence - self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat() - self.graph.edges[source_id, target_id]['updated_by'] = source_provider - return False # Add a new edge with raw attributes self.graph.add_edge(source_id, target_id, relationship_type=edge_label, - confidence_score=new_confidence, source_provider=source_provider, discovery_timestamp=datetime.now(timezone.utc).isoformat(), raw_data=raw_data or {}) @@ -137,11 +127,6 @@ class GraphManager: """Get all nodes of a specific type.""" return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value] - def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]: - """Get edges with confidence score above a given threshold.""" - return [(u, v, d) for u, v, d in self.graph.edges(data=True) - if d.get('confidence_score', 0) >= min_confidence] - def get_graph_data(self) -> Dict[str, Any]: """ Export graph data formatted for frontend visualization. @@ -177,9 +162,9 @@ class GraphManager: 'from': source, 'to': target, 'label': attrs.get('relationship_type', ''), - 'confidence_score': attrs.get('confidence_score', 0), 'source_provider': attrs.get('source_provider', ''), - 'discovery_timestamp': attrs.get('discovery_timestamp') + 'discovery_timestamp': attrs.get('discovery_timestamp'), + 'raw_data': attrs.get('raw_data', {}) }) return { @@ -188,24 +173,6 @@ class GraphManager: 'statistics': self.get_statistics()['basic_metrics'] } - def _get_confidence_distribution(self) -> Dict[str, int]: - """Get distribution of edge confidence scores with empty graph handling.""" - distribution = {'high': 0, 'medium': 0, 'low': 0} - - # FIXED: Handle empty graph case - if self.get_edge_count() == 0: - return distribution - - for _, _, data in self.graph.edges(data=True): - confidence = data.get('confidence_score', 0) - if confidence >= 0.8: - distribution['high'] += 1 - elif confidence >= 0.6: - distribution['medium'] += 1 - else: - distribution['low'] += 1 - return distribution - def get_statistics(self) -> Dict[str, Any]: """Get comprehensive statistics about the graph with proper empty graph handling.""" @@ -222,7 +189,6 @@ class GraphManager: }, 'node_type_distribution': {}, 'relationship_type_distribution': {}, - 'confidence_distribution': self._get_confidence_distribution(), 'provider_distribution': {} } diff --git a/core/logger.py b/core/logger.py index d6b0c8e..43d72bf 100644 --- a/core/logger.py +++ b/core/logger.py @@ -30,7 +30,6 @@ class RelationshipDiscovery: source_node: str target_node: str relationship_type: str - confidence_score: float provider: str raw_data: Dict[str, Any] discovery_method: str @@ -157,7 +156,7 @@ class ForensicLogger: self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}") def log_relationship_discovery(self, source_node: str, target_node: str, - relationship_type: str, confidence_score: float, + relationship_type: str, provider: str, raw_data: Dict[str, Any], discovery_method: str) -> None: """ @@ -167,7 +166,6 @@ class ForensicLogger: source_node: Source node identifier target_node: Target node identifier relationship_type: Type of relationship (e.g., 'SAN', 'A_Record') - confidence_score: Confidence score (0.0 to 1.0) provider: Provider that discovered this relationship raw_data: Raw data from provider response discovery_method: Method used to discover relationship @@ -177,7 +175,6 @@ class ForensicLogger: source_node=source_node, target_node=target_node, relationship_type=relationship_type, - confidence_score=confidence_score, provider=provider, raw_data=raw_data, discovery_method=discovery_method @@ -188,7 +185,7 @@ class ForensicLogger: self.logger.info( f"Relationship Discovered - {source_node} -> {target_node} " - f"({relationship_type}) - Confidence: {confidence_score:.2f} - Provider: {provider}" + f"({relationship_type}) - Provider: {provider}" ) def log_scan_start(self, target_domain: str, recursion_depth: int, @@ -238,7 +235,6 @@ class ForensicLogger: 'successful_requests': len([req for req in provider_requests if req.error is None]), 'failed_requests': len([req for req in provider_requests if req.error is not None]), 'relationships_discovered': len(provider_relationships), - 'avg_confidence': sum(rel.confidence_score for rel in provider_relationships) / len(provider_relationships) if provider_relationships else 0 } return { diff --git a/core/provider_result.py b/core/provider_result.py index 00f0579..1f1c46f 100644 --- a/core/provider_result.py +++ b/core/provider_result.py @@ -18,33 +18,19 @@ class StandardAttribute: value: Any type: str provider: str - confidence: float timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) metadata: Optional[Dict[str, Any]] = field(default_factory=dict) - def __post_init__(self): - """Validate the attribute after initialization.""" - if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0: - raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}") - - @dataclass class Relationship: """A unified data structure for a directional link between two nodes.""" source_node: str target_node: str relationship_type: str - confidence: float provider: str timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) raw_data: Optional[Dict[str, Any]] = field(default_factory=dict) - def __post_init__(self): - """Validate the relationship after initialization.""" - if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0: - raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}") - - @dataclass class ProviderResult: """A container for all data returned by a provider from a single query.""" @@ -52,8 +38,7 @@ class ProviderResult: relationships: List[Relationship] = field(default_factory=list) def add_attribute(self, target_node: str, name: str, value: Any, attr_type: str, - provider: str, confidence: float = 0.8, - metadata: Optional[Dict[str, Any]] = None) -> None: + provider: str, metadata: Optional[Dict[str, Any]] = None) -> None: """Helper method to add an attribute to the result.""" self.attributes.append(StandardAttribute( target_node=target_node, @@ -61,19 +46,16 @@ class ProviderResult: value=value, type=attr_type, provider=provider, - confidence=confidence, metadata=metadata or {} )) def add_relationship(self, source_node: str, target_node: str, relationship_type: str, - provider: str, confidence: float = 0.8, - raw_data: Optional[Dict[str, Any]] = None) -> None: + provider: str, raw_data: Optional[Dict[str, Any]] = None) -> None: """Helper method to add a relationship to the result.""" self.relationships.append(Relationship( source_node=source_node, target_node=target_node, relationship_type=relationship_type, - confidence=confidence, provider=provider, raw_data=raw_data or {} )) diff --git a/core/scanner.py b/core/scanner.py index e93b9b6..3bd6a9a 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -586,6 +586,7 @@ class Scanner: if self.status in [ScanStatus.FINALIZING, ScanStatus.COMPLETED, ScanStatus.STOPPED]: print(f"\n=== PHASE 2: Running correlation analysis ===") self._run_correlation_phase(max_depth, processed_tasks) + self._update_session_state() # Determine the final status *after* finalization. if self._is_stop_requested(): @@ -847,7 +848,6 @@ class Scanner: 'source_node': rel.source_node, 'target_node': rel.target_node, 'relationship_type': rel.relationship_type, - 'confidence': rel.confidence, 'provider': rel.provider, 'raw_data': rel.raw_data }) @@ -905,7 +905,6 @@ class Scanner: source_id=rel_data['source_node'], target_id=rel_data['target_node'], relationship_type=rel_data['relationship_type'], - confidence_score=rel_data['confidence'], source_provider=rel_data['provider'], raw_data=rel_data['raw_data'] ) @@ -931,7 +930,7 @@ class Scanner: # Re-enqueue the node for full processing is_ip = _is_valid_ip(node_id) - eligible_providers = self._get_eligible_providers(node_id, is_ip, False) + eligible_providers = self._get_eligible_providers(node_id, is_ip, False, is_extracted=True) for provider in eligible_providers: provider_name = provider.get_name() priority = self._get_priority(provider_name) @@ -1012,7 +1011,6 @@ class Scanner: self.graph.add_edge( visual_source, visual_target, relationship.relationship_type, - relationship.confidence, provider_name, relationship.raw_data ) @@ -1035,7 +1033,7 @@ class Scanner: for attribute in provider_result.attributes: attr_dict = { "name": attribute.name, "value": attribute.value, "type": attribute.type, - "provider": attribute.provider, "confidence": attribute.confidence, "metadata": attribute.metadata + "provider": attribute.provider, "metadata": attribute.metadata } attributes_by_node[attribute.target_node].append(attr_dict) @@ -1136,7 +1134,7 @@ class Scanner: self.logger.logger.warning(f"Error initializing provider states for {target}: {e}") - def _get_eligible_providers(self, target: str, is_ip: bool, dns_only: bool) -> List: + def _get_eligible_providers(self, target: str, is_ip: bool, dns_only: bool, is_extracted: bool = False) -> List: """ FIXED: Improved provider eligibility checking with better filtering. """ @@ -1148,7 +1146,7 @@ class Scanner: # Check if the target is part of a large entity is_in_large_entity = False - if self.graph.graph.has_node(target): + if self.graph.graph.has_node(target) and not is_extracted: metadata = self.graph.graph.nodes[target].get('metadata', {}) if 'large_entity_id' in metadata: is_in_large_entity = True diff --git a/providers/base_provider.py b/providers/base_provider.py index 6a8fa61..196b53f 100644 --- a/providers/base_provider.py +++ b/providers/base_provider.py @@ -229,7 +229,6 @@ class BaseProvider(ABC): def log_relationship_discovery(self, source_node: str, target_node: str, relationship_type: str, - confidence_score: float, raw_data: Dict[str, Any], discovery_method: str) -> None: """ @@ -239,7 +238,6 @@ class BaseProvider(ABC): source_node: Source node identifier target_node: Target node identifier relationship_type: Type of relationship - confidence_score: Confidence score raw_data: Raw data from provider discovery_method: Method used for discovery """ @@ -249,7 +247,6 @@ class BaseProvider(ABC): source_node=source_node, target_node=target_node, relationship_type=relationship_type, - confidence_score=confidence_score, provider=self.name, raw_data=raw_data, discovery_method=discovery_method diff --git a/providers/correlation_provider.py b/providers/correlation_provider.py index a6948b3..c34d8bd 100644 --- a/providers/correlation_provider.py +++ b/providers/correlation_provider.py @@ -1,7 +1,8 @@ -# DNScope/providers/correlation_provider.py +# dnsrecon-reduced/providers/correlation_provider.py import re from typing import Dict, Any, List +from datetime import datetime, timezone from .base_provider import BaseProvider from core.provider_result import ProviderResult @@ -10,6 +11,7 @@ from core.graph_manager import NodeType, GraphManager class CorrelationProvider(BaseProvider): """ A provider that finds correlations between nodes in the graph. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ def __init__(self, name: str = "correlation", session_config=None): @@ -22,6 +24,10 @@ class CorrelationProvider(BaseProvider): self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}') self.EXCLUDED_KEYS = [ 'cert_source', + 'a_records', + 'mx_records', + 'ns_records', + 'ptr_records', 'cert_issuer_ca_id', 'cert_common_name', 'cert_validity_period_days', @@ -36,6 +42,8 @@ class CorrelationProvider(BaseProvider): 'updated_timestamp', 'discovery_timestamp', 'query_timestamp', + 'shodan_ip_str', + 'shodan_a_record', ] def get_name(self) -> str: @@ -61,12 +69,14 @@ class CorrelationProvider(BaseProvider): def query_domain(self, domain: str) -> ProviderResult: """ Query the provider for information about a domain. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ return self._find_correlations(domain) def query_ip(self, ip: str) -> ProviderResult: """ Query the provider for information about an IP address. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ return self._find_correlations(ip) @@ -79,8 +89,10 @@ class CorrelationProvider(BaseProvider): def _find_correlations(self, node_id: str) -> ProviderResult: """ Find correlations for a given node with enhanced filtering and error handling. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring and list value processing. """ result = ProviderResult() + discovery_time = datetime.now(timezone.utc) # Enhanced safety checks if not self.graph or not self.graph.graph.has_node(node_id): @@ -103,38 +115,46 @@ class CorrelationProvider(BaseProvider): attr_value = attr.get('value') attr_provider = attr.get('provider', 'unknown') - # Enhanced filtering logic - should_exclude = self._should_exclude_attribute(attr_name, attr_value) - - if should_exclude: - continue + # Prepare a list of values to iterate over + values_to_process = [] + if isinstance(attr_value, list): + values_to_process.extend(attr_value) + else: + values_to_process.append(attr_value) - # Build correlation index - if attr_value not in self.correlation_index: - self.correlation_index[attr_value] = { - 'nodes': set(), - 'sources': [] + for value_item in values_to_process: + # Enhanced filtering logic + should_exclude = self._should_exclude_attribute(attr_name, value_item) + + if should_exclude: + continue + + # Build correlation index + if value_item not in self.correlation_index: + self.correlation_index[value_item] = { + 'nodes': set(), + 'sources': [] + } + + self.correlation_index[value_item]['nodes'].add(node_id) + + source_info = { + 'node_id': node_id, + 'provider': attr_provider, + 'attribute': attr_name, + 'path': f"{attr_provider}_{attr_name}" } - self.correlation_index[attr_value]['nodes'].add(node_id) + # Avoid duplicate sources + existing_sources = [s for s in self.correlation_index[value_item]['sources'] + if s['node_id'] == node_id and s['path'] == source_info['path']] + if not existing_sources: + self.correlation_index[value_item]['sources'].append(source_info) - source_info = { - 'node_id': node_id, - 'provider': attr_provider, - 'attribute': attr_name, - 'path': f"{attr_provider}_{attr_name}" - } - - # Avoid duplicate sources - existing_sources = [s for s in self.correlation_index[attr_value]['sources'] - if s['node_id'] == node_id and s['path'] == source_info['path']] - if not existing_sources: - self.correlation_index[attr_value]['sources'].append(source_info) - - # Create correlation if we have multiple nodes with this value - if len(self.correlation_index[attr_value]['nodes']) > 1: - self._create_correlation_relationships(attr_value, self.correlation_index[attr_value], result) - correlations_found += 1 + # Create correlation if we have multiple nodes with this value + if len(self.correlation_index[value_item]['nodes']) > 1: + self._create_correlation_relationships(value_item, self.correlation_index[value_item], result, discovery_time) + correlations_found += 1 # Log correlation results if correlations_found > 0: @@ -187,9 +207,11 @@ class CorrelationProvider(BaseProvider): return False - def _create_correlation_relationships(self, value: Any, correlation_data: Dict[str, Any], result: ProviderResult): + def _create_correlation_relationships(self, value: Any, correlation_data: Dict[str, Any], + result: ProviderResult, discovery_time: datetime): """ Create correlation relationships with enhanced deduplication and validation. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}" nodes = correlation_data['nodes'] @@ -216,7 +238,6 @@ class CorrelationProvider(BaseProvider): value=value, attr_type=str(type(value).__name__), provider=self.name, - confidence=0.9, metadata={ 'correlated_nodes': list(nodes), 'sources': sources, @@ -225,7 +246,7 @@ class CorrelationProvider(BaseProvider): } ) - # Create relationships with source validation + # Create relationships with source validation and enhanced timestamps created_relationships = set() for source in sources: @@ -240,19 +261,23 @@ class CorrelationProvider(BaseProvider): relationship_label = f"corr_{provider}_{attribute}" + # Enhanced raw_data with discovery timestamp for time-based edge coloring + raw_data = { + 'correlation_value': value, + 'original_attribute': attribute, + 'correlation_type': 'attribute_matching', + 'correlation_size': len(nodes), + 'discovery_timestamp': discovery_time.isoformat(), + 'relevance_timestamp': discovery_time.isoformat() # Correlation data is "fresh" when discovered + } + # Add the relationship to the result result.add_relationship( source_node=node_id, target_node=correlation_node_id, relationship_type=relationship_label, provider=self.name, - confidence=0.9, - raw_data={ - 'correlation_value': value, - 'original_attribute': attribute, - 'correlation_type': 'attribute_matching', - 'correlation_size': len(nodes) - } + raw_data=raw_data ) created_relationships.add(relationship_key) \ No newline at end of file diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index f7c0bf1..30958e4 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -18,6 +18,7 @@ class CrtShProvider(BaseProvider): Provider for querying crt.sh certificate transparency database. FIXED: Improved caching logic and error handling to prevent infinite retry loops. Returns standardized ProviderResult objects with caching support. + UPDATED: Enhanced with certificate timestamps for time-based edge coloring. """ def __init__(self, name=None, session_config=None): @@ -131,6 +132,7 @@ class CrtShProvider(BaseProvider): def query_domain(self, domain: str) -> ProviderResult: """ FIXED: Simplified and more robust domain querying with better error handling. + UPDATED: Enhanced with certificate timestamps for time-based edge coloring. """ if not _is_valid_domain(domain): return ProviderResult() @@ -245,7 +247,6 @@ class CrtShProvider(BaseProvider): target_node=rel_data.get("target_node", ""), relationship_type=rel_data.get("relationship_type", ""), provider=rel_data.get("provider", self.name), - confidence=float(rel_data.get("confidence", 0.8)), raw_data=rel_data.get("raw_data", {}) ) except (ValueError, TypeError) as e: @@ -265,7 +266,6 @@ class CrtShProvider(BaseProvider): value=attr_data.get("value"), attr_type=attr_data.get("type", "unknown"), provider=attr_data.get("provider", self.name), - confidence=float(attr_data.get("confidence", 0.9)), metadata=attr_data.get("metadata", {}) ) except (ValueError, TypeError) as e: @@ -293,7 +293,6 @@ class CrtShProvider(BaseProvider): "source_node": rel.source_node, "target_node": rel.target_node, "relationship_type": rel.relationship_type, - "confidence": rel.confidence, "provider": rel.provider, "raw_data": rel.raw_data } for rel in result.relationships @@ -305,7 +304,6 @@ class CrtShProvider(BaseProvider): "value": attr.value, "type": attr.type, "provider": attr.provider, - "confidence": attr.confidence, "metadata": attr.metadata } for attr in result.attributes ] @@ -372,6 +370,7 @@ class CrtShProvider(BaseProvider): """ Process certificates to create proper domain and CA nodes. FIXED: Better error handling and progress tracking. + UPDATED: Enhanced with certificate timestamps for time-based edge coloring. """ result = ProviderResult() @@ -391,8 +390,7 @@ class CrtShProvider(BaseProvider): name="crtsh_data_warning", value=incompleteness_warning, attr_type='metadata', - provider=self.name, - confidence=1.0 + provider=self.name ) all_discovered_domains = set() @@ -415,16 +413,28 @@ class CrtShProvider(BaseProvider): if cert_domains: all_discovered_domains.update(cert_domains) - # Create CA nodes for certificate issuers + # Create CA nodes for certificate issuers with timestamp issuer_name = self._parse_issuer_organization(cert_data.get('issuer_name', '')) if issuer_name and issuer_name not in processed_issuers: + # Enhanced raw_data with certificate timestamp for time-based edge coloring + issuer_raw_data = {'issuer_dn': cert_data.get('issuer_name', '')} + + # Add certificate issue date (not_before) as relevance timestamp + not_before = cert_data.get('not_before') + if not_before: + try: + not_before_date = self._parse_certificate_date(not_before) + issuer_raw_data['cert_not_before'] = not_before_date.isoformat() + issuer_raw_data['relevance_timestamp'] = not_before_date.isoformat() # Standardized field + except Exception as e: + self.logger.logger.debug(f"Failed to parse not_before date for issuer: {e}") + result.add_relationship( source_node=query_domain, target_node=issuer_name, relationship_type='crtsh_cert_issuer', provider=self.name, - confidence=0.95, - raw_data={'issuer_dn': cert_data.get('issuer_name', '')} + raw_data=issuer_raw_data ) processed_issuers.add(issuer_name) @@ -442,7 +452,6 @@ class CrtShProvider(BaseProvider): value=value, attr_type='certificate_data', provider=self.name, - confidence=0.9, metadata={'certificate_id': cert_data.get('id')} ) @@ -457,7 +466,7 @@ class CrtShProvider(BaseProvider): self.logger.logger.info(f"CrtSh query cancelled before relationship creation for domain: {query_domain}") return result - # Create selective relationships to avoid large entities + # Create selective relationships to avoid large entities with enhanced timestamps relationships_created = 0 for discovered_domain in all_discovered_domains: if discovered_domain == query_domain: @@ -467,25 +476,36 @@ class CrtShProvider(BaseProvider): continue if self._should_create_relationship(query_domain, discovered_domain): - confidence = self._calculate_domain_relationship_confidence( - query_domain, discovered_domain, [], all_discovered_domains + # Enhanced raw_data with certificate timestamp for domain relationships + domain_raw_data = {'relationship_type': 'certificate_discovery'} + + # Find the most recent certificate for this domain pair to use as timestamp + most_recent_cert = self._find_most_recent_cert_for_domains( + certificates, query_domain, discovered_domain ) + if most_recent_cert: + not_before = most_recent_cert.get('not_before') + if not_before: + try: + not_before_date = self._parse_certificate_date(not_before) + domain_raw_data['cert_not_before'] = not_before_date.isoformat() + domain_raw_data['relevance_timestamp'] = not_before_date.isoformat() + except Exception as e: + self.logger.logger.debug(f"Failed to parse not_before date for domain relationship: {e}") result.add_relationship( source_node=query_domain, target_node=discovered_domain, relationship_type='crtsh_san_certificate', provider=self.name, - confidence=confidence, - raw_data={'relationship_type': 'certificate_discovery'} + raw_data=domain_raw_data ) self.log_relationship_discovery( source_node=query_domain, target_node=discovered_domain, relationship_type='crtsh_san_certificate', - confidence_score=confidence, - raw_data={'relationship_type': 'certificate_discovery'}, + raw_data=domain_raw_data, discovery_method="certificate_transparency_analysis" ) relationships_created += 1 @@ -493,6 +513,31 @@ class CrtShProvider(BaseProvider): self.logger.logger.info(f"CrtSh processing completed for {query_domain}: processed {processed_certs}/{len(certificates)} certificates, {len(all_discovered_domains)} domains, {relationships_created} relationships") return result + def _find_most_recent_cert_for_domains(self, certificates: List[Dict[str, Any]], + domain1: str, domain2: str) -> Optional[Dict[str, Any]]: + """ + Find the most recent certificate that contains both domains. + Used for determining the relevance timestamp for domain relationships. + """ + most_recent_cert = None + most_recent_date = None + + for cert in certificates: + # Check if this certificate contains both domains + cert_domains = self._extract_domains_from_certificate(cert) + if domain1 in cert_domains and domain2 in cert_domains: + not_before = cert.get('not_before') + if not_before: + try: + cert_date = self._parse_certificate_date(not_before) + if most_recent_date is None or cert_date > most_recent_date: + most_recent_date = cert_date + most_recent_cert = cert + except Exception: + continue + + return most_recent_cert + # [Rest of the methods remain the same as in the original file] def _should_create_relationship(self, source_domain: str, target_domain: str) -> bool: """ @@ -664,25 +709,6 @@ class CrtShProvider(BaseProvider): return [d for d in final_domains if _is_valid_domain(d)] - def _calculate_domain_relationship_confidence(self, domain1: str, domain2: str, - shared_certificates: List[Dict[str, Any]], - all_discovered_domains: Set[str]) -> float: - """Calculate confidence score for domain relationship based on various factors.""" - base_confidence = 0.9 - - relationship_context = self._determine_relationship_context(domain2, domain1) - - if relationship_context == 'exact_match': - context_bonus = 0.0 - elif relationship_context == 'subdomain': - context_bonus = 0.1 - elif relationship_context == 'parent_domain': - context_bonus = 0.05 - else: - context_bonus = 0.0 - - final_confidence = base_confidence + context_bonus - return max(0.1, min(1.0, final_confidence)) def _determine_relationship_context(self, cert_domain: str, query_domain: str) -> str: """Determine the context of the relationship between certificate domain and query domain.""" diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 6187b0d..828dec0 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -2,6 +2,7 @@ from dns import resolver, reversename from typing import Dict +from datetime import datetime, timezone from .base_provider import BaseProvider from core.provider_result import ProviderResult from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version @@ -11,6 +12,7 @@ class DNSProvider(BaseProvider): """ Provider for standard DNS resolution and reverse DNS lookups. Now returns standardized ProviderResult objects with IPv4 and IPv6 support. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ def __init__(self, name=None, session_config=None): @@ -51,6 +53,7 @@ class DNSProvider(BaseProvider): """ Query DNS records for the domain to discover relationships and attributes. FIXED: Now creates separate attributes for each DNS record type. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. Args: domain: Domain to investigate @@ -62,11 +65,12 @@ class DNSProvider(BaseProvider): return ProviderResult() result = ProviderResult() + discovery_time = datetime.now(timezone.utc) # Query all record types - each gets its own attribute for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']: try: - self._query_record(domain, record_type, result) + self._query_record(domain, record_type, result, discovery_time) #except resolver.NoAnswer: # This is not an error, just a confirmation that the record doesn't exist. #self.logger.logger.debug(f"No {record_type} record found for {domain}") @@ -79,6 +83,7 @@ class DNSProvider(BaseProvider): def query_ip(self, ip: str) -> ProviderResult: """ Query reverse DNS for the IP address (supports both IPv4 and IPv6). + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. Args: ip: IP address to investigate (IPv4 or IPv6) @@ -91,6 +96,7 @@ class DNSProvider(BaseProvider): result = ProviderResult() ip_version = get_ip_version(ip) + discovery_time = datetime.now(timezone.utc) try: # Perform reverse DNS lookup (works for both IPv4 and IPv6) @@ -112,20 +118,24 @@ class DNSProvider(BaseProvider): relationship_type = 'dns_a_record' record_prefix = 'A' + # Enhanced raw_data with discovery timestamp for time-based edge coloring + raw_data = { + 'query_type': 'PTR', + 'ip_address': ip, + 'ip_version': ip_version, + 'hostname': hostname, + 'ttl': response.ttl, + 'discovery_timestamp': discovery_time.isoformat(), + 'relevance_timestamp': discovery_time.isoformat() # DNS data is "fresh" when discovered + } + # Add the relationship result.add_relationship( source_node=ip, target_node=hostname, relationship_type='dns_ptr_record', provider=self.name, - confidence=0.8, - raw_data={ - 'query_type': 'PTR', - 'ip_address': ip, - 'ip_version': ip_version, - 'hostname': hostname, - 'ttl': response.ttl - } + raw_data=raw_data ) # Add to PTR records list @@ -136,14 +146,7 @@ class DNSProvider(BaseProvider): source_node=ip, target_node=hostname, relationship_type='dns_ptr_record', - confidence_score=0.8, - raw_data={ - 'query_type': 'PTR', - 'ip_address': ip, - 'ip_version': ip_version, - 'hostname': hostname, - 'ttl': response.ttl - }, + raw_data=raw_data, discovery_method=f"reverse_dns_lookup_ipv{ip_version}" ) @@ -155,7 +158,6 @@ class DNSProvider(BaseProvider): value=ptr_records, attr_type='dns_record', provider=self.name, - confidence=0.8, metadata={'ttl': response.ttl, 'ip_version': ip_version} ) @@ -170,10 +172,11 @@ class DNSProvider(BaseProvider): return result - def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None: + def _query_record(self, domain: str, record_type: str, result: ProviderResult, discovery_time: datetime) -> None: """ FIXED: Query DNS records with unique attribute names for each record type. Enhanced to better handle IPv6 AAAA records. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ try: self.total_requests += 1 @@ -217,18 +220,20 @@ class DNSProvider(BaseProvider): if record_type in ['A', 'AAAA'] and _is_valid_ip(target): ip_version = get_ip_version(target) + # Enhanced raw_data with discovery timestamp for time-based edge coloring raw_data = { 'query_type': record_type, 'domain': domain, 'value': target, - 'ttl': response.ttl + 'ttl': response.ttl, + 'discovery_timestamp': discovery_time.isoformat(), + 'relevance_timestamp': discovery_time.isoformat() # DNS data is "fresh" when discovered } if ip_version: raw_data['ip_version'] = ip_version relationship_type = f"dns_{record_type.lower()}_record" - confidence = 0.8 # Add relationship result.add_relationship( @@ -236,7 +241,6 @@ class DNSProvider(BaseProvider): target_node=target, relationship_type=relationship_type, provider=self.name, - confidence=confidence, raw_data=raw_data ) @@ -252,7 +256,6 @@ class DNSProvider(BaseProvider): source_node=domain, target_node=target, relationship_type=relationship_type, - confidence_score=confidence, raw_data=raw_data, discovery_method=discovery_method ) @@ -276,7 +279,6 @@ class DNSProvider(BaseProvider): value=dns_records, attr_type='dns_record_list', provider=self.name, - confidence=0.8, metadata=metadata ) diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 11900be..0e1ac38 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -15,6 +15,7 @@ class ShodanProvider(BaseProvider): """ Provider for querying Shodan API for IP address information. Now returns standardized ProviderResult objects with caching support for IPv4 and IPv6. + UPDATED: Enhanced with last_seen timestamp for time-based edge coloring. """ def __init__(self, name=None, session_config=None): @@ -145,6 +146,7 @@ class ShodanProvider(BaseProvider): """ Query Shodan for information about an IP address (IPv4 or IPv6), with caching of processed data. FIXED: Proper 404 handling to prevent unnecessary retries. + UPDATED: Enhanced with last_seen timestamp extraction for time-based edge coloring. Args: ip: IP address to investigate (IPv4 or IPv6) @@ -304,7 +306,6 @@ class ShodanProvider(BaseProvider): target_node=rel_data["target_node"], relationship_type=rel_data["relationship_type"], provider=rel_data["provider"], - confidence=rel_data["confidence"], raw_data=rel_data.get("raw_data", {}) ) @@ -316,7 +317,6 @@ class ShodanProvider(BaseProvider): value=attr_data["value"], attr_type=attr_data["type"], provider=attr_data["provider"], - confidence=attr_data["confidence"], metadata=attr_data.get("metadata", {}) ) @@ -336,7 +336,6 @@ class ShodanProvider(BaseProvider): "source_node": rel.source_node, "target_node": rel.target_node, "relationship_type": rel.relationship_type, - "confidence": rel.confidence, "provider": rel.provider, "raw_data": rel.raw_data } for rel in result.relationships @@ -348,7 +347,6 @@ class ShodanProvider(BaseProvider): "value": attr.value, "type": attr.type, "provider": attr.provider, - "confidence": attr.confidence, "metadata": attr.metadata } for attr in result.attributes ] @@ -362,25 +360,40 @@ class ShodanProvider(BaseProvider): """ VERIFIED: Process Shodan data creating ISP nodes with ASN attributes and proper relationships. Enhanced to include IP version information for IPv6 addresses. + UPDATED: Enhanced with last_seen timestamp for time-based edge coloring. """ result = ProviderResult() # Determine IP version for metadata ip_version = get_ip_version(ip) + # Extract last_seen timestamp for time-based edge coloring + last_seen = data.get('last_seen') + # VERIFIED: Extract ISP information and create proper ISP node with ASN isp_name = data.get('org') asn_value = data.get('asn') if isp_name and asn_value: + # Enhanced raw_data with last_seen timestamp + raw_data = { + 'asn': asn_value, + 'shodan_org': isp_name, + 'ip_version': ip_version + } + + # Add last_seen timestamp if available + if last_seen: + raw_data['last_seen'] = last_seen + raw_data['relevance_timestamp'] = last_seen # Standardized field for time-based coloring + # Create relationship from IP to ISP result.add_relationship( source_node=ip, target_node=isp_name, relationship_type='shodan_isp', provider=self.name, - confidence=0.9, - raw_data={'asn': asn_value, 'shodan_org': isp_name, 'ip_version': ip_version} + raw_data=raw_data ) # Add ASN as attribute to the ISP node @@ -390,7 +403,6 @@ class ShodanProvider(BaseProvider): value=asn_value, attr_type='isp_info', provider=self.name, - confidence=0.9, metadata={'description': 'Autonomous System Number from Shodan', 'ip_version': ip_version} ) @@ -401,7 +413,6 @@ class ShodanProvider(BaseProvider): value=isp_name, attr_type='isp_info', provider=self.name, - confidence=0.9, metadata={'description': 'Organization name from Shodan', 'ip_version': ip_version} ) @@ -416,20 +427,24 @@ class ShodanProvider(BaseProvider): else: relationship_type = 'shodan_a_record' + # Enhanced raw_data with last_seen timestamp + hostname_raw_data = {**data, 'ip_version': ip_version} + if last_seen: + hostname_raw_data['last_seen'] = last_seen + hostname_raw_data['relevance_timestamp'] = last_seen + result.add_relationship( source_node=ip, target_node=hostname, relationship_type=relationship_type, provider=self.name, - confidence=0.8, - raw_data={**data, 'ip_version': ip_version} + raw_data=hostname_raw_data ) self.log_relationship_discovery( source_node=ip, target_node=hostname, relationship_type=relationship_type, - confidence_score=0.8, - raw_data={**data, 'ip_version': ip_version}, + raw_data=hostname_raw_data, discovery_method=f"shodan_host_lookup_ipv{ip_version}" ) elif key == 'ports': @@ -441,7 +456,6 @@ class ShodanProvider(BaseProvider): value=port, attr_type='shodan_network_info', provider=self.name, - confidence=0.9, metadata={'ip_version': ip_version} ) elif isinstance(value, (str, int, float, bool)) and value is not None: @@ -452,7 +466,6 @@ class ShodanProvider(BaseProvider): value=value, attr_type='shodan_info', provider=self.name, - confidence=0.9, metadata={'ip_version': ip_version} ) diff --git a/static/css/main.css b/static/css/main.css index ab788e1..18a74d1 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -326,6 +326,20 @@ input[type="text"]:focus, select:focus { animation: progressGlow 2s ease-in-out infinite alternate; } +.gradient-bar { + height: 4px; + background: linear-gradient(to right, #6b7280, #00bfff); + border-radius: 2px; + margin: 0.2rem 0; +} + +.gradient-labels { + display: flex; + justify-content: space-between; + font-size: 0.6rem; + color: #888; +} + @keyframes progressShimmer { 0% { transform: translateX(-100%); } 100% { transform: translateX(100%); } @@ -380,32 +394,59 @@ input[type="text"]:focus, select:focus { color: #999; } -/* Graph Controls */ +/* Enhanced graph controls layout */ .graph-controls { - position: absolute; - top: 8px; - right: 8px; - z-index: 10; display: flex; + flex-direction: column; gap: 0.3rem; + position: absolute; + top: 10px; + right: 10px; + background: rgba(26, 26, 26, 0.9); + padding: 0.5rem; + border-radius: 6px; + border: 1px solid #444; + box-shadow: 0 4px 15px rgba(0, 0, 0, 0.5); + z-index: 100; + min-width: 200px; } -.graph-control-btn, .btn-icon-small { - background: rgba(42, 42, 42, 0.9); +.graph-control-btn { + background: linear-gradient(135deg, #2a2a2a 0%, #1e1e1e 100%); border: 1px solid #555; color: #c7c7c7; - padding: 0.3rem 0.5rem; - font-family: 'Roboto Mono', monospace; - font-size: 0.7rem; + padding: 0.4rem 0.8rem; + border-radius: 4px; cursor: pointer; - transition: all 0.3s ease; + font-family: 'Roboto Mono', monospace; + font-size: 0.8rem; + transition: all 0.2s ease; + text-align: center; } -.graph-control-btn:hover, .btn-icon-small:hover { +.graph-control-btn:hover { + background: linear-gradient(135deg, #3a3a3a 0%, #2e2e2e 100%); border-color: #00ff41; color: #00ff41; } +.graph-control-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.manual-refresh-btn { + background: linear-gradient(135deg, #4a4a2a 0%, #3e3e1e 100%); + border-color: #ffaa00; + color: #ffaa00; +} + +.manual-refresh-btn:hover { + background: linear-gradient(135deg, #5a5a3a 0%, #4e4e2e 100%); + color: #ffcc33; + border-color: #ffcc33; +} + .graph-filter-panel { position: absolute; bottom: 8px; @@ -500,14 +541,6 @@ input[type="text"]:focus, select:focus { height: 2px; } -.legend-edge.high-confidence { - background: #00ff41; -} - -.legend-edge.medium-confidence { - background: #ff9900; -} - /* Provider Panel */ .provider-panel { grid-area: providers; @@ -987,11 +1020,6 @@ input[type="text"]:focus, select:focus { border-radius: 2px; } -.confidence-indicator { - font-size: 0.6rem; - letter-spacing: 1px; -} - .node-link-compact { color: #00aaff; text-decoration: none; @@ -1095,6 +1123,56 @@ input[type="text"]:focus, select:focus { border-left: 3px solid #00aaff; } +.time-control-container { + margin-bottom: 0.5rem; + padding: 0.5rem; + background: rgba(42, 42, 42, 0.3); + border-radius: 4px; + border: 1px solid #444; +} + +.time-control-label { + font-size: 0.8rem; + color: #c7c7c7; + margin-bottom: 0.3rem; + display: block; + font-family: 'Roboto Mono', monospace; +} + +.time-control-input { + width: 100%; + padding: 0.3rem; + background: #1a1a1a; + border: 1px solid #555; + border-radius: 3px; + color: #c7c7c7; + font-family: 'Roboto Mono', monospace; + font-size: 0.75rem; +} + +.time-control-input:focus { + outline: none; + border-color: #00ff41; + box-shadow: 0 0 5px rgba(0, 255, 65, 0.3); +} + +.time-gradient-info { + font-size: 0.7rem; + color: #999; + margin-top: 0.3rem; + text-align: center; + font-family: 'Roboto Mono', monospace; +} + +/* Edge color legend for time-based gradient */ +.time-gradient-legend { + margin-top: 0.5rem; + padding: 0.3rem; + background: rgba(26, 26, 26, 0.5); + border-radius: 3px; + border: 1px solid #333; +} + /* Settings Modal Specific */ .provider-toggle { appearance: none !important; @@ -1324,16 +1402,16 @@ input[type="password"]:focus { .provider-list { grid-template-columns: 1fr; } -} -.manual-refresh-btn { - background: rgba(92, 76, 44, 0.9) !important; /* Orange/amber background */ - border: 1px solid #7a6a3a !important; - color: #ffcc00 !important; /* Bright yellow text */ -} - -.manual-refresh-btn:hover { - border-color: #ffcc00 !important; - color: #fff !important; - background: rgba(112, 96, 54, 0.9) !important; + .graph-controls { + position: relative; + top: auto; + right: auto; + margin-bottom: 1rem; + min-width: auto; + } + + .time-control-input { + font-size: 0.7rem; + } } \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index a895b7b..31e1073 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -2,7 +2,7 @@ /** * Graph visualization module for DNScope * Handles network graph rendering using vis.js with proper large entity node hiding - * UPDATED: Added manual refresh button for polling optimization when graph becomes large + * UPDATED: Fixed time-based blue gradient edge coloring system and simplified logic. */ const contextMenuCSS = ` .graph-context-menu { @@ -22,12 +22,12 @@ const contextMenuCSS = ` .graph-context-menu ul { list-style: none; - padding: 0.5rem 0; + padding: 0.25rem 0; margin: 0; } .graph-context-menu ul li { - padding: 0.75rem 1rem; + padding: 0.5rem 0.75rem; cursor: pointer; transition: all 0.2s ease; display: flex; @@ -53,6 +53,44 @@ const contextMenuCSS = ` .graph-context-menu ul li:last-child { border-bottom: none; } + +.time-control-container { + margin-bottom: 0.5rem; + padding: 0.5rem; + background: rgba(42, 42, 42, 0.3); + border-radius: 4px; + border: 1px solid #444; +} + +.time-control-label { + font-size: 0.8rem; + color: #c7c7c7; + margin-bottom: 0.3rem; + display: block; +} + +.time-control-input { + width: 100%; + padding: 0.3rem; + background: #1a1a1a; + border: 1px solid #555; + border-radius: 3px; + color: #c7c7c7; + font-family: 'Roboto Mono', monospace; + font-size: 0.75rem; +} + +.time-control-input:focus { + outline: none; + border-color: #00ff41; +} + +.time-gradient-info { + font-size: 0.7rem; + color: #999; + margin-top: 0.3rem; + text-align: center; +} `; class GraphManager { @@ -68,13 +106,17 @@ class GraphManager { this.history = []; this.filterPanel = null; this.initialTargetIds = new Set(); - // Track large entity members for proper hiding this.largeEntityMembers = new Set(); this.isScanning = false; - - // Manual refresh button for polling optimization this.manualRefreshButton = null; - this.manualRefreshHandler = null; // Store the handler + this.manualRefreshHandler = null; + this.timeOfInterest = new Date(); + this.edgeTimestamps = new Map(); + + this.gradientColors = { + dark: '#6b7280', + light: '#00bfff' + }; this.options = { nodes: { @@ -178,19 +220,17 @@ class GraphManager { randomSeed: 2 } }; - if (typeof document !== 'undefined') { - const style = document.createElement('style'); - style.textContent = contextMenuCSS; - document.head.appendChild(style); - } + + if (typeof document !== 'undefined') { + const style = document.createElement('style'); + style.textContent = contextMenuCSS; + document.head.appendChild(style); + } this.createNodeInfoPopup(); this.createContextMenu(); document.body.addEventListener('click', () => this.hideContextMenu()); } - /** - * Create floating node info popup - */ createNodeInfoPopup() { this.nodeInfoPopup = document.createElement('div'); this.nodeInfoPopup.className = 'node-info-popup'; @@ -198,11 +238,7 @@ class GraphManager { document.body.appendChild(this.nodeInfoPopup); } - /** - * Create context menu - */ createContextMenu() { - // Remove existing context menu if it exists const existing = document.getElementById('graph-context-menu'); if (existing) { existing.remove(); @@ -213,7 +249,6 @@ class GraphManager { this.contextMenu.className = 'graph-context-menu'; this.contextMenu.style.display = 'none'; - // Prevent body click listener from firing when clicking the menu itself this.contextMenu.addEventListener('click', (event) => { event.stopPropagation(); }); @@ -221,31 +256,20 @@ class GraphManager { document.body.appendChild(this.contextMenu); } - /** - * Initialize the network graph - */ initialize() { - if (this.isInitialized) { - return; - } + if (this.isInitialized) return; try { - const data = { - nodes: this.nodes, - edges: this.edges - }; - + const data = { nodes: this.nodes, edges: this.edges }; this.network = new vis.Network(this.container, data, this.options); this.setupNetworkEvents(); this.isInitialized = true; - // Hide placeholder const placeholder = this.container.querySelector('.graph-placeholder'); if (placeholder) { placeholder.style.display = 'none'; } - // Add graph controls this.addGraphControls(); this.addFilterPanel(); @@ -256,14 +280,21 @@ class GraphManager { } } - /** - * Add interactive graph controls - * UPDATED: Added manual refresh button for polling optimization - */ addGraphControls() { const controlsContainer = document.createElement('div'); controlsContainer.className = 'graph-controls'; + + const currentDateTime = this.formatDateTimeForInput(this.timeOfInterest); + controlsContainer.innerHTML = ` +
+ + +
+ Dark: Old data | Light Blue: Recent data +
+
@@ -276,37 +307,117 @@ class GraphManager { this.container.appendChild(controlsContainer); - // Add control event listeners document.getElementById('graph-fit').addEventListener('click', () => this.fitView()); document.getElementById('graph-physics').addEventListener('click', () => this.togglePhysics()); document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering()); document.getElementById('graph-unhide').addEventListener('click', () => this.unhideAll()); document.getElementById('graph-revert').addEventListener('click', () => this.revertLastAction()); - // Manual refresh button - handler will be set by main app + document.getElementById('time-of-interest').addEventListener('change', (e) => { + this.timeOfInterest = new Date(e.target.value); + this.updateEdgeColors(); + }); + this.manualRefreshButton = document.getElementById('graph-manual-refresh'); - // If a handler was set before the button existed, attach it now if (this.manualRefreshButton && this.manualRefreshHandler) { this.manualRefreshButton.addEventListener('click', this.manualRefreshHandler); } } + + formatDateTimeForInput(date) { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + const hours = String(date.getHours()).padStart(2, '0'); + const minutes = String(date.getMinutes()).padStart(2, '0'); + return `${year}-${month}-${day}T${hours}:${minutes}`; + } + + extractEdgeTimestamp(edge) { + const rawData = edge.raw_data || {}; + + if (rawData.relevance_timestamp) { + return new Date(rawData.relevance_timestamp); + } + + if (edge.discovery_timestamp) { + return new Date(edge.discovery_timestamp); + } + + return new Date(); + } + + calculateTimeGradientColor(timestamp, maxTimeDiff) { + if (!timestamp || !this.timeOfInterest) { + return this.gradientColors.dark; + } + + const timeDiff = Math.abs(timestamp.getTime() - this.timeOfInterest.getTime()); + + if (maxTimeDiff === 0) { + return this.gradientColors.light; + } + + const gradientPosition = timeDiff / maxTimeDiff; + + return this.interpolateColor( + this.gradientColors.light, + this.gradientColors.dark, + gradientPosition + ); + } + + interpolateColor(color1, color2, factor) { + const hex1 = color1.replace('#', ''); + const hex2 = color2.replace('#', ''); + + const r1 = parseInt(hex1.substring(0, 2), 16); + const g1 = parseInt(hex1.substring(2, 4), 16); + const b1 = parseInt(hex1.substring(4, 6), 16); + + const r2 = parseInt(hex2.substring(0, 2), 16); + const g2 = parseInt(hex2.substring(2, 4), 16); + const b2 = parseInt(hex2.substring(4, 6), 16); + + const r = Math.round(r1 + (r2 - r1) * factor); + const g = Math.round(g1 + (g2 - g1) * factor); + const b = Math.round(b1 + (b2 - b1) * factor); + + return `#${r.toString(16).padStart(2, '0')}${g.toString(16).padStart(2, '0')}${b.toString(16).padStart(2, '0')}`; + } + + updateEdgeColors() { + const edgeUpdates = []; + let maxTimeDiff = 0; + this.edgeTimestamps.forEach((edgeTimestamp) => { + const diff = Math.abs(edgeTimestamp.getTime() - this.timeOfInterest.getTime()); + if (diff > maxTimeDiff) { + maxTimeDiff = diff; + } + }); + + this.edges.forEach((edge) => { + const timestamp = this.edgeTimestamps.get(edge.id); + const color = this.calculateTimeGradientColor(timestamp, maxTimeDiff); + + edgeUpdates.push({ + id: edge.id, + color: { color: color, highlight: '#00ff41', hover: '#ff9900' } + }); + }); + + if (edgeUpdates.length > 0) { + this.edges.update(edgeUpdates); + } + } - /** - * Set the manual refresh button click handler - * @param {Function} handler - Function to call when manual refresh is clicked - */ setManualRefreshHandler(handler) { this.manualRefreshHandler = handler; - // If the button already exists, attach the handler if (this.manualRefreshButton && typeof handler === 'function') { this.manualRefreshButton.addEventListener('click', handler); } } - /** - * Show or hide the manual refresh button - * @param {boolean} show - Whether to show the button - */ showManualRefreshButton(show) { if (this.manualRefreshButton) { this.manualRefreshButton.style.display = show ? 'inline-block' : 'none'; @@ -319,33 +430,20 @@ class GraphManager { this.container.appendChild(this.filterPanel); } - /** - * Setup network event handlers - */ setupNetworkEvents() { if (!this.network) return; - // FIXED: Right-click context menu this.container.addEventListener('contextmenu', (event) => { event.preventDefault(); - - // Get coordinates relative to the canvas - const pointer = { - x: event.offsetX, - y: event.offsetY - }; - + const pointer = { x: event.offsetX, y: event.offsetY }; const nodeId = this.network.getNodeAt(pointer); - if (nodeId) { - // Pass the original client event for positioning this.showContextMenu(nodeId, event); } else { this.hideContextMenu(); } }); - // Node click event with details this.network.on('click', (params) => { this.hideContextMenu(); if (params.nodes.length > 0) { @@ -364,27 +462,16 @@ class GraphManager { } }); - // Hover events this.network.on('hoverNode', (params) => { - const nodeId = params.node; - const node = this.nodes.get(nodeId); - if (node) { - this.highlightConnectedNodes(nodeId, true); - } - }); - - // Stabilization events with progress - this.network.on('stabilizationProgress', (params) => { - const progress = params.iterations / params.total; + this.highlightConnectedNodes(params.node, true); }); this.network.on('stabilizationIterationsDone', () => { this.onStabilizationComplete(); }); - // Click away to hide context menu document.addEventListener('click', (e) => { - if (!this.contextMenu.contains(e.target)) { + if (this.contextMenu && !this.contextMenu.contains(e.target)) { this.hideContextMenu(); } }); @@ -411,67 +498,71 @@ class GraphManager { if (!hasData) { this.nodes.clear(); this.edges.clear(); + this.edgeTimestamps.clear(); return; } const nodeMap = new Map(graphData.nodes.map(node => [node.id, node])); - // FIXED: Process all nodes first, then apply hiding logic correctly - const processedNodes = graphData.nodes.map(node => { - const processed = this.processNode(node); - - // FIXED: Only hide if node is still a large entity member - if (node.metadata && node.metadata.large_entity_id) { - processed.hidden = true; - } else { - // FIXED: Ensure extracted nodes are visible - processed.hidden = false; - } - - return processed; - }); - - const processedEdges = graphData.edges.map(edge => { + // --- START: TWO-PASS LOGIC FOR ACCURATE GRADIENTS --- + + // 1. First Pass: Re-route edges and gather all timestamps to find the time range + const rawEdges = graphData.edges.map(edge => { let fromNode = nodeMap.get(edge.from); let toNode = nodeMap.get(edge.to); let fromId = edge.from; let toId = edge.to; - // FIXED: Only re-route if nodes are STILL in large entities - if (fromNode && fromNode.metadata && fromNode.metadata.large_entity_id) { + if (fromNode?.metadata?.large_entity_id) { fromId = fromNode.metadata.large_entity_id; } - if (toNode && toNode.metadata && toNode.metadata.large_entity_id) { + if (toNode?.metadata?.large_entity_id) { toId = toNode.metadata.large_entity_id; } - // Avoid self-referencing edges from re-routing - if (fromId === toId) { - return null; + if (fromId === toId) return null; + return { ...edge, from: fromId, to: toId }; + }).filter(Boolean); + + this.edgeTimestamps.clear(); + rawEdges.forEach(edge => { + const edgeId = `${edge.from}-${edge.to}-${edge.label}`; + this.edgeTimestamps.set(edgeId, this.extractEdgeTimestamp(edge)); + }); + + // 2. Calculate the global maxTimeDiff for this update + let maxTimeDiff = 0; + this.edgeTimestamps.forEach((edgeTimestamp) => { + const diff = Math.abs(edgeTimestamp.getTime() - this.timeOfInterest.getTime()); + if (diff > maxTimeDiff) { + maxTimeDiff = diff; } + }); - const reRoutedEdge = { ...edge, from: fromId, to: toId }; - return this.processEdge(reRoutedEdge); - }).filter(Boolean); // Remove nulls from self-referencing edges + // 3. Second Pass: Process nodes and edges with the correct time context + const processedNodes = graphData.nodes.map(node => { + const processed = this.processNode(node); + processed.hidden = !!node.metadata?.large_entity_id; + return processed; + }); + const processedEdges = rawEdges.map(edge => this.processEdge(edge, maxTimeDiff)); - const existingNodeIds = this.nodes.getIds(); - const existingEdgeIds = this.edges.getIds(); + // --- END: TWO-PASS LOGIC --- - const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id)); - const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id)); - - // FIXED: Update all nodes to ensure extracted nodes become visible this.nodes.update(processedNodes); this.edges.update(processedEdges); this.updateFilterControls(); this.applyAllFilters(); + const newNodes = processedNodes.filter(node => !this.nodes.get(node.id)); + const newEdges = processedEdges.filter(edge => !this.edges.get(edge.id)); + if (newNodes.length > 0 || newEdges.length > 0) { setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100); } - if (this.nodes.length <= 10 || existingNodeIds.length === 0) { + if (this.nodes.length <= 10 || this.nodes.getIds().length === 0) { setTimeout(() => this.fitView(), 800); } @@ -480,6 +571,26 @@ class GraphManager { this.showError('Failed to update visualization'); } } + + processEdge(edge, maxTimeDiff) { + const edgeId = `${edge.from}-${edge.to}-${edge.label}`; + const timestamp = this.edgeTimestamps.get(edgeId); + const timeGradientColor = this.calculateTimeGradientColor(timestamp, maxTimeDiff); + + return { + id: edgeId, + from: edge.from, + to: edge.to, + label: edge.label, + title: this.createEdgeTooltip(edge), + color: { color: timeGradientColor, highlight: '#00ff41', hover: '#ff9900' }, + metadata: { + relationship_type: edge.label, + source_provider: edge.source_provider, + discovery_timestamp: edge.discovery_timestamp + } + }; + } analyzeCertificateInfo(attributes) { let hasCertificates = false; @@ -488,14 +599,10 @@ class GraphManager { for (const attr of attributes) { const attrName = (attr.name || '').toLowerCase(); - const attrProvider = (attr.provider || '').toLowerCase(); const attrValue = attr.value; - // Look for certificate attributes from crtsh provider - if (attrProvider === 'crtsh' || attrName.startsWith('cert_')) { + if (attrName.startsWith('cert_')) { hasCertificates = true; - - // Check certificate validity using raw attribute names if (attrName === 'cert_is_currently_valid') { if (attrValue === true) { hasValidCertificates = true; @@ -503,13 +610,6 @@ class GraphManager { hasExpiredCertificates = true; } } - // Check for expiry indicators - else if (attrName === 'cert_expires_soon' && attrValue === true) { - hasExpiredCertificates = true; - } - else if (attrName.includes('expired') && attrValue === true) { - hasExpiredCertificates = true; - } } } @@ -521,12 +621,6 @@ class GraphManager { }; } - /** - * UPDATED: Helper method to find an attribute by name in the standardized attributes list - * @param {Array} attributes - List of StandardAttribute objects - * @param {string} name - Attribute name to find - * @returns {Object|null} The attribute object if found, null otherwise - */ findAttributeByName(attributes, name) { if (!Array.isArray(attributes)) { return null; @@ -534,11 +628,6 @@ class GraphManager { return attributes.find(attr => attr.name === name) || null; } - /** - * UPDATED: Process node data with styling and metadata for the flat data model - * @param {Object} node - Raw node data with standardized attributes - * @returns {Object} Processed node data - */ processNode(node) { const processedNode = { id: node.id, @@ -556,32 +645,20 @@ class GraphManager { }; if (node.max_depth_reached) { - processedNode.borderColor = '#ff0000'; // Red border for max depth - } - - - // Add confidence-based styling - if (node.confidence) { - processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); + processedNode.borderColor = '#ff0000'; } - // FIXED: Certificate-based domain coloring if (node.type === 'domain' && Array.isArray(node.attributes)) { const certInfo = this.analyzeCertificateInfo(node.attributes); - if (certInfo.hasExpiredOnly) { - // Red for domains with only expired/invalid certificates processedNode.color = '#ff6b6b'; processedNode.borderColor = '#cc5555'; } else if (!certInfo.hasCertificates) { - // Grey for domains with no certificates processedNode.color = '#c7c7c7'; processedNode.borderColor = '#999999'; } - // Green for valid certificates (default color) } - // Handle merged correlation objects if (node.type === 'correlation_object') { const correlationValueAttr = this.findAttributeByName(node.attributes, 'correlation_value'); const value = correlationValueAttr ? correlationValueAttr.value : 'Unknown'; @@ -594,39 +671,6 @@ class GraphManager { return processedNode; } - /** - * Process edge data with styling and metadata - * @param {Object} edge - Raw edge data - * @returns {Object} Processed edge data - */ - processEdge(edge) { - const confidence = edge.confidence_score || 0; - const processedEdge = { - id: `${edge.from}-${edge.to}-${edge.label}`, - from: edge.from, - to: edge.to, - label: this.formatEdgeLabel(edge.label, confidence), - title: this.createEdgeTooltip(edge), - width: this.getEdgeWidth(confidence), - color: this.getEdgeColor(confidence), - dashes: confidence < 0.6 ? [5, 5] : false, - metadata: { - relationship_type: edge.label, - confidence_score: confidence, - source_provider: edge.source_provider, - discovery_timestamp: edge.discovery_timestamp - } - }; - - return processedEdge; - } - - /** - * Format node label for display - * @param {string} nodeId - Node identifier - * @param {string} nodeType - Node type - * @returns {string} Formatted label - */ formatNodeLabel(nodeId, nodeType) { if (typeof nodeId !== 'string') return ''; if (nodeId.length > 20) { @@ -635,319 +679,126 @@ class GraphManager { return nodeId; } - /** - * Format edge label for display - * @param {string} relationshipType - Type of relationship - * @param {number} confidence - Confidence score - * @returns {string} Formatted label - */ - formatEdgeLabel(relationshipType, confidence) { - if (!relationshipType) return ''; - - const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '●' : '○'; - return `${relationshipType} ${confidenceText}`; - } - - /** - * Get node color based on type - * @param {string} nodeType - Node type - * @returns {string} Color value - */ getNodeColor(nodeType) { const colors = { - 'domain': '#00ff41', // Green - 'ip': '#ff9900', // Amber - 'isp': '#00aaff', // Blue - 'ca': '#ff6b6b', // Red - 'large_entity': '#ff6b6b', // Red for large entities - 'correlation_object': '#9620c0ff' + 'domain': '#00ff41', 'ip': '#ff9900', 'isp': '#00aaff', + 'ca': '#ff6b6b', 'large_entity': '#ff6b6b', 'correlation_object': '#9620c0ff' }; return colors[nodeType] || '#ffffff'; } - /** - * Get node border color based on type - * @param {string} nodeType - Node type - * @returns {string} Border color value - */ getNodeBorderColor(nodeType) { const borderColors = { - 'domain': '#00aa2e', - 'ip': '#cc7700', - 'isp': '#0088cc', - 'ca': '#cc5555', - 'correlation_object': '#c235c9ff' + 'domain': '#00aa2e', 'ip': '#cc7700', 'isp': '#0088cc', + 'ca': '#cc5555', 'correlation_object': '#c235c9ff' }; return borderColors[nodeType] || '#666666'; } - /** - * Get node size based on type - * @param {string} nodeType - Node type - * @returns {number} Node size - */ getNodeSize(nodeType) { const sizes = { - 'domain': 12, - 'ip': 14, - 'isp': 16, - 'ca': 16, - 'correlation_object': 8, - 'large_entity': 25 + 'domain': 12, 'ip': 14, 'isp': 16, 'ca': 16, + 'correlation_object': 8, 'large_entity': 25 }; return sizes[nodeType] || 12; } - /** - * Get node shape based on type - * @param {string} nodeType - Node type - * @returns {string} Shape name - */ getNodeShape(nodeType) { const shapes = { - 'domain': 'dot', - 'ip': 'square', - 'isp': 'triangle', - 'ca': 'diamond', - 'correlation_object': 'hexagon', - 'large_entity': 'dot' + 'domain': 'dot', 'ip': 'square', 'isp': 'triangle', 'ca': 'diamond', + 'correlation_object': 'hexagon', 'large_entity': 'dot' }; return shapes[nodeType] || 'dot'; } - /** - * Get edge color based on confidence - * @param {number} confidence - Confidence score - * @returns {string} Edge color - */ - getEdgeColor(confidence) { - if (confidence >= 0.8) { - return '#00ff41'; // High confidence - green - } else if (confidence >= 0.6) { - return '#ff9900'; // Medium confidence - amber - } else { - return '#666666'; // Low confidence - gray - } - } - - /** - * Get edge width based on confidence - * @param {number} confidence - Confidence score - * @returns {number} Edge width - */ - getEdgeWidth(confidence) { - if (confidence >= 0.8) { - return 3; - } else if (confidence >= 0.6) { - return 2; - } else { - return 1; - } - } - - /** - * Create edge tooltip with correct provider information - * @param {Object} edge - Edge data - * @returns {string} HTML tooltip content - */ createEdgeTooltip(edge) { let tooltip = `
`; tooltip += `
${edge.label || 'Relationship'}
`; - tooltip += `
Confidence: ${(edge.confidence_score * 100).toFixed(1)}%
`; if (edge.source_provider) { tooltip += `
Provider: ${edge.source_provider}
`; } if (edge.discovery_timestamp) { - const date = new Date(edge.discovery_timestamp); - tooltip += `
Discovered: ${date.toLocaleString()}
`; + const discoveryDate = new Date(edge.discovery_timestamp); + tooltip += `
Discovered: ${discoveryDate.toLocaleString()}
`; + } + + const edgeId = `${edge.from}-${edge.to}-${edge.label}`; + const relevanceTimestamp = this.edgeTimestamps.get(edgeId); + if (relevanceTimestamp) { + tooltip += `
Data from: ${relevanceTimestamp.toLocaleString()}
`; } tooltip += `
`; return tooltip; } - /** - * Determine if node is important based on connections or metadata - * @param {Object} node - Node data - * @returns {boolean} True if node is important - */ - isImportantNode(node) { - // Mark nodes as important based on criteria - if (node.type === 'domain' && node.id.includes('www.')) return false; - if (node.metadata && node.metadata.connection_count > 3) return true; - if (node.type === 'asn') return true; - return false; - } - - /** - * Show node details in modal - * @param {Object} node - Node object - */ showNodeDetails(node) { - // Trigger custom event for main application to handle - const event = new CustomEvent('nodeSelected', { - detail: { node } - }); + const event = new CustomEvent('nodeSelected', { detail: { node } }); document.dispatchEvent(event); } - /** - * Hide node info popup - */ - hideNodeInfoPopup() { - if (this.nodeInfoPopup) { - this.nodeInfoPopup.style.display = 'none'; - } - } - - /** - * Highlight node connections - * @param {string} nodeId - Node to highlight - */ highlightNodeConnections(nodeId) { const connectedNodes = this.network.getConnectedNodes(nodeId); const connectedEdges = this.network.getConnectedEdges(nodeId); - // Update node colors - const nodeUpdates = connectedNodes.map(id => ({ - id: id, - borderColor: '#ff9900', - borderWidth: 3 - })); + const nodeUpdates = connectedNodes.map(id => ({ id: id, borderColor: '#ff9900', borderWidth: 3 })); + nodeUpdates.push({ id: nodeId, borderColor: '#00ff41', borderWidth: 4 }); - nodeUpdates.push({ - id: nodeId, - borderColor: '#00ff41', - borderWidth: 4 - }); - - // Update edge colors - const edgeUpdates = connectedEdges.map(id => ({ - id: id, - color: { color: '#ff9900' }, - width: 3 - })); + const edgeUpdates = connectedEdges.map(id => ({ id: id, color: { color: '#ff9900' }, width: 3 })); this.nodes.update(nodeUpdates); this.edges.update(edgeUpdates); - // Store for cleanup - this.highlightedElements = { - nodes: connectedNodes.concat([nodeId]), - edges: connectedEdges - }; + this.highlightedElements = { nodes: connectedNodes.concat([nodeId]), edges: connectedEdges }; } - /** - * Highlight connected nodes on hover - * @param {string} nodeId - Node ID - * @param {boolean} highlight - Whether to highlight or unhighlight - */ highlightConnectedNodes(nodeId, highlight) { const connectedNodes = this.network.getConnectedNodes(nodeId); const connectedEdges = this.network.getConnectedEdges(nodeId); - if (highlight) { - // Dim all other elements this.dimUnconnectedElements([nodeId, ...connectedNodes], connectedEdges); } } - /** - * Dim elements not connected to the specified nodes - * @param {Array} nodeIds - Node IDs to keep highlighted - * @param {Array} edgeIds - Edge IDs to keep highlighted - */ dimUnconnectedElements(nodeIds, edgeIds) { const allNodes = this.nodes.get(); const allEdges = this.edges.get(); - const nodeUpdates = allNodes.map(node => ({ - id: node.id, - opacity: nodeIds.includes(node.id) ? 1 : 0.3 - })); - - const edgeUpdates = allEdges.map(edge => ({ - id: edge.id, - opacity: edgeIds.includes(edge.id) ? 1 : 0.1 - })); + const nodeUpdates = allNodes.map(node => ({ id: node.id, opacity: nodeIds.includes(node.id) ? 1 : 0.3 })); + const edgeUpdates = allEdges.map(edge => ({ id: edge.id, opacity: edgeIds.includes(edge.id) ? 1 : 0.1 })); this.nodes.update(nodeUpdates); this.edges.update(edgeUpdates); } - /** - * Clear all highlights - */ clearHighlights() { if (this.highlightedElements) { - // Reset highlighted nodes const nodeUpdates = this.highlightedElements.nodes.map(id => { const originalNode = this.nodes.get(id); - return { - id: id, - borderColor: this.getNodeBorderColor(originalNode.type), - borderWidth: 2 - }; + return { id: id, borderColor: this.getNodeBorderColor(originalNode.type), borderWidth: 2 }; }); - // Reset highlighted edges const edgeUpdates = this.highlightedElements.edges.map(id => { - const originalEdge = this.edges.get(id); - return { - id: id, - color: this.getEdgeColor(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5), - width: this.getEdgeWidth(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5) - }; + const timestamp = this.edgeTimestamps.get(id); + const color = this.calculateTimeGradientColor(timestamp); + return { id: id, color: { color: color, highlight: '#00ff41', hover: '#ff9900' } }; }); this.nodes.update(nodeUpdates); this.edges.update(edgeUpdates); - this.highlightedElements = null; } } - /** - * Clear hover highlights - */ - clearHoverHighlights() { - const allNodes = this.nodes.get(); - const allEdges = this.edges.get(); - - const nodeUpdates = allNodes.map(node => ({ id: node.id, opacity: 1 })); - const edgeUpdates = allEdges.map(edge => ({ id: edge.id, opacity: 1 })); - - this.nodes.update(nodeUpdates); - this.edges.update(edgeUpdates); - } - - /** - * Highlight newly added elements - * @param {Array} newNodes - New nodes - * @param {Array} newEdges - New edges - */ highlightNewElements(newNodes, newEdges) { - // Briefly highlight new nodes - const nodeHighlights = newNodes.map(node => ({ - id: node.id, - borderColor: '#00ff41', - borderWidth: 4 - })); - - // Briefly highlight new edges - const edgeHighlights = newEdges.map(edge => ({ - id: edge.id, - color: '#00ff41', - width: 4 - })); + const nodeHighlights = newNodes.map(node => ({ id: node.id, borderColor: '#00ff41', borderWidth: 4 })); + const edgeHighlights = newEdges.map(edge => ({ id: edge.id, color: '#00ff41', width: 4 })); this.nodes.update(nodeHighlights); this.edges.update(edgeHighlights); - // Reset after animation setTimeout(() => { const nodeResets = newNodes.map(node => ({ id: node.id, @@ -955,163 +806,93 @@ class GraphManager { borderWidth: 2, })); - const edgeResets = newEdges.map(edge => ({ - id: edge.id, - color: this.getEdgeColor(edge.metadata ? edge.metadata.confidence_score : 0.5), - width: this.getEdgeWidth(edge.metadata ? edge.metadata.confidence_score : 0.5) - })); + const edgeResets = newEdges.map(edge => { + const timestamp = this.edgeTimestamps.get(edge.id); + const color = this.calculateTimeGradientColor(timestamp); + return { id: edge.id, color: { color: color, highlight: '#00ff41', hover: '#ff9900' } }; + }); this.nodes.update(nodeResets); this.edges.update(edgeResets); }, 2000); } - /** - * Handle stabilization completion - */ onStabilizationComplete() { console.log('Graph stabilization complete'); } - /** - * Focus view on specific node - * @param {string} nodeId - Node to focus on - */ focusOnNode(nodeId) { const nodePosition = this.network.getPositions([nodeId]); if (nodePosition[nodeId]) { this.network.moveTo({ position: nodePosition[nodeId], scale: 1.5, - animation: { - duration: 1000, - easingFunction: 'easeInOutQuart' - } + animation: { duration: 1000, easingFunction: 'easeInOutQuart' } }); } } - /** - * Toggle physics simulation - */ togglePhysics() { const currentPhysics = this.network.physics.physicsEnabled; this.network.setOptions({ physics: !currentPhysics }); - const button = document.getElementById('graph-physics'); if (button) { button.textContent = currentPhysics ? '[PHYSICS OFF]' : '[PHYSICS ON]'; - button.style.color = currentPhysics ? '#ff9900' : '#00ff41'; } } - /** - * Toggle node clustering - */ toggleClustering() { if (this.network.isCluster('domain-cluster')) { this.network.openCluster('domain-cluster'); } else { - const clusterOptions = { - joinCondition: (nodeOptions) => { - return nodeOptions.type === 'domain'; - }, - clusterNodeProperties: { - id: 'domain-cluster', - label: 'Domains', - shape: 'database', - color: '#00ff41', - borderWidth: 3, - } - }; - this.network.cluster(clusterOptions); - } - } - - /** - * Fit the view to show all nodes - */ - fitView() { - if (this.network) { - this.network.fit({ - animation: { - duration: 1000, - easingFunction: 'easeInOutQuad' - } + this.network.cluster({ + joinCondition: (nodeOptions) => nodeOptions.type === 'domain', + clusterNodeProperties: { id: 'domain-cluster', label: 'Domains', shape: 'database', color: '#00ff41' } }); } } - /** - * Clear the graph - */ + fitView() { + if (this.network) { + this.network.fit({ animation: { duration: 1000, easingFunction: 'easeInOutQuad' } }); + } + } + clear() { this.nodes.clear(); this.edges.clear(); + this.edgeTimestamps.clear(); this.history = []; - this.largeEntityMembers.clear(); - this.initialTargetIds.clear(); - - // Show placeholder const placeholder = this.container.querySelector('.graph-placeholder'); if (placeholder) { placeholder.style.display = 'flex'; } } - /** - * Show error message - * @param {string} message - Error message - */ showError(message) { const placeholder = this.container.querySelector('.graph-placeholder .placeholder-text'); if (placeholder) { placeholder.textContent = `Error: ${message}`; - placeholder.style.color = '#ff6b6b'; } } - /* * @param {Set} excludedNodeIds - Node IDs to exclude from analysis (for simulation) - * @param {Set} excludedEdgeTypes - Edge types to exclude from traversal - * @param {Set} excludedNodeTypes - Node types to exclude from traversal - * @returns {Object} Analysis results with reachable/unreachable nodes - */ analyzeGraphReachability(excludedNodeIds = new Set(), excludedEdgeTypes = new Set(), excludedNodeTypes = new Set()) { - console.log("Performing comprehensive reachability analysis..."); - - const analysis = { - reachableNodes: new Set(), - unreachableNodes: new Set(), - isolatedClusters: [], - affectedNodes: new Set() - }; - + const analysis = { reachableNodes: new Set(), unreachableNodes: new Set() }; if (this.nodes.length === 0) return analysis; - // Build adjacency list excluding specified elements const adjacencyList = {}; this.nodes.getIds().forEach(id => { - if (!excludedNodeIds.has(id)) { - adjacencyList[id] = []; - } + if (!excludedNodeIds.has(id)) adjacencyList[id] = []; }); this.edges.forEach(edge => { - const edgeType = edge.metadata?.relationship_type || ''; - if (!excludedEdgeTypes.has(edgeType) && - !excludedNodeIds.has(edge.from) && - !excludedNodeIds.has(edge.to)) { - - if (adjacencyList[edge.from]) { - adjacencyList[edge.from].push(edge.to); - } + if (!excludedEdgeTypes.has(edge.metadata?.relationship_type || '') && + !excludedNodeIds.has(edge.from) && !excludedNodeIds.has(edge.to)) { + if (adjacencyList[edge.from]) adjacencyList[edge.from].push(edge.to); } }); - // BFS traversal from initial targets const traversalQueue = []; - - // Start from initial targets that aren't excluded this.initialTargetIds.forEach(rootId => { if (!excludedNodeIds.has(rootId)) { const node = this.nodes.get(rootId); @@ -1124,11 +905,9 @@ class GraphManager { } }); - // BFS to find all reachable nodes let queueIndex = 0; while (queueIndex < traversalQueue.length) { const currentNode = traversalQueue[queueIndex++]; - for (const neighbor of (adjacencyList[currentNode] || [])) { if (!analysis.reachableNodes.has(neighbor)) { const node = this.nodes.get(neighbor); @@ -1140,115 +919,33 @@ class GraphManager { } } - // Identify unreachable nodes (maintaining forensic integrity) Object.keys(adjacencyList).forEach(nodeId => { if (!analysis.reachableNodes.has(nodeId)) { analysis.unreachableNodes.add(nodeId); } }); - // Find isolated clusters among unreachable nodes - analysis.isolatedClusters = this.findIsolatedClusters( - Array.from(analysis.unreachableNodes), - adjacencyList - ); - - /*console.log(`Reachability analysis complete:`, { - reachable: analysis.reachableNodes.size, - unreachable: analysis.unreachableNodes.size, - clusters: analysis.isolatedClusters.length - });*/ - return analysis; } - - /** - * Find isolated clusters within a set of nodes - * Used for forensic analysis to identify disconnected subgraphs - */ - findIsolatedClusters(nodeIds, adjacencyList) { - const visited = new Set(); - const clusters = []; - - for (const nodeId of nodeIds) { - if (!visited.has(nodeId)) { - const cluster = []; - const stack = [nodeId]; - - while (stack.length > 0) { - const current = stack.pop(); - if (!visited.has(current)) { - visited.add(current); - cluster.push(current); - - // Add unvisited neighbors within the unreachable set - for (const neighbor of (adjacencyList[current] || [])) { - if (nodeIds.includes(neighbor) && !visited.has(neighbor)) { - stack.push(neighbor); - } - } - } - } - - if (cluster.length > 0) { - clusters.push(cluster); - } - } - } - - return clusters; - } - - /** - * ENHANCED: Get comprehensive graph statistics with forensic information - * Updates the existing getStatistics() method - */ - getStatistics() { - const basicStats = { - nodeCount: this.nodes.length, - edgeCount: this.edges.length, - }; - - // Add forensic statistics - const visibleNodes = this.nodes.get({ filter: node => !node.hidden }); - const hiddenNodes = this.nodes.get({ filter: node => node.hidden }); - - return { - ...basicStats, - forensicStatistics: { - visibleNodes: visibleNodes.length, - hiddenNodes: hiddenNodes.length, - initialTargets: this.initialTargetIds.size, - integrityStatus: visibleNodes.length > 0 && this.initialTargetIds.size > 0 ? 'INTACT' : 'COMPROMISED' - } - }; - } updateFilterControls() { if (!this.filterPanel) return; const nodeTypes = new Set(this.nodes.get().map(n => n.type)); const edgeTypes = new Set(this.edges.get().map(e => e.metadata.relationship_type)); - // Wrap both columns in a single container with vertical layout let filterHTML = '
'; - - // Nodes section filterHTML += '

Nodes

'; nodeTypes.forEach(type => { const label = type === 'correlation_object' ? 'latent correlations' : type; - const isChecked = type !== 'correlation_object'; - filterHTML += ``; + filterHTML += ``; }); filterHTML += '
'; - // Edges section filterHTML += '

Edges

'; edgeTypes.forEach(type => { filterHTML += ``; }); - filterHTML += '
'; - - filterHTML += '
'; // Close filter-container + filterHTML += ''; this.filterPanel.innerHTML = filterHTML; this.filterPanel.querySelectorAll('input[type="checkbox"]').forEach(checkbox => { @@ -1256,14 +953,9 @@ class GraphManager { }); } - /** - * ENHANCED: Apply filters using consolidated reachability analysis - * Replaces the existing applyAllFilters() method - */ applyAllFilters() { if (this.nodes.length === 0) return; - // Get filter criteria from UI const excludedNodeTypes = new Set(); this.filterPanel?.querySelectorAll('input[data-filter-type="node"]:not(:checked)').forEach(cb => { excludedNodeTypes.add(cb.value); @@ -1274,15 +966,9 @@ class GraphManager { excludedEdgeTypes.add(cb.value); }); - // Perform comprehensive analysis const analysis = this.analyzeGraphReachability(new Set(), excludedEdgeTypes, excludedNodeTypes); - // Apply visibility updates - const nodeUpdates = this.nodes.map(node => ({ - id: node.id, - hidden: !analysis.reachableNodes.has(node.id) - })); - + const nodeUpdates = this.nodes.map(node => ({ id: node.id, hidden: !analysis.reachableNodes.has(node.id) })); const edgeUpdates = this.edges.map(edge => ({ id: edge.id, hidden: excludedEdgeTypes.has(edge.metadata?.relationship_type || '') || @@ -1292,357 +978,145 @@ class GraphManager { this.nodes.update(nodeUpdates); this.edges.update(edgeUpdates); - - console.log(`Enhanced filters applied. Visible nodes: ${analysis.reachableNodes.size}`); } - /** - * ENHANCED: Hide node with forensic integrity using reachability analysis - * Replaces the existing hideNodeAndOrphans() method - */ hideNodeWithReachabilityAnalysis(nodeId) { - console.log(`Hiding node ${nodeId} with reachability analysis...`); - - // Simulate hiding this node and analyze impact - const excludedNodes = new Set([nodeId]); - const analysis = this.analyzeGraphReachability(excludedNodes); - - // Nodes that will become unreachable (should be hidden) + const analysis = this.analyzeGraphReachability(new Set([nodeId])); const nodesToHide = [nodeId, ...Array.from(analysis.unreachableNodes)]; - - // Store history for potential revert - const historyData = { - nodeIds: nodesToHide, - operation: 'hide_with_reachability', - timestamp: Date.now() - }; + const historyData = { nodeIds: nodesToHide, operation: 'hide', timestamp: Date.now() }; const updates = nodesToHide.map(id => ({ id: id, hidden: true })); this.nodes.update(updates); this.addToHistory('hide', historyData); - - return { - hiddenNodes: nodesToHide, - isolatedClusters: analysis.isolatedClusters - }; } - /** - * ENHANCED: Delete node with forensic integrity using reachability analysis - * Replaces the existing deleteNodeAndOrphans() method - */ async deleteNodeWithReachabilityAnalysis(nodeId) { - console.log(`Deleting node ${nodeId} with reachability analysis...`); - - // Simulate deletion and analyze impact - const excludedNodes = new Set([nodeId]); - const analysis = this.analyzeGraphReachability(excludedNodes); - - // Nodes that will become unreachable (should be deleted) + const analysis = this.analyzeGraphReachability(new Set([nodeId])); const nodesToDelete = [nodeId, ...Array.from(analysis.unreachableNodes)]; - // Collect forensic data before deletion const historyData = { nodes: nodesToDelete.map(id => this.nodes.get(id)).filter(Boolean), edges: [], operation: 'delete_with_reachability', - timestamp: Date.now(), - forensicAnalysis: { - originalTarget: nodeId, - cascadeNodes: nodesToDelete.length - 1, - isolatedClusters: analysis.isolatedClusters.length, - clusterSizes: analysis.isolatedClusters.map(cluster => cluster.length) - } + timestamp: Date.now() }; - // Collect affected edges nodesToDelete.forEach(id => { const connectedEdgeIds = this.network.getConnectedEdges(id); - const edges = this.edges.get(connectedEdgeIds); - historyData.edges.push(...edges); + historyData.edges.push(...this.edges.get(connectedEdgeIds)); }); - - // Remove duplicates from edges historyData.edges = Array.from(new Map(historyData.edges.map(e => [e.id, e])).values()); - // Perform backend deletion with forensic logging - let operationFailed = false; - for (const targetNodeId of nodesToDelete) { try { - const response = await fetch(`/api/graph/node/${targetNodeId}`, { - method: 'DELETE', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - forensicContext: { - operation: 'reachability_cascade_delete', - originalTarget: nodeId, - analysisTimestamp: historyData.timestamp - } - }) - }); - - const result = await response.json(); - if (!result.success) { - console.error(`Backend deletion failed for node ${targetNodeId}:`, result.error); - operationFailed = true; - break; - } - - console.log(`Node ${targetNodeId} deleted from backend with forensic context`); + const response = await fetch(`/api/graph/node/${targetNodeId}`, { method: 'DELETE' }); + if (!response.ok) throw new Error(`Backend deletion failed for ${targetNodeId}`); this.nodes.remove({ id: targetNodeId }); - } catch (error) { - console.error(`API error during deletion of node ${targetNodeId}:`, error); - operationFailed = true; - break; + this.nodes.update(historyData.nodes); + this.edges.update(historyData.edges); + return { success: false, error: "Backend deletion failed, UI reverted" }; } } - - // Handle operation results - if (!operationFailed) { - this.addToHistory('delete', historyData); - return { - success: true, - deletedNodes: nodesToDelete, - forensicAnalysis: historyData.forensicAnalysis - }; - } else { - // Revert UI changes if backend operations failed - use update instead of add - console.log("Reverting UI changes due to backend failure"); - this.nodes.update(historyData.nodes); - this.edges.update(historyData.edges); - - return { - success: false, - error: "Backend deletion failed, UI reverted" - }; - } + this.addToHistory('delete', historyData); + return { success: true, deletedNodes: nodesToDelete }; } - /** - * Show context menu for a node - * @param {string} nodeId - The ID of the node - * @param {Event} event - The contextmenu event - */ showContextMenu(nodeId, event) { - console.log('Showing context menu for node:', nodeId); const node = this.nodes.get(nodeId); - - // Create menu items - let menuItems = ` - `; this.contextMenu.innerHTML = menuItems; - - // Position the menu this.contextMenu.style.left = `${event.clientX}px`; this.contextMenu.style.top = `${event.clientY}px`; this.contextMenu.style.display = 'block'; - // Ensure menu stays within viewport const rect = this.contextMenu.getBoundingClientRect(); - if (rect.right > window.innerWidth) { - this.contextMenu.style.left = `${event.clientX - rect.width}px`; - } - if (rect.bottom > window.innerHeight) { - this.contextMenu.style.top = `${event.clientY - rect.height}px`; - } + if (rect.right > window.innerWidth) this.contextMenu.style.left = `${event.clientX - rect.width}px`; + if (rect.bottom > window.innerHeight) this.contextMenu.style.top = `${event.clientY - rect.height}px`; - // Add event listeners to menu items this.contextMenu.querySelectorAll('li').forEach(item => { item.addEventListener('click', (e) => { - if (e.currentTarget.hasAttribute('disabled')) { // Prevent action if disabled - e.stopPropagation(); - return; - } + if (e.currentTarget.hasAttribute('disabled')) return; e.stopPropagation(); - const action = e.currentTarget.dataset.action; - const nodeId = e.currentTarget.dataset.nodeId; - this.performContextMenuAction(action, nodeId); + this.performContextMenuAction(e.currentTarget.dataset.action, e.currentTarget.dataset.nodeId); this.hideContextMenu(); }); }); } - /** - * Hide the context menu - */ hideContextMenu() { - if (this.contextMenu) { - this.contextMenu.style.display = 'none'; - } + if (this.contextMenu) this.contextMenu.style.display = 'none'; } - /** - * UPDATED: Enhanced context menu actions using new methods - * Updates the existing performContextMenuAction() method - */ performContextMenuAction(action, nodeId) { switch (action) { - case 'focus': - this.focusOnNode(nodeId); - break; - - case 'iterate': - const event = new CustomEvent('iterateScan', { - detail: { nodeId } - }); - document.dispatchEvent(event); - break; - - case 'hide': - // Use enhanced method with reachability analysis - this.hideNodeWithReachabilityAnalysis(nodeId); - break; - - case 'delete': - // Use enhanced method with reachability analysis - this.deleteNodeWithReachabilityAnalysis(nodeId); - break; - + case 'focus': this.focusOnNode(nodeId); break; + case 'iterate': document.dispatchEvent(new CustomEvent('iterateScan', { detail: { nodeId } })); break; + case 'hide': this.hideNodeWithReachabilityAnalysis(nodeId); break; + case 'delete': this.deleteNodeWithReachabilityAnalysis(nodeId); break; case 'details': const node = this.nodes.get(nodeId); - if (node) { - this.showNodeDetails(node); - } + if (node) this.showNodeDetails(node); break; - - default: - console.warn('Unknown action:', action); } } - /** - * Add an operation to the history stack - * @param {string} type - The type of operation ('hide', 'delete') - * @param {Object} data - The data needed to revert the operation - */ addToHistory(type, data) { this.history.push({ type, data }); } - /** - * Revert the last action - */ async revertLastAction() { const lastAction = this.history.pop(); - if (!lastAction) { - console.log('No actions to revert.'); - return; - } - + if (!lastAction) return; + switch (lastAction.type) { case 'hide': - // Revert hiding nodes by un-hiding them - const updates = lastAction.data.nodeIds.map(id => ({ id: id, hidden: false })); - this.nodes.update(updates); + this.nodes.update(lastAction.data.nodeIds.map(id => ({ id: id, hidden: false }))); break; case 'delete': try { const response = await fetch('/api/graph/revert', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, + headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(lastAction) }); - const result = await response.json(); - - if (result.success) { - console.log('Delete action reverted successfully on backend.'); - // Re-add all nodes and edges from the history to the local view - use update instead of add - this.nodes.update(lastAction.data.nodes); - this.edges.update(lastAction.data.edges); - } else { - console.error('Failed to revert delete action on backend:', result.error); - // Push the action back onto the history stack if the API call failed - this.history.push(lastAction); - } + if (!response.ok) throw new Error('Backend revert failed'); + this.nodes.update(lastAction.data.nodes); + this.edges.update(lastAction.data.edges); } catch (error) { - console.error('Error during revert API call:', error); this.history.push(lastAction); + this.showError('Failed to revert the last action.'); } break; } } - /** - * FIXED: Unhide all hidden nodes, excluding large entity members and disconnected nodes. - * This prevents orphaned large entity members from appearing as free-floating nodes. - */ unhideAll() { const allHiddenNodes = this.nodes.get({ filter: (node) => { - // Skip nodes that are part of a large entity - if (node.metadata && node.metadata.large_entity_id) { - return false; - } - - // Skip nodes that are not hidden - if (node.hidden !== true) { - return false; - } - - // Skip nodes that have no edges (would appear disconnected) - const nodeId = node.id; - const hasIncomingEdges = this.edges.get().some(edge => edge.to === nodeId && !edge.hidden); - const hasOutgoingEdges = this.edges.get().some(edge => edge.from === nodeId && !edge.hidden); - - if (!hasIncomingEdges && !hasOutgoingEdges) { - console.log(`Skipping disconnected node ${nodeId} from unhide`); - return false; - } - - return true; + if (node.metadata?.large_entity_id || node.hidden !== true) return false; + const hasVisibleEdges = this.edges.get().some(edge => (edge.to === node.id || edge.from === node.id) && !edge.hidden); + return hasVisibleEdges; } }); if (allHiddenNodes.length > 0) { - console.log(`Unhiding ${allHiddenNodes.length} nodes with valid connections`); - const updates = allHiddenNodes.map(node => ({ id: node.id, hidden: false })); - this.nodes.update(updates); - } else { - console.log('No eligible nodes to unhide'); + this.nodes.update(allHiddenNodes.map(node => ({ id: node.id, hidden: false }))); } } - } -// Export for use in main.js window.GraphManager = GraphManager; \ No newline at end of file diff --git a/static/js/main.js b/static/js/main.js index 5d3331c..6cf2ce1 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -224,12 +224,6 @@ class DNScopeApp { if (e.target === this.elements.settingsModal) this.hideSettingsModal(); }); } - if (this.elements.saveApiKeys) { - this.elements.saveApiKeys.removeEventListener('click', this.saveApiKeys); - } - if (this.elements.resetApiKeys) { - this.elements.resetApiKeys.removeEventListener('click', this.resetApiKeys); - } // Setup new handlers const saveSettingsBtn = document.getElementById('save-settings'); @@ -855,7 +849,7 @@ class DNScopeApp { // Do final graph update when scan completes console.log('Scan completed - performing final graph update'); - setTimeout(() => this.updateGraph(), 100); + setTimeout(() => this.updateGraph(), 1000); break; case 'failed': @@ -1722,17 +1716,9 @@ class DNScopeApp { return groups; } - formatEdgeLabel(relationshipType, confidence) { - if (!relationshipType) return ''; - - const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○'; - return `${relationshipType} ${confidenceText}`; - } - createEdgeTooltip(edge) { let tooltip = `
`; tooltip += `
${edge.label || 'Relationship'}
`; - tooltip += `
Confidence: ${(edge.confidence_score * 100).toFixed(1)}%
`; // UPDATED: Use raw provider name (no formatting) if (edge.source_provider) { @@ -1872,7 +1858,7 @@ class DNScopeApp { html += `
${innerNodeId} - @@ -1899,8 +1885,6 @@ class DNScopeApp { `; node.incoming_edges.forEach(edge => { - const confidence = edge.data.confidence_score || 0; - const confidenceClass = confidence >= 0.8 ? 'high' : confidence >= 0.6 ? 'medium' : 'low'; html += `
@@ -1909,9 +1893,6 @@ class DNScopeApp {
${edge.data.relationship_type} - - ${'●'.repeat(Math.ceil(confidence * 3))} -
`; @@ -1930,9 +1911,6 @@ class DNScopeApp { `; node.outgoing_edges.forEach(edge => { - const confidence = edge.data.confidence_score || 0; - const confidenceClass = confidence >= 0.8 ? 'high' : confidence >= 0.6 ? 'medium' : 'low'; - html += `
@@ -1940,9 +1918,6 @@ class DNScopeApp {
${edge.data.relationship_type} - - ${'●'.repeat(Math.ceil(confidence * 3))} -
`; @@ -2361,51 +2336,6 @@ class DNScopeApp { this.elements.settingsModal.style.display = 'none'; } } - - /** - * Save API Keys - */ - async saveApiKeys() { - const inputs = this.elements.apiKeyInputs.querySelectorAll('input'); - const keys = {}; - inputs.forEach(input => { - const provider = input.dataset.provider; - const value = input.value.trim(); - if (provider && value) { - keys[provider] = value; - } - }); - - if (Object.keys(keys).length === 0) { - this.showWarning('No API keys were entered.'); - return; - } - - try { - const response = await this.apiCall('/api/config/api-keys', 'POST', keys); - if (response.success) { - this.showSuccess(response.message); - this.hideSettingsModal(); - this.loadProviders(); // Refresh provider status - } else { - throw new Error(response.error || 'Failed to save API keys'); - } - } catch (error) { - this.showError(`Error saving API keys: ${error.message}`); - } - } - - /** - * Reset API Key fields - */ - resetApiKeys() { - const inputs = this.elements.apiKeyInputs.querySelectorAll('input'); - inputs.forEach(input => { - input.value = ''; - }); - } - - /** * Make API call to server diff --git a/utils/export_manager.py b/utils/export_manager.py index c8f1fba..9d8d795 100644 --- a/utils/export_manager.py +++ b/utils/export_manager.py @@ -188,7 +188,6 @@ class ExportManager: f" - Type: {domain_info['classification']}", f" - Connected IPs: {len(domain_info['ips'])}", f" - Certificate Status: {domain_info['cert_status']}", - f" - Relationship Confidence: {domain_info['avg_confidence']:.2f}", ]) if domain_info['security_notes']: @@ -247,11 +246,9 @@ class ExportManager: ]) for rel in key_relationships[:8]: # Top 8 relationships - confidence_desc = self._describe_confidence(rel['confidence']) report.extend([ f"â€ĸ {rel['source']} → {rel['target']}", f" - Relationship: {self._humanize_relationship_type(rel['type'])}", - f" - Evidence Strength: {confidence_desc} ({rel['confidence']:.2f})", f" - Discovery Method: {rel['provider']}", "" ]) @@ -291,21 +288,15 @@ class ExportManager: "Data Quality Assessment:", f"â€ĸ Total API Requests: {audit_trail.get('session_metadata', {}).get('total_requests', 0)}", f"â€ĸ Data Providers Used: {len(audit_trail.get('session_metadata', {}).get('providers_used', []))}", - f"â€ĸ Relationship Confidence Distribution:", ]) - # Confidence distribution - confidence_dist = self._calculate_confidence_distribution(edges) - for level, count in confidence_dist.items(): - percentage = (count / len(edges) * 100) if edges else 0 - report.extend([ - f" - {level.title()} Confidence (â‰Ĩ{self._get_confidence_threshold(level)}): {count} ({percentage:.1f}%)", - ]) + correlation_provider = next((p for p in scanner.providers if p.get_name() == 'correlation'), None) + correlation_count = len(correlation_provider.correlation_index) if correlation_provider else 0 report.extend([ "", "Correlation Analysis:", - f"â€ĸ Entity Correlations Identified: {len(scanner.graph.correlation_index)}", + f"â€ĸ Entity Correlations Identified: {correlation_count}", f"â€ĸ Cross-Reference Validation: {self._count_cross_validated_relationships(edges)} relationships verified by multiple sources", "" ]) @@ -375,9 +366,7 @@ class ExportManager: if len(connected_ips) > 5: security_notes.append("Multiple IP endpoints") - # Average confidence domain_edges = [e for e in edges if e['from'] == domain['id']] - avg_confidence = sum(e['confidence_score'] for e in domain_edges) / len(domain_edges) if domain_edges else 0 domain_analysis.append({ 'domain': domain['id'], @@ -385,7 +374,6 @@ class ExportManager: 'ips': connected_ips, 'cert_status': cert_status, 'security_notes': security_notes, - 'avg_confidence': avg_confidence }) # Sort by number of connections (most connected first) @@ -480,7 +468,7 @@ class ExportManager: def _identify_key_relationships(self, edges: List[Dict]) -> List[Dict[str, Any]]: """Identify the most significant relationships in the infrastructure.""" - # Score relationships by confidence and type importance + # Score relationships by type importance relationship_importance = { 'dns_a_record': 0.9, 'dns_aaaa_record': 0.9, @@ -491,23 +479,19 @@ class ExportManager: 'dns_ns_record': 0.7 } - scored_edges = [] + edges = [] for edge in edges: - base_confidence = edge.get('confidence_score', 0) type_weight = relationship_importance.get(edge.get('label', ''), 0.5) - combined_score = (base_confidence * 0.7) + (type_weight * 0.3) - scored_edges.append({ + edges.append({ 'source': edge['from'], 'target': edge['to'], 'type': edge.get('label', ''), - 'confidence': base_confidence, 'provider': edge.get('source_provider', ''), - 'score': combined_score }) # Return top relationships by score - return sorted(scored_edges, key=lambda x: x['score'], reverse=True) + return sorted(edges, key=lambda x: x['score'], reverse=True) def _analyze_certificate_infrastructure(self, nodes: List[Dict]) -> Dict[str, Any]: """Analyze certificate infrastructure across all domains.""" @@ -570,19 +554,6 @@ class ExportManager: else: return "Mixed Status" - def _describe_confidence(self, confidence: float) -> str: - """Convert confidence score to descriptive text.""" - if confidence >= 0.9: - return "Very High" - elif confidence >= 0.8: - return "High" - elif confidence >= 0.6: - return "Medium" - elif confidence >= 0.4: - return "Low" - else: - return "Very Low" - def _humanize_relationship_type(self, rel_type: str) -> str: """Convert technical relationship types to human-readable descriptions.""" type_map = { @@ -599,26 +570,6 @@ class ExportManager: } return type_map.get(rel_type, rel_type.replace('_', ' ').title()) - def _calculate_confidence_distribution(self, edges: List[Dict]) -> Dict[str, int]: - """Calculate confidence score distribution.""" - distribution = {'high': 0, 'medium': 0, 'low': 0} - - for edge in edges: - confidence = edge.get('confidence_score', 0) - if confidence >= 0.8: - distribution['high'] += 1 - elif confidence >= 0.6: - distribution['medium'] += 1 - else: - distribution['low'] += 1 - - return distribution - - def _get_confidence_threshold(self, level: str) -> str: - """Get confidence threshold for a level.""" - thresholds = {'high': '0.80', 'medium': '0.60', 'low': '0.00'} - return thresholds.get(level, '0.00') - def _count_cross_validated_relationships(self, edges: List[Dict]) -> int: """Count relationships verified by multiple providers.""" # Group edges by source-target pair