diff --git a/README.md b/README.md index 35826d0..e07c595 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,6 @@ For power users who require more in-depth information, DNScope can be configured * **In-Memory Graph Analysis**: Uses NetworkX for efficient relationship mapping. * **Real-Time Visualization**: The graph updates dynamically as the scan progresses. * **Forensic Logging**: A complete audit trail of all reconnaissance activities is maintained. - * **Confidence Scoring**: Relationships are weighted based on the reliability of the data source. * **Session Management**: Supports concurrent user sessions with isolated scanner instances. * **Extensible Provider Architecture**: Easily add new data sources to expand the tool's capabilities. * **Web-Based UI**: An intuitive and interactive web interface for managing scans and visualizing results. diff --git a/app.py b/app.py index f5a2a15..c9c475e 100644 --- a/app.py +++ b/app.py @@ -332,7 +332,6 @@ def revert_graph_action(): scanner.graph.add_edge( source_id=edge['from'], target_id=edge['to'], relationship_type=edge['metadata']['relationship_type'], - confidence_score=edge['metadata']['confidence_score'], source_provider=edge['metadata']['source_provider'], raw_data=edge.get('raw_data', {}) ) diff --git a/core/graph_manager.py b/core/graph_manager.py index cf7d1fa..735f50e 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -2,7 +2,7 @@ """ Graph data model for DNScope using NetworkX. -Manages in-memory graph storage with confidence scoring and forensic metadata. +Manages in-memory graph storage with forensic metadata. Now fully compatible with the unified ProviderResult data model. UPDATED: Fixed correlation exclusion keys to match actual attribute names. UPDATED: Removed export_json() method - now handled by ExportManager. @@ -31,7 +31,7 @@ class NodeType(Enum): class GraphManager: """ Thread-safe graph manager for DNScope infrastructure mapping. - Uses NetworkX for in-memory graph storage with confidence scoring. + Uses NetworkX for in-memory graph storage. Compatible with unified ProviderResult data model. """ @@ -83,7 +83,7 @@ class GraphManager: return is_new_node def add_edge(self, source_id: str, target_id: str, relationship_type: str, - confidence_score: float = 0.5, source_provider: str = "unknown", + source_provider: str = "unknown", raw_data: Optional[Dict[str, Any]] = None) -> bool: """ UPDATED: Add or update an edge between two nodes with raw relationship labels. @@ -91,23 +91,13 @@ class GraphManager: if not self.graph.has_node(source_id) or not self.graph.has_node(target_id): return False - new_confidence = confidence_score # UPDATED: Use raw relationship type - no formatting edge_label = relationship_type - - if self.graph.has_edge(source_id, target_id): - # If edge exists, update confidence if the new score is higher. - if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0): - self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence - self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat() - self.graph.edges[source_id, target_id]['updated_by'] = source_provider - return False # Add a new edge with raw attributes self.graph.add_edge(source_id, target_id, relationship_type=edge_label, - confidence_score=new_confidence, source_provider=source_provider, discovery_timestamp=datetime.now(timezone.utc).isoformat(), raw_data=raw_data or {}) @@ -137,11 +127,6 @@ class GraphManager: """Get all nodes of a specific type.""" return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value] - def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]: - """Get edges with confidence score above a given threshold.""" - return [(u, v, d) for u, v, d in self.graph.edges(data=True) - if d.get('confidence_score', 0) >= min_confidence] - def get_graph_data(self) -> Dict[str, Any]: """ Export graph data formatted for frontend visualization. @@ -177,7 +162,6 @@ class GraphManager: 'from': source, 'to': target, 'label': attrs.get('relationship_type', ''), - 'confidence_score': attrs.get('confidence_score', 0), 'source_provider': attrs.get('source_provider', ''), 'discovery_timestamp': attrs.get('discovery_timestamp') }) @@ -188,24 +172,6 @@ class GraphManager: 'statistics': self.get_statistics()['basic_metrics'] } - def _get_confidence_distribution(self) -> Dict[str, int]: - """Get distribution of edge confidence scores with empty graph handling.""" - distribution = {'high': 0, 'medium': 0, 'low': 0} - - # FIXED: Handle empty graph case - if self.get_edge_count() == 0: - return distribution - - for _, _, data in self.graph.edges(data=True): - confidence = data.get('confidence_score', 0) - if confidence >= 0.8: - distribution['high'] += 1 - elif confidence >= 0.6: - distribution['medium'] += 1 - else: - distribution['low'] += 1 - return distribution - def get_statistics(self) -> Dict[str, Any]: """Get comprehensive statistics about the graph with proper empty graph handling.""" @@ -222,7 +188,6 @@ class GraphManager: }, 'node_type_distribution': {}, 'relationship_type_distribution': {}, - 'confidence_distribution': self._get_confidence_distribution(), 'provider_distribution': {} } diff --git a/core/logger.py b/core/logger.py index d6b0c8e..43d72bf 100644 --- a/core/logger.py +++ b/core/logger.py @@ -30,7 +30,6 @@ class RelationshipDiscovery: source_node: str target_node: str relationship_type: str - confidence_score: float provider: str raw_data: Dict[str, Any] discovery_method: str @@ -157,7 +156,7 @@ class ForensicLogger: self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}") def log_relationship_discovery(self, source_node: str, target_node: str, - relationship_type: str, confidence_score: float, + relationship_type: str, provider: str, raw_data: Dict[str, Any], discovery_method: str) -> None: """ @@ -167,7 +166,6 @@ class ForensicLogger: source_node: Source node identifier target_node: Target node identifier relationship_type: Type of relationship (e.g., 'SAN', 'A_Record') - confidence_score: Confidence score (0.0 to 1.0) provider: Provider that discovered this relationship raw_data: Raw data from provider response discovery_method: Method used to discover relationship @@ -177,7 +175,6 @@ class ForensicLogger: source_node=source_node, target_node=target_node, relationship_type=relationship_type, - confidence_score=confidence_score, provider=provider, raw_data=raw_data, discovery_method=discovery_method @@ -188,7 +185,7 @@ class ForensicLogger: self.logger.info( f"Relationship Discovered - {source_node} -> {target_node} " - f"({relationship_type}) - Confidence: {confidence_score:.2f} - Provider: {provider}" + f"({relationship_type}) - Provider: {provider}" ) def log_scan_start(self, target_domain: str, recursion_depth: int, @@ -238,7 +235,6 @@ class ForensicLogger: 'successful_requests': len([req for req in provider_requests if req.error is None]), 'failed_requests': len([req for req in provider_requests if req.error is not None]), 'relationships_discovered': len(provider_relationships), - 'avg_confidence': sum(rel.confidence_score for rel in provider_relationships) / len(provider_relationships) if provider_relationships else 0 } return { diff --git a/core/provider_result.py b/core/provider_result.py index 00f0579..1f1c46f 100644 --- a/core/provider_result.py +++ b/core/provider_result.py @@ -18,33 +18,19 @@ class StandardAttribute: value: Any type: str provider: str - confidence: float timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) metadata: Optional[Dict[str, Any]] = field(default_factory=dict) - def __post_init__(self): - """Validate the attribute after initialization.""" - if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0: - raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}") - - @dataclass class Relationship: """A unified data structure for a directional link between two nodes.""" source_node: str target_node: str relationship_type: str - confidence: float provider: str timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) raw_data: Optional[Dict[str, Any]] = field(default_factory=dict) - def __post_init__(self): - """Validate the relationship after initialization.""" - if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0: - raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}") - - @dataclass class ProviderResult: """A container for all data returned by a provider from a single query.""" @@ -52,8 +38,7 @@ class ProviderResult: relationships: List[Relationship] = field(default_factory=list) def add_attribute(self, target_node: str, name: str, value: Any, attr_type: str, - provider: str, confidence: float = 0.8, - metadata: Optional[Dict[str, Any]] = None) -> None: + provider: str, metadata: Optional[Dict[str, Any]] = None) -> None: """Helper method to add an attribute to the result.""" self.attributes.append(StandardAttribute( target_node=target_node, @@ -61,19 +46,16 @@ class ProviderResult: value=value, type=attr_type, provider=provider, - confidence=confidence, metadata=metadata or {} )) def add_relationship(self, source_node: str, target_node: str, relationship_type: str, - provider: str, confidence: float = 0.8, - raw_data: Optional[Dict[str, Any]] = None) -> None: + provider: str, raw_data: Optional[Dict[str, Any]] = None) -> None: """Helper method to add a relationship to the result.""" self.relationships.append(Relationship( source_node=source_node, target_node=target_node, relationship_type=relationship_type, - confidence=confidence, provider=provider, raw_data=raw_data or {} )) diff --git a/core/scanner.py b/core/scanner.py index e93b9b6..3d88b26 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -847,7 +847,6 @@ class Scanner: 'source_node': rel.source_node, 'target_node': rel.target_node, 'relationship_type': rel.relationship_type, - 'confidence': rel.confidence, 'provider': rel.provider, 'raw_data': rel.raw_data }) @@ -905,7 +904,6 @@ class Scanner: source_id=rel_data['source_node'], target_id=rel_data['target_node'], relationship_type=rel_data['relationship_type'], - confidence_score=rel_data['confidence'], source_provider=rel_data['provider'], raw_data=rel_data['raw_data'] ) @@ -1012,7 +1010,6 @@ class Scanner: self.graph.add_edge( visual_source, visual_target, relationship.relationship_type, - relationship.confidence, provider_name, relationship.raw_data ) @@ -1035,7 +1032,7 @@ class Scanner: for attribute in provider_result.attributes: attr_dict = { "name": attribute.name, "value": attribute.value, "type": attribute.type, - "provider": attribute.provider, "confidence": attribute.confidence, "metadata": attribute.metadata + "provider": attribute.provider, "metadata": attribute.metadata } attributes_by_node[attribute.target_node].append(attr_dict) diff --git a/providers/base_provider.py b/providers/base_provider.py index 6a8fa61..196b53f 100644 --- a/providers/base_provider.py +++ b/providers/base_provider.py @@ -229,7 +229,6 @@ class BaseProvider(ABC): def log_relationship_discovery(self, source_node: str, target_node: str, relationship_type: str, - confidence_score: float, raw_data: Dict[str, Any], discovery_method: str) -> None: """ @@ -239,7 +238,6 @@ class BaseProvider(ABC): source_node: Source node identifier target_node: Target node identifier relationship_type: Type of relationship - confidence_score: Confidence score raw_data: Raw data from provider discovery_method: Method used for discovery """ @@ -249,7 +247,6 @@ class BaseProvider(ABC): source_node=source_node, target_node=target_node, relationship_type=relationship_type, - confidence_score=confidence_score, provider=self.name, raw_data=raw_data, discovery_method=discovery_method diff --git a/providers/correlation_provider.py b/providers/correlation_provider.py index a6948b3..db8cd1b 100644 --- a/providers/correlation_provider.py +++ b/providers/correlation_provider.py @@ -2,6 +2,7 @@ import re from typing import Dict, Any, List +from datetime import datetime, timezone from .base_provider import BaseProvider from core.provider_result import ProviderResult @@ -10,6 +11,7 @@ from core.graph_manager import NodeType, GraphManager class CorrelationProvider(BaseProvider): """ A provider that finds correlations between nodes in the graph. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ def __init__(self, name: str = "correlation", session_config=None): @@ -61,12 +63,14 @@ class CorrelationProvider(BaseProvider): def query_domain(self, domain: str) -> ProviderResult: """ Query the provider for information about a domain. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ return self._find_correlations(domain) def query_ip(self, ip: str) -> ProviderResult: """ Query the provider for information about an IP address. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ return self._find_correlations(ip) @@ -79,8 +83,10 @@ class CorrelationProvider(BaseProvider): def _find_correlations(self, node_id: str) -> ProviderResult: """ Find correlations for a given node with enhanced filtering and error handling. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ result = ProviderResult() + discovery_time = datetime.now(timezone.utc) # Enhanced safety checks if not self.graph or not self.graph.graph.has_node(node_id): @@ -133,7 +139,7 @@ class CorrelationProvider(BaseProvider): # Create correlation if we have multiple nodes with this value if len(self.correlation_index[attr_value]['nodes']) > 1: - self._create_correlation_relationships(attr_value, self.correlation_index[attr_value], result) + self._create_correlation_relationships(attr_value, self.correlation_index[attr_value], result, discovery_time) correlations_found += 1 # Log correlation results @@ -187,9 +193,11 @@ class CorrelationProvider(BaseProvider): return False - def _create_correlation_relationships(self, value: Any, correlation_data: Dict[str, Any], result: ProviderResult): + def _create_correlation_relationships(self, value: Any, correlation_data: Dict[str, Any], + result: ProviderResult, discovery_time: datetime): """ Create correlation relationships with enhanced deduplication and validation. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}" nodes = correlation_data['nodes'] @@ -216,7 +224,6 @@ class CorrelationProvider(BaseProvider): value=value, attr_type=str(type(value).__name__), provider=self.name, - confidence=0.9, metadata={ 'correlated_nodes': list(nodes), 'sources': sources, @@ -225,7 +232,7 @@ class CorrelationProvider(BaseProvider): } ) - # Create relationships with source validation + # Create relationships with source validation and enhanced timestamps created_relationships = set() for source in sources: @@ -240,19 +247,23 @@ class CorrelationProvider(BaseProvider): relationship_label = f"corr_{provider}_{attribute}" + # Enhanced raw_data with discovery timestamp for time-based edge coloring + raw_data = { + 'correlation_value': value, + 'original_attribute': attribute, + 'correlation_type': 'attribute_matching', + 'correlation_size': len(nodes), + 'discovery_timestamp': discovery_time.isoformat(), + 'relevance_timestamp': discovery_time.isoformat() # Correlation data is "fresh" when discovered + } + # Add the relationship to the result result.add_relationship( source_node=node_id, target_node=correlation_node_id, relationship_type=relationship_label, provider=self.name, - confidence=0.9, - raw_data={ - 'correlation_value': value, - 'original_attribute': attribute, - 'correlation_type': 'attribute_matching', - 'correlation_size': len(nodes) - } + raw_data=raw_data ) created_relationships.add(relationship_key) \ No newline at end of file diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index f7c0bf1..30958e4 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -18,6 +18,7 @@ class CrtShProvider(BaseProvider): Provider for querying crt.sh certificate transparency database. FIXED: Improved caching logic and error handling to prevent infinite retry loops. Returns standardized ProviderResult objects with caching support. + UPDATED: Enhanced with certificate timestamps for time-based edge coloring. """ def __init__(self, name=None, session_config=None): @@ -131,6 +132,7 @@ class CrtShProvider(BaseProvider): def query_domain(self, domain: str) -> ProviderResult: """ FIXED: Simplified and more robust domain querying with better error handling. + UPDATED: Enhanced with certificate timestamps for time-based edge coloring. """ if not _is_valid_domain(domain): return ProviderResult() @@ -245,7 +247,6 @@ class CrtShProvider(BaseProvider): target_node=rel_data.get("target_node", ""), relationship_type=rel_data.get("relationship_type", ""), provider=rel_data.get("provider", self.name), - confidence=float(rel_data.get("confidence", 0.8)), raw_data=rel_data.get("raw_data", {}) ) except (ValueError, TypeError) as e: @@ -265,7 +266,6 @@ class CrtShProvider(BaseProvider): value=attr_data.get("value"), attr_type=attr_data.get("type", "unknown"), provider=attr_data.get("provider", self.name), - confidence=float(attr_data.get("confidence", 0.9)), metadata=attr_data.get("metadata", {}) ) except (ValueError, TypeError) as e: @@ -293,7 +293,6 @@ class CrtShProvider(BaseProvider): "source_node": rel.source_node, "target_node": rel.target_node, "relationship_type": rel.relationship_type, - "confidence": rel.confidence, "provider": rel.provider, "raw_data": rel.raw_data } for rel in result.relationships @@ -305,7 +304,6 @@ class CrtShProvider(BaseProvider): "value": attr.value, "type": attr.type, "provider": attr.provider, - "confidence": attr.confidence, "metadata": attr.metadata } for attr in result.attributes ] @@ -372,6 +370,7 @@ class CrtShProvider(BaseProvider): """ Process certificates to create proper domain and CA nodes. FIXED: Better error handling and progress tracking. + UPDATED: Enhanced with certificate timestamps for time-based edge coloring. """ result = ProviderResult() @@ -391,8 +390,7 @@ class CrtShProvider(BaseProvider): name="crtsh_data_warning", value=incompleteness_warning, attr_type='metadata', - provider=self.name, - confidence=1.0 + provider=self.name ) all_discovered_domains = set() @@ -415,16 +413,28 @@ class CrtShProvider(BaseProvider): if cert_domains: all_discovered_domains.update(cert_domains) - # Create CA nodes for certificate issuers + # Create CA nodes for certificate issuers with timestamp issuer_name = self._parse_issuer_organization(cert_data.get('issuer_name', '')) if issuer_name and issuer_name not in processed_issuers: + # Enhanced raw_data with certificate timestamp for time-based edge coloring + issuer_raw_data = {'issuer_dn': cert_data.get('issuer_name', '')} + + # Add certificate issue date (not_before) as relevance timestamp + not_before = cert_data.get('not_before') + if not_before: + try: + not_before_date = self._parse_certificate_date(not_before) + issuer_raw_data['cert_not_before'] = not_before_date.isoformat() + issuer_raw_data['relevance_timestamp'] = not_before_date.isoformat() # Standardized field + except Exception as e: + self.logger.logger.debug(f"Failed to parse not_before date for issuer: {e}") + result.add_relationship( source_node=query_domain, target_node=issuer_name, relationship_type='crtsh_cert_issuer', provider=self.name, - confidence=0.95, - raw_data={'issuer_dn': cert_data.get('issuer_name', '')} + raw_data=issuer_raw_data ) processed_issuers.add(issuer_name) @@ -442,7 +452,6 @@ class CrtShProvider(BaseProvider): value=value, attr_type='certificate_data', provider=self.name, - confidence=0.9, metadata={'certificate_id': cert_data.get('id')} ) @@ -457,7 +466,7 @@ class CrtShProvider(BaseProvider): self.logger.logger.info(f"CrtSh query cancelled before relationship creation for domain: {query_domain}") return result - # Create selective relationships to avoid large entities + # Create selective relationships to avoid large entities with enhanced timestamps relationships_created = 0 for discovered_domain in all_discovered_domains: if discovered_domain == query_domain: @@ -467,25 +476,36 @@ class CrtShProvider(BaseProvider): continue if self._should_create_relationship(query_domain, discovered_domain): - confidence = self._calculate_domain_relationship_confidence( - query_domain, discovered_domain, [], all_discovered_domains + # Enhanced raw_data with certificate timestamp for domain relationships + domain_raw_data = {'relationship_type': 'certificate_discovery'} + + # Find the most recent certificate for this domain pair to use as timestamp + most_recent_cert = self._find_most_recent_cert_for_domains( + certificates, query_domain, discovered_domain ) + if most_recent_cert: + not_before = most_recent_cert.get('not_before') + if not_before: + try: + not_before_date = self._parse_certificate_date(not_before) + domain_raw_data['cert_not_before'] = not_before_date.isoformat() + domain_raw_data['relevance_timestamp'] = not_before_date.isoformat() + except Exception as e: + self.logger.logger.debug(f"Failed to parse not_before date for domain relationship: {e}") result.add_relationship( source_node=query_domain, target_node=discovered_domain, relationship_type='crtsh_san_certificate', provider=self.name, - confidence=confidence, - raw_data={'relationship_type': 'certificate_discovery'} + raw_data=domain_raw_data ) self.log_relationship_discovery( source_node=query_domain, target_node=discovered_domain, relationship_type='crtsh_san_certificate', - confidence_score=confidence, - raw_data={'relationship_type': 'certificate_discovery'}, + raw_data=domain_raw_data, discovery_method="certificate_transparency_analysis" ) relationships_created += 1 @@ -493,6 +513,31 @@ class CrtShProvider(BaseProvider): self.logger.logger.info(f"CrtSh processing completed for {query_domain}: processed {processed_certs}/{len(certificates)} certificates, {len(all_discovered_domains)} domains, {relationships_created} relationships") return result + def _find_most_recent_cert_for_domains(self, certificates: List[Dict[str, Any]], + domain1: str, domain2: str) -> Optional[Dict[str, Any]]: + """ + Find the most recent certificate that contains both domains. + Used for determining the relevance timestamp for domain relationships. + """ + most_recent_cert = None + most_recent_date = None + + for cert in certificates: + # Check if this certificate contains both domains + cert_domains = self._extract_domains_from_certificate(cert) + if domain1 in cert_domains and domain2 in cert_domains: + not_before = cert.get('not_before') + if not_before: + try: + cert_date = self._parse_certificate_date(not_before) + if most_recent_date is None or cert_date > most_recent_date: + most_recent_date = cert_date + most_recent_cert = cert + except Exception: + continue + + return most_recent_cert + # [Rest of the methods remain the same as in the original file] def _should_create_relationship(self, source_domain: str, target_domain: str) -> bool: """ @@ -664,25 +709,6 @@ class CrtShProvider(BaseProvider): return [d for d in final_domains if _is_valid_domain(d)] - def _calculate_domain_relationship_confidence(self, domain1: str, domain2: str, - shared_certificates: List[Dict[str, Any]], - all_discovered_domains: Set[str]) -> float: - """Calculate confidence score for domain relationship based on various factors.""" - base_confidence = 0.9 - - relationship_context = self._determine_relationship_context(domain2, domain1) - - if relationship_context == 'exact_match': - context_bonus = 0.0 - elif relationship_context == 'subdomain': - context_bonus = 0.1 - elif relationship_context == 'parent_domain': - context_bonus = 0.05 - else: - context_bonus = 0.0 - - final_confidence = base_confidence + context_bonus - return max(0.1, min(1.0, final_confidence)) def _determine_relationship_context(self, cert_domain: str, query_domain: str) -> str: """Determine the context of the relationship between certificate domain and query domain.""" diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 6187b0d..828dec0 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -2,6 +2,7 @@ from dns import resolver, reversename from typing import Dict +from datetime import datetime, timezone from .base_provider import BaseProvider from core.provider_result import ProviderResult from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version @@ -11,6 +12,7 @@ class DNSProvider(BaseProvider): """ Provider for standard DNS resolution and reverse DNS lookups. Now returns standardized ProviderResult objects with IPv4 and IPv6 support. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ def __init__(self, name=None, session_config=None): @@ -51,6 +53,7 @@ class DNSProvider(BaseProvider): """ Query DNS records for the domain to discover relationships and attributes. FIXED: Now creates separate attributes for each DNS record type. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. Args: domain: Domain to investigate @@ -62,11 +65,12 @@ class DNSProvider(BaseProvider): return ProviderResult() result = ProviderResult() + discovery_time = datetime.now(timezone.utc) # Query all record types - each gets its own attribute for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']: try: - self._query_record(domain, record_type, result) + self._query_record(domain, record_type, result, discovery_time) #except resolver.NoAnswer: # This is not an error, just a confirmation that the record doesn't exist. #self.logger.logger.debug(f"No {record_type} record found for {domain}") @@ -79,6 +83,7 @@ class DNSProvider(BaseProvider): def query_ip(self, ip: str) -> ProviderResult: """ Query reverse DNS for the IP address (supports both IPv4 and IPv6). + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. Args: ip: IP address to investigate (IPv4 or IPv6) @@ -91,6 +96,7 @@ class DNSProvider(BaseProvider): result = ProviderResult() ip_version = get_ip_version(ip) + discovery_time = datetime.now(timezone.utc) try: # Perform reverse DNS lookup (works for both IPv4 and IPv6) @@ -112,20 +118,24 @@ class DNSProvider(BaseProvider): relationship_type = 'dns_a_record' record_prefix = 'A' + # Enhanced raw_data with discovery timestamp for time-based edge coloring + raw_data = { + 'query_type': 'PTR', + 'ip_address': ip, + 'ip_version': ip_version, + 'hostname': hostname, + 'ttl': response.ttl, + 'discovery_timestamp': discovery_time.isoformat(), + 'relevance_timestamp': discovery_time.isoformat() # DNS data is "fresh" when discovered + } + # Add the relationship result.add_relationship( source_node=ip, target_node=hostname, relationship_type='dns_ptr_record', provider=self.name, - confidence=0.8, - raw_data={ - 'query_type': 'PTR', - 'ip_address': ip, - 'ip_version': ip_version, - 'hostname': hostname, - 'ttl': response.ttl - } + raw_data=raw_data ) # Add to PTR records list @@ -136,14 +146,7 @@ class DNSProvider(BaseProvider): source_node=ip, target_node=hostname, relationship_type='dns_ptr_record', - confidence_score=0.8, - raw_data={ - 'query_type': 'PTR', - 'ip_address': ip, - 'ip_version': ip_version, - 'hostname': hostname, - 'ttl': response.ttl - }, + raw_data=raw_data, discovery_method=f"reverse_dns_lookup_ipv{ip_version}" ) @@ -155,7 +158,6 @@ class DNSProvider(BaseProvider): value=ptr_records, attr_type='dns_record', provider=self.name, - confidence=0.8, metadata={'ttl': response.ttl, 'ip_version': ip_version} ) @@ -170,10 +172,11 @@ class DNSProvider(BaseProvider): return result - def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None: + def _query_record(self, domain: str, record_type: str, result: ProviderResult, discovery_time: datetime) -> None: """ FIXED: Query DNS records with unique attribute names for each record type. Enhanced to better handle IPv6 AAAA records. + UPDATED: Enhanced with discovery timestamps for time-based edge coloring. """ try: self.total_requests += 1 @@ -217,18 +220,20 @@ class DNSProvider(BaseProvider): if record_type in ['A', 'AAAA'] and _is_valid_ip(target): ip_version = get_ip_version(target) + # Enhanced raw_data with discovery timestamp for time-based edge coloring raw_data = { 'query_type': record_type, 'domain': domain, 'value': target, - 'ttl': response.ttl + 'ttl': response.ttl, + 'discovery_timestamp': discovery_time.isoformat(), + 'relevance_timestamp': discovery_time.isoformat() # DNS data is "fresh" when discovered } if ip_version: raw_data['ip_version'] = ip_version relationship_type = f"dns_{record_type.lower()}_record" - confidence = 0.8 # Add relationship result.add_relationship( @@ -236,7 +241,6 @@ class DNSProvider(BaseProvider): target_node=target, relationship_type=relationship_type, provider=self.name, - confidence=confidence, raw_data=raw_data ) @@ -252,7 +256,6 @@ class DNSProvider(BaseProvider): source_node=domain, target_node=target, relationship_type=relationship_type, - confidence_score=confidence, raw_data=raw_data, discovery_method=discovery_method ) @@ -276,7 +279,6 @@ class DNSProvider(BaseProvider): value=dns_records, attr_type='dns_record_list', provider=self.name, - confidence=0.8, metadata=metadata ) diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py index 11900be..0e1ac38 100644 --- a/providers/shodan_provider.py +++ b/providers/shodan_provider.py @@ -15,6 +15,7 @@ class ShodanProvider(BaseProvider): """ Provider for querying Shodan API for IP address information. Now returns standardized ProviderResult objects with caching support for IPv4 and IPv6. + UPDATED: Enhanced with last_seen timestamp for time-based edge coloring. """ def __init__(self, name=None, session_config=None): @@ -145,6 +146,7 @@ class ShodanProvider(BaseProvider): """ Query Shodan for information about an IP address (IPv4 or IPv6), with caching of processed data. FIXED: Proper 404 handling to prevent unnecessary retries. + UPDATED: Enhanced with last_seen timestamp extraction for time-based edge coloring. Args: ip: IP address to investigate (IPv4 or IPv6) @@ -304,7 +306,6 @@ class ShodanProvider(BaseProvider): target_node=rel_data["target_node"], relationship_type=rel_data["relationship_type"], provider=rel_data["provider"], - confidence=rel_data["confidence"], raw_data=rel_data.get("raw_data", {}) ) @@ -316,7 +317,6 @@ class ShodanProvider(BaseProvider): value=attr_data["value"], attr_type=attr_data["type"], provider=attr_data["provider"], - confidence=attr_data["confidence"], metadata=attr_data.get("metadata", {}) ) @@ -336,7 +336,6 @@ class ShodanProvider(BaseProvider): "source_node": rel.source_node, "target_node": rel.target_node, "relationship_type": rel.relationship_type, - "confidence": rel.confidence, "provider": rel.provider, "raw_data": rel.raw_data } for rel in result.relationships @@ -348,7 +347,6 @@ class ShodanProvider(BaseProvider): "value": attr.value, "type": attr.type, "provider": attr.provider, - "confidence": attr.confidence, "metadata": attr.metadata } for attr in result.attributes ] @@ -362,25 +360,40 @@ class ShodanProvider(BaseProvider): """ VERIFIED: Process Shodan data creating ISP nodes with ASN attributes and proper relationships. Enhanced to include IP version information for IPv6 addresses. + UPDATED: Enhanced with last_seen timestamp for time-based edge coloring. """ result = ProviderResult() # Determine IP version for metadata ip_version = get_ip_version(ip) + # Extract last_seen timestamp for time-based edge coloring + last_seen = data.get('last_seen') + # VERIFIED: Extract ISP information and create proper ISP node with ASN isp_name = data.get('org') asn_value = data.get('asn') if isp_name and asn_value: + # Enhanced raw_data with last_seen timestamp + raw_data = { + 'asn': asn_value, + 'shodan_org': isp_name, + 'ip_version': ip_version + } + + # Add last_seen timestamp if available + if last_seen: + raw_data['last_seen'] = last_seen + raw_data['relevance_timestamp'] = last_seen # Standardized field for time-based coloring + # Create relationship from IP to ISP result.add_relationship( source_node=ip, target_node=isp_name, relationship_type='shodan_isp', provider=self.name, - confidence=0.9, - raw_data={'asn': asn_value, 'shodan_org': isp_name, 'ip_version': ip_version} + raw_data=raw_data ) # Add ASN as attribute to the ISP node @@ -390,7 +403,6 @@ class ShodanProvider(BaseProvider): value=asn_value, attr_type='isp_info', provider=self.name, - confidence=0.9, metadata={'description': 'Autonomous System Number from Shodan', 'ip_version': ip_version} ) @@ -401,7 +413,6 @@ class ShodanProvider(BaseProvider): value=isp_name, attr_type='isp_info', provider=self.name, - confidence=0.9, metadata={'description': 'Organization name from Shodan', 'ip_version': ip_version} ) @@ -416,20 +427,24 @@ class ShodanProvider(BaseProvider): else: relationship_type = 'shodan_a_record' + # Enhanced raw_data with last_seen timestamp + hostname_raw_data = {**data, 'ip_version': ip_version} + if last_seen: + hostname_raw_data['last_seen'] = last_seen + hostname_raw_data['relevance_timestamp'] = last_seen + result.add_relationship( source_node=ip, target_node=hostname, relationship_type=relationship_type, provider=self.name, - confidence=0.8, - raw_data={**data, 'ip_version': ip_version} + raw_data=hostname_raw_data ) self.log_relationship_discovery( source_node=ip, target_node=hostname, relationship_type=relationship_type, - confidence_score=0.8, - raw_data={**data, 'ip_version': ip_version}, + raw_data=hostname_raw_data, discovery_method=f"shodan_host_lookup_ipv{ip_version}" ) elif key == 'ports': @@ -441,7 +456,6 @@ class ShodanProvider(BaseProvider): value=port, attr_type='shodan_network_info', provider=self.name, - confidence=0.9, metadata={'ip_version': ip_version} ) elif isinstance(value, (str, int, float, bool)) and value is not None: @@ -452,7 +466,6 @@ class ShodanProvider(BaseProvider): value=value, attr_type='shodan_info', provider=self.name, - confidence=0.9, metadata={'ip_version': ip_version} ) diff --git a/static/css/main.css b/static/css/main.css index ab788e1..f0aae48 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -326,6 +326,20 @@ input[type="text"]:focus, select:focus { animation: progressGlow 2s ease-in-out infinite alternate; } +.gradient-bar { + height: 4px; + background: linear-gradient(to right, #6b7280, #00bfff); + border-radius: 2px; + margin: 0.2rem 0; +} + +.gradient-labels { + display: flex; + justify-content: space-between; + font-size: 0.6rem; + color: #888; +} + @keyframes progressShimmer { 0% { transform: translateX(-100%); } 100% { transform: translateX(100%); } @@ -380,32 +394,59 @@ input[type="text"]:focus, select:focus { color: #999; } -/* Graph Controls */ +/* Enhanced graph controls layout */ .graph-controls { - position: absolute; - top: 8px; - right: 8px; - z-index: 10; display: flex; + flex-direction: column; gap: 0.3rem; + position: absolute; + top: 10px; + left: 10px; + background: rgba(26, 26, 26, 0.9); + padding: 0.5rem; + border-radius: 6px; + border: 1px solid #444; + box-shadow: 0 4px 15px rgba(0, 0, 0, 0.5); + z-index: 100; + min-width: 200px; } -.graph-control-btn, .btn-icon-small { - background: rgba(42, 42, 42, 0.9); +.graph-control-btn { + background: linear-gradient(135deg, #2a2a2a 0%, #1e1e1e 100%); border: 1px solid #555; color: #c7c7c7; - padding: 0.3rem 0.5rem; - font-family: 'Roboto Mono', monospace; - font-size: 0.7rem; + padding: 0.4rem 0.8rem; + border-radius: 4px; cursor: pointer; - transition: all 0.3s ease; + font-family: 'Roboto Mono', monospace; + font-size: 0.8rem; + transition: all 0.2s ease; + text-align: center; } -.graph-control-btn:hover, .btn-icon-small:hover { +.graph-control-btn:hover { + background: linear-gradient(135deg, #3a3a3a 0%, #2e2e2e 100%); border-color: #00ff41; color: #00ff41; } +.graph-control-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.manual-refresh-btn { + background: linear-gradient(135deg, #4a4a2a 0%, #3e3e1e 100%); + border-color: #ffaa00; + color: #ffaa00; +} + +.manual-refresh-btn:hover { + background: linear-gradient(135deg, #5a5a3a 0%, #4e4e2e 100%); + color: #ffcc33; + border-color: #ffcc33; +} + .graph-filter-panel { position: absolute; bottom: 8px; @@ -500,14 +541,6 @@ input[type="text"]:focus, select:focus { height: 2px; } -.legend-edge.high-confidence { - background: #00ff41; -} - -.legend-edge.medium-confidence { - background: #ff9900; -} - /* Provider Panel */ .provider-panel { grid-area: providers; @@ -987,11 +1020,6 @@ input[type="text"]:focus, select:focus { border-radius: 2px; } -.confidence-indicator { - font-size: 0.6rem; - letter-spacing: 1px; -} - .node-link-compact { color: #00aaff; text-decoration: none; @@ -1095,6 +1123,56 @@ input[type="text"]:focus, select:focus { border-left: 3px solid #00aaff; } +.time-control-container { + margin-bottom: 0.5rem; + padding: 0.5rem; + background: rgba(42, 42, 42, 0.3); + border-radius: 4px; + border: 1px solid #444; +} + +.time-control-label { + font-size: 0.8rem; + color: #c7c7c7; + margin-bottom: 0.3rem; + display: block; + font-family: 'Roboto Mono', monospace; +} + +.time-control-input { + width: 100%; + padding: 0.3rem; + background: #1a1a1a; + border: 1px solid #555; + border-radius: 3px; + color: #c7c7c7; + font-family: 'Roboto Mono', monospace; + font-size: 0.75rem; +} + +.time-control-input:focus { + outline: none; + border-color: #00ff41; + box-shadow: 0 0 5px rgba(0, 255, 65, 0.3); +} + +.time-gradient-info { + font-size: 0.7rem; + color: #999; + margin-top: 0.3rem; + text-align: center; + font-family: 'Roboto Mono', monospace; +} + +/* Edge color legend for time-based gradient */ +.time-gradient-legend { + margin-top: 0.5rem; + padding: 0.3rem; + background: rgba(26, 26, 26, 0.5); + border-radius: 3px; + border: 1px solid #333; +} + /* Settings Modal Specific */ .provider-toggle { appearance: none !important; @@ -1324,16 +1402,16 @@ input[type="password"]:focus { .provider-list { grid-template-columns: 1fr; } -} -.manual-refresh-btn { - background: rgba(92, 76, 44, 0.9) !important; /* Orange/amber background */ - border: 1px solid #7a6a3a !important; - color: #ffcc00 !important; /* Bright yellow text */ -} - -.manual-refresh-btn:hover { - border-color: #ffcc00 !important; - color: #fff !important; - background: rgba(112, 96, 54, 0.9) !important; + .graph-controls { + position: relative; + top: auto; + left: auto; + margin-bottom: 1rem; + min-width: auto; + } + + .time-control-input { + font-size: 0.7rem; + } } \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index a895b7b..d2cec72 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -2,7 +2,7 @@ /** * Graph visualization module for DNScope * Handles network graph rendering using vis.js with proper large entity node hiding - * UPDATED: Added manual refresh button for polling optimization when graph becomes large + * UPDATED: Added time-based blue gradient edge coloring system */ const contextMenuCSS = ` .graph-context-menu { @@ -53,6 +53,44 @@ const contextMenuCSS = ` .graph-context-menu ul li:last-child { border-bottom: none; } + +.time-control-container { + margin-bottom: 0.5rem; + padding: 0.5rem; + background: rgba(42, 42, 42, 0.3); + border-radius: 4px; + border: 1px solid #444; +} + +.time-control-label { + font-size: 0.8rem; + color: #c7c7c7; + margin-bottom: 0.3rem; + display: block; +} + +.time-control-input { + width: 100%; + padding: 0.3rem; + background: #1a1a1a; + border: 1px solid #555; + border-radius: 3px; + color: #c7c7c7; + font-family: 'Roboto Mono', monospace; + font-size: 0.75rem; +} + +.time-control-input:focus { + outline: none; + border-color: #00ff41; +} + +.time-gradient-info { + font-size: 0.7rem; + color: #999; + margin-top: 0.3rem; + text-align: center; +} `; class GraphManager { @@ -76,6 +114,16 @@ class GraphManager { this.manualRefreshButton = null; this.manualRefreshHandler = null; // Store the handler + // Time-based gradient settings + this.timeOfInterest = new Date(); // Default to now + this.edgeTimestamps = new Map(); // Store edge ID -> timestamp mapping + + // Gradient colors: grey-ish dark to retina-melting light blue + this.gradientColors = { + dark: '#6b7280', // Grey-ish dark + light: '#00bfff' // Retina-melting light blue + }; + this.options = { nodes: { shape: 'dot', @@ -257,13 +305,25 @@ class GraphManager { } /** - * Add interactive graph controls - * UPDATED: Added manual refresh button for polling optimization + * Add interactive graph controls with time of interest control + * UPDATED: Added time-based edge coloring controls */ addGraphControls() { const controlsContainer = document.createElement('div'); controlsContainer.className = 'graph-controls'; + + // Format current date/time for the input + const currentDateTime = this.formatDateTimeForInput(this.timeOfInterest); + controlsContainer.innerHTML = ` +
+ + +
+ Dark: Old data | Light Blue: Recent data +
+
@@ -283,6 +343,13 @@ class GraphManager { document.getElementById('graph-unhide').addEventListener('click', () => this.unhideAll()); document.getElementById('graph-revert').addEventListener('click', () => this.revertLastAction()); + // Time of interest control + document.getElementById('time-of-interest').addEventListener('change', (e) => { + this.timeOfInterest = new Date(e.target.value); + console.log('Time of interest updated:', this.timeOfInterest); + this.updateEdgeColors(); + }); + // Manual refresh button - handler will be set by main app this.manualRefreshButton = document.getElementById('graph-manual-refresh'); // If a handler was set before the button existed, attach it now @@ -290,6 +357,150 @@ class GraphManager { this.manualRefreshButton.addEventListener('click', this.manualRefreshHandler); } } + + /** + * Format date for datetime-local input + */ + formatDateTimeForInput(date) { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + const hours = String(date.getHours()).padStart(2, '0'); + const minutes = String(date.getMinutes()).padStart(2, '0'); + return `${year}-${month}-${day}T${hours}:${minutes}`; + } + + /** + * Extract relevant timestamp from edge raw_data based on provider + */ + extractEdgeTimestamp(edge) { + const rawData = edge.raw_data || {}; + const provider = edge.source_provider || ''; + + // Check for standardized relevance_timestamp first + if (rawData.relevance_timestamp) { + return new Date(rawData.relevance_timestamp); + } + + // Provider-specific timestamp extraction + switch (provider.toLowerCase()) { + case 'shodan': + // Use last_seen timestamp for Shodan + if (rawData.last_seen) { + return new Date(rawData.last_seen); + } + break; + + case 'crtsh': + // Use certificate issue date (not_before) for certificates + if (rawData.cert_not_before) { + return new Date(rawData.cert_not_before); + } + break; + + case 'dns': + case 'correlation': + default: + // Use discovery timestamp for DNS and correlation + if (edge.discovery_timestamp) { + return new Date(edge.discovery_timestamp); + } + break; + } + + // Fallback to discovery timestamp or current time + if (edge.discovery_timestamp) { + return new Date(edge.discovery_timestamp); + } + + return new Date(); // Default to now if no timestamp available + } + + /** + * Calculate time-based blue gradient color + */ + calculateTimeGradientColor(timestamp) { + if (!timestamp || !this.timeOfInterest) { + return this.gradientColors.dark; // Default to dark grey + } + + // Calculate time difference in milliseconds + const timeDiff = Math.abs(timestamp.getTime() - this.timeOfInterest.getTime()); + + // Find maximum time difference across all edges for normalization + let maxTimeDiff = 0; + this.edgeTimestamps.forEach((edgeTimestamp) => { + const diff = Math.abs(edgeTimestamp.getTime() - this.timeOfInterest.getTime()); + if (diff > maxTimeDiff) { + maxTimeDiff = diff; + } + }); + + if (maxTimeDiff === 0) { + return this.gradientColors.light; // All timestamps are the same + } + + // Calculate gradient position (0 = closest to time of interest, 1 = furthest) + const gradientPosition = timeDiff / maxTimeDiff; + + // Interpolate between light blue (close) and dark grey (far) + return this.interpolateColor( + this.gradientColors.light, // Close to time of interest + this.gradientColors.dark, // Far from time of interest + gradientPosition + ); + } + + /** + * Interpolate between two hex colors + */ + interpolateColor(color1, color2, factor) { + // Parse hex colors + const hex1 = color1.replace('#', ''); + const hex2 = color2.replace('#', ''); + + const r1 = parseInt(hex1.substring(0, 2), 16); + const g1 = parseInt(hex1.substring(2, 4), 16); + const b1 = parseInt(hex1.substring(4, 6), 16); + + const r2 = parseInt(hex2.substring(0, 2), 16); + const g2 = parseInt(hex2.substring(2, 4), 16); + const b2 = parseInt(hex2.substring(4, 6), 16); + + // Interpolate + const r = Math.round(r1 + (r2 - r1) * factor); + const g = Math.round(g1 + (g2 - g1) * factor); + const b = Math.round(b1 + (b2 - b1) * factor); + + // Convert back to hex + return `#${r.toString(16).padStart(2, '0')}${g.toString(16).padStart(2, '0')}${b.toString(16).padStart(2, '0')}`; + } + + /** + * Update all edge colors based on current time of interest + */ + updateEdgeColors() { + const edgeUpdates = []; + + this.edges.forEach((edge) => { + const timestamp = this.edgeTimestamps.get(edge.id); + const color = this.calculateTimeGradientColor(timestamp); + + edgeUpdates.push({ + id: edge.id, + color: { + color: color, + highlight: '#00ff41', + hover: '#ff9900' + } + }); + }); + + if (edgeUpdates.length > 0) { + this.edges.update(edgeUpdates); + console.log(`Updated ${edgeUpdates.length} edge colors based on time gradient`); + } + } /** * Set the manual refresh button click handler @@ -411,6 +622,7 @@ class GraphManager { if (!hasData) { this.nodes.clear(); this.edges.clear(); + this.edgeTimestamps.clear(); return; } @@ -464,6 +676,9 @@ class GraphManager { this.nodes.update(processedNodes); this.edges.update(processedEdges); + // Update edge timestamps and colors for time-based gradient + this.updateEdgeTimestampsAndColors(graphData.edges); + this.updateFilterControls(); this.applyAllFilters(); @@ -481,6 +696,21 @@ class GraphManager { } } + /** + * Update edge timestamps and apply time-based gradient colors + */ + updateEdgeTimestampsAndColors(edgeData) { + // Extract timestamps from raw edge data + edgeData.forEach(edge => { + const edgeId = `${edge.from}-${edge.to}-${edge.label}`; + const timestamp = this.extractEdgeTimestamp(edge); + this.edgeTimestamps.set(edgeId, timestamp); + }); + + // Update edge colors based on new timestamps + this.updateEdgeColors(); + } + analyzeCertificateInfo(attributes) { let hasCertificates = false; let hasValidCertificates = false; @@ -558,12 +788,6 @@ class GraphManager { if (node.max_depth_reached) { processedNode.borderColor = '#ff0000'; // Red border for max depth } - - - // Add confidence-based styling - if (node.confidence) { - processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); - } // FIXED: Certificate-based domain coloring if (node.type === 'domain' && Array.isArray(node.attributes)) { @@ -595,24 +819,33 @@ class GraphManager { } /** - * Process edge data with styling and metadata + * Process edge data with styling, metadata, and time-based gradient colors * @param {Object} edge - Raw edge data * @returns {Object} Processed edge data */ processEdge(edge) { - const confidence = edge.confidence_score || 0; + const edgeId = `${edge.from}-${edge.to}-${edge.label}`; + + // Extract timestamp for this edge + const timestamp = this.extractEdgeTimestamp(edge); + this.edgeTimestamps.set(edgeId, timestamp); + + // Calculate time-based gradient color + const timeGradientColor = this.calculateTimeGradientColor(timestamp); + const processedEdge = { - id: `${edge.from}-${edge.to}-${edge.label}`, + id: edgeId, from: edge.from, to: edge.to, - label: this.formatEdgeLabel(edge.label, confidence), + label: edge.label, // Correctly access the label directly title: this.createEdgeTooltip(edge), - width: this.getEdgeWidth(confidence), - color: this.getEdgeColor(confidence), - dashes: confidence < 0.6 ? [5, 5] : false, + color: { + color: timeGradientColor, + highlight: '#00ff41', + hover: '#ff9900' + }, metadata: { relationship_type: edge.label, - confidence_score: confidence, source_provider: edge.source_provider, discovery_timestamp: edge.discovery_timestamp } @@ -635,18 +868,7 @@ class GraphManager { return nodeId; } - /** - * Format edge label for display - * @param {string} relationshipType - Type of relationship - * @param {number} confidence - Confidence score - * @returns {string} Formatted label - */ - formatEdgeLabel(relationshipType, confidence) { - if (!relationshipType) return ''; - const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '●' : '○'; - return `${relationshipType} ${confidenceText}`; - } /** * Get node color based on type @@ -716,44 +938,13 @@ class GraphManager { } /** - * Get edge color based on confidence - * @param {number} confidence - Confidence score - * @returns {string} Edge color - */ - getEdgeColor(confidence) { - if (confidence >= 0.8) { - return '#00ff41'; // High confidence - green - } else if (confidence >= 0.6) { - return '#ff9900'; // Medium confidence - amber - } else { - return '#666666'; // Low confidence - gray - } - } - - /** - * Get edge width based on confidence - * @param {number} confidence - Confidence score - * @returns {number} Edge width - */ - getEdgeWidth(confidence) { - if (confidence >= 0.8) { - return 3; - } else if (confidence >= 0.6) { - return 2; - } else { - return 1; - } - } - - /** - * Create edge tooltip with correct provider information + * Create edge tooltip with correct provider information and timestamp * @param {Object} edge - Edge data * @returns {string} HTML tooltip content */ createEdgeTooltip(edge) { let tooltip = `
`; tooltip += `
${edge.label || 'Relationship'}
`; - tooltip += `
Confidence: ${(edge.confidence_score * 100).toFixed(1)}%
`; if (edge.source_provider) { tooltip += `
Provider: ${edge.source_provider}
`; @@ -764,6 +955,13 @@ class GraphManager { tooltip += `
Discovered: ${date.toLocaleString()}
`; } + // Add timestamp information for time-based coloring + const edgeId = `${edge.from}-${edge.to}-${edge.label}`; + const timestamp = this.edgeTimestamps.get(edgeId); + if (timestamp) { + tooltip += `
Data from: ${timestamp.toLocaleString()}
`; + } + tooltip += `
`; return tooltip; } @@ -893,13 +1091,17 @@ class GraphManager { }; }); - // Reset highlighted edges + // Reset highlighted edges to time-based colors const edgeUpdates = this.highlightedElements.edges.map(id => { - const originalEdge = this.edges.get(id); + const timestamp = this.edgeTimestamps.get(id); + const color = this.calculateTimeGradientColor(timestamp); return { id: id, - color: this.getEdgeColor(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5), - width: this.getEdgeWidth(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5) + color: { + color: color, + highlight: '#00ff41', + hover: '#ff9900' + } }; }); @@ -955,11 +1157,19 @@ class GraphManager { borderWidth: 2, })); - const edgeResets = newEdges.map(edge => ({ - id: edge.id, - color: this.getEdgeColor(edge.metadata ? edge.metadata.confidence_score : 0.5), - width: this.getEdgeWidth(edge.metadata ? edge.metadata.confidence_score : 0.5) - })); + // Reset edges to time-based colors + const edgeResets = newEdges.map(edge => { + const timestamp = this.edgeTimestamps.get(edge.id); + const color = this.calculateTimeGradientColor(timestamp); + return { + id: edge.id, + color: { + color: color, + highlight: '#00ff41', + hover: '#ff9900' + } + }; + }); this.nodes.update(nodeResets); this.edges.update(edgeResets); @@ -1048,6 +1258,7 @@ class GraphManager { clear() { this.nodes.clear(); this.edges.clear(); + this.edgeTimestamps.clear(); this.history = []; this.largeEntityMembers.clear(); this.initialTargetIds.clear(); diff --git a/static/js/main.js b/static/js/main.js index 5d3331c..6a12a87 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -1722,17 +1722,9 @@ class DNScopeApp { return groups; } - formatEdgeLabel(relationshipType, confidence) { - if (!relationshipType) return ''; - - const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○'; - return `${relationshipType} ${confidenceText}`; - } - createEdgeTooltip(edge) { let tooltip = `
`; tooltip += `
${edge.label || 'Relationship'}
`; - tooltip += `
Confidence: ${(edge.confidence_score * 100).toFixed(1)}%
`; // UPDATED: Use raw provider name (no formatting) if (edge.source_provider) { @@ -1872,7 +1864,7 @@ class DNScopeApp { html += `
${innerNodeId} - @@ -1899,8 +1891,6 @@ class DNScopeApp { `; node.incoming_edges.forEach(edge => { - const confidence = edge.data.confidence_score || 0; - const confidenceClass = confidence >= 0.8 ? 'high' : confidence >= 0.6 ? 'medium' : 'low'; html += `
@@ -1909,9 +1899,6 @@ class DNScopeApp {
${edge.data.relationship_type} - - ${'●'.repeat(Math.ceil(confidence * 3))} -
`; @@ -1930,9 +1917,6 @@ class DNScopeApp { `; node.outgoing_edges.forEach(edge => { - const confidence = edge.data.confidence_score || 0; - const confidenceClass = confidence >= 0.8 ? 'high' : confidence >= 0.6 ? 'medium' : 'low'; - html += `
@@ -1940,9 +1924,6 @@ class DNScopeApp {
${edge.data.relationship_type} - - ${'●'.repeat(Math.ceil(confidence * 3))} -
`; diff --git a/utils/export_manager.py b/utils/export_manager.py index c8f1fba..322d3ff 100644 --- a/utils/export_manager.py +++ b/utils/export_manager.py @@ -188,7 +188,6 @@ class ExportManager: f" - Type: {domain_info['classification']}", f" - Connected IPs: {len(domain_info['ips'])}", f" - Certificate Status: {domain_info['cert_status']}", - f" - Relationship Confidence: {domain_info['avg_confidence']:.2f}", ]) if domain_info['security_notes']: @@ -247,11 +246,9 @@ class ExportManager: ]) for rel in key_relationships[:8]: # Top 8 relationships - confidence_desc = self._describe_confidence(rel['confidence']) report.extend([ f"• {rel['source']} → {rel['target']}", f" - Relationship: {self._humanize_relationship_type(rel['type'])}", - f" - Evidence Strength: {confidence_desc} ({rel['confidence']:.2f})", f" - Discovery Method: {rel['provider']}", "" ]) @@ -291,17 +288,8 @@ class ExportManager: "Data Quality Assessment:", f"• Total API Requests: {audit_trail.get('session_metadata', {}).get('total_requests', 0)}", f"• Data Providers Used: {len(audit_trail.get('session_metadata', {}).get('providers_used', []))}", - f"• Relationship Confidence Distribution:", ]) - # Confidence distribution - confidence_dist = self._calculate_confidence_distribution(edges) - for level, count in confidence_dist.items(): - percentage = (count / len(edges) * 100) if edges else 0 - report.extend([ - f" - {level.title()} Confidence (≥{self._get_confidence_threshold(level)}): {count} ({percentage:.1f}%)", - ]) - report.extend([ "", "Correlation Analysis:", @@ -375,9 +363,7 @@ class ExportManager: if len(connected_ips) > 5: security_notes.append("Multiple IP endpoints") - # Average confidence domain_edges = [e for e in edges if e['from'] == domain['id']] - avg_confidence = sum(e['confidence_score'] for e in domain_edges) / len(domain_edges) if domain_edges else 0 domain_analysis.append({ 'domain': domain['id'], @@ -385,7 +371,6 @@ class ExportManager: 'ips': connected_ips, 'cert_status': cert_status, 'security_notes': security_notes, - 'avg_confidence': avg_confidence }) # Sort by number of connections (most connected first) @@ -480,7 +465,7 @@ class ExportManager: def _identify_key_relationships(self, edges: List[Dict]) -> List[Dict[str, Any]]: """Identify the most significant relationships in the infrastructure.""" - # Score relationships by confidence and type importance + # Score relationships by type importance relationship_importance = { 'dns_a_record': 0.9, 'dns_aaaa_record': 0.9, @@ -493,15 +478,12 @@ class ExportManager: scored_edges = [] for edge in edges: - base_confidence = edge.get('confidence_score', 0) type_weight = relationship_importance.get(edge.get('label', ''), 0.5) - combined_score = (base_confidence * 0.7) + (type_weight * 0.3) scored_edges.append({ 'source': edge['from'], 'target': edge['to'], 'type': edge.get('label', ''), - 'confidence': base_confidence, 'provider': edge.get('source_provider', ''), 'score': combined_score }) @@ -570,19 +552,6 @@ class ExportManager: else: return "Mixed Status" - def _describe_confidence(self, confidence: float) -> str: - """Convert confidence score to descriptive text.""" - if confidence >= 0.9: - return "Very High" - elif confidence >= 0.8: - return "High" - elif confidence >= 0.6: - return "Medium" - elif confidence >= 0.4: - return "Low" - else: - return "Very Low" - def _humanize_relationship_type(self, rel_type: str) -> str: """Convert technical relationship types to human-readable descriptions.""" type_map = { @@ -599,26 +568,6 @@ class ExportManager: } return type_map.get(rel_type, rel_type.replace('_', ' ').title()) - def _calculate_confidence_distribution(self, edges: List[Dict]) -> Dict[str, int]: - """Calculate confidence score distribution.""" - distribution = {'high': 0, 'medium': 0, 'low': 0} - - for edge in edges: - confidence = edge.get('confidence_score', 0) - if confidence >= 0.8: - distribution['high'] += 1 - elif confidence >= 0.6: - distribution['medium'] += 1 - else: - distribution['low'] += 1 - - return distribution - - def _get_confidence_threshold(self, level: str) -> str: - """Get confidence threshold for a level.""" - thresholds = {'high': '0.80', 'medium': '0.60', 'low': '0.00'} - return thresholds.get(level, '0.00') - def _count_cross_validated_relationships(self, edges: List[Dict]) -> int: """Count relationships verified by multiple providers.""" # Group edges by source-target pair