gradient-test #4
@ -22,7 +22,6 @@ For power users who require more in-depth information, DNScope can be configured
|
||||
* **In-Memory Graph Analysis**: Uses NetworkX for efficient relationship mapping.
|
||||
* **Real-Time Visualization**: The graph updates dynamically as the scan progresses.
|
||||
* **Forensic Logging**: A complete audit trail of all reconnaissance activities is maintained.
|
||||
* **Confidence Scoring**: Relationships are weighted based on the reliability of the data source.
|
||||
* **Session Management**: Supports concurrent user sessions with isolated scanner instances.
|
||||
* **Extensible Provider Architecture**: Easily add new data sources to expand the tool's capabilities.
|
||||
* **Web-Based UI**: An intuitive and interactive web interface for managing scans and visualizing results.
|
||||
|
||||
1
app.py
1
app.py
@ -332,7 +332,6 @@ def revert_graph_action():
|
||||
scanner.graph.add_edge(
|
||||
source_id=edge['from'], target_id=edge['to'],
|
||||
relationship_type=edge['metadata']['relationship_type'],
|
||||
confidence_score=edge['metadata']['confidence_score'],
|
||||
source_provider=edge['metadata']['source_provider'],
|
||||
raw_data=edge.get('raw_data', {})
|
||||
)
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
"""
|
||||
Graph data model for DNScope using NetworkX.
|
||||
Manages in-memory graph storage with confidence scoring and forensic metadata.
|
||||
Manages in-memory graph storage with forensic metadata.
|
||||
Now fully compatible with the unified ProviderResult data model.
|
||||
UPDATED: Fixed correlation exclusion keys to match actual attribute names.
|
||||
UPDATED: Removed export_json() method - now handled by ExportManager.
|
||||
@ -31,7 +31,7 @@ class NodeType(Enum):
|
||||
class GraphManager:
|
||||
"""
|
||||
Thread-safe graph manager for DNScope infrastructure mapping.
|
||||
Uses NetworkX for in-memory graph storage with confidence scoring.
|
||||
Uses NetworkX for in-memory graph storage.
|
||||
Compatible with unified ProviderResult data model.
|
||||
"""
|
||||
|
||||
@ -83,7 +83,7 @@ class GraphManager:
|
||||
return is_new_node
|
||||
|
||||
def add_edge(self, source_id: str, target_id: str, relationship_type: str,
|
||||
confidence_score: float = 0.5, source_provider: str = "unknown",
|
||||
source_provider: str = "unknown",
|
||||
raw_data: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""
|
||||
UPDATED: Add or update an edge between two nodes with raw relationship labels.
|
||||
@ -91,23 +91,13 @@ class GraphManager:
|
||||
if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
|
||||
return False
|
||||
|
||||
new_confidence = confidence_score
|
||||
|
||||
# UPDATED: Use raw relationship type - no formatting
|
||||
edge_label = relationship_type
|
||||
|
||||
if self.graph.has_edge(source_id, target_id):
|
||||
# If edge exists, update confidence if the new score is higher.
|
||||
if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0):
|
||||
self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence
|
||||
self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat()
|
||||
self.graph.edges[source_id, target_id]['updated_by'] = source_provider
|
||||
return False
|
||||
|
||||
# Add a new edge with raw attributes
|
||||
self.graph.add_edge(source_id, target_id,
|
||||
relationship_type=edge_label,
|
||||
confidence_score=new_confidence,
|
||||
source_provider=source_provider,
|
||||
discovery_timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
raw_data=raw_data or {})
|
||||
@ -137,11 +127,6 @@ class GraphManager:
|
||||
"""Get all nodes of a specific type."""
|
||||
return [n for n, d in self.graph.nodes(data=True) if d.get('type') == node_type.value]
|
||||
|
||||
def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]:
|
||||
"""Get edges with confidence score above a given threshold."""
|
||||
return [(u, v, d) for u, v, d in self.graph.edges(data=True)
|
||||
if d.get('confidence_score', 0) >= min_confidence]
|
||||
|
||||
def get_graph_data(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Export graph data formatted for frontend visualization.
|
||||
@ -177,7 +162,6 @@ class GraphManager:
|
||||
'from': source,
|
||||
'to': target,
|
||||
'label': attrs.get('relationship_type', ''),
|
||||
'confidence_score': attrs.get('confidence_score', 0),
|
||||
'source_provider': attrs.get('source_provider', ''),
|
||||
'discovery_timestamp': attrs.get('discovery_timestamp')
|
||||
})
|
||||
@ -188,24 +172,6 @@ class GraphManager:
|
||||
'statistics': self.get_statistics()['basic_metrics']
|
||||
}
|
||||
|
||||
def _get_confidence_distribution(self) -> Dict[str, int]:
|
||||
"""Get distribution of edge confidence scores with empty graph handling."""
|
||||
distribution = {'high': 0, 'medium': 0, 'low': 0}
|
||||
|
||||
# FIXED: Handle empty graph case
|
||||
if self.get_edge_count() == 0:
|
||||
return distribution
|
||||
|
||||
for _, _, data in self.graph.edges(data=True):
|
||||
confidence = data.get('confidence_score', 0)
|
||||
if confidence >= 0.8:
|
||||
distribution['high'] += 1
|
||||
elif confidence >= 0.6:
|
||||
distribution['medium'] += 1
|
||||
else:
|
||||
distribution['low'] += 1
|
||||
return distribution
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive statistics about the graph with proper empty graph handling."""
|
||||
|
||||
@ -222,7 +188,6 @@ class GraphManager:
|
||||
},
|
||||
'node_type_distribution': {},
|
||||
'relationship_type_distribution': {},
|
||||
'confidence_distribution': self._get_confidence_distribution(),
|
||||
'provider_distribution': {}
|
||||
}
|
||||
|
||||
|
||||
@ -30,7 +30,6 @@ class RelationshipDiscovery:
|
||||
source_node: str
|
||||
target_node: str
|
||||
relationship_type: str
|
||||
confidence_score: float
|
||||
provider: str
|
||||
raw_data: Dict[str, Any]
|
||||
discovery_method: str
|
||||
@ -157,7 +156,7 @@ class ForensicLogger:
|
||||
self.logger.info(f"API Request - {provider}: {url} - Status: {status_code}")
|
||||
|
||||
def log_relationship_discovery(self, source_node: str, target_node: str,
|
||||
relationship_type: str, confidence_score: float,
|
||||
relationship_type: str,
|
||||
provider: str, raw_data: Dict[str, Any],
|
||||
discovery_method: str) -> None:
|
||||
"""
|
||||
@ -167,7 +166,6 @@ class ForensicLogger:
|
||||
source_node: Source node identifier
|
||||
target_node: Target node identifier
|
||||
relationship_type: Type of relationship (e.g., 'SAN', 'A_Record')
|
||||
confidence_score: Confidence score (0.0 to 1.0)
|
||||
provider: Provider that discovered this relationship
|
||||
raw_data: Raw data from provider response
|
||||
discovery_method: Method used to discover relationship
|
||||
@ -177,7 +175,6 @@ class ForensicLogger:
|
||||
source_node=source_node,
|
||||
target_node=target_node,
|
||||
relationship_type=relationship_type,
|
||||
confidence_score=confidence_score,
|
||||
provider=provider,
|
||||
raw_data=raw_data,
|
||||
discovery_method=discovery_method
|
||||
@ -188,7 +185,7 @@ class ForensicLogger:
|
||||
|
||||
self.logger.info(
|
||||
f"Relationship Discovered - {source_node} -> {target_node} "
|
||||
f"({relationship_type}) - Confidence: {confidence_score:.2f} - Provider: {provider}"
|
||||
f"({relationship_type}) - Provider: {provider}"
|
||||
)
|
||||
|
||||
def log_scan_start(self, target_domain: str, recursion_depth: int,
|
||||
@ -238,7 +235,6 @@ class ForensicLogger:
|
||||
'successful_requests': len([req for req in provider_requests if req.error is None]),
|
||||
'failed_requests': len([req for req in provider_requests if req.error is not None]),
|
||||
'relationships_discovered': len(provider_relationships),
|
||||
'avg_confidence': sum(rel.confidence_score for rel in provider_relationships) / len(provider_relationships) if provider_relationships else 0
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@ -18,33 +18,19 @@ class StandardAttribute:
|
||||
value: Any
|
||||
type: str
|
||||
provider: str
|
||||
confidence: float
|
||||
timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
metadata: Optional[Dict[str, Any]] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate the attribute after initialization."""
|
||||
if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0:
|
||||
raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Relationship:
|
||||
"""A unified data structure for a directional link between two nodes."""
|
||||
source_node: str
|
||||
target_node: str
|
||||
relationship_type: str
|
||||
confidence: float
|
||||
provider: str
|
||||
timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
raw_data: Optional[Dict[str, Any]] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate the relationship after initialization."""
|
||||
if not isinstance(self.confidence, (int, float)) or not 0.0 <= self.confidence <= 1.0:
|
||||
raise ValueError(f"Confidence must be between 0.0 and 1.0, got {self.confidence}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProviderResult:
|
||||
"""A container for all data returned by a provider from a single query."""
|
||||
@ -52,8 +38,7 @@ class ProviderResult:
|
||||
relationships: List[Relationship] = field(default_factory=list)
|
||||
|
||||
def add_attribute(self, target_node: str, name: str, value: Any, attr_type: str,
|
||||
provider: str, confidence: float = 0.8,
|
||||
metadata: Optional[Dict[str, Any]] = None) -> None:
|
||||
provider: str, metadata: Optional[Dict[str, Any]] = None) -> None:
|
||||
"""Helper method to add an attribute to the result."""
|
||||
self.attributes.append(StandardAttribute(
|
||||
target_node=target_node,
|
||||
@ -61,19 +46,16 @@ class ProviderResult:
|
||||
value=value,
|
||||
type=attr_type,
|
||||
provider=provider,
|
||||
confidence=confidence,
|
||||
metadata=metadata or {}
|
||||
))
|
||||
|
||||
def add_relationship(self, source_node: str, target_node: str, relationship_type: str,
|
||||
provider: str, confidence: float = 0.8,
|
||||
raw_data: Optional[Dict[str, Any]] = None) -> None:
|
||||
provider: str, raw_data: Optional[Dict[str, Any]] = None) -> None:
|
||||
"""Helper method to add a relationship to the result."""
|
||||
self.relationships.append(Relationship(
|
||||
source_node=source_node,
|
||||
target_node=target_node,
|
||||
relationship_type=relationship_type,
|
||||
confidence=confidence,
|
||||
provider=provider,
|
||||
raw_data=raw_data or {}
|
||||
))
|
||||
|
||||
@ -847,7 +847,6 @@ class Scanner:
|
||||
'source_node': rel.source_node,
|
||||
'target_node': rel.target_node,
|
||||
'relationship_type': rel.relationship_type,
|
||||
'confidence': rel.confidence,
|
||||
'provider': rel.provider,
|
||||
'raw_data': rel.raw_data
|
||||
})
|
||||
@ -905,7 +904,6 @@ class Scanner:
|
||||
source_id=rel_data['source_node'],
|
||||
target_id=rel_data['target_node'],
|
||||
relationship_type=rel_data['relationship_type'],
|
||||
confidence_score=rel_data['confidence'],
|
||||
source_provider=rel_data['provider'],
|
||||
raw_data=rel_data['raw_data']
|
||||
)
|
||||
@ -1012,7 +1010,6 @@ class Scanner:
|
||||
self.graph.add_edge(
|
||||
visual_source, visual_target,
|
||||
relationship.relationship_type,
|
||||
relationship.confidence,
|
||||
provider_name,
|
||||
relationship.raw_data
|
||||
)
|
||||
@ -1035,7 +1032,7 @@ class Scanner:
|
||||
for attribute in provider_result.attributes:
|
||||
attr_dict = {
|
||||
"name": attribute.name, "value": attribute.value, "type": attribute.type,
|
||||
"provider": attribute.provider, "confidence": attribute.confidence, "metadata": attribute.metadata
|
||||
"provider": attribute.provider, "metadata": attribute.metadata
|
||||
}
|
||||
attributes_by_node[attribute.target_node].append(attr_dict)
|
||||
|
||||
|
||||
@ -229,7 +229,6 @@ class BaseProvider(ABC):
|
||||
|
||||
def log_relationship_discovery(self, source_node: str, target_node: str,
|
||||
relationship_type: str,
|
||||
confidence_score: float,
|
||||
raw_data: Dict[str, Any],
|
||||
discovery_method: str) -> None:
|
||||
"""
|
||||
@ -239,7 +238,6 @@ class BaseProvider(ABC):
|
||||
source_node: Source node identifier
|
||||
target_node: Target node identifier
|
||||
relationship_type: Type of relationship
|
||||
confidence_score: Confidence score
|
||||
raw_data: Raw data from provider
|
||||
discovery_method: Method used for discovery
|
||||
"""
|
||||
@ -249,7 +247,6 @@ class BaseProvider(ABC):
|
||||
source_node=source_node,
|
||||
target_node=target_node,
|
||||
relationship_type=relationship_type,
|
||||
confidence_score=confidence_score,
|
||||
provider=self.name,
|
||||
raw_data=raw_data,
|
||||
discovery_method=discovery_method
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
import re
|
||||
from typing import Dict, Any, List
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .base_provider import BaseProvider
|
||||
from core.provider_result import ProviderResult
|
||||
@ -10,6 +11,7 @@ from core.graph_manager import NodeType, GraphManager
|
||||
class CorrelationProvider(BaseProvider):
|
||||
"""
|
||||
A provider that finds correlations between nodes in the graph.
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str = "correlation", session_config=None):
|
||||
@ -61,12 +63,14 @@ class CorrelationProvider(BaseProvider):
|
||||
def query_domain(self, domain: str) -> ProviderResult:
|
||||
"""
|
||||
Query the provider for information about a domain.
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
"""
|
||||
return self._find_correlations(domain)
|
||||
|
||||
def query_ip(self, ip: str) -> ProviderResult:
|
||||
"""
|
||||
Query the provider for information about an IP address.
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
"""
|
||||
return self._find_correlations(ip)
|
||||
|
||||
@ -79,8 +83,10 @@ class CorrelationProvider(BaseProvider):
|
||||
def _find_correlations(self, node_id: str) -> ProviderResult:
|
||||
"""
|
||||
Find correlations for a given node with enhanced filtering and error handling.
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
"""
|
||||
result = ProviderResult()
|
||||
discovery_time = datetime.now(timezone.utc)
|
||||
|
||||
# Enhanced safety checks
|
||||
if not self.graph or not self.graph.graph.has_node(node_id):
|
||||
@ -133,7 +139,7 @@ class CorrelationProvider(BaseProvider):
|
||||
|
||||
# Create correlation if we have multiple nodes with this value
|
||||
if len(self.correlation_index[attr_value]['nodes']) > 1:
|
||||
self._create_correlation_relationships(attr_value, self.correlation_index[attr_value], result)
|
||||
self._create_correlation_relationships(attr_value, self.correlation_index[attr_value], result, discovery_time)
|
||||
correlations_found += 1
|
||||
|
||||
# Log correlation results
|
||||
@ -187,9 +193,11 @@ class CorrelationProvider(BaseProvider):
|
||||
|
||||
return False
|
||||
|
||||
def _create_correlation_relationships(self, value: Any, correlation_data: Dict[str, Any], result: ProviderResult):
|
||||
def _create_correlation_relationships(self, value: Any, correlation_data: Dict[str, Any],
|
||||
result: ProviderResult, discovery_time: datetime):
|
||||
"""
|
||||
Create correlation relationships with enhanced deduplication and validation.
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
"""
|
||||
correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}"
|
||||
nodes = correlation_data['nodes']
|
||||
@ -216,7 +224,6 @@ class CorrelationProvider(BaseProvider):
|
||||
value=value,
|
||||
attr_type=str(type(value).__name__),
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
metadata={
|
||||
'correlated_nodes': list(nodes),
|
||||
'sources': sources,
|
||||
@ -225,7 +232,7 @@ class CorrelationProvider(BaseProvider):
|
||||
}
|
||||
)
|
||||
|
||||
# Create relationships with source validation
|
||||
# Create relationships with source validation and enhanced timestamps
|
||||
created_relationships = set()
|
||||
|
||||
for source in sources:
|
||||
@ -240,19 +247,23 @@ class CorrelationProvider(BaseProvider):
|
||||
|
||||
relationship_label = f"corr_{provider}_{attribute}"
|
||||
|
||||
# Enhanced raw_data with discovery timestamp for time-based edge coloring
|
||||
raw_data = {
|
||||
'correlation_value': value,
|
||||
'original_attribute': attribute,
|
||||
'correlation_type': 'attribute_matching',
|
||||
'correlation_size': len(nodes),
|
||||
'discovery_timestamp': discovery_time.isoformat(),
|
||||
'relevance_timestamp': discovery_time.isoformat() # Correlation data is "fresh" when discovered
|
||||
}
|
||||
|
||||
# Add the relationship to the result
|
||||
result.add_relationship(
|
||||
source_node=node_id,
|
||||
target_node=correlation_node_id,
|
||||
relationship_type=relationship_label,
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
raw_data={
|
||||
'correlation_value': value,
|
||||
'original_attribute': attribute,
|
||||
'correlation_type': 'attribute_matching',
|
||||
'correlation_size': len(nodes)
|
||||
}
|
||||
raw_data=raw_data
|
||||
)
|
||||
|
||||
created_relationships.add(relationship_key)
|
||||
@ -18,6 +18,7 @@ class CrtShProvider(BaseProvider):
|
||||
Provider for querying crt.sh certificate transparency database.
|
||||
FIXED: Improved caching logic and error handling to prevent infinite retry loops.
|
||||
Returns standardized ProviderResult objects with caching support.
|
||||
UPDATED: Enhanced with certificate timestamps for time-based edge coloring.
|
||||
"""
|
||||
|
||||
def __init__(self, name=None, session_config=None):
|
||||
@ -131,6 +132,7 @@ class CrtShProvider(BaseProvider):
|
||||
def query_domain(self, domain: str) -> ProviderResult:
|
||||
"""
|
||||
FIXED: Simplified and more robust domain querying with better error handling.
|
||||
UPDATED: Enhanced with certificate timestamps for time-based edge coloring.
|
||||
"""
|
||||
if not _is_valid_domain(domain):
|
||||
return ProviderResult()
|
||||
@ -245,7 +247,6 @@ class CrtShProvider(BaseProvider):
|
||||
target_node=rel_data.get("target_node", ""),
|
||||
relationship_type=rel_data.get("relationship_type", ""),
|
||||
provider=rel_data.get("provider", self.name),
|
||||
confidence=float(rel_data.get("confidence", 0.8)),
|
||||
raw_data=rel_data.get("raw_data", {})
|
||||
)
|
||||
except (ValueError, TypeError) as e:
|
||||
@ -265,7 +266,6 @@ class CrtShProvider(BaseProvider):
|
||||
value=attr_data.get("value"),
|
||||
attr_type=attr_data.get("type", "unknown"),
|
||||
provider=attr_data.get("provider", self.name),
|
||||
confidence=float(attr_data.get("confidence", 0.9)),
|
||||
metadata=attr_data.get("metadata", {})
|
||||
)
|
||||
except (ValueError, TypeError) as e:
|
||||
@ -293,7 +293,6 @@ class CrtShProvider(BaseProvider):
|
||||
"source_node": rel.source_node,
|
||||
"target_node": rel.target_node,
|
||||
"relationship_type": rel.relationship_type,
|
||||
"confidence": rel.confidence,
|
||||
"provider": rel.provider,
|
||||
"raw_data": rel.raw_data
|
||||
} for rel in result.relationships
|
||||
@ -305,7 +304,6 @@ class CrtShProvider(BaseProvider):
|
||||
"value": attr.value,
|
||||
"type": attr.type,
|
||||
"provider": attr.provider,
|
||||
"confidence": attr.confidence,
|
||||
"metadata": attr.metadata
|
||||
} for attr in result.attributes
|
||||
]
|
||||
@ -372,6 +370,7 @@ class CrtShProvider(BaseProvider):
|
||||
"""
|
||||
Process certificates to create proper domain and CA nodes.
|
||||
FIXED: Better error handling and progress tracking.
|
||||
UPDATED: Enhanced with certificate timestamps for time-based edge coloring.
|
||||
"""
|
||||
result = ProviderResult()
|
||||
|
||||
@ -391,8 +390,7 @@ class CrtShProvider(BaseProvider):
|
||||
name="crtsh_data_warning",
|
||||
value=incompleteness_warning,
|
||||
attr_type='metadata',
|
||||
provider=self.name,
|
||||
confidence=1.0
|
||||
provider=self.name
|
||||
)
|
||||
|
||||
all_discovered_domains = set()
|
||||
@ -415,16 +413,28 @@ class CrtShProvider(BaseProvider):
|
||||
if cert_domains:
|
||||
all_discovered_domains.update(cert_domains)
|
||||
|
||||
# Create CA nodes for certificate issuers
|
||||
# Create CA nodes for certificate issuers with timestamp
|
||||
issuer_name = self._parse_issuer_organization(cert_data.get('issuer_name', ''))
|
||||
if issuer_name and issuer_name not in processed_issuers:
|
||||
# Enhanced raw_data with certificate timestamp for time-based edge coloring
|
||||
issuer_raw_data = {'issuer_dn': cert_data.get('issuer_name', '')}
|
||||
|
||||
# Add certificate issue date (not_before) as relevance timestamp
|
||||
not_before = cert_data.get('not_before')
|
||||
if not_before:
|
||||
try:
|
||||
not_before_date = self._parse_certificate_date(not_before)
|
||||
issuer_raw_data['cert_not_before'] = not_before_date.isoformat()
|
||||
issuer_raw_data['relevance_timestamp'] = not_before_date.isoformat() # Standardized field
|
||||
except Exception as e:
|
||||
self.logger.logger.debug(f"Failed to parse not_before date for issuer: {e}")
|
||||
|
||||
result.add_relationship(
|
||||
source_node=query_domain,
|
||||
target_node=issuer_name,
|
||||
relationship_type='crtsh_cert_issuer',
|
||||
provider=self.name,
|
||||
confidence=0.95,
|
||||
raw_data={'issuer_dn': cert_data.get('issuer_name', '')}
|
||||
raw_data=issuer_raw_data
|
||||
)
|
||||
processed_issuers.add(issuer_name)
|
||||
|
||||
@ -442,7 +452,6 @@ class CrtShProvider(BaseProvider):
|
||||
value=value,
|
||||
attr_type='certificate_data',
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
metadata={'certificate_id': cert_data.get('id')}
|
||||
)
|
||||
|
||||
@ -457,7 +466,7 @@ class CrtShProvider(BaseProvider):
|
||||
self.logger.logger.info(f"CrtSh query cancelled before relationship creation for domain: {query_domain}")
|
||||
return result
|
||||
|
||||
# Create selective relationships to avoid large entities
|
||||
# Create selective relationships to avoid large entities with enhanced timestamps
|
||||
relationships_created = 0
|
||||
for discovered_domain in all_discovered_domains:
|
||||
if discovered_domain == query_domain:
|
||||
@ -467,25 +476,36 @@ class CrtShProvider(BaseProvider):
|
||||
continue
|
||||
|
||||
if self._should_create_relationship(query_domain, discovered_domain):
|
||||
confidence = self._calculate_domain_relationship_confidence(
|
||||
query_domain, discovered_domain, [], all_discovered_domains
|
||||
# Enhanced raw_data with certificate timestamp for domain relationships
|
||||
domain_raw_data = {'relationship_type': 'certificate_discovery'}
|
||||
|
||||
# Find the most recent certificate for this domain pair to use as timestamp
|
||||
most_recent_cert = self._find_most_recent_cert_for_domains(
|
||||
certificates, query_domain, discovered_domain
|
||||
)
|
||||
if most_recent_cert:
|
||||
not_before = most_recent_cert.get('not_before')
|
||||
if not_before:
|
||||
try:
|
||||
not_before_date = self._parse_certificate_date(not_before)
|
||||
domain_raw_data['cert_not_before'] = not_before_date.isoformat()
|
||||
domain_raw_data['relevance_timestamp'] = not_before_date.isoformat()
|
||||
except Exception as e:
|
||||
self.logger.logger.debug(f"Failed to parse not_before date for domain relationship: {e}")
|
||||
|
||||
result.add_relationship(
|
||||
source_node=query_domain,
|
||||
target_node=discovered_domain,
|
||||
relationship_type='crtsh_san_certificate',
|
||||
provider=self.name,
|
||||
confidence=confidence,
|
||||
raw_data={'relationship_type': 'certificate_discovery'}
|
||||
raw_data=domain_raw_data
|
||||
)
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=query_domain,
|
||||
target_node=discovered_domain,
|
||||
relationship_type='crtsh_san_certificate',
|
||||
confidence_score=confidence,
|
||||
raw_data={'relationship_type': 'certificate_discovery'},
|
||||
raw_data=domain_raw_data,
|
||||
discovery_method="certificate_transparency_analysis"
|
||||
)
|
||||
relationships_created += 1
|
||||
@ -493,6 +513,31 @@ class CrtShProvider(BaseProvider):
|
||||
self.logger.logger.info(f"CrtSh processing completed for {query_domain}: processed {processed_certs}/{len(certificates)} certificates, {len(all_discovered_domains)} domains, {relationships_created} relationships")
|
||||
return result
|
||||
|
||||
def _find_most_recent_cert_for_domains(self, certificates: List[Dict[str, Any]],
|
||||
domain1: str, domain2: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Find the most recent certificate that contains both domains.
|
||||
Used for determining the relevance timestamp for domain relationships.
|
||||
"""
|
||||
most_recent_cert = None
|
||||
most_recent_date = None
|
||||
|
||||
for cert in certificates:
|
||||
# Check if this certificate contains both domains
|
||||
cert_domains = self._extract_domains_from_certificate(cert)
|
||||
if domain1 in cert_domains and domain2 in cert_domains:
|
||||
not_before = cert.get('not_before')
|
||||
if not_before:
|
||||
try:
|
||||
cert_date = self._parse_certificate_date(not_before)
|
||||
if most_recent_date is None or cert_date > most_recent_date:
|
||||
most_recent_date = cert_date
|
||||
most_recent_cert = cert
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return most_recent_cert
|
||||
|
||||
# [Rest of the methods remain the same as in the original file]
|
||||
def _should_create_relationship(self, source_domain: str, target_domain: str) -> bool:
|
||||
"""
|
||||
@ -664,25 +709,6 @@ class CrtShProvider(BaseProvider):
|
||||
|
||||
return [d for d in final_domains if _is_valid_domain(d)]
|
||||
|
||||
def _calculate_domain_relationship_confidence(self, domain1: str, domain2: str,
|
||||
shared_certificates: List[Dict[str, Any]],
|
||||
all_discovered_domains: Set[str]) -> float:
|
||||
"""Calculate confidence score for domain relationship based on various factors."""
|
||||
base_confidence = 0.9
|
||||
|
||||
relationship_context = self._determine_relationship_context(domain2, domain1)
|
||||
|
||||
if relationship_context == 'exact_match':
|
||||
context_bonus = 0.0
|
||||
elif relationship_context == 'subdomain':
|
||||
context_bonus = 0.1
|
||||
elif relationship_context == 'parent_domain':
|
||||
context_bonus = 0.05
|
||||
else:
|
||||
context_bonus = 0.0
|
||||
|
||||
final_confidence = base_confidence + context_bonus
|
||||
return max(0.1, min(1.0, final_confidence))
|
||||
|
||||
def _determine_relationship_context(self, cert_domain: str, query_domain: str) -> str:
|
||||
"""Determine the context of the relationship between certificate domain and query domain."""
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
from dns import resolver, reversename
|
||||
from typing import Dict
|
||||
from datetime import datetime, timezone
|
||||
from .base_provider import BaseProvider
|
||||
from core.provider_result import ProviderResult
|
||||
from utils.helpers import _is_valid_ip, _is_valid_domain, get_ip_version
|
||||
@ -11,6 +12,7 @@ class DNSProvider(BaseProvider):
|
||||
"""
|
||||
Provider for standard DNS resolution and reverse DNS lookups.
|
||||
Now returns standardized ProviderResult objects with IPv4 and IPv6 support.
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
"""
|
||||
|
||||
def __init__(self, name=None, session_config=None):
|
||||
@ -51,6 +53,7 @@ class DNSProvider(BaseProvider):
|
||||
"""
|
||||
Query DNS records for the domain to discover relationships and attributes.
|
||||
FIXED: Now creates separate attributes for each DNS record type.
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
|
||||
Args:
|
||||
domain: Domain to investigate
|
||||
@ -62,11 +65,12 @@ class DNSProvider(BaseProvider):
|
||||
return ProviderResult()
|
||||
|
||||
result = ProviderResult()
|
||||
discovery_time = datetime.now(timezone.utc)
|
||||
|
||||
# Query all record types - each gets its own attribute
|
||||
for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']:
|
||||
try:
|
||||
self._query_record(domain, record_type, result)
|
||||
self._query_record(domain, record_type, result, discovery_time)
|
||||
#except resolver.NoAnswer:
|
||||
# This is not an error, just a confirmation that the record doesn't exist.
|
||||
#self.logger.logger.debug(f"No {record_type} record found for {domain}")
|
||||
@ -79,6 +83,7 @@ class DNSProvider(BaseProvider):
|
||||
def query_ip(self, ip: str) -> ProviderResult:
|
||||
"""
|
||||
Query reverse DNS for the IP address (supports both IPv4 and IPv6).
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
|
||||
Args:
|
||||
ip: IP address to investigate (IPv4 or IPv6)
|
||||
@ -91,6 +96,7 @@ class DNSProvider(BaseProvider):
|
||||
|
||||
result = ProviderResult()
|
||||
ip_version = get_ip_version(ip)
|
||||
discovery_time = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
# Perform reverse DNS lookup (works for both IPv4 and IPv6)
|
||||
@ -112,20 +118,24 @@ class DNSProvider(BaseProvider):
|
||||
relationship_type = 'dns_a_record'
|
||||
record_prefix = 'A'
|
||||
|
||||
# Enhanced raw_data with discovery timestamp for time-based edge coloring
|
||||
raw_data = {
|
||||
'query_type': 'PTR',
|
||||
'ip_address': ip,
|
||||
'ip_version': ip_version,
|
||||
'hostname': hostname,
|
||||
'ttl': response.ttl,
|
||||
'discovery_timestamp': discovery_time.isoformat(),
|
||||
'relevance_timestamp': discovery_time.isoformat() # DNS data is "fresh" when discovered
|
||||
}
|
||||
|
||||
# Add the relationship
|
||||
result.add_relationship(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type='dns_ptr_record',
|
||||
provider=self.name,
|
||||
confidence=0.8,
|
||||
raw_data={
|
||||
'query_type': 'PTR',
|
||||
'ip_address': ip,
|
||||
'ip_version': ip_version,
|
||||
'hostname': hostname,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
raw_data=raw_data
|
||||
)
|
||||
|
||||
# Add to PTR records list
|
||||
@ -136,14 +146,7 @@ class DNSProvider(BaseProvider):
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type='dns_ptr_record',
|
||||
confidence_score=0.8,
|
||||
raw_data={
|
||||
'query_type': 'PTR',
|
||||
'ip_address': ip,
|
||||
'ip_version': ip_version,
|
||||
'hostname': hostname,
|
||||
'ttl': response.ttl
|
||||
},
|
||||
raw_data=raw_data,
|
||||
discovery_method=f"reverse_dns_lookup_ipv{ip_version}"
|
||||
)
|
||||
|
||||
@ -155,7 +158,6 @@ class DNSProvider(BaseProvider):
|
||||
value=ptr_records,
|
||||
attr_type='dns_record',
|
||||
provider=self.name,
|
||||
confidence=0.8,
|
||||
metadata={'ttl': response.ttl, 'ip_version': ip_version}
|
||||
)
|
||||
|
||||
@ -170,10 +172,11 @@ class DNSProvider(BaseProvider):
|
||||
|
||||
return result
|
||||
|
||||
def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None:
|
||||
def _query_record(self, domain: str, record_type: str, result: ProviderResult, discovery_time: datetime) -> None:
|
||||
"""
|
||||
FIXED: Query DNS records with unique attribute names for each record type.
|
||||
Enhanced to better handle IPv6 AAAA records.
|
||||
UPDATED: Enhanced with discovery timestamps for time-based edge coloring.
|
||||
"""
|
||||
try:
|
||||
self.total_requests += 1
|
||||
@ -217,18 +220,20 @@ class DNSProvider(BaseProvider):
|
||||
if record_type in ['A', 'AAAA'] and _is_valid_ip(target):
|
||||
ip_version = get_ip_version(target)
|
||||
|
||||
# Enhanced raw_data with discovery timestamp for time-based edge coloring
|
||||
raw_data = {
|
||||
'query_type': record_type,
|
||||
'domain': domain,
|
||||
'value': target,
|
||||
'ttl': response.ttl
|
||||
'ttl': response.ttl,
|
||||
'discovery_timestamp': discovery_time.isoformat(),
|
||||
'relevance_timestamp': discovery_time.isoformat() # DNS data is "fresh" when discovered
|
||||
}
|
||||
|
||||
if ip_version:
|
||||
raw_data['ip_version'] = ip_version
|
||||
|
||||
relationship_type = f"dns_{record_type.lower()}_record"
|
||||
confidence = 0.8
|
||||
|
||||
# Add relationship
|
||||
result.add_relationship(
|
||||
@ -236,7 +241,6 @@ class DNSProvider(BaseProvider):
|
||||
target_node=target,
|
||||
relationship_type=relationship_type,
|
||||
provider=self.name,
|
||||
confidence=confidence,
|
||||
raw_data=raw_data
|
||||
)
|
||||
|
||||
@ -252,7 +256,6 @@ class DNSProvider(BaseProvider):
|
||||
source_node=domain,
|
||||
target_node=target,
|
||||
relationship_type=relationship_type,
|
||||
confidence_score=confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method=discovery_method
|
||||
)
|
||||
@ -276,7 +279,6 @@ class DNSProvider(BaseProvider):
|
||||
value=dns_records,
|
||||
attr_type='dns_record_list',
|
||||
provider=self.name,
|
||||
confidence=0.8,
|
||||
metadata=metadata
|
||||
)
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@ class ShodanProvider(BaseProvider):
|
||||
"""
|
||||
Provider for querying Shodan API for IP address information.
|
||||
Now returns standardized ProviderResult objects with caching support for IPv4 and IPv6.
|
||||
UPDATED: Enhanced with last_seen timestamp for time-based edge coloring.
|
||||
"""
|
||||
|
||||
def __init__(self, name=None, session_config=None):
|
||||
@ -145,6 +146,7 @@ class ShodanProvider(BaseProvider):
|
||||
"""
|
||||
Query Shodan for information about an IP address (IPv4 or IPv6), with caching of processed data.
|
||||
FIXED: Proper 404 handling to prevent unnecessary retries.
|
||||
UPDATED: Enhanced with last_seen timestamp extraction for time-based edge coloring.
|
||||
|
||||
Args:
|
||||
ip: IP address to investigate (IPv4 or IPv6)
|
||||
@ -304,7 +306,6 @@ class ShodanProvider(BaseProvider):
|
||||
target_node=rel_data["target_node"],
|
||||
relationship_type=rel_data["relationship_type"],
|
||||
provider=rel_data["provider"],
|
||||
confidence=rel_data["confidence"],
|
||||
raw_data=rel_data.get("raw_data", {})
|
||||
)
|
||||
|
||||
@ -316,7 +317,6 @@ class ShodanProvider(BaseProvider):
|
||||
value=attr_data["value"],
|
||||
attr_type=attr_data["type"],
|
||||
provider=attr_data["provider"],
|
||||
confidence=attr_data["confidence"],
|
||||
metadata=attr_data.get("metadata", {})
|
||||
)
|
||||
|
||||
@ -336,7 +336,6 @@ class ShodanProvider(BaseProvider):
|
||||
"source_node": rel.source_node,
|
||||
"target_node": rel.target_node,
|
||||
"relationship_type": rel.relationship_type,
|
||||
"confidence": rel.confidence,
|
||||
"provider": rel.provider,
|
||||
"raw_data": rel.raw_data
|
||||
} for rel in result.relationships
|
||||
@ -348,7 +347,6 @@ class ShodanProvider(BaseProvider):
|
||||
"value": attr.value,
|
||||
"type": attr.type,
|
||||
"provider": attr.provider,
|
||||
"confidence": attr.confidence,
|
||||
"metadata": attr.metadata
|
||||
} for attr in result.attributes
|
||||
]
|
||||
@ -362,25 +360,40 @@ class ShodanProvider(BaseProvider):
|
||||
"""
|
||||
VERIFIED: Process Shodan data creating ISP nodes with ASN attributes and proper relationships.
|
||||
Enhanced to include IP version information for IPv6 addresses.
|
||||
UPDATED: Enhanced with last_seen timestamp for time-based edge coloring.
|
||||
"""
|
||||
result = ProviderResult()
|
||||
|
||||
# Determine IP version for metadata
|
||||
ip_version = get_ip_version(ip)
|
||||
|
||||
# Extract last_seen timestamp for time-based edge coloring
|
||||
last_seen = data.get('last_seen')
|
||||
|
||||
# VERIFIED: Extract ISP information and create proper ISP node with ASN
|
||||
isp_name = data.get('org')
|
||||
asn_value = data.get('asn')
|
||||
|
||||
if isp_name and asn_value:
|
||||
# Enhanced raw_data with last_seen timestamp
|
||||
raw_data = {
|
||||
'asn': asn_value,
|
||||
'shodan_org': isp_name,
|
||||
'ip_version': ip_version
|
||||
}
|
||||
|
||||
# Add last_seen timestamp if available
|
||||
if last_seen:
|
||||
raw_data['last_seen'] = last_seen
|
||||
raw_data['relevance_timestamp'] = last_seen # Standardized field for time-based coloring
|
||||
|
||||
# Create relationship from IP to ISP
|
||||
result.add_relationship(
|
||||
source_node=ip,
|
||||
target_node=isp_name,
|
||||
relationship_type='shodan_isp',
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
raw_data={'asn': asn_value, 'shodan_org': isp_name, 'ip_version': ip_version}
|
||||
raw_data=raw_data
|
||||
)
|
||||
|
||||
# Add ASN as attribute to the ISP node
|
||||
@ -390,7 +403,6 @@ class ShodanProvider(BaseProvider):
|
||||
value=asn_value,
|
||||
attr_type='isp_info',
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
metadata={'description': 'Autonomous System Number from Shodan', 'ip_version': ip_version}
|
||||
)
|
||||
|
||||
@ -401,7 +413,6 @@ class ShodanProvider(BaseProvider):
|
||||
value=isp_name,
|
||||
attr_type='isp_info',
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
metadata={'description': 'Organization name from Shodan', 'ip_version': ip_version}
|
||||
)
|
||||
|
||||
@ -416,20 +427,24 @@ class ShodanProvider(BaseProvider):
|
||||
else:
|
||||
relationship_type = 'shodan_a_record'
|
||||
|
||||
# Enhanced raw_data with last_seen timestamp
|
||||
hostname_raw_data = {**data, 'ip_version': ip_version}
|
||||
if last_seen:
|
||||
hostname_raw_data['last_seen'] = last_seen
|
||||
hostname_raw_data['relevance_timestamp'] = last_seen
|
||||
|
||||
result.add_relationship(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type=relationship_type,
|
||||
provider=self.name,
|
||||
confidence=0.8,
|
||||
raw_data={**data, 'ip_version': ip_version}
|
||||
raw_data=hostname_raw_data
|
||||
)
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type=relationship_type,
|
||||
confidence_score=0.8,
|
||||
raw_data={**data, 'ip_version': ip_version},
|
||||
raw_data=hostname_raw_data,
|
||||
discovery_method=f"shodan_host_lookup_ipv{ip_version}"
|
||||
)
|
||||
elif key == 'ports':
|
||||
@ -441,7 +456,6 @@ class ShodanProvider(BaseProvider):
|
||||
value=port,
|
||||
attr_type='shodan_network_info',
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
metadata={'ip_version': ip_version}
|
||||
)
|
||||
elif isinstance(value, (str, int, float, bool)) and value is not None:
|
||||
@ -452,7 +466,6 @@ class ShodanProvider(BaseProvider):
|
||||
value=value,
|
||||
attr_type='shodan_info',
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
metadata={'ip_version': ip_version}
|
||||
)
|
||||
|
||||
|
||||
@ -326,6 +326,20 @@ input[type="text"]:focus, select:focus {
|
||||
animation: progressGlow 2s ease-in-out infinite alternate;
|
||||
}
|
||||
|
||||
.gradient-bar {
|
||||
height: 4px;
|
||||
background: linear-gradient(to right, #6b7280, #00bfff);
|
||||
border-radius: 2px;
|
||||
margin: 0.2rem 0;
|
||||
}
|
||||
|
||||
.gradient-labels {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
font-size: 0.6rem;
|
||||
color: #888;
|
||||
}
|
||||
|
||||
@keyframes progressShimmer {
|
||||
0% { transform: translateX(-100%); }
|
||||
100% { transform: translateX(100%); }
|
||||
@ -380,32 +394,59 @@ input[type="text"]:focus, select:focus {
|
||||
color: #999;
|
||||
}
|
||||
|
||||
/* Graph Controls */
|
||||
/* Enhanced graph controls layout */
|
||||
.graph-controls {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 8px;
|
||||
z-index: 10;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.3rem;
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
left: 10px;
|
||||
background: rgba(26, 26, 26, 0.9);
|
||||
padding: 0.5rem;
|
||||
border-radius: 6px;
|
||||
border: 1px solid #444;
|
||||
box-shadow: 0 4px 15px rgba(0, 0, 0, 0.5);
|
||||
z-index: 100;
|
||||
min-width: 200px;
|
||||
}
|
||||
|
||||
.graph-control-btn, .btn-icon-small {
|
||||
background: rgba(42, 42, 42, 0.9);
|
||||
.graph-control-btn {
|
||||
background: linear-gradient(135deg, #2a2a2a 0%, #1e1e1e 100%);
|
||||
border: 1px solid #555;
|
||||
color: #c7c7c7;
|
||||
padding: 0.3rem 0.5rem;
|
||||
font-family: 'Roboto Mono', monospace;
|
||||
font-size: 0.7rem;
|
||||
padding: 0.4rem 0.8rem;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s ease;
|
||||
font-family: 'Roboto Mono', monospace;
|
||||
font-size: 0.8rem;
|
||||
transition: all 0.2s ease;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.graph-control-btn:hover, .btn-icon-small:hover {
|
||||
.graph-control-btn:hover {
|
||||
background: linear-gradient(135deg, #3a3a3a 0%, #2e2e2e 100%);
|
||||
border-color: #00ff41;
|
||||
color: #00ff41;
|
||||
}
|
||||
|
||||
.graph-control-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.manual-refresh-btn {
|
||||
background: linear-gradient(135deg, #4a4a2a 0%, #3e3e1e 100%);
|
||||
border-color: #ffaa00;
|
||||
color: #ffaa00;
|
||||
}
|
||||
|
||||
.manual-refresh-btn:hover {
|
||||
background: linear-gradient(135deg, #5a5a3a 0%, #4e4e2e 100%);
|
||||
color: #ffcc33;
|
||||
border-color: #ffcc33;
|
||||
}
|
||||
|
||||
.graph-filter-panel {
|
||||
position: absolute;
|
||||
bottom: 8px;
|
||||
@ -500,14 +541,6 @@ input[type="text"]:focus, select:focus {
|
||||
height: 2px;
|
||||
}
|
||||
|
||||
.legend-edge.high-confidence {
|
||||
background: #00ff41;
|
||||
}
|
||||
|
||||
.legend-edge.medium-confidence {
|
||||
background: #ff9900;
|
||||
}
|
||||
|
||||
/* Provider Panel */
|
||||
.provider-panel {
|
||||
grid-area: providers;
|
||||
@ -987,11 +1020,6 @@ input[type="text"]:focus, select:focus {
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
.confidence-indicator {
|
||||
font-size: 0.6rem;
|
||||
letter-spacing: 1px;
|
||||
}
|
||||
|
||||
.node-link-compact {
|
||||
color: #00aaff;
|
||||
text-decoration: none;
|
||||
@ -1095,6 +1123,56 @@ input[type="text"]:focus, select:focus {
|
||||
border-left: 3px solid #00aaff;
|
||||
}
|
||||
|
||||
.time-control-container {
|
||||
margin-bottom: 0.5rem;
|
||||
padding: 0.5rem;
|
||||
background: rgba(42, 42, 42, 0.3);
|
||||
border-radius: 4px;
|
||||
border: 1px solid #444;
|
||||
}
|
||||
|
||||
.time-control-label {
|
||||
font-size: 0.8rem;
|
||||
color: #c7c7c7;
|
||||
margin-bottom: 0.3rem;
|
||||
display: block;
|
||||
font-family: 'Roboto Mono', monospace;
|
||||
}
|
||||
|
||||
.time-control-input {
|
||||
width: 100%;
|
||||
padding: 0.3rem;
|
||||
background: #1a1a1a;
|
||||
border: 1px solid #555;
|
||||
border-radius: 3px;
|
||||
color: #c7c7c7;
|
||||
font-family: 'Roboto Mono', monospace;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.time-control-input:focus {
|
||||
outline: none;
|
||||
border-color: #00ff41;
|
||||
box-shadow: 0 0 5px rgba(0, 255, 65, 0.3);
|
||||
}
|
||||
|
||||
.time-gradient-info {
|
||||
font-size: 0.7rem;
|
||||
color: #999;
|
||||
margin-top: 0.3rem;
|
||||
text-align: center;
|
||||
font-family: 'Roboto Mono', monospace;
|
||||
}
|
||||
|
||||
/* Edge color legend for time-based gradient */
|
||||
.time-gradient-legend {
|
||||
margin-top: 0.5rem;
|
||||
padding: 0.3rem;
|
||||
background: rgba(26, 26, 26, 0.5);
|
||||
border-radius: 3px;
|
||||
border: 1px solid #333;
|
||||
}
|
||||
|
||||
/* Settings Modal Specific */
|
||||
.provider-toggle {
|
||||
appearance: none !important;
|
||||
@ -1324,16 +1402,16 @@ input[type="password"]:focus {
|
||||
.provider-list {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
.manual-refresh-btn {
|
||||
background: rgba(92, 76, 44, 0.9) !important; /* Orange/amber background */
|
||||
border: 1px solid #7a6a3a !important;
|
||||
color: #ffcc00 !important; /* Bright yellow text */
|
||||
}
|
||||
.graph-controls {
|
||||
position: relative;
|
||||
top: auto;
|
||||
left: auto;
|
||||
margin-bottom: 1rem;
|
||||
min-width: auto;
|
||||
}
|
||||
|
||||
.manual-refresh-btn:hover {
|
||||
border-color: #ffcc00 !important;
|
||||
color: #fff !important;
|
||||
background: rgba(112, 96, 54, 0.9) !important;
|
||||
.time-control-input {
|
||||
font-size: 0.7rem;
|
||||
}
|
||||
}
|
||||
@ -2,7 +2,7 @@
|
||||
/**
|
||||
* Graph visualization module for DNScope
|
||||
* Handles network graph rendering using vis.js with proper large entity node hiding
|
||||
* UPDATED: Added manual refresh button for polling optimization when graph becomes large
|
||||
* UPDATED: Added time-based blue gradient edge coloring system
|
||||
*/
|
||||
const contextMenuCSS = `
|
||||
.graph-context-menu {
|
||||
@ -53,6 +53,44 @@ const contextMenuCSS = `
|
||||
.graph-context-menu ul li:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.time-control-container {
|
||||
margin-bottom: 0.5rem;
|
||||
padding: 0.5rem;
|
||||
background: rgba(42, 42, 42, 0.3);
|
||||
border-radius: 4px;
|
||||
border: 1px solid #444;
|
||||
}
|
||||
|
||||
.time-control-label {
|
||||
font-size: 0.8rem;
|
||||
color: #c7c7c7;
|
||||
margin-bottom: 0.3rem;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.time-control-input {
|
||||
width: 100%;
|
||||
padding: 0.3rem;
|
||||
background: #1a1a1a;
|
||||
border: 1px solid #555;
|
||||
border-radius: 3px;
|
||||
color: #c7c7c7;
|
||||
font-family: 'Roboto Mono', monospace;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.time-control-input:focus {
|
||||
outline: none;
|
||||
border-color: #00ff41;
|
||||
}
|
||||
|
||||
.time-gradient-info {
|
||||
font-size: 0.7rem;
|
||||
color: #999;
|
||||
margin-top: 0.3rem;
|
||||
text-align: center;
|
||||
}
|
||||
`;
|
||||
|
||||
class GraphManager {
|
||||
@ -76,6 +114,16 @@ class GraphManager {
|
||||
this.manualRefreshButton = null;
|
||||
this.manualRefreshHandler = null; // Store the handler
|
||||
|
||||
// Time-based gradient settings
|
||||
this.timeOfInterest = new Date(); // Default to now
|
||||
this.edgeTimestamps = new Map(); // Store edge ID -> timestamp mapping
|
||||
|
||||
// Gradient colors: grey-ish dark to retina-melting light blue
|
||||
this.gradientColors = {
|
||||
dark: '#6b7280', // Grey-ish dark
|
||||
light: '#00bfff' // Retina-melting light blue
|
||||
};
|
||||
|
||||
this.options = {
|
||||
nodes: {
|
||||
shape: 'dot',
|
||||
@ -257,13 +305,25 @@ class GraphManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Add interactive graph controls
|
||||
* UPDATED: Added manual refresh button for polling optimization
|
||||
* Add interactive graph controls with time of interest control
|
||||
* UPDATED: Added time-based edge coloring controls
|
||||
*/
|
||||
addGraphControls() {
|
||||
const controlsContainer = document.createElement('div');
|
||||
controlsContainer.className = 'graph-controls';
|
||||
|
||||
// Format current date/time for the input
|
||||
const currentDateTime = this.formatDateTimeForInput(this.timeOfInterest);
|
||||
|
||||
controlsContainer.innerHTML = `
|
||||
<div class="time-control-container">
|
||||
<label class="time-control-label">Time of Interest (for edge coloring)</label>
|
||||
<input type="datetime-local" id="time-of-interest" class="time-control-input"
|
||||
value="${currentDateTime}" title="Reference time for edge color gradient">
|
||||
<div class="time-gradient-info">
|
||||
Dark: Old data | Light Blue: Recent data
|
||||
</div>
|
||||
</div>
|
||||
<button class="graph-control-btn" id="graph-fit" title="Fit to Screen">[FIT]</button>
|
||||
<button class="graph-control-btn" id="graph-physics" title="Toggle Physics">[PHYSICS]</button>
|
||||
<button class="graph-control-btn" id="graph-cluster" title="Cluster Nodes">[CLUSTER]</button>
|
||||
@ -283,6 +343,13 @@ class GraphManager {
|
||||
document.getElementById('graph-unhide').addEventListener('click', () => this.unhideAll());
|
||||
document.getElementById('graph-revert').addEventListener('click', () => this.revertLastAction());
|
||||
|
||||
// Time of interest control
|
||||
document.getElementById('time-of-interest').addEventListener('change', (e) => {
|
||||
this.timeOfInterest = new Date(e.target.value);
|
||||
console.log('Time of interest updated:', this.timeOfInterest);
|
||||
this.updateEdgeColors();
|
||||
});
|
||||
|
||||
// Manual refresh button - handler will be set by main app
|
||||
this.manualRefreshButton = document.getElementById('graph-manual-refresh');
|
||||
// If a handler was set before the button existed, attach it now
|
||||
@ -291,6 +358,150 @@ class GraphManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format date for datetime-local input
|
||||
*/
|
||||
formatDateTimeForInput(date) {
|
||||
const year = date.getFullYear();
|
||||
const month = String(date.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(date.getDate()).padStart(2, '0');
|
||||
const hours = String(date.getHours()).padStart(2, '0');
|
||||
const minutes = String(date.getMinutes()).padStart(2, '0');
|
||||
return `${year}-${month}-${day}T${hours}:${minutes}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract relevant timestamp from edge raw_data based on provider
|
||||
*/
|
||||
extractEdgeTimestamp(edge) {
|
||||
const rawData = edge.raw_data || {};
|
||||
const provider = edge.source_provider || '';
|
||||
|
||||
// Check for standardized relevance_timestamp first
|
||||
if (rawData.relevance_timestamp) {
|
||||
return new Date(rawData.relevance_timestamp);
|
||||
}
|
||||
|
||||
// Provider-specific timestamp extraction
|
||||
switch (provider.toLowerCase()) {
|
||||
case 'shodan':
|
||||
// Use last_seen timestamp for Shodan
|
||||
if (rawData.last_seen) {
|
||||
return new Date(rawData.last_seen);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'crtsh':
|
||||
// Use certificate issue date (not_before) for certificates
|
||||
if (rawData.cert_not_before) {
|
||||
return new Date(rawData.cert_not_before);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'dns':
|
||||
case 'correlation':
|
||||
default:
|
||||
// Use discovery timestamp for DNS and correlation
|
||||
if (edge.discovery_timestamp) {
|
||||
return new Date(edge.discovery_timestamp);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Fallback to discovery timestamp or current time
|
||||
if (edge.discovery_timestamp) {
|
||||
return new Date(edge.discovery_timestamp);
|
||||
}
|
||||
|
||||
return new Date(); // Default to now if no timestamp available
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate time-based blue gradient color
|
||||
*/
|
||||
calculateTimeGradientColor(timestamp) {
|
||||
if (!timestamp || !this.timeOfInterest) {
|
||||
return this.gradientColors.dark; // Default to dark grey
|
||||
}
|
||||
|
||||
// Calculate time difference in milliseconds
|
||||
const timeDiff = Math.abs(timestamp.getTime() - this.timeOfInterest.getTime());
|
||||
|
||||
// Find maximum time difference across all edges for normalization
|
||||
let maxTimeDiff = 0;
|
||||
this.edgeTimestamps.forEach((edgeTimestamp) => {
|
||||
const diff = Math.abs(edgeTimestamp.getTime() - this.timeOfInterest.getTime());
|
||||
if (diff > maxTimeDiff) {
|
||||
maxTimeDiff = diff;
|
||||
}
|
||||
});
|
||||
|
||||
if (maxTimeDiff === 0) {
|
||||
return this.gradientColors.light; // All timestamps are the same
|
||||
}
|
||||
|
||||
// Calculate gradient position (0 = closest to time of interest, 1 = furthest)
|
||||
const gradientPosition = timeDiff / maxTimeDiff;
|
||||
|
||||
// Interpolate between light blue (close) and dark grey (far)
|
||||
return this.interpolateColor(
|
||||
this.gradientColors.light, // Close to time of interest
|
||||
this.gradientColors.dark, // Far from time of interest
|
||||
gradientPosition
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpolate between two hex colors
|
||||
*/
|
||||
interpolateColor(color1, color2, factor) {
|
||||
// Parse hex colors
|
||||
const hex1 = color1.replace('#', '');
|
||||
const hex2 = color2.replace('#', '');
|
||||
|
||||
const r1 = parseInt(hex1.substring(0, 2), 16);
|
||||
const g1 = parseInt(hex1.substring(2, 4), 16);
|
||||
const b1 = parseInt(hex1.substring(4, 6), 16);
|
||||
|
||||
const r2 = parseInt(hex2.substring(0, 2), 16);
|
||||
const g2 = parseInt(hex2.substring(2, 4), 16);
|
||||
const b2 = parseInt(hex2.substring(4, 6), 16);
|
||||
|
||||
// Interpolate
|
||||
const r = Math.round(r1 + (r2 - r1) * factor);
|
||||
const g = Math.round(g1 + (g2 - g1) * factor);
|
||||
const b = Math.round(b1 + (b2 - b1) * factor);
|
||||
|
||||
// Convert back to hex
|
||||
return `#${r.toString(16).padStart(2, '0')}${g.toString(16).padStart(2, '0')}${b.toString(16).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update all edge colors based on current time of interest
|
||||
*/
|
||||
updateEdgeColors() {
|
||||
const edgeUpdates = [];
|
||||
|
||||
this.edges.forEach((edge) => {
|
||||
const timestamp = this.edgeTimestamps.get(edge.id);
|
||||
const color = this.calculateTimeGradientColor(timestamp);
|
||||
|
||||
edgeUpdates.push({
|
||||
id: edge.id,
|
||||
color: {
|
||||
color: color,
|
||||
highlight: '#00ff41',
|
||||
hover: '#ff9900'
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
if (edgeUpdates.length > 0) {
|
||||
this.edges.update(edgeUpdates);
|
||||
console.log(`Updated ${edgeUpdates.length} edge colors based on time gradient`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the manual refresh button click handler
|
||||
* @param {Function} handler - Function to call when manual refresh is clicked
|
||||
@ -411,6 +622,7 @@ class GraphManager {
|
||||
if (!hasData) {
|
||||
this.nodes.clear();
|
||||
this.edges.clear();
|
||||
this.edgeTimestamps.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
@ -464,6 +676,9 @@ class GraphManager {
|
||||
this.nodes.update(processedNodes);
|
||||
this.edges.update(processedEdges);
|
||||
|
||||
// Update edge timestamps and colors for time-based gradient
|
||||
this.updateEdgeTimestampsAndColors(graphData.edges);
|
||||
|
||||
this.updateFilterControls();
|
||||
this.applyAllFilters();
|
||||
|
||||
@ -481,6 +696,21 @@ class GraphManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update edge timestamps and apply time-based gradient colors
|
||||
*/
|
||||
updateEdgeTimestampsAndColors(edgeData) {
|
||||
// Extract timestamps from raw edge data
|
||||
edgeData.forEach(edge => {
|
||||
const edgeId = `${edge.from}-${edge.to}-${edge.label}`;
|
||||
const timestamp = this.extractEdgeTimestamp(edge);
|
||||
this.edgeTimestamps.set(edgeId, timestamp);
|
||||
});
|
||||
|
||||
// Update edge colors based on new timestamps
|
||||
this.updateEdgeColors();
|
||||
}
|
||||
|
||||
analyzeCertificateInfo(attributes) {
|
||||
let hasCertificates = false;
|
||||
let hasValidCertificates = false;
|
||||
@ -559,12 +789,6 @@ class GraphManager {
|
||||
processedNode.borderColor = '#ff0000'; // Red border for max depth
|
||||
}
|
||||
|
||||
|
||||
// Add confidence-based styling
|
||||
if (node.confidence) {
|
||||
processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5));
|
||||
}
|
||||
|
||||
// FIXED: Certificate-based domain coloring
|
||||
if (node.type === 'domain' && Array.isArray(node.attributes)) {
|
||||
const certInfo = this.analyzeCertificateInfo(node.attributes);
|
||||
@ -595,24 +819,33 @@ class GraphManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Process edge data with styling and metadata
|
||||
* Process edge data with styling, metadata, and time-based gradient colors
|
||||
* @param {Object} edge - Raw edge data
|
||||
* @returns {Object} Processed edge data
|
||||
*/
|
||||
processEdge(edge) {
|
||||
const confidence = edge.confidence_score || 0;
|
||||
const edgeId = `${edge.from}-${edge.to}-${edge.label}`;
|
||||
|
||||
// Extract timestamp for this edge
|
||||
const timestamp = this.extractEdgeTimestamp(edge);
|
||||
this.edgeTimestamps.set(edgeId, timestamp);
|
||||
|
||||
// Calculate time-based gradient color
|
||||
const timeGradientColor = this.calculateTimeGradientColor(timestamp);
|
||||
|
||||
const processedEdge = {
|
||||
id: `${edge.from}-${edge.to}-${edge.label}`,
|
||||
id: edgeId,
|
||||
from: edge.from,
|
||||
to: edge.to,
|
||||
label: this.formatEdgeLabel(edge.label, confidence),
|
||||
label: edge.label, // Correctly access the label directly
|
||||
title: this.createEdgeTooltip(edge),
|
||||
width: this.getEdgeWidth(confidence),
|
||||
color: this.getEdgeColor(confidence),
|
||||
dashes: confidence < 0.6 ? [5, 5] : false,
|
||||
color: {
|
||||
color: timeGradientColor,
|
||||
highlight: '#00ff41',
|
||||
hover: '#ff9900'
|
||||
},
|
||||
metadata: {
|
||||
relationship_type: edge.label,
|
||||
confidence_score: confidence,
|
||||
source_provider: edge.source_provider,
|
||||
discovery_timestamp: edge.discovery_timestamp
|
||||
}
|
||||
@ -635,18 +868,7 @@ class GraphManager {
|
||||
return nodeId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format edge label for display
|
||||
* @param {string} relationshipType - Type of relationship
|
||||
* @param {number} confidence - Confidence score
|
||||
* @returns {string} Formatted label
|
||||
*/
|
||||
formatEdgeLabel(relationshipType, confidence) {
|
||||
if (!relationshipType) return '';
|
||||
|
||||
const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '●' : '○';
|
||||
return `${relationshipType} ${confidenceText}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get node color based on type
|
||||
@ -716,44 +938,13 @@ class GraphManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edge color based on confidence
|
||||
* @param {number} confidence - Confidence score
|
||||
* @returns {string} Edge color
|
||||
*/
|
||||
getEdgeColor(confidence) {
|
||||
if (confidence >= 0.8) {
|
||||
return '#00ff41'; // High confidence - green
|
||||
} else if (confidence >= 0.6) {
|
||||
return '#ff9900'; // Medium confidence - amber
|
||||
} else {
|
||||
return '#666666'; // Low confidence - gray
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get edge width based on confidence
|
||||
* @param {number} confidence - Confidence score
|
||||
* @returns {number} Edge width
|
||||
*/
|
||||
getEdgeWidth(confidence) {
|
||||
if (confidence >= 0.8) {
|
||||
return 3;
|
||||
} else if (confidence >= 0.6) {
|
||||
return 2;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create edge tooltip with correct provider information
|
||||
* Create edge tooltip with correct provider information and timestamp
|
||||
* @param {Object} edge - Edge data
|
||||
* @returns {string} HTML tooltip content
|
||||
*/
|
||||
createEdgeTooltip(edge) {
|
||||
let tooltip = `<div style="font-family: 'Roboto Mono', monospace; font-size: 11px;">`;
|
||||
tooltip += `<div style="color: #00ff41; font-weight: bold; margin-bottom: 4px;">${edge.label || 'Relationship'}</div>`;
|
||||
tooltip += `<div style="color: #999; margin-bottom: 2px;">Confidence: ${(edge.confidence_score * 100).toFixed(1)}%</div>`;
|
||||
|
||||
if (edge.source_provider) {
|
||||
tooltip += `<div style="color: #999; margin-bottom: 2px;">Provider: ${edge.source_provider}</div>`;
|
||||
@ -764,6 +955,13 @@ class GraphManager {
|
||||
tooltip += `<div style="color: #666; font-size: 10px;">Discovered: ${date.toLocaleString()}</div>`;
|
||||
}
|
||||
|
||||
// Add timestamp information for time-based coloring
|
||||
const edgeId = `${edge.from}-${edge.to}-${edge.label}`;
|
||||
const timestamp = this.edgeTimestamps.get(edgeId);
|
||||
if (timestamp) {
|
||||
tooltip += `<div style="color: #888; font-size: 10px;">Data from: ${timestamp.toLocaleString()}</div>`;
|
||||
}
|
||||
|
||||
tooltip += `</div>`;
|
||||
return tooltip;
|
||||
}
|
||||
@ -893,13 +1091,17 @@ class GraphManager {
|
||||
};
|
||||
});
|
||||
|
||||
// Reset highlighted edges
|
||||
// Reset highlighted edges to time-based colors
|
||||
const edgeUpdates = this.highlightedElements.edges.map(id => {
|
||||
const originalEdge = this.edges.get(id);
|
||||
const timestamp = this.edgeTimestamps.get(id);
|
||||
const color = this.calculateTimeGradientColor(timestamp);
|
||||
return {
|
||||
id: id,
|
||||
color: this.getEdgeColor(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5),
|
||||
width: this.getEdgeWidth(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5)
|
||||
color: {
|
||||
color: color,
|
||||
highlight: '#00ff41',
|
||||
hover: '#ff9900'
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
@ -955,11 +1157,19 @@ class GraphManager {
|
||||
borderWidth: 2,
|
||||
}));
|
||||
|
||||
const edgeResets = newEdges.map(edge => ({
|
||||
// Reset edges to time-based colors
|
||||
const edgeResets = newEdges.map(edge => {
|
||||
const timestamp = this.edgeTimestamps.get(edge.id);
|
||||
const color = this.calculateTimeGradientColor(timestamp);
|
||||
return {
|
||||
id: edge.id,
|
||||
color: this.getEdgeColor(edge.metadata ? edge.metadata.confidence_score : 0.5),
|
||||
width: this.getEdgeWidth(edge.metadata ? edge.metadata.confidence_score : 0.5)
|
||||
}));
|
||||
color: {
|
||||
color: color,
|
||||
highlight: '#00ff41',
|
||||
hover: '#ff9900'
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
this.nodes.update(nodeResets);
|
||||
this.edges.update(edgeResets);
|
||||
@ -1048,6 +1258,7 @@ class GraphManager {
|
||||
clear() {
|
||||
this.nodes.clear();
|
||||
this.edges.clear();
|
||||
this.edgeTimestamps.clear();
|
||||
this.history = [];
|
||||
this.largeEntityMembers.clear();
|
||||
this.initialTargetIds.clear();
|
||||
|
||||
@ -1722,17 +1722,9 @@ class DNScopeApp {
|
||||
return groups;
|
||||
}
|
||||
|
||||
formatEdgeLabel(relationshipType, confidence) {
|
||||
if (!relationshipType) return '';
|
||||
|
||||
const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○';
|
||||
return `${relationshipType} ${confidenceText}`;
|
||||
}
|
||||
|
||||
createEdgeTooltip(edge) {
|
||||
let tooltip = `<div style="font-family: 'Roboto Mono', monospace; font-size: 11px;">`;
|
||||
tooltip += `<div style="color: #00ff41; font-weight: bold; margin-bottom: 4px;">${edge.label || 'Relationship'}</div>`;
|
||||
tooltip += `<div style="color: #999; margin-bottom: 2px;">Confidence: ${(edge.confidence_score * 100).toFixed(1)}%</div>`;
|
||||
|
||||
// UPDATED: Use raw provider name (no formatting)
|
||||
if (edge.source_provider) {
|
||||
@ -1872,7 +1864,7 @@ class DNScopeApp {
|
||||
html += `
|
||||
<div class="relationship-compact-item">
|
||||
<span class="node-link-compact" data-node-id="${innerNodeId}">${innerNodeId}</span>
|
||||
<button class="btn-icon-small extract-node-btn"
|
||||
<button class="graph-control-btn extract-node-btn"
|
||||
title="Extract to graph"
|
||||
data-large-entity-id="${largeEntityId}"
|
||||
data-node-id="${innerNodeId}">[+]</button>
|
||||
@ -1899,8 +1891,6 @@ class DNScopeApp {
|
||||
`;
|
||||
|
||||
node.incoming_edges.forEach(edge => {
|
||||
const confidence = edge.data.confidence_score || 0;
|
||||
const confidenceClass = confidence >= 0.8 ? 'high' : confidence >= 0.6 ? 'medium' : 'low';
|
||||
|
||||
html += `
|
||||
<div class="relationship-item">
|
||||
@ -1909,9 +1899,6 @@ class DNScopeApp {
|
||||
</div>
|
||||
<div class="relationship-type">
|
||||
<span class="relation-label">${edge.data.relationship_type}</span>
|
||||
<span class="confidence-indicator confidence-${confidenceClass}" title="Confidence: ${(confidence * 100).toFixed(1)}%">
|
||||
${'●'.repeat(Math.ceil(confidence * 3))}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
@ -1930,9 +1917,6 @@ class DNScopeApp {
|
||||
`;
|
||||
|
||||
node.outgoing_edges.forEach(edge => {
|
||||
const confidence = edge.data.confidence_score || 0;
|
||||
const confidenceClass = confidence >= 0.8 ? 'high' : confidence >= 0.6 ? 'medium' : 'low';
|
||||
|
||||
html += `
|
||||
<div class="relationship-item">
|
||||
<div class="relationship-target node-link" data-node-id="${edge.to}">
|
||||
@ -1940,9 +1924,6 @@ class DNScopeApp {
|
||||
</div>
|
||||
<div class="relationship-type">
|
||||
<span class="relation-label">${edge.data.relationship_type}</span>
|
||||
<span class="confidence-indicator confidence-${confidenceClass}" title="Confidence: ${(confidence * 100).toFixed(1)}%">
|
||||
${'●'.repeat(Math.ceil(confidence * 3))}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
@ -188,7 +188,6 @@ class ExportManager:
|
||||
f" - Type: {domain_info['classification']}",
|
||||
f" - Connected IPs: {len(domain_info['ips'])}",
|
||||
f" - Certificate Status: {domain_info['cert_status']}",
|
||||
f" - Relationship Confidence: {domain_info['avg_confidence']:.2f}",
|
||||
])
|
||||
|
||||
if domain_info['security_notes']:
|
||||
@ -247,11 +246,9 @@ class ExportManager:
|
||||
])
|
||||
|
||||
for rel in key_relationships[:8]: # Top 8 relationships
|
||||
confidence_desc = self._describe_confidence(rel['confidence'])
|
||||
report.extend([
|
||||
f"• {rel['source']} → {rel['target']}",
|
||||
f" - Relationship: {self._humanize_relationship_type(rel['type'])}",
|
||||
f" - Evidence Strength: {confidence_desc} ({rel['confidence']:.2f})",
|
||||
f" - Discovery Method: {rel['provider']}",
|
||||
""
|
||||
])
|
||||
@ -291,15 +288,6 @@ class ExportManager:
|
||||
"Data Quality Assessment:",
|
||||
f"• Total API Requests: {audit_trail.get('session_metadata', {}).get('total_requests', 0)}",
|
||||
f"• Data Providers Used: {len(audit_trail.get('session_metadata', {}).get('providers_used', []))}",
|
||||
f"• Relationship Confidence Distribution:",
|
||||
])
|
||||
|
||||
# Confidence distribution
|
||||
confidence_dist = self._calculate_confidence_distribution(edges)
|
||||
for level, count in confidence_dist.items():
|
||||
percentage = (count / len(edges) * 100) if edges else 0
|
||||
report.extend([
|
||||
f" - {level.title()} Confidence (≥{self._get_confidence_threshold(level)}): {count} ({percentage:.1f}%)",
|
||||
])
|
||||
|
||||
report.extend([
|
||||
@ -375,9 +363,7 @@ class ExportManager:
|
||||
if len(connected_ips) > 5:
|
||||
security_notes.append("Multiple IP endpoints")
|
||||
|
||||
# Average confidence
|
||||
domain_edges = [e for e in edges if e['from'] == domain['id']]
|
||||
avg_confidence = sum(e['confidence_score'] for e in domain_edges) / len(domain_edges) if domain_edges else 0
|
||||
|
||||
domain_analysis.append({
|
||||
'domain': domain['id'],
|
||||
@ -385,7 +371,6 @@ class ExportManager:
|
||||
'ips': connected_ips,
|
||||
'cert_status': cert_status,
|
||||
'security_notes': security_notes,
|
||||
'avg_confidence': avg_confidence
|
||||
})
|
||||
|
||||
# Sort by number of connections (most connected first)
|
||||
@ -480,7 +465,7 @@ class ExportManager:
|
||||
|
||||
def _identify_key_relationships(self, edges: List[Dict]) -> List[Dict[str, Any]]:
|
||||
"""Identify the most significant relationships in the infrastructure."""
|
||||
# Score relationships by confidence and type importance
|
||||
# Score relationships by type importance
|
||||
relationship_importance = {
|
||||
'dns_a_record': 0.9,
|
||||
'dns_aaaa_record': 0.9,
|
||||
@ -493,15 +478,12 @@ class ExportManager:
|
||||
|
||||
scored_edges = []
|
||||
for edge in edges:
|
||||
base_confidence = edge.get('confidence_score', 0)
|
||||
type_weight = relationship_importance.get(edge.get('label', ''), 0.5)
|
||||
combined_score = (base_confidence * 0.7) + (type_weight * 0.3)
|
||||
|
||||
scored_edges.append({
|
||||
'source': edge['from'],
|
||||
'target': edge['to'],
|
||||
'type': edge.get('label', ''),
|
||||
'confidence': base_confidence,
|
||||
'provider': edge.get('source_provider', ''),
|
||||
'score': combined_score
|
||||
})
|
||||
@ -570,19 +552,6 @@ class ExportManager:
|
||||
else:
|
||||
return "Mixed Status"
|
||||
|
||||
def _describe_confidence(self, confidence: float) -> str:
|
||||
"""Convert confidence score to descriptive text."""
|
||||
if confidence >= 0.9:
|
||||
return "Very High"
|
||||
elif confidence >= 0.8:
|
||||
return "High"
|
||||
elif confidence >= 0.6:
|
||||
return "Medium"
|
||||
elif confidence >= 0.4:
|
||||
return "Low"
|
||||
else:
|
||||
return "Very Low"
|
||||
|
||||
def _humanize_relationship_type(self, rel_type: str) -> str:
|
||||
"""Convert technical relationship types to human-readable descriptions."""
|
||||
type_map = {
|
||||
@ -599,26 +568,6 @@ class ExportManager:
|
||||
}
|
||||
return type_map.get(rel_type, rel_type.replace('_', ' ').title())
|
||||
|
||||
def _calculate_confidence_distribution(self, edges: List[Dict]) -> Dict[str, int]:
|
||||
"""Calculate confidence score distribution."""
|
||||
distribution = {'high': 0, 'medium': 0, 'low': 0}
|
||||
|
||||
for edge in edges:
|
||||
confidence = edge.get('confidence_score', 0)
|
||||
if confidence >= 0.8:
|
||||
distribution['high'] += 1
|
||||
elif confidence >= 0.6:
|
||||
distribution['medium'] += 1
|
||||
else:
|
||||
distribution['low'] += 1
|
||||
|
||||
return distribution
|
||||
|
||||
def _get_confidence_threshold(self, level: str) -> str:
|
||||
"""Get confidence threshold for a level."""
|
||||
thresholds = {'high': '0.80', 'medium': '0.60', 'low': '0.00'}
|
||||
return thresholds.get(level, '0.00')
|
||||
|
||||
def _count_cross_validated_relationships(self, edges: List[Dict]) -> int:
|
||||
"""Count relationships verified by multiple providers."""
|
||||
# Group edges by source-target pair
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user