it
This commit is contained in:
overcuriousity
parent
db2101d814
commit
2d485c5703
@ -21,6 +21,7 @@ class Config:
|
||||
self.default_recursion_depth = 2
|
||||
self.default_timeout = 30
|
||||
self.max_concurrent_requests = 5
|
||||
self.large_entity_threshold = 100
|
||||
|
||||
# Rate limiting settings (requests per minute)
|
||||
self.rate_limits = {
|
||||
|
||||
@ -9,6 +9,7 @@ from datetime import datetime
|
||||
from typing import Dict, List, Any, Optional, Tuple, Set
|
||||
from enum import Enum
|
||||
from datetime import timezone
|
||||
from collections import defaultdict
|
||||
|
||||
import networkx as nx
|
||||
|
||||
@ -24,14 +25,28 @@ class NodeType(Enum):
|
||||
|
||||
class RelationshipType(Enum):
|
||||
"""Enumeration of supported relationship types with confidence scores."""
|
||||
SAN_CERTIFICATE = ("san", 0.9) # Certificate SAN relationships
|
||||
A_RECORD = ("a_record", 0.8) # A/AAAA record relationships
|
||||
CNAME_RECORD = ("cname", 0.8) # CNAME relationships
|
||||
PASSIVE_DNS = ("passive_dns", 0.6) # Passive DNS relationships
|
||||
ASN_MEMBERSHIP = ("asn", 0.7) # ASN relationships
|
||||
MX_RECORD = ("mx_record", 0.7) # MX record relationships
|
||||
NS_RECORD = ("ns_record", 0.7) # NS record relationships
|
||||
|
||||
SAN_CERTIFICATE = ("san", 0.9)
|
||||
A_RECORD = ("a_record", 0.8)
|
||||
AAAA_RECORD = ("aaaa_record", 0.8)
|
||||
CNAME_RECORD = ("cname", 0.8)
|
||||
MX_RECORD = ("mx_record", 0.7)
|
||||
NS_RECORD = ("ns_record", 0.7)
|
||||
PTR_RECORD = ("ptr_record", 0.8)
|
||||
SOA_RECORD = ("soa_record", 0.7)
|
||||
TXT_RECORD = ("txt_record", 0.7)
|
||||
SRV_RECORD = ("srv_record", 0.7)
|
||||
CAA_RECORD = ("caa_record", 0.7)
|
||||
DNSKEY_RECORD = ("dnskey_record", 0.7)
|
||||
DS_RECORD = ("ds_record", 0.7)
|
||||
RRSIG_RECORD = ("rrsig_record", 0.7)
|
||||
SSHFP_RECORD = ("sshfp_record", 0.7)
|
||||
TLSA_RECORD = ("tlsa_record", 0.7)
|
||||
NAPTR_RECORD = ("naptr_record", 0.7)
|
||||
SPF_RECORD = ("spf_record", 0.7)
|
||||
PASSIVE_DNS = ("passive_dns", 0.6)
|
||||
ASN_MEMBERSHIP = ("asn", 0.7)
|
||||
|
||||
|
||||
def __init__(self, relationship_name: str, default_confidence: float):
|
||||
self.relationship_name = relationship_name
|
||||
self.default_confidence = default_confidence
|
||||
@ -42,24 +57,24 @@ class GraphManager:
|
||||
Thread-safe graph manager for DNSRecon infrastructure mapping.
|
||||
Uses NetworkX for in-memory graph storage with confidence scoring.
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize empty directed graph."""
|
||||
self.graph = nx.DiGraph()
|
||||
# self.lock = threading.Lock()
|
||||
self.creation_time = datetime.now(timezone.utc).isoformat()
|
||||
self.last_modified = self.creation_time
|
||||
|
||||
def add_node(self, node_id: str, node_type: NodeType,
|
||||
|
||||
def add_node(self, node_id: str, node_type: NodeType,
|
||||
metadata: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""
|
||||
Add a node to the graph.
|
||||
|
||||
|
||||
Args:
|
||||
node_id: Unique identifier for the node
|
||||
node_type: Type of the node (Domain, IP, Certificate, ASN)
|
||||
metadata: Additional metadata for the node
|
||||
|
||||
|
||||
Returns:
|
||||
bool: True if node was added, False if it already exists
|
||||
"""
|
||||
@ -70,33 +85,33 @@ class GraphManager:
|
||||
existing_metadata.update(metadata)
|
||||
self.graph.nodes[node_id]['metadata'] = existing_metadata
|
||||
return False
|
||||
|
||||
|
||||
node_attributes = {
|
||||
'type': node_type.value,
|
||||
'added_timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'metadata': metadata or {}
|
||||
}
|
||||
|
||||
|
||||
self.graph.add_node(node_id, **node_attributes)
|
||||
self.last_modified = datetime.now(timezone.utc).isoformat()
|
||||
return True
|
||||
|
||||
def add_edge(self, source_id: str, target_id: str,
|
||||
|
||||
def add_edge(self, source_id: str, target_id: str,
|
||||
relationship_type: RelationshipType,
|
||||
confidence_score: Optional[float] = None,
|
||||
source_provider: str = "unknown",
|
||||
raw_data: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""
|
||||
Add an edge between two nodes.
|
||||
|
||||
|
||||
Args:
|
||||
source_id: Source node identifier
|
||||
target_id: Target node identifier
|
||||
target_id: Target node identifier
|
||||
relationship_type: Type of relationship
|
||||
confidence_score: Custom confidence score (overrides default)
|
||||
source_provider: Provider that discovered this relationship
|
||||
raw_data: Raw data from provider response
|
||||
|
||||
|
||||
Returns:
|
||||
bool: True if edge was added, False if it already exists
|
||||
"""
|
||||
@ -112,14 +127,14 @@ class GraphManager:
|
||||
# Update confidence score if new score is higher
|
||||
existing_confidence = self.graph.edges[source_id, target_id]['confidence_score']
|
||||
new_confidence = confidence_score or relationship_type.default_confidence
|
||||
|
||||
|
||||
if new_confidence > existing_confidence:
|
||||
self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence
|
||||
self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat()
|
||||
self.graph.edges[source_id, target_id]['updated_by'] = source_provider
|
||||
|
||||
|
||||
return False
|
||||
|
||||
|
||||
edge_attributes = {
|
||||
'relationship_type': relationship_type.relationship_name,
|
||||
'confidence_score': confidence_score or relationship_type.default_confidence,
|
||||
@ -127,7 +142,7 @@ class GraphManager:
|
||||
'discovery_timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'raw_data': raw_data or {}
|
||||
}
|
||||
|
||||
|
||||
self.graph.add_edge(source_id, target_id, **edge_attributes)
|
||||
self.last_modified = datetime.now(timezone.utc).isoformat()
|
||||
return True
|
||||
@ -136,19 +151,19 @@ class GraphManager:
|
||||
"""Get total number of nodes in the graph."""
|
||||
#with self.lock:
|
||||
return self.graph.number_of_nodes()
|
||||
|
||||
|
||||
def get_edge_count(self) -> int:
|
||||
"""Get total number of edges in the graph."""
|
||||
#with self.lock:
|
||||
return self.graph.number_of_edges()
|
||||
|
||||
|
||||
def get_nodes_by_type(self, node_type: NodeType) -> List[str]:
|
||||
"""
|
||||
Get all nodes of a specific type.
|
||||
|
||||
|
||||
Args:
|
||||
node_type: Type of nodes to retrieve
|
||||
|
||||
|
||||
Returns:
|
||||
List of node identifiers
|
||||
"""
|
||||
@ -157,32 +172,32 @@ class GraphManager:
|
||||
node_id for node_id, attributes in self.graph.nodes(data=True)
|
||||
if attributes.get('type') == node_type.value
|
||||
]
|
||||
|
||||
|
||||
def get_neighbors(self, node_id: str) -> List[str]:
|
||||
"""
|
||||
Get all neighboring nodes (both incoming and outgoing).
|
||||
|
||||
|
||||
Args:
|
||||
node_id: Node identifier
|
||||
|
||||
|
||||
Returns:
|
||||
List of neighboring node identifiers
|
||||
"""
|
||||
#with self.lock:
|
||||
if not self.graph.has_node(node_id):
|
||||
return []
|
||||
|
||||
|
||||
predecessors = list(self.graph.predecessors(node_id))
|
||||
successors = list(self.graph.successors(node_id))
|
||||
return list(set(predecessors + successors))
|
||||
|
||||
|
||||
def get_high_confidence_edges(self, min_confidence: float = 0.8) -> List[Tuple[str, str, Dict]]:
|
||||
"""
|
||||
Get edges with confidence score above threshold.
|
||||
|
||||
|
||||
Args:
|
||||
min_confidence: Minimum confidence threshold
|
||||
|
||||
|
||||
Returns:
|
||||
List of tuples (source, target, attributes)
|
||||
"""
|
||||
@ -192,18 +207,49 @@ class GraphManager:
|
||||
for source, target, attributes in self.graph.edges(data=True)
|
||||
if attributes.get('confidence_score', 0) >= min_confidence
|
||||
]
|
||||
|
||||
|
||||
def get_graph_data(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Export graph data for visualization.
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary containing nodes and edges for frontend visualization
|
||||
"""
|
||||
#with self.lock:
|
||||
nodes = []
|
||||
edges = []
|
||||
|
||||
|
||||
# Create a dictionary to hold aggregated data for each node
|
||||
node_details = defaultdict(lambda: defaultdict(list))
|
||||
|
||||
for source, target, attributes in self.graph.edges(data=True):
|
||||
provider = attributes.get('source_provider', 'unknown')
|
||||
raw_data = attributes.get('raw_data', {})
|
||||
|
||||
if provider == 'dns':
|
||||
record_type = raw_data.get('query_type', 'UNKNOWN')
|
||||
value = raw_data.get('value', target)
|
||||
# DNS data is always about the source node of the query
|
||||
node_details[source]['dns_records'].append(f"{record_type}: {value}")
|
||||
|
||||
elif provider == 'crtsh':
|
||||
# Data from crt.sh are domain names found in certificates (SANs)
|
||||
node_details[source]['related_domains_san'].append(target)
|
||||
|
||||
elif provider == 'shodan':
|
||||
# Shodan data is about the IP, which can be either the source or target
|
||||
source_node_type = self.graph.nodes[source].get('type')
|
||||
target_node_type = self.graph.nodes[target].get('type')
|
||||
|
||||
if source_node_type == 'ip':
|
||||
node_details[source]['shodan'] = raw_data
|
||||
elif target_node_type == 'ip':
|
||||
node_details[target]['shodan'] = raw_data
|
||||
|
||||
elif provider == 'virustotal':
|
||||
# VirusTotal data is about the source node of the query
|
||||
node_details[source]['virustotal'] = raw_data
|
||||
|
||||
# Format nodes for visualization
|
||||
for node_id, attributes in self.graph.nodes(data=True):
|
||||
node_data = {
|
||||
@ -213,7 +259,18 @@ class GraphManager:
|
||||
'metadata': attributes.get('metadata', {}),
|
||||
'added_timestamp': attributes.get('added_timestamp')
|
||||
}
|
||||
|
||||
|
||||
# Add the aggregated details to the metadata
|
||||
if node_id in node_details:
|
||||
for key, value in node_details[node_id].items():
|
||||
# Use a set to avoid adding duplicate entries to lists
|
||||
if key in node_data['metadata'] and isinstance(node_data['metadata'][key], list):
|
||||
existing_values = set(node_data['metadata'][key])
|
||||
new_values = [v for v in value if v not in existing_values]
|
||||
node_data['metadata'][key].extend(new_values)
|
||||
else:
|
||||
node_data['metadata'][key] = value
|
||||
|
||||
# Color coding by type - now returns color objects for enhanced visualization
|
||||
type_colors = {
|
||||
'domain': {
|
||||
@ -239,18 +296,24 @@ class GraphManager:
|
||||
'border': '#0088cc',
|
||||
'highlight': {'background': '#44ccff', 'border': '#00aaff'},
|
||||
'hover': {'background': '#22bbff', 'border': '#0099dd'}
|
||||
},
|
||||
'large_entity': {
|
||||
'background': '#ff6b6b',
|
||||
'border': '#cc3a3a',
|
||||
'highlight': {'background': '#ff8c8c', 'border': '#ff6b6b'},
|
||||
'hover': {'background': '#ff7a7a', 'border': '#dd4a4a'}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
node_color_config = type_colors.get(attributes.get('type', 'unknown'), type_colors['domain'])
|
||||
node_data['color'] = node_color_config
|
||||
|
||||
|
||||
# Pass the has_valid_cert metadata for styling
|
||||
if 'metadata' in attributes and 'has_valid_cert' in attributes['metadata']:
|
||||
node_data['has_valid_cert'] = attributes['metadata']['has_valid_cert']
|
||||
|
||||
nodes.append(node_data)
|
||||
|
||||
|
||||
# Format edges for visualization
|
||||
for source, target, attributes in self.graph.edges(data=True):
|
||||
edge_data = {
|
||||
@ -261,7 +324,7 @@ class GraphManager:
|
||||
'source_provider': attributes.get('source_provider', ''),
|
||||
'discovery_timestamp': attributes.get('discovery_timestamp')
|
||||
}
|
||||
|
||||
|
||||
# Enhanced edge styling based on confidence
|
||||
confidence = attributes.get('confidence_score', 0)
|
||||
if confidence >= 0.8:
|
||||
@ -275,7 +338,7 @@ class GraphManager:
|
||||
elif confidence >= 0.6:
|
||||
edge_data['color'] = {
|
||||
'color': '#ff9900',
|
||||
'highlight': '#ffbb44',
|
||||
'highlight': '#ffbb44',
|
||||
'hover': '#ffaa22',
|
||||
'inherit': False
|
||||
}
|
||||
@ -288,13 +351,13 @@ class GraphManager:
|
||||
'inherit': False
|
||||
}
|
||||
edge_data['width'] = 2
|
||||
|
||||
|
||||
# Add dashed line for low confidence
|
||||
if confidence < 0.6:
|
||||
edge_data['dashes'] = [5, 5]
|
||||
|
||||
|
||||
edges.append(edge_data)
|
||||
|
||||
|
||||
return {
|
||||
'nodes': nodes,
|
||||
'edges': edges,
|
||||
@ -305,18 +368,18 @@ class GraphManager:
|
||||
'last_modified': self.last_modified
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def export_json(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Export complete graph data as JSON for download.
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary containing complete graph data with metadata
|
||||
"""
|
||||
#with self.lock:
|
||||
# Get basic graph data
|
||||
graph_data = self.get_graph_data()
|
||||
|
||||
|
||||
# Add comprehensive metadata
|
||||
export_data = {
|
||||
'export_metadata': {
|
||||
@ -339,13 +402,13 @@ class GraphManager:
|
||||
],
|
||||
'confidence_distribution': self._get_confidence_distribution()
|
||||
}
|
||||
|
||||
|
||||
return export_data
|
||||
|
||||
|
||||
def _get_confidence_distribution(self) -> Dict[str, int]:
|
||||
"""Get distribution of confidence scores."""
|
||||
distribution = {'high': 0, 'medium': 0, 'low': 0}
|
||||
|
||||
|
||||
for _, _, attributes in self.graph.edges(data=True):
|
||||
confidence = attributes.get('confidence_score', 0)
|
||||
if confidence >= 0.8:
|
||||
@ -354,13 +417,13 @@ class GraphManager:
|
||||
distribution['medium'] += 1
|
||||
else:
|
||||
distribution['low'] += 1
|
||||
|
||||
|
||||
return distribution
|
||||
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get comprehensive graph statistics.
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary containing various graph metrics
|
||||
"""
|
||||
@ -377,26 +440,26 @@ class GraphManager:
|
||||
'confidence_distribution': self._get_confidence_distribution(),
|
||||
'provider_distribution': {}
|
||||
}
|
||||
|
||||
|
||||
# Node type distribution
|
||||
for node_type in NodeType:
|
||||
count = len(self.get_nodes_by_type(node_type))
|
||||
stats['node_type_distribution'][node_type.value] = count
|
||||
|
||||
|
||||
# Relationship type distribution
|
||||
for _, _, attributes in self.graph.edges(data=True):
|
||||
rel_type = attributes.get('relationship_type', 'unknown')
|
||||
stats['relationship_type_distribution'][rel_type] = \
|
||||
stats['relationship_type_distribution'].get(rel_type, 0) + 1
|
||||
|
||||
|
||||
# Provider distribution
|
||||
for _, _, attributes in self.graph.edges(data=True):
|
||||
provider = attributes.get('source_provider', 'unknown')
|
||||
stats['provider_distribution'][provider] = \
|
||||
stats['provider_distribution'].get(provider, 0) + 1
|
||||
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all nodes and edges from the graph."""
|
||||
#with self.lock:
|
||||
|
||||
@ -8,6 +8,7 @@ import time
|
||||
import traceback
|
||||
from typing import List, Set, Dict, Any, Optional, Tuple
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError
|
||||
from collections import defaultdict
|
||||
|
||||
from core.graph_manager import GraphManager, NodeType, RelationshipType
|
||||
from core.logger import get_forensic_logger, new_session
|
||||
@ -334,9 +335,7 @@ class Scanner:
|
||||
print(f"Querying {len(self.providers)} providers for domain: {domain}")
|
||||
discovered_domains = set()
|
||||
discovered_ips = set()
|
||||
|
||||
# Define a threshold for creating a "large entity" node
|
||||
LARGE_ENTITY_THRESHOLD = 50
|
||||
relationships_by_type = defaultdict(list)
|
||||
|
||||
if not self.providers or self.stop_event.is_set():
|
||||
return discovered_domains, discovered_ips
|
||||
@ -355,35 +354,72 @@ class Scanner:
|
||||
relationships = future.result()
|
||||
print(f"Provider {provider.get_name()} returned {len(relationships)} relationships")
|
||||
|
||||
# Check if the number of relationships exceeds the threshold
|
||||
if len(relationships) > LARGE_ENTITY_THRESHOLD:
|
||||
# Create a single "large entity" node
|
||||
large_entity_id = f"large_entity_{provider.get_name()}_{domain}"
|
||||
self.graph.add_node(large_entity_id, NodeType.LARGE_ENTITY, metadata={'count': len(relationships), 'provider': provider.get_name()})
|
||||
self.graph.add_edge(domain, large_entity_id, RelationshipType.PASSIVE_DNS, 1.0, provider.get_name(), {})
|
||||
print(f"Created large entity node for {domain} from {provider.get_name()} with {len(relationships)} relationships")
|
||||
continue # Skip adding individual nodes
|
||||
for rel in relationships:
|
||||
relationships_by_type[rel[2]].append(rel)
|
||||
|
||||
for source, target, rel_type, confidence, raw_data in relationships:
|
||||
if self._is_valid_ip(target):
|
||||
target_node_type = NodeType.IP
|
||||
discovered_ips.add(target)
|
||||
elif self._is_valid_domain(target):
|
||||
target_node_type = NodeType.DOMAIN
|
||||
discovered_domains.add(target)
|
||||
else:
|
||||
target_node_type = NodeType.ASN if target.startswith('AS') else NodeType.CERTIFICATE
|
||||
|
||||
self.graph.add_node(source, NodeType.DOMAIN)
|
||||
self.graph.add_node(target, target_node_type)
|
||||
if self.graph.add_edge(source, target, rel_type, confidence, provider.get_name(), raw_data):
|
||||
print(f"Added relationship: {source} -> {target} ({rel_type.relationship_name})")
|
||||
except (Exception, CancelledError) as e:
|
||||
print(f"Provider {provider.get_name()} failed for {domain}: {e}")
|
||||
|
||||
for rel_type, relationships in relationships_by_type.items():
|
||||
if len(relationships) > config.large_entity_threshold and rel_type == RelationshipType.SAN_CERTIFICATE:
|
||||
self._handle_large_entity(domain, relationships, rel_type, provider.get_name())
|
||||
else:
|
||||
for source, target, rel_type, confidence, raw_data in relationships:
|
||||
# Determine if the target should create a new node
|
||||
create_node = rel_type in [
|
||||
RelationshipType.A_RECORD,
|
||||
RelationshipType.AAAA_RECORD,
|
||||
RelationshipType.CNAME_RECORD,
|
||||
RelationshipType.MX_RECORD,
|
||||
RelationshipType.NS_RECORD,
|
||||
RelationshipType.PTR_RECORD,
|
||||
RelationshipType.SAN_CERTIFICATE
|
||||
]
|
||||
|
||||
# Determine if the target should be subject to recursion
|
||||
recurse = rel_type in [
|
||||
RelationshipType.A_RECORD,
|
||||
RelationshipType.AAAA_RECORD,
|
||||
RelationshipType.CNAME_RECORD,
|
||||
RelationshipType.MX_RECORD,
|
||||
RelationshipType.SAN_CERTIFICATE
|
||||
]
|
||||
|
||||
if create_node:
|
||||
target_node_type = NodeType.IP if self._is_valid_ip(target) else NodeType.DOMAIN
|
||||
self.graph.add_node(target, target_node_type)
|
||||
if self.graph.add_edge(source, target, rel_type, confidence, provider.get_name(), raw_data):
|
||||
print(f"Added relationship: {source} -> {target} ({rel_type.relationship_name})")
|
||||
else:
|
||||
# For records that don't create nodes, we still want to log the relationship
|
||||
self.logger.log_relationship_discovery(
|
||||
source_node=source,
|
||||
target_node=target,
|
||||
relationship_type=rel_type.relationship_name,
|
||||
confidence_score=confidence,
|
||||
provider=provider.name,
|
||||
raw_data=raw_data,
|
||||
discovery_method=f"dns_{rel_type.name.lower()}_record"
|
||||
)
|
||||
|
||||
if recurse:
|
||||
if self._is_valid_ip(target):
|
||||
discovered_ips.add(target)
|
||||
elif self._is_valid_domain(target):
|
||||
discovered_domains.add(target)
|
||||
|
||||
print(f"Domain {domain}: discovered {len(discovered_domains)} domains, {len(discovered_ips)} IPs")
|
||||
return discovered_domains, discovered_ips
|
||||
|
||||
def _handle_large_entity(self, source_domain: str, relationships: list, rel_type: RelationshipType, provider_name: str):
|
||||
"""
|
||||
Handles the creation of a large entity node when a threshold is exceeded.
|
||||
"""
|
||||
print(f"Large number of {rel_type.name} relationships for {source_domain}. Creating a large entity node.")
|
||||
entity_name = f"Large collection of {rel_type.name} for {source_domain}"
|
||||
self.graph.add_node(entity_name, NodeType.LARGE_ENTITY, metadata={"count": len(relationships)})
|
||||
self.graph.add_edge(source_domain, entity_name, rel_type, 0.9, provider_name, {"info": "Aggregated node"})
|
||||
|
||||
def _query_providers_for_ip(self, ip: str) -> None:
|
||||
"""
|
||||
Query all enabled providers for information about an IP address.
|
||||
|
||||
@ -13,7 +13,7 @@ class DNSProvider(BaseProvider):
|
||||
Provider for standard DNS resolution and reverse DNS lookups.
|
||||
Discovers domain-to-IP and IP-to-domain relationships through DNS records.
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize DNS provider with appropriate rate limiting."""
|
||||
super().__init__(
|
||||
@ -21,77 +21,66 @@ class DNSProvider(BaseProvider):
|
||||
rate_limit=100, # DNS queries can be faster
|
||||
timeout=10
|
||||
)
|
||||
|
||||
|
||||
# Configure DNS resolver
|
||||
self.resolver = dns.resolver.Resolver()
|
||||
self.resolver.timeout = 5
|
||||
self.resolver.lifetime = 10
|
||||
|
||||
|
||||
def get_name(self) -> str:
|
||||
"""Return the provider name."""
|
||||
return "dns"
|
||||
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""DNS is always available - no API key required."""
|
||||
return True
|
||||
|
||||
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query DNS records for the domain to discover relationships.
|
||||
|
||||
|
||||
Args:
|
||||
domain: Domain to investigate
|
||||
|
||||
|
||||
Returns:
|
||||
List of relationships discovered from DNS analysis
|
||||
"""
|
||||
if not self._is_valid_domain(domain):
|
||||
return []
|
||||
|
||||
|
||||
relationships = []
|
||||
|
||||
# Query A records
|
||||
relationships.extend(self._query_a_records(domain))
|
||||
|
||||
# Query AAAA records (IPv6)
|
||||
relationships.extend(self._query_aaaa_records(domain))
|
||||
|
||||
# Query CNAME records
|
||||
relationships.extend(self._query_cname_records(domain))
|
||||
|
||||
# Query MX records
|
||||
relationships.extend(self._query_mx_records(domain))
|
||||
|
||||
# Query NS records
|
||||
relationships.extend(self._query_ns_records(domain))
|
||||
|
||||
|
||||
# Query all record types
|
||||
for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA', 'DNSKEY', 'DS', 'RRSIG', 'SSHFP', 'TLSA', 'NAPTR', 'SPF']:
|
||||
relationships.extend(self._query_record(domain, record_type))
|
||||
|
||||
return relationships
|
||||
|
||||
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query reverse DNS for the IP address.
|
||||
|
||||
|
||||
Args:
|
||||
ip: IP address to investigate
|
||||
|
||||
|
||||
Returns:
|
||||
List of relationships discovered from reverse DNS
|
||||
"""
|
||||
if not self._is_valid_ip(ip):
|
||||
return []
|
||||
|
||||
|
||||
relationships = []
|
||||
|
||||
|
||||
try:
|
||||
# Perform reverse DNS lookup
|
||||
self.total_requests += 1
|
||||
reverse_name = dns.reversename.from_address(ip)
|
||||
response = self.resolver.resolve(reverse_name, 'PTR')
|
||||
self.successful_requests += 1
|
||||
|
||||
|
||||
for ptr_record in response:
|
||||
hostname = str(ptr_record).rstrip('.')
|
||||
|
||||
|
||||
if self._is_valid_domain(hostname):
|
||||
raw_data = {
|
||||
'query_type': 'PTR',
|
||||
@ -99,255 +88,90 @@ class DNSProvider(BaseProvider):
|
||||
'hostname': hostname,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
|
||||
relationships.append((
|
||||
ip,
|
||||
hostname,
|
||||
RelationshipType.A_RECORD, # Reverse relationship
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
RelationshipType.PTR_RECORD,
|
||||
RelationshipType.PTR_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
relationship_type=RelationshipType.PTR_RECORD,
|
||||
confidence_score=RelationshipType.PTR_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="reverse_dns_lookup"
|
||||
)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
self.failed_requests += 1
|
||||
self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}")
|
||||
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_a_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query A records for the domain."""
|
||||
|
||||
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query a specific type of DNS record for the domain.
|
||||
"""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
self.total_requests += 1
|
||||
response = self.resolver.resolve(domain, 'A')
|
||||
response = self.resolver.resolve(domain, record_type)
|
||||
self.successful_requests += 1
|
||||
|
||||
for a_record in response:
|
||||
ip_address = str(a_record)
|
||||
|
||||
raw_data = {
|
||||
'query_type': 'A',
|
||||
'domain': domain,
|
||||
'ip_address': ip_address,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
ip_address,
|
||||
RelationshipType.A_RECORD,
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ip_address,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_a_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.failed_requests += 1
|
||||
self.logger.logger.debug(f"A record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_aaaa_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query AAAA records (IPv6) for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
self.total_requests += 1
|
||||
response = self.resolver.resolve(domain, 'AAAA')
|
||||
self.successful_requests += 1
|
||||
|
||||
for aaaa_record in response:
|
||||
ip_address = str(aaaa_record)
|
||||
|
||||
raw_data = {
|
||||
'query_type': 'AAAA',
|
||||
'domain': domain,
|
||||
'ip_address': ip_address,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
ip_address,
|
||||
RelationshipType.A_RECORD, # Using same type for IPv6
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ip_address,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_aaaa_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.failed_requests += 1
|
||||
self.logger.logger.debug(f"AAAA record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_cname_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query CNAME records for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
self.total_requests += 1
|
||||
response = self.resolver.resolve(domain, 'CNAME')
|
||||
self.successful_requests += 1
|
||||
|
||||
for cname_record in response:
|
||||
target_domain = str(cname_record).rstrip('.')
|
||||
|
||||
if self._is_valid_domain(target_domain):
|
||||
|
||||
for record in response:
|
||||
target = ""
|
||||
if record_type in ['A', 'AAAA']:
|
||||
target = str(record)
|
||||
elif record_type in ['CNAME', 'NS', 'PTR']:
|
||||
target = str(record.target).rstrip('.')
|
||||
elif record_type == 'MX':
|
||||
target = str(record.exchange).rstrip('.')
|
||||
elif record_type == 'SOA':
|
||||
target = str(record.mname).rstrip('.')
|
||||
elif record_type in ['TXT', 'SPF']:
|
||||
target = b' '.join(record.strings).decode('utf-8', 'ignore')
|
||||
elif record_type == 'SRV':
|
||||
target = str(record.target).rstrip('.')
|
||||
elif record_type == 'CAA':
|
||||
target = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
|
||||
else:
|
||||
target = str(record)
|
||||
|
||||
|
||||
if target:
|
||||
raw_data = {
|
||||
'query_type': 'CNAME',
|
||||
'source_domain': domain,
|
||||
'target_domain': target_domain,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
target_domain,
|
||||
RelationshipType.CNAME_RECORD,
|
||||
RelationshipType.CNAME_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=target_domain,
|
||||
relationship_type=RelationshipType.CNAME_RECORD,
|
||||
confidence_score=RelationshipType.CNAME_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_cname_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.failed_requests += 1
|
||||
self.logger.logger.debug(f"CNAME record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_mx_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query MX records for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
self.total_requests += 1
|
||||
response = self.resolver.resolve(domain, 'MX')
|
||||
self.successful_requests += 1
|
||||
|
||||
for mx_record in response:
|
||||
mx_host = str(mx_record.exchange).rstrip('.')
|
||||
|
||||
if self._is_valid_domain(mx_host):
|
||||
raw_data = {
|
||||
'query_type': 'MX',
|
||||
'query_type': record_type,
|
||||
'domain': domain,
|
||||
'mx_host': mx_host,
|
||||
'priority': mx_record.preference,
|
||||
'value': target,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
mx_host,
|
||||
RelationshipType.MX_RECORD,
|
||||
RelationshipType.MX_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=mx_host,
|
||||
relationship_type=RelationshipType.MX_RECORD,
|
||||
confidence_score=RelationshipType.MX_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_mx_record"
|
||||
)
|
||||
|
||||
try:
|
||||
relationship_type_enum = getattr(RelationshipType, f"{record_type}_RECORD")
|
||||
relationships.append((
|
||||
domain,
|
||||
target,
|
||||
relationship_type_enum,
|
||||
relationship_type_enum.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=target,
|
||||
relationship_type=relationship_type_enum,
|
||||
confidence_score=relationship_type_enum.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method=f"dns_{record_type.lower()}_record"
|
||||
)
|
||||
except AttributeError:
|
||||
self.logger.logger.error(f"Unsupported record type '{record_type}' encountered for domain {domain}")
|
||||
|
||||
except Exception as e:
|
||||
self.failed_requests += 1
|
||||
self.logger.logger.debug(f"MX record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_ns_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query NS records for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
self.total_requests += 1
|
||||
response = self.resolver.resolve(domain, 'NS')
|
||||
self.successful_requests += 1
|
||||
|
||||
for ns_record in response:
|
||||
ns_host = str(ns_record).rstrip('.')
|
||||
|
||||
if self._is_valid_domain(ns_host):
|
||||
raw_data = {
|
||||
'query_type': 'NS',
|
||||
'domain': domain,
|
||||
'ns_host': ns_host,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
ns_host,
|
||||
RelationshipType.NS_RECORD,
|
||||
RelationshipType.NS_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ns_host,
|
||||
relationship_type=RelationshipType.NS_RECORD,
|
||||
confidence_score=RelationshipType.NS_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_ns_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.failed_requests += 1
|
||||
self.logger.logger.debug(f"NS record query failed for {domain}: {e}")
|
||||
|
||||
self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
@ -653,6 +653,7 @@ input[type="text"]:focus, select:focus {
|
||||
.detail-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-bottom: 0.75rem;
|
||||
padding-bottom: 0.25rem;
|
||||
border-bottom: 1px solid #333;
|
||||
@ -668,6 +669,23 @@ input[type="text"]:focus, select:focus {
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.copy-btn {
|
||||
background: none;
|
||||
border: none;
|
||||
color: #666;
|
||||
cursor: pointer;
|
||||
font-size: 1rem;
|
||||
margin-left: 10px;
|
||||
}
|
||||
|
||||
.copy-btn:hover {
|
||||
color: #00ff41;
|
||||
}
|
||||
|
||||
.status-icon {
|
||||
margin-left: 5px;
|
||||
}
|
||||
|
||||
/* Responsive Design */
|
||||
@media (max-width: 768px) {
|
||||
.main-content {
|
||||
|
||||
@ -724,29 +724,82 @@ class DNSReconApp {
|
||||
*/
|
||||
showNodeModal(nodeId, node) {
|
||||
if (!this.elements.nodeModal) return;
|
||||
|
||||
|
||||
if (this.elements.modalTitle) {
|
||||
this.elements.modalTitle.textContent = `Node Details: ${nodeId}`;
|
||||
}
|
||||
|
||||
|
||||
let detailsHtml = '';
|
||||
detailsHtml += `<div class="detail-row"><span class="detail-label">Identifier:</span><span class="detail-value">${nodeId}</span></div>`;
|
||||
detailsHtml += `<div class="detail-row"><span class="detail-label">Type:</span><span class="detail-value">${node.metadata.type || node.type || 'Unknown'}</span></div>`;
|
||||
|
||||
if (node.metadata) {
|
||||
for (const [key, value] of Object.entries(node.metadata)) {
|
||||
if (key !== 'type') {
|
||||
detailsHtml += `<div class="detail-row"><span class="detail-label">${this.formatLabel(key)}:</span><span class="detail-value">${this.formatValue(value)}</span></div>`;
|
||||
}
|
||||
const createDetailRow = (label, value) => {
|
||||
const baseId = `detail-${label.replace(/[^a-zA-Z0-9]/g, '-')}`;
|
||||
|
||||
// Handle empty or undefined values by showing N/A
|
||||
if (value === null || value === undefined || (Array.isArray(value) && value.length === 0)) {
|
||||
return `
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">${label} <span class="status-icon text-warning">✗</span></span>
|
||||
<span class="detail-value">N/A</span>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
// If value is an array, create a row for each item
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item, index) => {
|
||||
const itemId = `${baseId}-${index}`;
|
||||
// Only show the label for the first item in the list
|
||||
const itemLabel = index === 0 ? label : '';
|
||||
return `
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">${itemLabel}</span>
|
||||
<span class="detail-value" id="${itemId}">${this.formatValue(item)}</span>
|
||||
<button class="copy-btn" onclick="copyToClipboard('${itemId}')" title="Copy">📋</button>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
// Handle objects and other primitive values in a single row
|
||||
else {
|
||||
const valueId = `${baseId}-0`;
|
||||
return `
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">${label} <span class="status-icon text-success">✓</span></span>
|
||||
<span class="detail-value" id="${valueId}">${this.formatValue(value)}</span>
|
||||
<button class="copy-btn" onclick="copyToClipboard('${valueId}')" title="Copy">📋</button>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
};
|
||||
|
||||
const metadata = node.metadata || {};
|
||||
|
||||
switch (node.type) {
|
||||
case 'domain':
|
||||
detailsHtml += createDetailRow('DNS Records', metadata.dns_records);
|
||||
detailsHtml += createDetailRow('Related Domains (SAN)', metadata.related_domains_san);
|
||||
detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
|
||||
detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
|
||||
break;
|
||||
case 'ip':
|
||||
detailsHtml += createDetailRow('DNS Records', metadata.dns_records);
|
||||
detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
|
||||
detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
|
||||
break;
|
||||
case 'certificate':
|
||||
detailsHtml += createDetailRow('Certificate Hash', metadata.hash);
|
||||
detailsHtml += createDetailRow('SANs', metadata.sans);
|
||||
detailsHtml += createDetailRow('Issuer', metadata.issuer);
|
||||
detailsHtml += createDetailRow('Validity', `From: ${metadata.not_before || 'N/A'} To: ${metadata.not_after || 'N/A'}`);
|
||||
break;
|
||||
case 'asn':
|
||||
detailsHtml += createDetailRow('ASN', metadata.asn);
|
||||
detailsHtml += createDetailRow('Description', metadata.description);
|
||||
break;
|
||||
case 'large_entity':
|
||||
detailsHtml += createDetailRow('Discovered Domains', metadata.domains);
|
||||
break;
|
||||
}
|
||||
|
||||
// Add timestamps if available
|
||||
if (node.added_timestamp) {
|
||||
const addedDate = new Date(node.added_timestamp);
|
||||
detailsHtml += `<div class="detail-row"><span class="detail-label">Added:</span><span class="detail-value">${addedDate.toLocaleString()}</span></div>`;
|
||||
}
|
||||
|
||||
|
||||
if (this.elements.modalDetails) {
|
||||
this.elements.modalDetails.innerHTML = detailsHtml;
|
||||
}
|
||||
@ -982,12 +1035,13 @@ class DNSReconApp {
|
||||
* @returns {string} Formatted value
|
||||
*/
|
||||
formatValue(value) {
|
||||
if (Array.isArray(value)) {
|
||||
return value.join(', ');
|
||||
} else if (typeof value === 'object') {
|
||||
return JSON.stringify(value, null, 2);
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
// Use <pre> for nicely formatted JSON
|
||||
return `<pre>${JSON.stringify(value, null, 2)}</pre>`;
|
||||
} else {
|
||||
return String(value);
|
||||
// Escape HTML to prevent XSS issues with string values
|
||||
const strValue = String(value);
|
||||
return strValue.replace(/</g, "<").replace(/>/g, ">");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -218,6 +218,18 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function copyToClipboard(elementId) {
|
||||
const element = document.getElementById(elementId);
|
||||
const textToCopy = element.innerText;
|
||||
navigator.clipboard.writeText(textToCopy).then(() => {
|
||||
// Optional: Show a success message
|
||||
console.log('Copied to clipboard');
|
||||
}).catch(err => {
|
||||
console.error('Failed to copy: ', err);
|
||||
});
|
||||
}
|
||||
</script>
|
||||
<script src="{{ url_for('static', filename='js/graph.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/main.js') }}"></script>
|
||||
</body>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user