This commit is contained in:
overcuriousity 2025-09-11 00:00:00 +02:00
parent db2101d814
commit 2d485c5703
7 changed files with 373 additions and 365 deletions

View File

@ -21,6 +21,7 @@ class Config:
self.default_recursion_depth = 2
self.default_timeout = 30
self.max_concurrent_requests = 5
self.large_entity_threshold = 100
# Rate limiting settings (requests per minute)
self.rate_limits = {

View File

@ -9,6 +9,7 @@ from datetime import datetime
from typing import Dict, List, Any, Optional, Tuple, Set
from enum import Enum
from datetime import timezone
from collections import defaultdict
import networkx as nx
@ -24,13 +25,27 @@ class NodeType(Enum):
class RelationshipType(Enum):
"""Enumeration of supported relationship types with confidence scores."""
SAN_CERTIFICATE = ("san", 0.9) # Certificate SAN relationships
A_RECORD = ("a_record", 0.8) # A/AAAA record relationships
CNAME_RECORD = ("cname", 0.8) # CNAME relationships
PASSIVE_DNS = ("passive_dns", 0.6) # Passive DNS relationships
ASN_MEMBERSHIP = ("asn", 0.7) # ASN relationships
MX_RECORD = ("mx_record", 0.7) # MX record relationships
NS_RECORD = ("ns_record", 0.7) # NS record relationships
SAN_CERTIFICATE = ("san", 0.9)
A_RECORD = ("a_record", 0.8)
AAAA_RECORD = ("aaaa_record", 0.8)
CNAME_RECORD = ("cname", 0.8)
MX_RECORD = ("mx_record", 0.7)
NS_RECORD = ("ns_record", 0.7)
PTR_RECORD = ("ptr_record", 0.8)
SOA_RECORD = ("soa_record", 0.7)
TXT_RECORD = ("txt_record", 0.7)
SRV_RECORD = ("srv_record", 0.7)
CAA_RECORD = ("caa_record", 0.7)
DNSKEY_RECORD = ("dnskey_record", 0.7)
DS_RECORD = ("ds_record", 0.7)
RRSIG_RECORD = ("rrsig_record", 0.7)
SSHFP_RECORD = ("sshfp_record", 0.7)
TLSA_RECORD = ("tlsa_record", 0.7)
NAPTR_RECORD = ("naptr_record", 0.7)
SPF_RECORD = ("spf_record", 0.7)
PASSIVE_DNS = ("passive_dns", 0.6)
ASN_MEMBERSHIP = ("asn", 0.7)
def __init__(self, relationship_name: str, default_confidence: float):
self.relationship_name = relationship_name
@ -204,6 +219,37 @@ class GraphManager:
nodes = []
edges = []
# Create a dictionary to hold aggregated data for each node
node_details = defaultdict(lambda: defaultdict(list))
for source, target, attributes in self.graph.edges(data=True):
provider = attributes.get('source_provider', 'unknown')
raw_data = attributes.get('raw_data', {})
if provider == 'dns':
record_type = raw_data.get('query_type', 'UNKNOWN')
value = raw_data.get('value', target)
# DNS data is always about the source node of the query
node_details[source]['dns_records'].append(f"{record_type}: {value}")
elif provider == 'crtsh':
# Data from crt.sh are domain names found in certificates (SANs)
node_details[source]['related_domains_san'].append(target)
elif provider == 'shodan':
# Shodan data is about the IP, which can be either the source or target
source_node_type = self.graph.nodes[source].get('type')
target_node_type = self.graph.nodes[target].get('type')
if source_node_type == 'ip':
node_details[source]['shodan'] = raw_data
elif target_node_type == 'ip':
node_details[target]['shodan'] = raw_data
elif provider == 'virustotal':
# VirusTotal data is about the source node of the query
node_details[source]['virustotal'] = raw_data
# Format nodes for visualization
for node_id, attributes in self.graph.nodes(data=True):
node_data = {
@ -214,6 +260,17 @@ class GraphManager:
'added_timestamp': attributes.get('added_timestamp')
}
# Add the aggregated details to the metadata
if node_id in node_details:
for key, value in node_details[node_id].items():
# Use a set to avoid adding duplicate entries to lists
if key in node_data['metadata'] and isinstance(node_data['metadata'][key], list):
existing_values = set(node_data['metadata'][key])
new_values = [v for v in value if v not in existing_values]
node_data['metadata'][key].extend(new_values)
else:
node_data['metadata'][key] = value
# Color coding by type - now returns color objects for enhanced visualization
type_colors = {
'domain': {
@ -239,6 +296,12 @@ class GraphManager:
'border': '#0088cc',
'highlight': {'background': '#44ccff', 'border': '#00aaff'},
'hover': {'background': '#22bbff', 'border': '#0099dd'}
},
'large_entity': {
'background': '#ff6b6b',
'border': '#cc3a3a',
'highlight': {'background': '#ff8c8c', 'border': '#ff6b6b'},
'hover': {'background': '#ff7a7a', 'border': '#dd4a4a'}
}
}

View File

@ -8,6 +8,7 @@ import time
import traceback
from typing import List, Set, Dict, Any, Optional, Tuple
from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError
from collections import defaultdict
from core.graph_manager import GraphManager, NodeType, RelationshipType
from core.logger import get_forensic_logger, new_session
@ -334,9 +335,7 @@ class Scanner:
print(f"Querying {len(self.providers)} providers for domain: {domain}")
discovered_domains = set()
discovered_ips = set()
# Define a threshold for creating a "large entity" node
LARGE_ENTITY_THRESHOLD = 50
relationships_by_type = defaultdict(list)
if not self.providers or self.stop_event.is_set():
return discovered_domains, discovered_ips
@ -355,35 +354,72 @@ class Scanner:
relationships = future.result()
print(f"Provider {provider.get_name()} returned {len(relationships)} relationships")
# Check if the number of relationships exceeds the threshold
if len(relationships) > LARGE_ENTITY_THRESHOLD:
# Create a single "large entity" node
large_entity_id = f"large_entity_{provider.get_name()}_{domain}"
self.graph.add_node(large_entity_id, NodeType.LARGE_ENTITY, metadata={'count': len(relationships), 'provider': provider.get_name()})
self.graph.add_edge(domain, large_entity_id, RelationshipType.PASSIVE_DNS, 1.0, provider.get_name(), {})
print(f"Created large entity node for {domain} from {provider.get_name()} with {len(relationships)} relationships")
continue # Skip adding individual nodes
for rel in relationships:
relationships_by_type[rel[2]].append(rel)
for source, target, rel_type, confidence, raw_data in relationships:
if self._is_valid_ip(target):
target_node_type = NodeType.IP
discovered_ips.add(target)
elif self._is_valid_domain(target):
target_node_type = NodeType.DOMAIN
discovered_domains.add(target)
else:
target_node_type = NodeType.ASN if target.startswith('AS') else NodeType.CERTIFICATE
self.graph.add_node(source, NodeType.DOMAIN)
self.graph.add_node(target, target_node_type)
if self.graph.add_edge(source, target, rel_type, confidence, provider.get_name(), raw_data):
print(f"Added relationship: {source} -> {target} ({rel_type.relationship_name})")
except (Exception, CancelledError) as e:
print(f"Provider {provider.get_name()} failed for {domain}: {e}")
for rel_type, relationships in relationships_by_type.items():
if len(relationships) > config.large_entity_threshold and rel_type == RelationshipType.SAN_CERTIFICATE:
self._handle_large_entity(domain, relationships, rel_type, provider.get_name())
else:
for source, target, rel_type, confidence, raw_data in relationships:
# Determine if the target should create a new node
create_node = rel_type in [
RelationshipType.A_RECORD,
RelationshipType.AAAA_RECORD,
RelationshipType.CNAME_RECORD,
RelationshipType.MX_RECORD,
RelationshipType.NS_RECORD,
RelationshipType.PTR_RECORD,
RelationshipType.SAN_CERTIFICATE
]
# Determine if the target should be subject to recursion
recurse = rel_type in [
RelationshipType.A_RECORD,
RelationshipType.AAAA_RECORD,
RelationshipType.CNAME_RECORD,
RelationshipType.MX_RECORD,
RelationshipType.SAN_CERTIFICATE
]
if create_node:
target_node_type = NodeType.IP if self._is_valid_ip(target) else NodeType.DOMAIN
self.graph.add_node(target, target_node_type)
if self.graph.add_edge(source, target, rel_type, confidence, provider.get_name(), raw_data):
print(f"Added relationship: {source} -> {target} ({rel_type.relationship_name})")
else:
# For records that don't create nodes, we still want to log the relationship
self.logger.log_relationship_discovery(
source_node=source,
target_node=target,
relationship_type=rel_type.relationship_name,
confidence_score=confidence,
provider=provider.name,
raw_data=raw_data,
discovery_method=f"dns_{rel_type.name.lower()}_record"
)
if recurse:
if self._is_valid_ip(target):
discovered_ips.add(target)
elif self._is_valid_domain(target):
discovered_domains.add(target)
print(f"Domain {domain}: discovered {len(discovered_domains)} domains, {len(discovered_ips)} IPs")
return discovered_domains, discovered_ips
def _handle_large_entity(self, source_domain: str, relationships: list, rel_type: RelationshipType, provider_name: str):
"""
Handles the creation of a large entity node when a threshold is exceeded.
"""
print(f"Large number of {rel_type.name} relationships for {source_domain}. Creating a large entity node.")
entity_name = f"Large collection of {rel_type.name} for {source_domain}"
self.graph.add_node(entity_name, NodeType.LARGE_ENTITY, metadata={"count": len(relationships)})
self.graph.add_edge(source_domain, entity_name, rel_type, 0.9, provider_name, {"info": "Aggregated node"})
def _query_providers_for_ip(self, ip: str) -> None:
"""
Query all enabled providers for information about an IP address.

View File

@ -50,20 +50,9 @@ class DNSProvider(BaseProvider):
relationships = []
# Query A records
relationships.extend(self._query_a_records(domain))
# Query AAAA records (IPv6)
relationships.extend(self._query_aaaa_records(domain))
# Query CNAME records
relationships.extend(self._query_cname_records(domain))
# Query MX records
relationships.extend(self._query_mx_records(domain))
# Query NS records
relationships.extend(self._query_ns_records(domain))
# Query all record types
for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA', 'DNSKEY', 'DS', 'RRSIG', 'SSHFP', 'TLSA', 'NAPTR', 'SPF']:
relationships.extend(self._query_record(domain, record_type))
return relationships
@ -103,16 +92,16 @@ class DNSProvider(BaseProvider):
relationships.append((
ip,
hostname,
RelationshipType.A_RECORD, # Reverse relationship
RelationshipType.A_RECORD.default_confidence,
RelationshipType.PTR_RECORD,
RelationshipType.PTR_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
relationship_type=RelationshipType.PTR_RECORD,
confidence_score=RelationshipType.PTR_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="reverse_dns_lookup"
)
@ -123,231 +112,66 @@ class DNSProvider(BaseProvider):
return relationships
def _query_a_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query A records for the domain."""
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query a specific type of DNS record for the domain.
"""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
self.total_requests += 1
response = self.resolver.resolve(domain, 'A')
response = self.resolver.resolve(domain, record_type)
self.successful_requests += 1
for a_record in response:
ip_address = str(a_record)
for record in response:
target = ""
if record_type in ['A', 'AAAA']:
target = str(record)
elif record_type in ['CNAME', 'NS', 'PTR']:
target = str(record.target).rstrip('.')
elif record_type == 'MX':
target = str(record.exchange).rstrip('.')
elif record_type == 'SOA':
target = str(record.mname).rstrip('.')
elif record_type in ['TXT', 'SPF']:
target = b' '.join(record.strings).decode('utf-8', 'ignore')
elif record_type == 'SRV':
target = str(record.target).rstrip('.')
elif record_type == 'CAA':
target = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
else:
target = str(record)
raw_data = {
'query_type': 'A',
'domain': domain,
'ip_address': ip_address,
'ttl': response.ttl
}
relationships.append((
domain,
ip_address,
RelationshipType.A_RECORD,
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ip_address,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_a_record"
)
except Exception as e:
self.failed_requests += 1
self.logger.logger.debug(f"A record query failed for {domain}: {e}")
return relationships
def _query_aaaa_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query AAAA records (IPv6) for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
self.total_requests += 1
response = self.resolver.resolve(domain, 'AAAA')
self.successful_requests += 1
for aaaa_record in response:
ip_address = str(aaaa_record)
raw_data = {
'query_type': 'AAAA',
'domain': domain,
'ip_address': ip_address,
'ttl': response.ttl
}
relationships.append((
domain,
ip_address,
RelationshipType.A_RECORD, # Using same type for IPv6
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ip_address,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_aaaa_record"
)
except Exception as e:
self.failed_requests += 1
self.logger.logger.debug(f"AAAA record query failed for {domain}: {e}")
return relationships
def _query_cname_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query CNAME records for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
self.total_requests += 1
response = self.resolver.resolve(domain, 'CNAME')
self.successful_requests += 1
for cname_record in response:
target_domain = str(cname_record).rstrip('.')
if self._is_valid_domain(target_domain):
if target:
raw_data = {
'query_type': 'CNAME',
'source_domain': domain,
'target_domain': target_domain,
'ttl': response.ttl
}
relationships.append((
domain,
target_domain,
RelationshipType.CNAME_RECORD,
RelationshipType.CNAME_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=target_domain,
relationship_type=RelationshipType.CNAME_RECORD,
confidence_score=RelationshipType.CNAME_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_cname_record"
)
except Exception as e:
self.failed_requests += 1
self.logger.logger.debug(f"CNAME record query failed for {domain}: {e}")
return relationships
def _query_mx_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query MX records for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
self.total_requests += 1
response = self.resolver.resolve(domain, 'MX')
self.successful_requests += 1
for mx_record in response:
mx_host = str(mx_record.exchange).rstrip('.')
if self._is_valid_domain(mx_host):
raw_data = {
'query_type': 'MX',
'query_type': record_type,
'domain': domain,
'mx_host': mx_host,
'priority': mx_record.preference,
'value': target,
'ttl': response.ttl
}
try:
relationship_type_enum = getattr(RelationshipType, f"{record_type}_RECORD")
relationships.append((
domain,
target,
relationship_type_enum,
relationship_type_enum.default_confidence,
raw_data
))
relationships.append((
domain,
mx_host,
RelationshipType.MX_RECORD,
RelationshipType.MX_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=mx_host,
relationship_type=RelationshipType.MX_RECORD,
confidence_score=RelationshipType.MX_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_mx_record"
)
self.log_relationship_discovery(
source_node=domain,
target_node=target,
relationship_type=relationship_type_enum,
confidence_score=relationship_type_enum.default_confidence,
raw_data=raw_data,
discovery_method=f"dns_{record_type.lower()}_record"
)
except AttributeError:
self.logger.logger.error(f"Unsupported record type '{record_type}' encountered for domain {domain}")
except Exception as e:
self.failed_requests += 1
self.logger.logger.debug(f"MX record query failed for {domain}: {e}")
return relationships
def _query_ns_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query NS records for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
self.total_requests += 1
response = self.resolver.resolve(domain, 'NS')
self.successful_requests += 1
for ns_record in response:
ns_host = str(ns_record).rstrip('.')
if self._is_valid_domain(ns_host):
raw_data = {
'query_type': 'NS',
'domain': domain,
'ns_host': ns_host,
'ttl': response.ttl
}
relationships.append((
domain,
ns_host,
RelationshipType.NS_RECORD,
RelationshipType.NS_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ns_host,
relationship_type=RelationshipType.NS_RECORD,
confidence_score=RelationshipType.NS_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_ns_record"
)
except Exception as e:
self.failed_requests += 1
self.logger.logger.debug(f"NS record query failed for {domain}: {e}")
self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
return relationships

View File

@ -653,6 +653,7 @@ input[type="text"]:focus, select:focus {
.detail-row {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.75rem;
padding-bottom: 0.25rem;
border-bottom: 1px solid #333;
@ -668,6 +669,23 @@ input[type="text"]:focus, select:focus {
word-break: break-word;
}
.copy-btn {
background: none;
border: none;
color: #666;
cursor: pointer;
font-size: 1rem;
margin-left: 10px;
}
.copy-btn:hover {
color: #00ff41;
}
.status-icon {
margin-left: 5px;
}
/* Responsive Design */
@media (max-width: 768px) {
.main-content {

View File

@ -730,21 +730,74 @@ class DNSReconApp {
}
let detailsHtml = '';
detailsHtml += `<div class="detail-row"><span class="detail-label">Identifier:</span><span class="detail-value">${nodeId}</span></div>`;
detailsHtml += `<div class="detail-row"><span class="detail-label">Type:</span><span class="detail-value">${node.metadata.type || node.type || 'Unknown'}</span></div>`;
const createDetailRow = (label, value) => {
const baseId = `detail-${label.replace(/[^a-zA-Z0-9]/g, '-')}`;
if (node.metadata) {
for (const [key, value] of Object.entries(node.metadata)) {
if (key !== 'type') {
detailsHtml += `<div class="detail-row"><span class="detail-label">${this.formatLabel(key)}:</span><span class="detail-value">${this.formatValue(value)}</span></div>`;
}
// Handle empty or undefined values by showing N/A
if (value === null || value === undefined || (Array.isArray(value) && value.length === 0)) {
return `
<div class="detail-row">
<span class="detail-label">${label} <span class="status-icon text-warning"></span></span>
<span class="detail-value">N/A</span>
</div>
`;
}
}
// Add timestamps if available
if (node.added_timestamp) {
const addedDate = new Date(node.added_timestamp);
detailsHtml += `<div class="detail-row"><span class="detail-label">Added:</span><span class="detail-value">${addedDate.toLocaleString()}</span></div>`;
// If value is an array, create a row for each item
if (Array.isArray(value)) {
return value.map((item, index) => {
const itemId = `${baseId}-${index}`;
// Only show the label for the first item in the list
const itemLabel = index === 0 ? label : '';
return `
<div class="detail-row">
<span class="detail-label">${itemLabel}</span>
<span class="detail-value" id="${itemId}">${this.formatValue(item)}</span>
<button class="copy-btn" onclick="copyToClipboard('${itemId}')" title="Copy">📋</button>
</div>
`;
}).join('');
}
// Handle objects and other primitive values in a single row
else {
const valueId = `${baseId}-0`;
return `
<div class="detail-row">
<span class="detail-label">${label} <span class="status-icon text-success"></span></span>
<span class="detail-value" id="${valueId}">${this.formatValue(value)}</span>
<button class="copy-btn" onclick="copyToClipboard('${valueId}')" title="Copy">📋</button>
</div>
`;
}
};
const metadata = node.metadata || {};
switch (node.type) {
case 'domain':
detailsHtml += createDetailRow('DNS Records', metadata.dns_records);
detailsHtml += createDetailRow('Related Domains (SAN)', metadata.related_domains_san);
detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
break;
case 'ip':
detailsHtml += createDetailRow('DNS Records', metadata.dns_records);
detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
break;
case 'certificate':
detailsHtml += createDetailRow('Certificate Hash', metadata.hash);
detailsHtml += createDetailRow('SANs', metadata.sans);
detailsHtml += createDetailRow('Issuer', metadata.issuer);
detailsHtml += createDetailRow('Validity', `From: ${metadata.not_before || 'N/A'} To: ${metadata.not_after || 'N/A'}`);
break;
case 'asn':
detailsHtml += createDetailRow('ASN', metadata.asn);
detailsHtml += createDetailRow('Description', metadata.description);
break;
case 'large_entity':
detailsHtml += createDetailRow('Discovered Domains', metadata.domains);
break;
}
if (this.elements.modalDetails) {
@ -982,12 +1035,13 @@ class DNSReconApp {
* @returns {string} Formatted value
*/
formatValue(value) {
if (Array.isArray(value)) {
return value.join(', ');
} else if (typeof value === 'object') {
return JSON.stringify(value, null, 2);
if (typeof value === 'object' && value !== null) {
// Use <pre> for nicely formatted JSON
return `<pre>${JSON.stringify(value, null, 2)}</pre>`;
} else {
return String(value);
// Escape HTML to prevent XSS issues with string values
const strValue = String(value);
return strValue.replace(/</g, "&lt;").replace(/>/g, "&gt;");
}
}

View File

@ -218,6 +218,18 @@
</div>
</div>
<script>
function copyToClipboard(elementId) {
const element = document.getElementById(elementId);
const textToCopy = element.innerText;
navigator.clipboard.writeText(textToCopy).then(() => {
// Optional: Show a success message
console.log('Copied to clipboard');
}).catch(err => {
console.error('Failed to copy: ', err);
});
}
</script>
<script src="{{ url_for('static', filename='js/graph.js') }}"></script>
<script src="{{ url_for('static', filename='js/main.js') }}"></script>
</body>