mstoeck3/dnscope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

it

Browse Source
This commit is contained in:
overcuriousity 2025-09-11 20:37:43 +02:00
parent 0021bbc696
commit b47e679992
5 changed files with 138 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app.py
View File

@ -90,6 +90,7 @@ def start_scan():
target_domain = data['target_domain'].strip() target_domain = data['target_domain'].strip()
max_depth = data.get('max_depth', config.default_recursion_depth) max_depth = data.get('max_depth', config.default_recursion_depth)
clear_graph = data.get('clear_graph', True)
print(f"Parsed - target_domain: '{target_domain}', max_depth: {max_depth}") print(f"Parsed - target_domain: '{target_domain}', max_depth: {max_depth}")
@ -131,7 +132,7 @@ def start_scan():
# Start scan # Start scan
print(f"Calling start_scan on scanner {id(scanner)}...") print(f"Calling start_scan on scanner {id(scanner)}...")
success = scanner.start_scan(target_domain, max_depth) success = scanner.start_scan(target_domain, max_depth, clear_graph=clear_graph)
print(f"scanner.start_scan returned: {success}") print(f"scanner.start_scan returned: {success}")
print(f"Scanner status after start attempt: {scanner.status}") print(f"Scanner status after start attempt: {scanner.status}")

419
core/scanner.py
View File

@ -143,7 +143,7 @@ class Scanner:
print("Session configuration updated") print("Session configuration updated")
def start_scan(self, target_domain: str, max_depth: int = 2) -> bool: def start_scan(self, target_domain: str, max_depth: int = 2, clear_graph: bool = True) -> bool:
""" """
Start a new reconnaissance scan with concurrent processing. Start a new reconnaissance scan with concurrent processing.
Enhanced with better debugging and state validation. Enhanced with better debugging and state validation.
@ -183,9 +183,8 @@ class Scanner:
print(f"WARNING: Could not stop existing thread in scanner {id(self)}") print(f"WARNING: Could not stop existing thread in scanner {id(self)}")
return False return False
# Reset state if clear_graph:
print(f"Resetting scanner {id(self)} state...") self.graph.clear()
self.graph.clear()
self.current_target = target_domain.lower().strip() self.current_target = target_domain.lower().strip()
self.max_depth = max_depth self.max_depth = max_depth
self.current_depth = 0 self.current_depth = 0
@ -240,9 +239,8 @@ class Scanner:
self.graph.add_node(target_domain, NodeType.DOMAIN) self.graph.add_node(target_domain, NodeType.DOMAIN)
# BFS-style exploration # BFS-style exploration
current_level_domains = {target_domain} current_level_targets = {target_domain}
processed_domains = set() processed_targets = set()
all_discovered_ips = set()
print("Starting BFS exploration...") print("Starting BFS exploration...")
@ -252,46 +250,26 @@ class Scanner:
break break
self.current_depth = depth self.current_depth = depth
print(f"Processing depth level {depth} with {len(current_level_domains)} domains") print(f"Processing depth level {depth} with {len(current_level_targets)} targets")
if not current_level_domains: if not current_level_targets:
print("No domains to process at this level") print("No targets to process at this level")
break break
self.total_indicators_found += len(current_level_domains) self.total_indicators_found += len(current_level_targets)
next_level_domains = set()
target_results = self._process_targets_concurrent(current_level_targets, processed_targets)
domain_results = self._process_domains_concurrent(current_level_domains, processed_domains)
for domain, discovered_domains, discovered_ips in domain_results:
if self.stop_event.is_set():
break
processed_domains.add(domain)
all_discovered_ips.update(discovered_ips)
next_level_targets = set()
for target, new_targets in target_results:
processed_targets.add(target)
if depth < max_depth: if depth < max_depth:
for discovered_domain in discovered_domains: for new_target in new_targets:
if discovered_domain not in processed_domains: if new_target not in processed_targets:
next_level_domains.add(discovered_domain) next_level_targets.add(new_target)
print(f"Adding {discovered_domain} to next level from domain query")
current_level_targets = next_level_targets
if self.stop_event.is_set(): print(f"Completed depth {depth}, {len(next_level_targets)} targets for next level")
break
if all_discovered_ips:
print(f"Processing {len(all_discovered_ips)} discovered IP addresses")
# MODIFICATION START: Capture new domains from IP processing
new_domains_from_ips = self._process_ips_concurrent(all_discovered_ips)
if depth < max_depth:
for new_domain in new_domains_from_ips:
if new_domain not in processed_domains:
next_level_domains.add(new_domain)
print(f"Adding {new_domain} to next level from IP query")
# MODIFICATION END
current_level_domains = next_level_domains
print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level")
except Exception as e: except Exception as e:
print(f"ERROR: Scan execution failed with error: {e}") print(f"ERROR: Scan execution failed with error: {e}")
@ -313,86 +291,57 @@ class Scanner:
print("Final scan statistics:") print("Final scan statistics:")
print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}") print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}")
print(f" - Total edges: {stats['basic_metrics']['total_edges']}") print(f" - Total edges: {stats['basic_metrics']['total_edges']}")
print(f" - Domains processed: {len(processed_domains)}") print(f" - Targets processed: {len(processed_targets)}")
print(f" - IPs discovered: {len(all_discovered_ips)}")
def _process_domains_concurrent(self, domains: Set[str], processed_domains: Set[str]) -> List[Tuple[str, Set[str], Set[str]]]: def _process_targets_concurrent(self, targets: Set[str], processed_targets: Set[str]) -> List[Tuple[str, Set[str]]]:
""" """Process multiple targets (domains or IPs) concurrently using a thread pool."""
Process multiple domains concurrently using thread pool.
"""
results = [] results = []
domains_to_process = domains - processed_domains targets_to_process = targets - processed_targets
if not domains_to_process: if not targets_to_process:
return results return results
print(f"Processing {len(domains_to_process)} domains concurrently with {self.max_workers} workers") print(f"Processing {len(targets_to_process)} targets concurrently with {self.max_workers} workers")
future_to_domain = { future_to_target = {
self.executor.submit(self._query_providers_for_domain, domain): domain self.executor.submit(self._query_providers_for_target, target): target
for domain in domains_to_process for target in targets_to_process
} }
for future in as_completed(future_to_domain): for future in as_completed(future_to_target):
if self.stop_event.is_set(): if self.stop_event.is_set():
future.cancel() future.cancel()
continue continue
domain = future_to_domain[future] target = future_to_target[future]
try: try:
discovered_domains, discovered_ips = future.result() new_targets = future.result()
results.append((domain, discovered_domains, discovered_ips)) results.append((target, new_targets))
self.indicators_processed += 1 self.indicators_processed += 1
print(f"Completed processing domain: {domain} ({len(discovered_domains)} domains, {len(discovered_ips)} IPs)") print(f"Completed processing target: {target} (found {len(new_targets)} new targets)")
except (Exception, CancelledError) as e: except (Exception, CancelledError) as e:
print(f"Error processing domain {domain}: {e}") print(f"Error processing target {target}: {e}")
return results return results
def _process_ips_concurrent(self, ips: Set[str]) -> Set[str]: # MODIFICATION: Changed return type def _query_providers_for_target(self, target: str) -> Set[str]:
""" """
Process multiple IP addresses concurrently. Query all enabled providers for information about a target (domain or IP) and collect comprehensive metadata.
"""
all_discovered_domains = set() # NEW: Set to aggregate all results
if not ips or self.stop_event.is_set():
return all_discovered_domains # MODIFICATION: Return the new set
print(f"Processing {len(ips)} IP addresses concurrently")
future_to_ip = {
self.executor.submit(self._query_providers_for_ip, ip): ip
for ip in ips
}
for future in as_completed(future_to_ip):
if self.stop_event.is_set():
future.cancel()
continue
ip = future_to_ip[future]
try:
# NEW: Get the set of domains from the future's result and update our aggregate set
discovered_domains_from_ip = future.result()
all_discovered_domains.update(discovered_domains_from_ip)
print(f"Completed processing IP: {ip}, found {len(discovered_domains_from_ip)} new domains.")
except (Exception, CancelledError) as e:
print(f"Error processing IP {ip}: {e}")
return all_discovered_domains # MODIFICATION: Return the final aggregated set
def _query_providers_for_domain(self, domain: str) -> Tuple[Set[str], Set[str]]:
"""
Query all enabled providers for information about a domain and collect comprehensive metadata.
Creates appropriate node types and relationships based on discovered data. Creates appropriate node types and relationships based on discovered data.
""" """
print(f"Querying {len(self.providers)} providers for domain: {domain}") is_ip = _is_valid_ip(target)
discovered_domains = set() target_type = NodeType.IP if is_ip else NodeType.DOMAIN
discovered_ips = set() print(f"Querying {len(self.providers)} providers for {target_type.value}: {target}")
new_targets = set()
all_relationships = [] all_relationships = []
if not self.providers or self.stop_event.is_set(): if not self.providers or self.stop_event.is_set():
return discovered_domains, discovered_ips return new_targets
# Step 1: Query all providers and gather all relationships
with ThreadPoolExecutor(max_workers=len(self.providers)) as provider_executor: with ThreadPoolExecutor(max_workers=len(self.providers)) as provider_executor:
future_to_provider = { future_to_provider = {
provider_executor.submit(self._safe_provider_query_domain, provider, domain): provider provider_executor.submit(
self._safe_provider_query, provider, target, is_ip
): provider
for provider in self.providers for provider in self.providers
} }
@ -404,38 +353,36 @@ class Scanner:
provider = future_to_provider[future] provider = future_to_provider[future]
try: try:
relationships = future.result() relationships = future.result()
print(f"Provider {provider.get_name()} returned {len(relationships)} relationships for {domain}") print(f"Provider {provider.get_name()} returned {len(relationships)} relationships for {target}")
for rel in relationships: for rel in relationships:
source, target, rel_type, confidence, raw_data = rel source, rel_target, rel_type, confidence, raw_data = rel
enhanced_rel = (source, target, rel_type, confidence, raw_data, provider.get_name()) enhanced_rel = (source, rel_target, rel_type, confidence, raw_data, provider.get_name())
all_relationships.append(enhanced_rel) all_relationships.append(enhanced_rel)
except (Exception, CancelledError) as e: except (Exception, CancelledError) as e:
print(f"Provider {provider.get_name()} failed for {domain}: {e}") print(f"Provider {provider.get_name()} failed for {target}: {e}")
# NEW Step 2: Group all targets by type and identify large entities # NEW Step 2: Group all targets by type and identify large entities
discovered_targets_by_type = defaultdict(set) discovered_targets_by_type = defaultdict(set)
for _, target, _, _, _, _ in all_relationships: for _, rel_target, _, _, _, _ in all_relationships:
if _is_valid_domain(target): if _is_valid_domain(rel_target):
discovered_targets_by_type[NodeType.DOMAIN].add(target) discovered_targets_by_type[NodeType.DOMAIN].add(rel_target)
elif _is_valid_ip(target): elif _is_valid_ip(rel_target):
discovered_targets_by_type[NodeType.IP].add(target) discovered_targets_by_type[NodeType.IP].add(rel_target)
targets_to_skip = set() targets_to_skip = set()
for node_type, targets in discovered_targets_by_type.items(): for node_type, targets in discovered_targets_by_type.items():
if len(targets) > self.config.large_entity_threshold: if len(targets) > self.config.large_entity_threshold:
print(f"Large number of {node_type.value}s ({len(targets)}) found for {domain}. Creating a large entity node.") print(f"Large number of {node_type.value}s ({len(targets)}) found for {target}. Creating a large entity node.")
# We can use the first relationship's type and provider for the large entity node
first_rel = next((r for r in all_relationships if r[1] in targets), None) first_rel = next((r for r in all_relationships if r[1] in targets), None)
if first_rel: if first_rel:
self._handle_large_entity(domain, list(targets), first_rel[2], first_rel[5]) self._handle_large_entity(target, list(targets), first_rel[2], first_rel[5])
targets_to_skip.update(targets) targets_to_skip.update(targets)
# Step 3: Process all relationships to create/update nodes and edges # Step 3: Process all relationships to create/update nodes and edges
domain_metadata = defaultdict(lambda: defaultdict(list)) target_metadata = defaultdict(lambda: defaultdict(list))
dns_records_to_create = {} dns_records_to_create = {}
for source, target, rel_type, confidence, raw_data, provider_name in all_relationships: for source, rel_target, rel_type, confidence, raw_data, provider_name in all_relationships:
if self.stop_event.is_set(): if self.stop_event.is_set():
break break
@ -444,33 +391,31 @@ class Scanner:
domain_certs = raw_data.get('domain_certificates', {}) domain_certs = raw_data.get('domain_certificates', {})
for cert_domain, cert_summary in domain_certs.items(): for cert_domain, cert_summary in domain_certs.items():
if _is_valid_domain(cert_domain) and cert_domain not in targets_to_skip: if _is_valid_domain(cert_domain) and cert_domain not in targets_to_skip:
# Create the node with its metadata. If node exists, metadata is updated.
self.graph.add_node(cert_domain, NodeType.DOMAIN, metadata={'certificate_data': cert_summary}) self.graph.add_node(cert_domain, NodeType.DOMAIN, metadata={'certificate_data': cert_summary})
# General metadata collection for the source domain # General metadata collection
self._collect_node_metadata(source, provider_name, rel_type, target, raw_data, domain_metadata[source]) self._collect_node_metadata(source, provider_name, rel_type, rel_target, raw_data, target_metadata[source])
# Add nodes and edges to the graph # Add nodes and edges to the graph
if target in targets_to_skip: if rel_target in targets_to_skip:
continue continue
if _is_valid_ip(target): if _is_valid_ip(rel_target):
self.graph.add_node(target, NodeType.IP) self.graph.add_node(rel_target, NodeType.IP)
if self.graph.add_edge(source, target, rel_type, confidence, provider_name, raw_data): if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
print(f"Added IP relationship: {source} -> {target} ({rel_type.relationship_name})") print(f"Added IP relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
if rel_type in [RelationshipType.A_RECORD, RelationshipType.AAAA_RECORD]: if rel_type in [RelationshipType.A_RECORD, RelationshipType.AAAA_RECORD]:
discovered_ips.add(target) new_targets.add(rel_target)
elif target.startswith('AS') and target[2:].isdigit(): elif rel_target.startswith('AS') and rel_target[2:].isdigit():
self.graph.add_node(target, NodeType.ASN) self.graph.add_node(rel_target, NodeType.ASN)
if self.graph.add_edge(source, target, rel_type, confidence, provider_name, raw_data): if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
print(f"Added ASN relationship: {source} -> {target} ({rel_type.relationship_name})") print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
elif _is_valid_domain(target): elif _is_valid_domain(rel_target):
# Ensure the target node exists before adding an edge self.graph.add_node(rel_target, NodeType.DOMAIN)
self.graph.add_node(target, NodeType.DOMAIN) if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
if self.graph.add_edge(source, target, rel_type, confidence, provider_name, raw_data): print(f"Added domain relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
print(f"Added domain relationship: {source} -> {target} ({rel_type.relationship_name})")
recurse_types = [ recurse_types = [
RelationshipType.CNAME_RECORD, RelationshipType.MX_RECORD, RelationshipType.CNAME_RECORD, RelationshipType.MX_RECORD,
@ -478,7 +423,7 @@ class Scanner:
RelationshipType.PASSIVE_DNS RelationshipType.PASSIVE_DNS
] ]
if rel_type in recurse_types: if rel_type in recurse_types:
discovered_domains.add(target) new_targets.add(rel_target)
else: else:
# Handle DNS record content # Handle DNS record content
@ -491,7 +436,7 @@ class Scanner:
] ]
if rel_type in dns_record_types: if rel_type in dns_record_types:
record_type = rel_type.relationship_name.upper().replace('_RECORD', '') record_type = rel_type.relationship_name.upper().replace('_RECORD', '')
record_content = target.strip() record_content = rel_target.strip()
content_hash = hash(record_content) & 0x7FFFFFFF content_hash = hash(record_content) & 0x7FFFFFFF
dns_record_id = f"{record_type}:{content_hash}" dns_record_id = f"{record_type}:{content_hash}"
@ -502,9 +447,9 @@ class Scanner:
} }
dns_records_to_create[dns_record_id]['domains'].add(source) dns_records_to_create[dns_record_id]['domains'].add(source)
# Step 4: Update the source domain node with its collected metadata # Step 4: Update the source node with its collected metadata
if domain in domain_metadata: if target in target_metadata:
self.graph.add_node(domain, NodeType.DOMAIN, metadata=dict(domain_metadata[domain])) self.graph.add_node(target, target_type, metadata=dict(target_metadata[target]))
# Step 5: Create DNS record nodes and edges # Step 5: Create DNS record nodes and edges
for dns_record_id, record_info in dns_records_to_create.items(): for dns_record_id, record_info in dns_records_to_create.items():
@ -519,9 +464,8 @@ class Scanner:
self.graph.add_edge(domain_name, dns_record_id, RelationshipType.DNS_RECORD, self.graph.add_edge(domain_name, dns_record_id, RelationshipType.DNS_RECORD,
record_info['confidence'], record_info['provider_name'], record_info['confidence'], record_info['provider_name'],
record_info['raw_data']) record_info['raw_data'])
print(f"Domain {domain}: discovered {len(discovered_domains)} domains, {len(discovered_ips)} IPs") return new_targets
return discovered_domains, discovered_ips
def _collect_node_metadata(self, node_id: str, provider_name: str, rel_type: RelationshipType, def _collect_node_metadata(self, node_id: str, provider_name: str, rel_type: RelationshipType,
target: str, raw_data: Dict[str, Any], metadata: Dict[str, Any]) -> None: target: str, raw_data: Dict[str, Any], metadata: Dict[str, Any]) -> None:
@ -532,189 +476,41 @@ class Scanner:
record_type = raw_data.get('query_type', 'UNKNOWN') record_type = raw_data.get('query_type', 'UNKNOWN')
value = raw_data.get('value', target) value = raw_data.get('value', target)
# For non-infrastructure DNS records, store the full content
if record_type in ['TXT', 'SPF', 'CAA']: if record_type in ['TXT', 'SPF', 'CAA']:
dns_entry = f"{record_type}: {value}" dns_entry = f"{record_type}: {value}"
else: else:
dns_entry = f"{record_type}: {value}" dns_entry = f"{record_type}: {value}"
if dns_entry not in metadata['dns_records']: if dns_entry not in metadata.get('dns_records', []):
metadata['dns_records'].append(dns_entry) metadata.setdefault('dns_records', []).append(dns_entry)
elif provider_name == 'crtsh': elif provider_name == 'crtsh':
if rel_type == RelationshipType.SAN_CERTIFICATE: if rel_type == RelationshipType.SAN_CERTIFICATE:
# Handle certificate data storage on domain nodes
domain_certs = raw_data.get('domain_certificates', {}) domain_certs = raw_data.get('domain_certificates', {})
# Store certificate information for this domain
if node_id in domain_certs: if node_id in domain_certs:
cert_summary = domain_certs[node_id] cert_summary = domain_certs[node_id]
# Update domain metadata with certificate information
metadata['certificate_data'] = cert_summary metadata['certificate_data'] = cert_summary
metadata['has_valid_cert'] = cert_summary.get('has_valid_cert', False) metadata['has_valid_cert'] = cert_summary.get('has_valid_cert', False)
# Add related domains from shared certificates
if target not in metadata.get('related_domains_san', []): if target not in metadata.get('related_domains_san', []):
if 'related_domains_san' not in metadata: metadata.setdefault('related_domains_san', []).append(target)
metadata['related_domains_san'] = []
metadata['related_domains_san'].append(target)
# Store shared certificate details for forensic analysis
shared_certs = raw_data.get('shared_certificates', []) shared_certs = raw_data.get('shared_certificates', [])
if shared_certs and 'shared_certificate_details' not in metadata: if shared_certs and 'shared_certificate_details' not in metadata:
metadata['shared_certificate_details'] = shared_certs metadata['shared_certificate_details'] = shared_certs
elif provider_name == 'shodan': elif provider_name == 'shodan':
# Merge Shodan data (avoid overwriting)
for key, value in raw_data.items(): for key, value in raw_data.items():
if key not in metadata['shodan'] or not metadata['shodan'][key]: if key not in metadata.get('shodan', {}) or not metadata.get('shodan', {}).get(key):
metadata['shodan'][key] = value metadata.setdefault('shodan', {})[key] = value
elif provider_name == 'virustotal': elif provider_name == 'virustotal':
# Merge VirusTotal data
for key, value in raw_data.items(): for key, value in raw_data.items():
if key not in metadata['virustotal'] or not metadata['virustotal'][key]: if key not in metadata.get('virustotal', {}) or not metadata.get('virustotal', {}).get(key):
metadata['virustotal'][key] = value metadata.setdefault('virustotal', {})[key] = value
# Add passive DNS entries
if rel_type == RelationshipType.PASSIVE_DNS: if rel_type == RelationshipType.PASSIVE_DNS:
passive_entry = f"Passive DNS: {target}" passive_entry = f"Passive DNS: {target}"
if passive_entry not in metadata['passive_dns']: if passive_entry not in metadata.get('passive_dns', []):
metadata['passive_dns'].append(passive_entry) metadata.setdefault('passive_dns', []).append(passive_entry)
def _handle_large_entity(self, source_domain: str, relationships: list, rel_type: RelationshipType, provider_name: str):
"""
Handles the creation of a large entity node when a threshold is exceeded.
"""
print(f"Large number of {rel_type.name} relationships for {source_domain}. Creating a large entity node.")
entity_name = f"Large collection of {rel_type.name} for {source_domain}"
self.graph.add_node(entity_name, NodeType.LARGE_ENTITY, metadata={"count": len(relationships)})
self.graph.add_edge(source_domain, entity_name, rel_type, 0.9, provider_name, {"info": "Aggregated node"})
def _query_providers_for_ip(self, ip: str) -> Set[str]:
"""
Query all enabled providers for information about an IP address and collect comprehensive metadata.
"""
print(f"Querying {len(self.providers)} providers for IP: {ip}")
discovered_hostnames = set()
if not self.providers or self.stop_event.is_set():
return discovered_hostnames
# Comprehensive metadata collection for this IP
ip_metadata = {
'dns_records': [],
'passive_dns': [],
'shodan': {},
'virustotal': {},
'asn_data': {},
'hostnames': [],
}
all_relationships = [] # Store relationships with provider info
with ThreadPoolExecutor(max_workers=len(self.providers)) as provider_executor:
future_to_provider = {
provider_executor.submit(self._safe_provider_query_ip, provider, ip): provider
for provider in self.providers
}
for future in as_completed(future_to_provider):
if self.stop_event.is_set():
future.cancel()
continue
provider = future_to_provider[future]
try:
relationships = future.result()
print(f"Provider {provider.get_name()} returned {len(relationships)} relationships for IP {ip}")
for source, target, rel_type, confidence, raw_data in relationships:
enhanced_rel = (source, target, rel_type, confidence, raw_data, provider.get_name())
all_relationships.append(enhanced_rel)
self._collect_ip_metadata(ip, provider.get_name(), rel_type, target, raw_data, ip_metadata)
except (Exception, CancelledError) as e:
print(f"Provider {provider.get_name()} failed for IP {ip}: {e}")
# NEW: Group all targets by type and identify large entities
discovered_targets_by_type = defaultdict(set)
for _, target, _, _, _, _ in all_relationships:
if _is_valid_domain(target):
discovered_targets_by_type[NodeType.DOMAIN].add(target)
# THE FIX IS HERE: Initialize the set before using it.
targets_to_skip = set()
for node_type, targets in discovered_targets_by_type.items():
if len(targets) > self.config.large_entity_threshold:
print(f"Large number of {node_type.value}s ({len(targets)}) found for IP {ip}. Creating a large entity node.")
first_rel = next((r for r in all_relationships if r[1] in targets), None)
if first_rel:
self._handle_large_entity(ip, list(targets), first_rel[2], first_rel[5])
targets_to_skip.update(targets)
# Update the IP node with comprehensive metadata
self.graph.add_node(ip, NodeType.IP, metadata=ip_metadata)
# Process relationships with correct provider attribution
for source, target, rel_type, confidence, raw_data, provider_name in all_relationships:
if target in targets_to_skip:
continue
if _is_valid_domain(target):
target_node_type = NodeType.DOMAIN
discovered_hostnames.add(target)
elif target.startswith('AS'):
target_node_type = NodeType.ASN
else:
target_node_type = NodeType.IP
self.graph.add_node(target, target_node_type)
if self.graph.add_edge(source, target, rel_type, confidence, provider_name, raw_data):
print(f"Added IP relationship: {source} -> {target} ({rel_type.relationship_name}) from {provider_name}")
return discovered_hostnames
def _collect_ip_metadata(self, ip: str, provider_name: str, rel_type: RelationshipType,
target: str, raw_data: Dict[str, Any], metadata: Dict[str, Any]) -> None:
"""
Collect and organize metadata for an IP node based on provider responses.
"""
if provider_name == 'dns':
if rel_type == RelationshipType.PTR_RECORD:
reverse_entry = f"PTR: {target}"
if reverse_entry not in metadata['dns_records']:
metadata['dns_records'].append(reverse_entry)
if target not in metadata['hostnames']:
metadata['hostnames'].append(target)
elif provider_name == 'shodan':
# Merge Shodan data
for key, value in raw_data.items():
if key not in metadata['shodan'] or not metadata['shodan'][key]:
metadata['shodan'][key] = value
# Collect hostname information
if 'hostname' in raw_data and raw_data['hostname'] not in metadata['hostnames']:
metadata['hostnames'].append(raw_data['hostname'])
if 'hostnames' in raw_data:
for hostname in raw_data['hostnames']:
if hostname not in metadata['hostnames']:
metadata['hostnames'].append(hostname)
elif provider_name == 'virustotal':
# Merge VirusTotal data
for key, value in raw_data.items():
if key not in metadata['virustotal'] or not metadata['virustotal'][key]:
metadata['virustotal'][key] = value
# Add passive DNS entries
if rel_type == RelationshipType.PASSIVE_DNS:
passive_entry = f"Passive DNS: {target}"
if passive_entry not in metadata['passive_dns']:
metadata['passive_dns'].append(passive_entry)
# Handle ASN relationships
if rel_type == RelationshipType.ASN_MEMBERSHIP: if rel_type == RelationshipType.ASN_MEMBERSHIP:
metadata['asn_data'] = { metadata['asn_data'] = {
'asn': target, 'asn': target,
@ -723,25 +519,26 @@ class Scanner:
'country': raw_data.get('country', '') 'country': raw_data.get('country', '')
} }
def _handle_large_entity(self, source: str, targets: list, rel_type: RelationshipType, provider_name: str):
"""
Handles the creation of a large entity node when a threshold is exceeded.
"""
print(f"Large number of {rel_type.name} relationships for {source}. Creating a large entity node.")
entity_name = f"Large collection of {rel_type.name} for {source}"
self.graph.add_node(entity_name, NodeType.LARGE_ENTITY, metadata={"count": len(targets)})
self.graph.add_edge(source, entity_name, rel_type, 0.9, provider_name, {"info": "Aggregated node"})
def _safe_provider_query_domain(self, provider, domain: str): def _safe_provider_query(self, provider, target: str, is_ip: bool) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Safely query provider for domain with error handling.""" """Safely query a provider for a target with error handling."""
if self.stop_event.is_set(): if self.stop_event.is_set():
return [] return []
try: try:
return provider.query_domain(domain) if is_ip:
return provider.query_ip(target)
else:
return provider.query_domain(target)
except Exception as e: except Exception as e:
print(f"Provider {provider.get_name()} query_domain failed: {e}") print(f"Provider {provider.get_name()} query failed for {target}: {e}")
return []
def _safe_provider_query_ip(self, provider, ip: str):
"""Safely query provider for IP with error handling."""
if self.stop_event.is_set():
return []
try:
return provider.query_ip(ip)
except Exception as e:
print(f"Provider {provider.get_name()} query_ip failed: {e}")
return [] return []
def stop_scan(self) -> bool: def stop_scan(self) -> bool:
@ -752,10 +549,8 @@ class Scanner:
if self.status == ScanStatus.RUNNING: if self.status == ScanStatus.RUNNING:
print("=== INITIATING IMMEDIATE SCAN TERMINATION ===") print("=== INITIATING IMMEDIATE SCAN TERMINATION ===")
# Signal all threads to stop
self.stop_event.set() self.stop_event.set()
# Close HTTP sessions in all providers to terminate ongoing requests
for provider in self.providers: for provider in self.providers:
try: try:
if hasattr(provider, 'session'): if hasattr(provider, 'session'):
@ -764,12 +559,10 @@ class Scanner:
except Exception as e: except Exception as e:
print(f"Error closing session for {provider.get_name()}: {e}") print(f"Error closing session for {provider.get_name()}: {e}")
# Shutdown executor immediately with cancel_futures=True
if self.executor: if self.executor:
print("Shutting down executor with immediate cancellation...") print("Shutting down executor with immediate cancellation...")
self.executor.shutdown(wait=False, cancel_futures=True) self.executor.shutdown(wait=False, cancel_futures=True)
# Give threads a moment to respond to cancellation, then force status change
threading.Timer(2.0, self._force_stop_completion).start() threading.Timer(2.0, self._force_stop_completion).start()
print("Immediate termination requested - ongoing requests will be cancelled") print("Immediate termination requested - ongoing requests will be cancelled")

4
static/js/graph.js
View File

@ -447,7 +447,7 @@ class GraphManager {
'ip': '#ff9900', // Amber 'ip': '#ff9900', // Amber
'asn': '#00aaff', // Blue 'asn': '#00aaff', // Blue
'large_entity': '#ff6b6b', // Red for large entities 'large_entity': '#ff6b6b', // Red for large entities
'dns_record': '#999999' 'dns_record': '#9620c0ff'
}; };
return colors[nodeType] || '#ffffff'; return colors[nodeType] || '#ffffff';
} }
@ -463,7 +463,7 @@ class GraphManager {
'domain': '#00aa2e', 'domain': '#00aa2e',
'ip': '#cc7700', 'ip': '#cc7700',
'asn': '#0088cc', 'asn': '#0088cc',
'dns_record': '#999999' 'dns_record': '#c235c9ff'
}; };
return borderColors[nodeType] || '#666666'; return borderColors[nodeType] || '#666666';
} }

31
static/js/main.js
View File

@ -54,6 +54,7 @@ class DNSReconApp {
targetDomain: document.getElementById('target-domain'), targetDomain: document.getElementById('target-domain'),
maxDepth: document.getElementById('max-depth'), maxDepth: document.getElementById('max-depth'),
startScan: document.getElementById('start-scan'), startScan: document.getElementById('start-scan'),
addToGraph: document.getElementById('add-to-graph'),
stopScan: document.getElementById('stop-scan'), stopScan: document.getElementById('stop-scan'),
exportResults: document.getElementById('export-results'), exportResults: document.getElementById('export-results'),
configureApiKeys: document.getElementById('configure-api-keys'), configureApiKeys: document.getElementById('configure-api-keys'),
@ -136,6 +137,11 @@ class DNSReconApp {
e.preventDefault(); e.preventDefault();
this.startScan(); this.startScan();
}); });
this.elements.addToGraph.addEventListener('click', (e) => {
e.preventDefault();
this.startScan(false);
});
this.elements.stopScan.addEventListener('click', (e) => { this.elements.stopScan.addEventListener('click', (e) => {
console.log('Stop scan button clicked'); console.log('Stop scan button clicked');
@ -230,7 +236,7 @@ class DNSReconApp {
/** /**
* Start a reconnaissance scan * Start a reconnaissance scan
*/ */
async startScan() { async startScan(clearGraph = true) {
console.log('=== STARTING SCAN ==='); console.log('=== STARTING SCAN ===');
try { try {
@ -262,7 +268,8 @@ class DNSReconApp {
const requestData = { const requestData = {
target_domain: targetDomain, target_domain: targetDomain,
max_depth: maxDepth max_depth: maxDepth,
clear_graph: clearGraph
}; };
console.log('Request data:', requestData); console.log('Request data:', requestData);
@ -273,12 +280,12 @@ class DNSReconApp {
if (response.success) { if (response.success) {
this.currentSessionId = response.scan_id; this.currentSessionId = response.scan_id;
console.log('Starting polling with session ID:', this.currentSessionId);
this.startPolling(); this.startPolling();
this.showSuccess('Reconnaissance scan started successfully'); this.showSuccess('Reconnaissance scan started successfully');
// Clear previous graph if (clearGraph) {
this.graphManager.clear(); this.graphManager.clear();
}
console.log(`Scan started for ${targetDomain} with depth ${maxDepth}`); console.log(`Scan started for ${targetDomain} with depth ${maxDepth}`);
@ -627,6 +634,10 @@ class DNSReconApp {
this.elements.startScan.disabled = true; this.elements.startScan.disabled = true;
this.elements.startScan.classList.add('loading'); this.elements.startScan.classList.add('loading');
} }
if (this.elements.addToGraph) {
this.elements.addToGraph.disabled = true;
this.elements.addToGraph.classList.add('loading');
}
if (this.elements.stopScan) { if (this.elements.stopScan) {
this.elements.stopScan.disabled = false; this.elements.stopScan.disabled = false;
this.elements.stopScan.classList.remove('loading'); this.elements.stopScan.classList.remove('loading');
@ -645,6 +656,10 @@ class DNSReconApp {
this.elements.startScan.disabled = false; this.elements.startScan.disabled = false;
this.elements.startScan.classList.remove('loading'); this.elements.startScan.classList.remove('loading');
} }
if (this.elements.addToGraph) {
this.elements.addToGraph.disabled = false;
this.elements.addToGraph.classList.remove('loading');
}
if (this.elements.stopScan) { if (this.elements.stopScan) {
this.elements.stopScan.disabled = true; this.elements.stopScan.disabled = true;
} }
@ -735,7 +750,7 @@ class DNSReconApp {
if (!this.elements.nodeModal) return; if (!this.elements.nodeModal) return;
if (this.elements.modalTitle) { if (this.elements.modalTitle) {
this.elements.modalTitle.textContent = `Node Details: ${nodeId}`; this.elements.modalTitle.textContent = `Node Details`;
} }
let detailsHtml = ''; let detailsHtml = '';
@ -781,7 +796,7 @@ class DNSReconApp {
const metadata = node.metadata || {}; const metadata = node.metadata || {};
// General Node Info // General Node Info
detailsHtml += createDetailRow('Node Type', node.type); detailsHtml += createDetailRow('Node Descriptor', node.nodeId);
// Display data based on node type // Display data based on node type
switch (node.type) { switch (node.type) {

6
templates/index.html
View File

@ -52,6 +52,10 @@
<span class="btn-icon">[RUN]</span> <span class="btn-icon">[RUN]</span>
<span>Start Reconnaissance</span> <span>Start Reconnaissance</span>
</button> </button>
<button id="add-to-graph" class="btn btn-primary">
<span class="btn-icon">[ADD]</span>
<span>Add to Graph</span>
</button>
<button id="stop-scan" class="btn btn-secondary" disabled> <button id="stop-scan" class="btn btn-secondary" disabled>
<span class="btn-icon">[STOP]</span> <span class="btn-icon">[STOP]</span>
<span>Terminate Scan</span> <span>Terminate Scan</span>
@ -135,7 +139,7 @@
</div> </div>
<div class="legend-item"> <div class="legend-item">
<div class="legend-color" style="background-color: #c7c7c7;"></div> <div class="legend-color" style="background-color: #c7c7c7;"></div>
<span>Certificates</span> <span>Domain (invalid cert)</span>
</div> </div>
<div class="legend-item"> <div class="legend-item">
<div class="legend-color" style="background-color: #9d4edd;"></div> <div class="legend-color" style="background-color: #9d4edd;"></div>