461 lines
16 KiB
Python
461 lines
16 KiB
Python
"""
|
|
Main scanning orchestrator for DNSRecon.
|
|
Coordinates data gathering from multiple providers and builds the infrastructure graph.
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
import traceback
|
|
from typing import List, Set, Dict, Any, Optional
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
from core.graph_manager import GraphManager, NodeType, RelationshipType
|
|
from core.logger import get_forensic_logger, new_session
|
|
from providers.crtsh_provider import CrtShProvider
|
|
from config import config
|
|
|
|
|
|
class ScanStatus:
|
|
"""Enumeration of scan statuses."""
|
|
IDLE = "idle"
|
|
RUNNING = "running"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
STOPPED = "stopped"
|
|
|
|
|
|
class Scanner:
|
|
"""
|
|
Main scanning orchestrator for DNSRecon passive reconnaissance.
|
|
Manages multi-provider data gathering and graph construction.
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""Initialize scanner with default providers and empty graph."""
|
|
print("Initializing Scanner instance...")
|
|
|
|
try:
|
|
from providers.base_provider import BaseProvider
|
|
self.graph = GraphManager()
|
|
self.providers: List[BaseProvider] = []
|
|
self.status = ScanStatus.IDLE
|
|
self.current_target = None
|
|
self.current_depth = 0
|
|
self.max_depth = 2
|
|
self.stop_requested = False
|
|
self.scan_thread = None
|
|
|
|
# Scanning progress tracking
|
|
self.total_indicators_found = 0
|
|
self.indicators_processed = 0
|
|
self.current_indicator = ""
|
|
|
|
# Initialize providers
|
|
print("Calling _initialize_providers...")
|
|
self._initialize_providers()
|
|
|
|
# Initialize logger
|
|
print("Initializing forensic logger...")
|
|
self.logger = get_forensic_logger()
|
|
|
|
print("Scanner initialization complete")
|
|
|
|
except Exception as e:
|
|
print(f"ERROR: Scanner initialization failed: {e}")
|
|
traceback.print_exc()
|
|
raise
|
|
|
|
def _initialize_providers(self) -> None:
|
|
"""Initialize available providers based on configuration."""
|
|
self.providers = []
|
|
|
|
print("Initializing providers...")
|
|
|
|
# Always add free providers
|
|
if config.is_provider_enabled('crtsh'):
|
|
try:
|
|
crtsh_provider = CrtShProvider()
|
|
if crtsh_provider.is_available():
|
|
self.providers.append(crtsh_provider)
|
|
print("✓ CrtSh provider initialized successfully")
|
|
else:
|
|
print("✗ CrtSh provider is not available")
|
|
except Exception as e:
|
|
print(f"✗ Failed to initialize CrtSh provider: {e}")
|
|
traceback.print_exc()
|
|
|
|
print(f"Initialized {len(self.providers)} providers")
|
|
|
|
def _debug_threads(self):
|
|
"""Debug function to show current threads."""
|
|
print("=== THREAD DEBUG INFO ===")
|
|
for t in threading.enumerate():
|
|
print(f"Thread: {t.name} | Alive: {t.is_alive()} | Daemon: {t.daemon}")
|
|
print("=== END THREAD DEBUG ===")
|
|
|
|
def start_scan(self, target_domain: str, max_depth: int = 2) -> bool:
|
|
"""
|
|
Start a new reconnaissance scan.
|
|
|
|
Args:
|
|
target_domain: Initial domain to investigate
|
|
max_depth: Maximum recursion depth
|
|
|
|
Returns:
|
|
bool: True if scan started successfully
|
|
"""
|
|
print(f"Scanner.start_scan called with target='{target_domain}', depth={max_depth}")
|
|
|
|
try:
|
|
print("Checking current status...")
|
|
self._debug_threads()
|
|
|
|
if self.status == ScanStatus.RUNNING:
|
|
print("Scan already running, rejecting new scan")
|
|
return False
|
|
|
|
# Check if we have any providers
|
|
if not self.providers:
|
|
print("No providers available, cannot start scan")
|
|
return False
|
|
|
|
print(f"Current status: {self.status}, Providers: {len(self.providers)}")
|
|
|
|
# Stop any existing scan thread
|
|
if self.scan_thread and self.scan_thread.is_alive():
|
|
print("Stopping existing scan thread...")
|
|
self.stop_requested = True
|
|
self.scan_thread.join(timeout=2.0)
|
|
if self.scan_thread.is_alive():
|
|
print("WARNING: Could not stop existing thread")
|
|
return False
|
|
|
|
# Reset state
|
|
print("Resetting scanner state...")
|
|
#print("Running graph.clear()")
|
|
#self.graph.clear()
|
|
print("running self.current_target = target_domain.lower().strip()")
|
|
self.current_target = target_domain.lower().strip()
|
|
self.max_depth = max_depth
|
|
self.current_depth = 0
|
|
self.stop_requested = False
|
|
self.total_indicators_found = 0
|
|
self.indicators_processed = 0
|
|
self.current_indicator = self.current_target
|
|
|
|
# Start new forensic session
|
|
print("Starting new forensic session...")
|
|
self.logger = new_session()
|
|
|
|
# FOR DEBUGGING: Run scan synchronously instead of in thread
|
|
print("Running scan synchronously for debugging...")
|
|
self._execute_scan_sync(self.current_target, max_depth)
|
|
return True
|
|
|
|
except Exception as e:
|
|
print(f"ERROR: Exception in start_scan: {e}")
|
|
traceback.print_exc()
|
|
return False
|
|
|
|
def stop_scan(self) -> bool:
|
|
"""
|
|
Request scan termination.
|
|
|
|
Returns:
|
|
bool: True if stop request was accepted
|
|
"""
|
|
try:
|
|
if self.status == ScanStatus.RUNNING:
|
|
self.stop_requested = True
|
|
print("Scan stop requested")
|
|
return True
|
|
print("No active scan to stop")
|
|
return False
|
|
except Exception as e:
|
|
print(f"ERROR: Exception in stop_scan: {e}")
|
|
traceback.print_exc()
|
|
return False
|
|
|
|
def get_scan_status(self) -> Dict[str, Any]:
|
|
"""
|
|
Get current scan status and progress.
|
|
|
|
Returns:
|
|
Dictionary containing scan status information
|
|
"""
|
|
try:
|
|
return {
|
|
'status': self.status,
|
|
'target_domain': self.current_target,
|
|
'current_depth': self.current_depth,
|
|
'max_depth': self.max_depth,
|
|
'current_indicator': self.current_indicator,
|
|
'total_indicators_found': self.total_indicators_found,
|
|
'indicators_processed': self.indicators_processed,
|
|
'progress_percentage': self._calculate_progress(),
|
|
'enabled_providers': [provider.get_name() for provider in self.providers],
|
|
'graph_statistics': self.graph.get_statistics()
|
|
}
|
|
except Exception as e:
|
|
print(f"ERROR: Exception in get_scan_status: {e}")
|
|
traceback.print_exc()
|
|
return {
|
|
'status': 'error',
|
|
'target_domain': None,
|
|
'current_depth': 0,
|
|
'max_depth': 0,
|
|
'current_indicator': '',
|
|
'total_indicators_found': 0,
|
|
'indicators_processed': 0,
|
|
'progress_percentage': 0.0,
|
|
'enabled_providers': [],
|
|
'graph_statistics': {}
|
|
}
|
|
|
|
def _calculate_progress(self) -> float:
|
|
"""Calculate scan progress percentage."""
|
|
if self.total_indicators_found == 0:
|
|
return 0.0
|
|
return min(100.0, (self.indicators_processed / self.total_indicators_found) * 100)
|
|
|
|
def _execute_scan_sync(self, target_domain: str, max_depth: int) -> None:
|
|
"""
|
|
Execute the reconnaissance scan synchronously (for debugging).
|
|
|
|
Args:
|
|
target_domain: Target domain to investigate
|
|
max_depth: Maximum recursion depth
|
|
"""
|
|
print(f"_execute_scan_sync started for {target_domain} with depth {max_depth}")
|
|
|
|
try:
|
|
print("Setting status to RUNNING")
|
|
self.status = ScanStatus.RUNNING
|
|
|
|
# Log scan start
|
|
enabled_providers = [provider.get_name() for provider in self.providers]
|
|
self.logger.log_scan_start(target_domain, max_depth, enabled_providers)
|
|
print(f"Logged scan start with providers: {enabled_providers}")
|
|
|
|
# Initialize with target domain
|
|
print(f"Adding target domain '{target_domain}' as initial node")
|
|
self.graph.add_node(target_domain, NodeType.DOMAIN)
|
|
|
|
# BFS-style exploration with depth limiting
|
|
current_level_domains = {target_domain}
|
|
processed_domains = set()
|
|
|
|
print(f"Starting BFS exploration...")
|
|
|
|
for depth in range(max_depth + 1):
|
|
if self.stop_requested:
|
|
print(f"Stop requested at depth {depth}")
|
|
break
|
|
|
|
self.current_depth = depth
|
|
print(f"Processing depth level {depth} with {len(current_level_domains)} domains")
|
|
|
|
if not current_level_domains:
|
|
print("No domains to process at this level")
|
|
break
|
|
|
|
# Update progress tracking
|
|
self.total_indicators_found += len(current_level_domains)
|
|
next_level_domains = set()
|
|
|
|
# Process domains at current depth level
|
|
for domain in current_level_domains:
|
|
if self.stop_requested:
|
|
print(f"Stop requested while processing domain {domain}")
|
|
break
|
|
|
|
if domain in processed_domains:
|
|
print(f"Domain {domain} already processed, skipping")
|
|
continue
|
|
|
|
print(f"Processing domain: {domain}")
|
|
self.current_indicator = domain
|
|
self.indicators_processed += 1
|
|
|
|
# Query all providers for this domain
|
|
discovered_domains = self._query_providers_for_domain(domain)
|
|
print(f"Discovered {len(discovered_domains)} new domains from {domain}")
|
|
|
|
# Add discovered domains to next level if not at max depth
|
|
if depth < max_depth:
|
|
for discovered_domain in discovered_domains:
|
|
if discovered_domain not in processed_domains:
|
|
next_level_domains.add(discovered_domain)
|
|
print(f"Adding {discovered_domain} to next level")
|
|
|
|
processed_domains.add(domain)
|
|
|
|
current_level_domains = next_level_domains
|
|
print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level")
|
|
|
|
# Finalize scan
|
|
if self.stop_requested:
|
|
self.status = ScanStatus.STOPPED
|
|
print("Scan completed with STOPPED status")
|
|
else:
|
|
self.status = ScanStatus.COMPLETED
|
|
print("Scan completed with COMPLETED status")
|
|
|
|
self.logger.log_scan_complete()
|
|
|
|
# Print final statistics
|
|
stats = self.graph.get_statistics()
|
|
print(f"Final scan statistics:")
|
|
print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}")
|
|
print(f" - Total edges: {stats['basic_metrics']['total_edges']}")
|
|
print(f" - Domains processed: {len(processed_domains)}")
|
|
|
|
except Exception as e:
|
|
print(f"ERROR: Scan execution failed with error: {e}")
|
|
traceback.print_exc()
|
|
self.status = ScanStatus.FAILED
|
|
self.logger.logger.error(f"Scan failed: {e}")
|
|
|
|
def _query_providers_for_domain(self, domain: str) -> Set[str]:
|
|
"""
|
|
Query all enabled providers for information about a domain.
|
|
|
|
Args:
|
|
domain: Domain to investigate
|
|
|
|
Returns:
|
|
Set of newly discovered domains
|
|
"""
|
|
print(f"Querying {len(self.providers)} providers for domain: {domain}")
|
|
discovered_domains = set()
|
|
|
|
if not self.providers:
|
|
print("No providers available")
|
|
return discovered_domains
|
|
|
|
# Query providers sequentially for debugging
|
|
for provider in self.providers:
|
|
if self.stop_requested:
|
|
print("Stop requested, cancelling provider queries")
|
|
break
|
|
|
|
try:
|
|
print(f"Querying provider: {provider.get_name()}")
|
|
relationships = provider.query_domain(domain)
|
|
print(f"Provider {provider.get_name()} returned {len(relationships)} relationships")
|
|
|
|
for source, target, rel_type, confidence, raw_data in relationships:
|
|
print(f"Processing relationship: {source} -> {target} ({rel_type.relationship_name})")
|
|
|
|
# Add target node to graph if it doesn't exist
|
|
self.graph.add_node(target, NodeType.DOMAIN)
|
|
|
|
# Add relationship
|
|
success = self.graph.add_edge(
|
|
source, target, rel_type, confidence,
|
|
provider.get_name(), raw_data
|
|
)
|
|
|
|
if success:
|
|
print(f"Added new relationship: {source} -> {target}")
|
|
else:
|
|
print(f"Relationship already exists or failed to add: {source} -> {target}")
|
|
|
|
discovered_domains.add(target)
|
|
|
|
except Exception as e:
|
|
print(f"Provider {provider.get_name()} failed for {domain}: {e}")
|
|
traceback.print_exc()
|
|
self.logger.logger.error(f"Provider {provider.get_name()} failed for {domain}: {e}")
|
|
|
|
print(f"Total unique domains discovered: {len(discovered_domains)}")
|
|
return discovered_domains
|
|
|
|
def get_graph_data(self) -> Dict[str, Any]:
|
|
"""
|
|
Get current graph data for visualization.
|
|
|
|
Returns:
|
|
Graph data formatted for frontend
|
|
"""
|
|
return self.graph.get_graph_data()
|
|
|
|
def export_results(self) -> Dict[str, Any]:
|
|
"""
|
|
Export complete scan results including graph and audit trail.
|
|
|
|
Returns:
|
|
Dictionary containing complete scan results
|
|
"""
|
|
# Get graph data
|
|
graph_data = self.graph.export_json()
|
|
|
|
# Get forensic audit trail
|
|
audit_trail = self.logger.export_audit_trail()
|
|
|
|
# Get provider statistics
|
|
provider_stats = {}
|
|
for provider in self.providers:
|
|
provider_stats[provider.get_name()] = provider.get_statistics()
|
|
|
|
# Combine all results
|
|
export_data = {
|
|
'scan_metadata': {
|
|
'target_domain': self.current_target,
|
|
'max_depth': self.max_depth,
|
|
'final_status': self.status,
|
|
'total_indicators_processed': self.indicators_processed,
|
|
'enabled_providers': list(provider_stats.keys())
|
|
},
|
|
'graph_data': graph_data,
|
|
'forensic_audit': audit_trail,
|
|
'provider_statistics': provider_stats,
|
|
'scan_summary': self.logger.get_forensic_summary()
|
|
}
|
|
|
|
return export_data
|
|
|
|
def remove_provider(self, provider_name: str) -> bool:
|
|
"""
|
|
Remove a provider from the scanner.
|
|
|
|
Args:
|
|
provider_name: Name of provider to remove
|
|
|
|
Returns:
|
|
bool: True if provider was removed
|
|
"""
|
|
for i, provider in enumerate(self.providers):
|
|
if provider.get_name() == provider_name:
|
|
self.providers.pop(i)
|
|
return True
|
|
return False
|
|
|
|
def get_provider_statistics(self) -> Dict[str, Dict[str, Any]]:
|
|
"""
|
|
Get statistics for all providers.
|
|
|
|
Returns:
|
|
Dictionary mapping provider names to their statistics
|
|
"""
|
|
stats = {}
|
|
for provider in self.providers:
|
|
stats[provider.get_name()] = provider.get_statistics()
|
|
return stats
|
|
|
|
|
|
class ScannerProxy:
|
|
def __init__(self):
|
|
self._scanner = None
|
|
print("ScannerProxy initialized")
|
|
|
|
def __getattr__(self, name):
|
|
if self._scanner is None:
|
|
print("Creating new Scanner instance...")
|
|
self._scanner = Scanner()
|
|
print("Scanner instance created")
|
|
return getattr(self._scanner, name)
|
|
|
|
|
|
# Global scanner instance
|
|
scanner = ScannerProxy() |