""" Main scanning orchestrator for DNSRecon. Coordinates data gathering from multiple providers and builds the infrastructure graph. """ import threading import time import traceback from typing import List, Set, Dict, Any, Optional from concurrent.futures import ThreadPoolExecutor, as_completed from core.graph_manager import GraphManager, NodeType, RelationshipType from core.logger import get_forensic_logger, new_session from providers.crtsh_provider import CrtShProvider from config import config class ScanStatus: """Enumeration of scan statuses.""" IDLE = "idle" RUNNING = "running" COMPLETED = "completed" FAILED = "failed" STOPPED = "stopped" class Scanner: """ Main scanning orchestrator for DNSRecon passive reconnaissance. Manages multi-provider data gathering and graph construction. """ def __init__(self): """Initialize scanner with default providers and empty graph.""" print("Initializing Scanner instance...") try: from providers.base_provider import BaseProvider self.graph = GraphManager() self.providers: List[BaseProvider] = [] self.status = ScanStatus.IDLE self.current_target = None self.current_depth = 0 self.max_depth = 2 self.stop_requested = False self.scan_thread = None # Scanning progress tracking self.total_indicators_found = 0 self.indicators_processed = 0 self.current_indicator = "" # Initialize providers print("Calling _initialize_providers...") self._initialize_providers() # Initialize logger print("Initializing forensic logger...") self.logger = get_forensic_logger() print("Scanner initialization complete") except Exception as e: print(f"ERROR: Scanner initialization failed: {e}") traceback.print_exc() raise def _initialize_providers(self) -> None: """Initialize available providers based on configuration.""" self.providers = [] print("Initializing providers...") # Always add free providers if config.is_provider_enabled('crtsh'): try: crtsh_provider = CrtShProvider() if crtsh_provider.is_available(): self.providers.append(crtsh_provider) print("✓ CrtSh provider initialized successfully") else: print("✗ CrtSh provider is not available") except Exception as e: print(f"✗ Failed to initialize CrtSh provider: {e}") traceback.print_exc() print(f"Initialized {len(self.providers)} providers") def _debug_threads(self): """Debug function to show current threads.""" print("=== THREAD DEBUG INFO ===") for t in threading.enumerate(): print(f"Thread: {t.name} | Alive: {t.is_alive()} | Daemon: {t.daemon}") print("=== END THREAD DEBUG ===") def start_scan(self, target_domain: str, max_depth: int = 2) -> bool: """ Start a new reconnaissance scan. Args: target_domain: Initial domain to investigate max_depth: Maximum recursion depth Returns: bool: True if scan started successfully """ print(f"Scanner.start_scan called with target='{target_domain}', depth={max_depth}") try: print("Checking current status...") self._debug_threads() if self.status == ScanStatus.RUNNING: print("Scan already running, rejecting new scan") return False # Check if we have any providers if not self.providers: print("No providers available, cannot start scan") return False print(f"Current status: {self.status}, Providers: {len(self.providers)}") # Stop any existing scan thread if self.scan_thread and self.scan_thread.is_alive(): print("Stopping existing scan thread...") self.stop_requested = True self.scan_thread.join(timeout=2.0) if self.scan_thread.is_alive(): print("WARNING: Could not stop existing thread") return False # Reset state print("Resetting scanner state...") #print("Running graph.clear()") #self.graph.clear() print("running self.current_target = target_domain.lower().strip()") self.current_target = target_domain.lower().strip() self.max_depth = max_depth self.current_depth = 0 self.stop_requested = False self.total_indicators_found = 0 self.indicators_processed = 0 self.current_indicator = self.current_target # Start new forensic session print("Starting new forensic session...") self.logger = new_session() # FOR DEBUGGING: Run scan synchronously instead of in thread print("Running scan synchronously for debugging...") self._execute_scan_sync(self.current_target, max_depth) return True except Exception as e: print(f"ERROR: Exception in start_scan: {e}") traceback.print_exc() return False def stop_scan(self) -> bool: """ Request scan termination. Returns: bool: True if stop request was accepted """ try: if self.status == ScanStatus.RUNNING: self.stop_requested = True print("Scan stop requested") return True print("No active scan to stop") return False except Exception as e: print(f"ERROR: Exception in stop_scan: {e}") traceback.print_exc() return False def get_scan_status(self) -> Dict[str, Any]: """ Get current scan status and progress. Returns: Dictionary containing scan status information """ try: return { 'status': self.status, 'target_domain': self.current_target, 'current_depth': self.current_depth, 'max_depth': self.max_depth, 'current_indicator': self.current_indicator, 'total_indicators_found': self.total_indicators_found, 'indicators_processed': self.indicators_processed, 'progress_percentage': self._calculate_progress(), 'enabled_providers': [provider.get_name() for provider in self.providers], 'graph_statistics': self.graph.get_statistics() } except Exception as e: print(f"ERROR: Exception in get_scan_status: {e}") traceback.print_exc() return { 'status': 'error', 'target_domain': None, 'current_depth': 0, 'max_depth': 0, 'current_indicator': '', 'total_indicators_found': 0, 'indicators_processed': 0, 'progress_percentage': 0.0, 'enabled_providers': [], 'graph_statistics': {} } def _calculate_progress(self) -> float: """Calculate scan progress percentage.""" if self.total_indicators_found == 0: return 0.0 return min(100.0, (self.indicators_processed / self.total_indicators_found) * 100) def _execute_scan_sync(self, target_domain: str, max_depth: int) -> None: """ Execute the reconnaissance scan synchronously (for debugging). Args: target_domain: Target domain to investigate max_depth: Maximum recursion depth """ print(f"_execute_scan_sync started for {target_domain} with depth {max_depth}") try: print("Setting status to RUNNING") self.status = ScanStatus.RUNNING # Log scan start enabled_providers = [provider.get_name() for provider in self.providers] self.logger.log_scan_start(target_domain, max_depth, enabled_providers) print(f"Logged scan start with providers: {enabled_providers}") # Initialize with target domain print(f"Adding target domain '{target_domain}' as initial node") self.graph.add_node(target_domain, NodeType.DOMAIN) # BFS-style exploration with depth limiting current_level_domains = {target_domain} processed_domains = set() print(f"Starting BFS exploration...") for depth in range(max_depth + 1): if self.stop_requested: print(f"Stop requested at depth {depth}") break self.current_depth = depth print(f"Processing depth level {depth} with {len(current_level_domains)} domains") if not current_level_domains: print("No domains to process at this level") break # Update progress tracking self.total_indicators_found += len(current_level_domains) next_level_domains = set() # Process domains at current depth level for domain in current_level_domains: if self.stop_requested: print(f"Stop requested while processing domain {domain}") break if domain in processed_domains: print(f"Domain {domain} already processed, skipping") continue print(f"Processing domain: {domain}") self.current_indicator = domain self.indicators_processed += 1 # Query all providers for this domain discovered_domains = self._query_providers_for_domain(domain) print(f"Discovered {len(discovered_domains)} new domains from {domain}") # Add discovered domains to next level if not at max depth if depth < max_depth: for discovered_domain in discovered_domains: if discovered_domain not in processed_domains: next_level_domains.add(discovered_domain) print(f"Adding {discovered_domain} to next level") processed_domains.add(domain) current_level_domains = next_level_domains print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level") # Finalize scan if self.stop_requested: self.status = ScanStatus.STOPPED print("Scan completed with STOPPED status") else: self.status = ScanStatus.COMPLETED print("Scan completed with COMPLETED status") self.logger.log_scan_complete() # Print final statistics stats = self.graph.get_statistics() print(f"Final scan statistics:") print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}") print(f" - Total edges: {stats['basic_metrics']['total_edges']}") print(f" - Domains processed: {len(processed_domains)}") except Exception as e: print(f"ERROR: Scan execution failed with error: {e}") traceback.print_exc() self.status = ScanStatus.FAILED self.logger.logger.error(f"Scan failed: {e}") def _query_providers_for_domain(self, domain: str) -> Set[str]: """ Query all enabled providers for information about a domain. Args: domain: Domain to investigate Returns: Set of newly discovered domains """ print(f"Querying {len(self.providers)} providers for domain: {domain}") discovered_domains = set() if not self.providers: print("No providers available") return discovered_domains # Query providers sequentially for debugging for provider in self.providers: if self.stop_requested: print("Stop requested, cancelling provider queries") break try: print(f"Querying provider: {provider.get_name()}") relationships = provider.query_domain(domain) print(f"Provider {provider.get_name()} returned {len(relationships)} relationships") for source, target, rel_type, confidence, raw_data in relationships: print(f"Processing relationship: {source} -> {target} ({rel_type.relationship_name})") # Add target node to graph if it doesn't exist self.graph.add_node(target, NodeType.DOMAIN) # Add relationship success = self.graph.add_edge( source, target, rel_type, confidence, provider.get_name(), raw_data ) if success: print(f"Added new relationship: {source} -> {target}") else: print(f"Relationship already exists or failed to add: {source} -> {target}") discovered_domains.add(target) except Exception as e: print(f"Provider {provider.get_name()} failed for {domain}: {e}") traceback.print_exc() self.logger.logger.error(f"Provider {provider.get_name()} failed for {domain}: {e}") print(f"Total unique domains discovered: {len(discovered_domains)}") return discovered_domains def get_graph_data(self) -> Dict[str, Any]: """ Get current graph data for visualization. Returns: Graph data formatted for frontend """ return self.graph.get_graph_data() def export_results(self) -> Dict[str, Any]: """ Export complete scan results including graph and audit trail. Returns: Dictionary containing complete scan results """ # Get graph data graph_data = self.graph.export_json() # Get forensic audit trail audit_trail = self.logger.export_audit_trail() # Get provider statistics provider_stats = {} for provider in self.providers: provider_stats[provider.get_name()] = provider.get_statistics() # Combine all results export_data = { 'scan_metadata': { 'target_domain': self.current_target, 'max_depth': self.max_depth, 'final_status': self.status, 'total_indicators_processed': self.indicators_processed, 'enabled_providers': list(provider_stats.keys()) }, 'graph_data': graph_data, 'forensic_audit': audit_trail, 'provider_statistics': provider_stats, 'scan_summary': self.logger.get_forensic_summary() } return export_data def remove_provider(self, provider_name: str) -> bool: """ Remove a provider from the scanner. Args: provider_name: Name of provider to remove Returns: bool: True if provider was removed """ for i, provider in enumerate(self.providers): if provider.get_name() == provider_name: self.providers.pop(i) return True return False def get_provider_statistics(self) -> Dict[str, Dict[str, Any]]: """ Get statistics for all providers. Returns: Dictionary mapping provider names to their statistics """ stats = {} for provider in self.providers: stats[provider.get_name()] = provider.get_statistics() return stats class ScannerProxy: def __init__(self): self._scanner = None print("ScannerProxy initialized") def __getattr__(self, name): if self._scanner is None: print("Creating new Scanner instance...") self._scanner = Scanner() print("Scanner instance created") return getattr(self._scanner, name) # Global scanner instance scanner = ScannerProxy()