progress

app.py

@@ -6,7 +6,7 @@ Provides REST API endpoints and serves the web interface.
 import json
 import traceback
 from flask import Flask, render_template, request, jsonify, send_file
-from datetime import datetime
+from datetime import datetime, timezone
 import io
 
 from core.scanner import scanner
@@ -173,7 +173,7 @@ def export_results():
         results = scanner.export_results()
         
         # Create filename with timestamp
-        timestamp = datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S')
+        timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')
         target = scanner.current_target or 'unknown'
         filename = f"dnsrecon_{target}_{timestamp}.json"
         
@@ -284,12 +284,22 @@ def set_api_keys():
 
 @app.route('/api/health', methods=['GET'])
 def health_check():
-    """Health check endpoint."""
+    """Health check endpoint with enhanced Phase 2 information."""
     return jsonify({
         'success': True,
         'status': 'healthy',
         'timestamp': datetime.now(datetime.UTC).isoformat(),
-        'version': '1.0.0-phase1'
+        'version': '1.0.0-phase2',
+        'phase': 2,
+        'features': {
+            'multi_provider': True,
+            'concurrent_processing': True,
+            'real_time_updates': True,
+            'api_key_management': True,
+            'enhanced_visualization': True,
+            'retry_logic': True
+        },
+        'providers_available': len(scanner.providers) if hasattr(scanner, 'providers') else 0
     })
 
 

config.py

@@ -111,6 +111,8 @@ class Config:
         # Override default settings from environment
         self.default_recursion_depth = int(os.getenv('DEFAULT_RECURSION_DEPTH', '2'))
         self.flask_debug = os.getenv('FLASK_DEBUG', 'True').lower() == 'true'
+        self.default_timeout = 30
+        self.max_concurrent_requests = 5
 
 
 # Global configuration instance

core/__init__.py

@@ -1,6 +1,7 @@
 """
 Core modules for DNSRecon passive reconnaissance tool.
 Contains graph management, scanning orchestration, and forensic logging.
+Phase 2: Enhanced with concurrent processing and real-time capabilities.
 """
 
 from .graph_manager import GraphManager, NodeType, RelationshipType
@@ -19,4 +20,4 @@ __all__ = [
     'new_session'
 ]
 
-__version__ = "1.0.0-phase1"
+__version__ = "1.0.0-phase2"

core/graph_manager.py

@@ -8,6 +8,7 @@ import threading
 from datetime import datetime
 from typing import Dict, List, Any, Optional, Tuple, Set
 from enum import Enum
+from datetime import timezone
 
 import networkx as nx
 
@@ -45,7 +46,7 @@ class GraphManager:
         """Initialize empty directed graph."""
         self.graph = nx.DiGraph()
         # self.lock = threading.Lock()
-        self.creation_time = datetime.now(datetime.UTC).isoformat()
+        self.creation_time = datetime.now(timezone.utc).isoformat()
         self.last_modified = self.creation_time
         
     def add_node(self, node_id: str, node_type: NodeType, 
@@ -71,12 +72,12 @@ class GraphManager:
         
         node_attributes = {
             'type': node_type.value,
-            'added_timestamp': datetime.now(datetime.UTC).isoformat(),
+            'added_timestamp': datetime.now(timezone.utc).isoformat(),
             'metadata': metadata or {}
         }
         
         self.graph.add_node(node_id, **node_attributes)
-        self.last_modified = datetime.now(datetime.UTC).isoformat()
+        self.last_modified = datetime.now(timezone.utc).isoformat()
         return True
     
     def add_edge(self, source_id: str, target_id: str, 
@@ -111,7 +112,7 @@ class GraphManager:
             
             if new_confidence > existing_confidence:
                 self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence
-                self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(datetime.UTC).isoformat()
+                self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat()
                 self.graph.edges[source_id, target_id]['updated_by'] = source_provider
             
             return False
@@ -120,12 +121,12 @@ class GraphManager:
             'relationship_type': relationship_type.relationship_name,
             'confidence_score': confidence_score or relationship_type.default_confidence,
             'source_provider': source_provider,
-            'discovery_timestamp': datetime.now(datetime.UTC).isoformat(),
+            'discovery_timestamp': datetime.now(timezone.utc).isoformat(),
             'raw_data': raw_data or {}
         }
         
         self.graph.add_edge(source_id, target_id, **edge_attributes)
-        self.last_modified = datetime.now(datetime.UTC).isoformat()
+        self.last_modified = datetime.now(timezone.utc).isoformat()
         return True
 
     def get_node_count(self) -> int:
@@ -210,14 +211,36 @@ class GraphManager:
                 'added_timestamp': attributes.get('added_timestamp')
             }
             
-            # Color coding by type
+            # Color coding by type - now returns color objects for enhanced visualization
             type_colors = {
-                'domain': '#00ff41',     # Green for domains
+                'domain': {
-                'ip': '#ff9900',         # Amber for IPs
+                    'background': '#00ff41',
-                'certificate': '#c7c7c7', # Gray for certificates
+                    'border': '#00aa2e',
-                'asn': '#00aaff'         # Blue for ASNs
+                    'highlight': {'background': '#44ff75', 'border': '#00ff41'},
+                    'hover': {'background': '#22ff63', 'border': '#00cc35'}
+                },
+                'ip': {
+                    'background': '#ff9900',
+                    'border': '#cc7700',
+                    'highlight': {'background': '#ffbb44', 'border': '#ff9900'},
+                    'hover': {'background': '#ffaa22', 'border': '#dd8800'}
+                },
+                'certificate': {
+                    'background': '#c7c7c7',
+                    'border': '#999999',
+                    'highlight': {'background': '#e0e0e0', 'border': '#c7c7c7'},
+                    'hover': {'background': '#d4d4d4', 'border': '#aaaaaa'}
+                },
+                'asn': {
+                    'background': '#00aaff',
+                    'border': '#0088cc',
+                    'highlight': {'background': '#44ccff', 'border': '#00aaff'},
+                    'hover': {'background': '#22bbff', 'border': '#0099dd'}
                 }
-            node_data['color'] = type_colors.get(attributes.get('type'), '#ffffff')
+            }
+            
+            node_color_config = type_colors.get(attributes.get('type', 'unknown'), type_colors['domain'])
+            node_data['color'] = node_color_config
             nodes.append(node_data)
         
         # Format edges for visualization
@@ -231,17 +254,36 @@ class GraphManager:
                 'discovery_timestamp': attributes.get('discovery_timestamp')
             }
             
-            # Edge styling based on confidence
+            # Enhanced edge styling based on confidence
             confidence = attributes.get('confidence_score', 0)
             if confidence >= 0.8:
-                edge_data['color'] = '#00ff41'  # Green for high confidence
+                edge_data['color'] = {
-                edge_data['width'] = 3
+                    'color': '#00ff41',
+                    'highlight': '#44ff75',
+                    'hover': '#22ff63',
+                    'inherit': False
+                }
+                edge_data['width'] = 4
             elif confidence >= 0.6:
-                edge_data['color'] = '#ff9900'  # Amber for medium confidence
+                edge_data['color'] = {
-                edge_data['width'] = 2
+                    'color': '#ff9900',
+                    'highlight': '#ffbb44', 
+                    'hover': '#ffaa22',
+                    'inherit': False
+                }
+                edge_data['width'] = 3
             else:
-                edge_data['color'] = '#444444'  # Dark gray for low confidence
+                edge_data['color'] = {
-                edge_data['width'] = 1
+                    'color': '#666666',
+                    'highlight': '#888888',
+                    'hover': '#777777',
+                    'inherit': False
+                }
+                edge_data['width'] = 2
+            
+            # Add dashed line for low confidence
+            if confidence < 0.6:
+                edge_data['dashes'] = [5, 5]
             
             edges.append(edge_data)
         
@@ -270,7 +312,7 @@ class GraphManager:
         # Add comprehensive metadata
         export_data = {
             'export_metadata': {
-                'export_timestamp': datetime.now(datetime.UTC).isoformat(),
+                'export_timestamp': datetime.now(timezone.utc).isoformat(),
                 'graph_creation_time': self.creation_time,
                 'last_modified': self.last_modified,
                 'total_nodes': self.graph.number_of_nodes(),
@@ -351,5 +393,5 @@ class GraphManager:
         """Clear all nodes and edges from the graph."""
         #with self.lock:
         self.graph.clear()
-        self.creation_time = datetime.now(datetime.UTC).isoformat()
+        self.creation_time = datetime.now(timezone.utc).isoformat()
         self.last_modified = self.creation_time

core/logger.py

@@ -9,6 +9,7 @@ import threading
 from datetime import datetime
 from typing import Dict, Any, Optional, List
 from dataclasses import dataclass, asdict
+from datetime import timezone
 
 
 @dataclass
@@ -60,7 +61,7 @@ class ForensicLogger:
         self.relationships: List[RelationshipDiscovery] = []
         self.session_metadata = {
             'session_id': self.session_id,
-            'start_time': datetime.now(datetime.UTC).isoformat(),
+            'start_time': datetime.now(timezone.utc).isoformat(),
             'end_time': None,
             'total_requests': 0,
             'total_relationships': 0,
@@ -85,7 +86,7 @@ class ForensicLogger:
     
     def _generate_session_id(self) -> str:
         """Generate unique session identifier."""
-        return f"dnsrecon_{datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S')}"
+        return f"dnsrecon_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
     
     def log_api_request(self, provider: str, url: str, method: str = "GET", 
                        status_code: Optional[int] = None, 
@@ -110,7 +111,7 @@ class ForensicLogger:
         """
         #with self.lock:
         api_request = APIRequest(
-            timestamp=datetime.now(datetime.UTC).isoformat(),
+            timestamp=datetime.now(timezone.utc).isoformat(),
             provider=provider,
             url=url,
             method=method,
@@ -153,7 +154,7 @@ class ForensicLogger:
         """
         #with self.lock:
         relationship = RelationshipDiscovery(
-            timestamp=datetime.now(datetime.UTC).isoformat(),
+            timestamp=datetime.now(timezone.utc).isoformat(),
             source_node=source_node,
             target_node=target_node,
             relationship_type=relationship_type,
@@ -183,7 +184,7 @@ class ForensicLogger:
     def log_scan_complete(self) -> None:
         """Log the completion of a reconnaissance scan."""
         #with self.lock:
-        self.session_metadata['end_time'] = datetime.now(datetime.UTC).isoformat()
+        self.session_metadata['end_time'] = datetime.now(timezone.utc).isoformat()
         self.session_metadata['providers_used'] = list(self.session_metadata['providers_used'])
         self.session_metadata['target_domains'] = list(self.session_metadata['target_domains'])
         
@@ -203,7 +204,7 @@ class ForensicLogger:
             'session_metadata': self.session_metadata.copy(),
             'api_requests': [asdict(req) for req in self.api_requests],
             'relationships': [asdict(rel) for rel in self.relationships],
-            'export_timestamp': datetime.now(datetime.UTC).isoformat()
+            'export_timestamp': datetime.now(timezone.utc).isoformat()
         }
     
     def get_forensic_summary(self) -> Dict[str, Any]:
@@ -239,7 +240,7 @@ class ForensicLogger:
     def _calculate_session_duration(self) -> float:
         """Calculate session duration in minutes."""
         if not self.session_metadata['end_time']:
-            end_time = datetime.now(datetime.UTC)
+            end_time = datetime.now(timezone.utc)
         else:
             end_time = datetime.fromisoformat(self.session_metadata['end_time'])
         

core/scanner.py

@@ -6,12 +6,15 @@ Coordinates data gathering from multiple providers and builds the infrastructure
 import threading
 import time
 import traceback
-from typing import List, Set, Dict, Any, Optional
+from typing import List, Set, Dict, Any, Optional, Tuple
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from core.graph_manager import GraphManager, NodeType, RelationshipType
 from core.logger import get_forensic_logger, new_session
 from providers.crtsh_provider import CrtShProvider
+from providers.dns_provider import DNSProvider
+from providers.shodan_provider import ShodanProvider
+from providers.virustotal_provider import VirusTotalProvider
 from config import config
 
 
@@ -27,17 +30,16 @@ class ScanStatus:
 class Scanner:
     """
     Main scanning orchestrator for DNSRecon passive reconnaissance.
-    Manages multi-provider data gathering and graph construction.
+    Manages multi-provider data gathering and graph construction with concurrent processing.
     """
 
     def __init__(self):
-        """Initialize scanner with default providers and empty graph."""
+        """Initialize scanner with all available providers and empty graph."""
         print("Initializing Scanner instance...")
         
         try:
-            from providers.base_provider import BaseProvider
             self.graph = GraphManager()
-            self.providers: List[BaseProvider] = []
+            self.providers = []
             self.status = ScanStatus.IDLE
             self.current_target = None
             self.current_depth = 0
@@ -50,6 +52,9 @@ class Scanner:
             self.indicators_processed = 0
             self.current_indicator = ""
             
+            # Concurrent processing configuration
+            self.max_workers = config.max_concurrent_requests
+
             # Initialize providers
             print("Calling _initialize_providers...")
             self._initialize_providers()
@@ -66,36 +71,54 @@ class Scanner:
             raise
 
     def _initialize_providers(self) -> None:
-        """Initialize available providers based on configuration."""
+        """Initialize all available providers based on configuration."""
         self.providers = []
 
         print("Initializing providers...")
 
         # Always add free providers
-        if config.is_provider_enabled('crtsh'):
+        free_providers = [
+            ('crtsh', CrtShProvider),
+            ('dns', DNSProvider)
+        ]
+        
+        for provider_name, provider_class in free_providers:
+            if config.is_provider_enabled(provider_name):
                 try:
-                crtsh_provider = CrtShProvider()
+                    provider = provider_class()
-                if crtsh_provider.is_available():
+                    if provider.is_available():
-                    self.providers.append(crtsh_provider)
+                        self.providers.append(provider)
-                    print("✓ CrtSh provider initialized successfully")
+                        print(f"✓ {provider_name.title()} provider initialized successfully")
                     else:
-                    print("✗ CrtSh provider is not available")
+                        print(f"✗ {provider_name.title()} provider is not available")
                 except Exception as e:
-                print(f"✗ Failed to initialize CrtSh provider: {e}")
+                    print(f"✗ Failed to initialize {provider_name.title()} provider: {e}")
+                    traceback.print_exc()
+
+        # Add API key-dependent providers
+        api_providers = [
+            ('shodan', ShodanProvider),
+            ('virustotal', VirusTotalProvider)
+        ]
+        
+        for provider_name, provider_class in api_providers:
+            if config.is_provider_enabled(provider_name):
+                try:
+                    provider = provider_class()
+                    if provider.is_available():
+                        self.providers.append(provider)
+                        print(f"✓ {provider_name.title()} provider initialized successfully")
+                    else:
+                        print(f"✗ {provider_name.title()} provider is not available (API key required)")
+                except Exception as e:
+                    print(f"✗ Failed to initialize {provider_name.title()} provider: {e}")
                     traceback.print_exc()
 
         print(f"Initialized {len(self.providers)} providers")
 
-    def _debug_threads(self):
-        """Debug function to show current threads."""
-        print("=== THREAD DEBUG INFO ===")
-        for t in threading.enumerate():
-            print(f"Thread: {t.name} | Alive: {t.is_alive()} | Daemon: {t.daemon}")
-        print("=== END THREAD DEBUG ===")
-
     def start_scan(self, target_domain: str, max_depth: int = 2) -> bool:
         """
-        Start a new reconnaissance scan.
+        Start a new reconnaissance scan with concurrent processing.
 
         Args:
             target_domain: Initial domain to investigate
@@ -107,9 +130,6 @@ class Scanner:
         print(f"Scanner.start_scan called with target='{target_domain}', depth={max_depth}")
         
         try:
-            print("Checking current status...")
-            self._debug_threads()
-            
             if self.status == ScanStatus.RUNNING:
                 print("Scan already running, rejecting new scan")
                 return False
@@ -119,8 +139,6 @@ class Scanner:
                 print("No providers available, cannot start scan")
                 return False
 
-            print(f"Current status: {self.status}, Providers: {len(self.providers)}")
-
             # Stop any existing scan thread
             if self.scan_thread and self.scan_thread.is_alive():
                 print("Stopping existing scan thread...")
@@ -132,9 +150,7 @@ class Scanner:
 
             # Reset state
             print("Resetting scanner state...")
-            #print("Running graph.clear()")
+            self.graph.clear()
-            #self.graph.clear()
-            print("running self.current_target = target_domain.lower().strip()")
             self.current_target = target_domain.lower().strip()
             self.max_depth = max_depth
             self.current_depth = 0
@@ -147,9 +163,15 @@ class Scanner:
             print("Starting new forensic session...")
             self.logger = new_session()
 
-            # FOR DEBUGGING: Run scan synchronously instead of in thread
+            # Start scan in separate thread for Phase 2
-            print("Running scan synchronously for debugging...")
+            print("Starting scan thread...")
-            self._execute_scan_sync(self.current_target, max_depth)
+            self.scan_thread = threading.Thread(
+                target=self._execute_scan_async,
+                args=(self.current_target, max_depth),
+                daemon=True
+            )
+            self.scan_thread.start()
+            
             return True
                 
         except Exception as e:
@@ -157,6 +179,321 @@ class Scanner:
             traceback.print_exc()
             return False
 
+    def _execute_scan_async(self, target_domain: str, max_depth: int) -> None:
+        """
+        Execute the reconnaissance scan asynchronously with concurrent provider queries.
+
+        Args:
+            target_domain: Target domain to investigate
+            max_depth: Maximum recursion depth
+        """
+        print(f"_execute_scan_async started for {target_domain} with depth {max_depth}")
+
+        try:
+            print("Setting status to RUNNING")
+            self.status = ScanStatus.RUNNING
+
+            # Log scan start
+            enabled_providers = [provider.get_name() for provider in self.providers]
+            self.logger.log_scan_start(target_domain, max_depth, enabled_providers)
+            print(f"Logged scan start with providers: {enabled_providers}")
+
+            # Initialize with target domain
+            print(f"Adding target domain '{target_domain}' as initial node")
+            self.graph.add_node(target_domain, NodeType.DOMAIN)
+
+            # BFS-style exploration with depth limiting and concurrent processing
+            current_level_domains = {target_domain}
+            processed_domains = set()
+            all_discovered_ips = set()
+
+            print(f"Starting BFS exploration...")
+
+            for depth in range(max_depth + 1):
+                if self.stop_requested:
+                    print(f"Stop requested at depth {depth}")
+                    break
+
+                self.current_depth = depth
+                print(f"Processing depth level {depth} with {len(current_level_domains)} domains")
+
+                if not current_level_domains:
+                    print("No domains to process at this level")
+                    break
+
+                # Update progress tracking
+                self.total_indicators_found += len(current_level_domains)
+                next_level_domains = set()
+
+                # Process domains at current depth level with concurrent queries
+                domain_results = self._process_domains_concurrent(current_level_domains, processed_domains)
+                
+                for domain, discovered_domains, discovered_ips in domain_results:
+                    if self.stop_requested:
+                        break
+                    
+                    processed_domains.add(domain)
+                    all_discovered_ips.update(discovered_ips)
+                    
+                    # Add discovered domains to next level if not at max depth
+                    if depth < max_depth:
+                        for discovered_domain in discovered_domains:
+                            if discovered_domain not in processed_domains:
+                                next_level_domains.add(discovered_domain)
+                                print(f"Adding {discovered_domain} to next level")
+
+                # Process discovered IPs concurrently
+                if all_discovered_ips:
+                    print(f"Processing {len(all_discovered_ips)} discovered IP addresses")
+                    self._process_ips_concurrent(all_discovered_ips)
+
+                current_level_domains = next_level_domains
+                print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level")
+
+            # Finalize scan
+            if self.stop_requested:
+                self.status = ScanStatus.STOPPED
+                print("Scan completed with STOPPED status")
+            else:
+                self.status = ScanStatus.COMPLETED
+                print("Scan completed with COMPLETED status")
+
+            self.logger.log_scan_complete()
+
+            # Print final statistics
+            stats = self.graph.get_statistics()
+            print(f"Final scan statistics:")
+            print(f"  - Total nodes: {stats['basic_metrics']['total_nodes']}")
+            print(f"  - Total edges: {stats['basic_metrics']['total_edges']}")
+            print(f"  - Domains processed: {len(processed_domains)}")
+            print(f"  - IPs discovered: {len(all_discovered_ips)}")
+
+        except Exception as e:
+            print(f"ERROR: Scan execution failed with error: {e}")
+            traceback.print_exc()
+            self.status = ScanStatus.FAILED
+            self.logger.logger.error(f"Scan failed: {e}")
+
+    def _process_domains_concurrent(self, domains: Set[str], processed_domains: Set[str]) -> List[Tuple[str, Set[str], Set[str]]]:
+        """
+        Process multiple domains concurrently using thread pool.
+        
+        Args:
+            domains: Set of domains to process
+            processed_domains: Set of already processed domains
+            
+        Returns:
+            List of tuples (domain, discovered_domains, discovered_ips)
+        """
+        results = []
+        
+        # Filter out already processed domains
+        domains_to_process = domains - processed_domains
+        
+        if not domains_to_process:
+            return results
+        
+        print(f"Processing {len(domains_to_process)} domains concurrently with {self.max_workers} workers")
+        
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            # Submit all domain processing tasks
+            future_to_domain = {
+                executor.submit(self._query_providers_for_domain, domain): domain 
+                for domain in domains_to_process
+            }
+            
+            # Collect results as they complete
+            for future in as_completed(future_to_domain):
+                if self.stop_requested:
+                    break
+                    
+                domain = future_to_domain[future]
+                
+                try:
+                    discovered_domains, discovered_ips = future.result()
+                    results.append((domain, discovered_domains, discovered_ips))
+                    self.indicators_processed += 1
+                    print(f"Completed processing domain: {domain} ({len(discovered_domains)} domains, {len(discovered_ips)} IPs)")
+                    
+                except Exception as e:
+                    print(f"Error processing domain {domain}: {e}")
+                    traceback.print_exc()
+        
+        return results
+
+    def _process_ips_concurrent(self, ips: Set[str]) -> None:
+        """
+        Process multiple IP addresses concurrently.
+        
+        Args:
+            ips: Set of IP addresses to process
+        """
+        if not ips:
+            return
+            
+        print(f"Processing {len(ips)} IP addresses concurrently")
+        
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            # Submit all IP processing tasks
+            future_to_ip = {
+                executor.submit(self._query_providers_for_ip, ip): ip 
+                for ip in ips
+            }
+            
+            # Collect results as they complete
+            for future in as_completed(future_to_ip):
+                if self.stop_requested:
+                    break
+                    
+                ip = future_to_ip[future]
+                
+                try:
+                    future.result()  # Just wait for completion
+                    print(f"Completed processing IP: {ip}")
+                    
+                except Exception as e:
+                    print(f"Error processing IP {ip}: {e}")
+                    traceback.print_exc()
+
+    def _query_providers_for_domain(self, domain: str) -> Tuple[Set[str], Set[str]]:
+        """
+        Query all enabled providers for information about a domain.
+
+        Args:
+            domain: Domain to investigate
+
+        Returns:
+            Tuple of (discovered_domains, discovered_ips)
+        """
+        print(f"Querying {len(self.providers)} providers for domain: {domain}")
+        discovered_domains = set()
+        discovered_ips = set()
+
+        if not self.providers:
+            print("No providers available")
+            return discovered_domains, discovered_ips
+
+        # Query providers concurrently for better performance
+        with ThreadPoolExecutor(max_workers=len(self.providers)) as executor:
+            # Submit queries for all providers
+            future_to_provider = {
+                executor.submit(self._safe_provider_query_domain, provider, domain): provider 
+                for provider in self.providers
+            }
+            
+            # Collect results as they complete
+            for future in as_completed(future_to_provider):
+                if self.stop_requested:
+                    break
+                    
+                provider = future_to_provider[future]
+                
+                try:
+                    relationships = future.result()
+                    print(f"Provider {provider.get_name()} returned {len(relationships)} relationships")
+                    
+                    for source, target, rel_type, confidence, raw_data in relationships:
+                        # Determine node type based on target
+                        if self._is_valid_ip(target):
+                            target_node_type = NodeType.IP
+                            discovered_ips.add(target)
+                        elif self._is_valid_domain(target):
+                            target_node_type = NodeType.DOMAIN
+                            discovered_domains.add(target)
+                        else:
+                            # Could be ASN or certificate
+                            target_node_type = NodeType.ASN if target.startswith('AS') else NodeType.CERTIFICATE
+                        
+                        # Add nodes and relationship to graph
+                        self.graph.add_node(source, NodeType.DOMAIN)
+                        self.graph.add_node(target, target_node_type)
+                        
+                        success = self.graph.add_edge(
+                            source, target, rel_type, confidence,
+                            provider.get_name(), raw_data
+                        )
+                        
+                        if success:
+                            print(f"Added relationship: {source} -> {target} ({rel_type.relationship_name})")
+                
+                except Exception as e:
+                    print(f"Provider {provider.get_name()} failed for {domain}: {e}")
+
+        print(f"Domain {domain}: discovered {len(discovered_domains)} domains, {len(discovered_ips)} IPs")
+        return discovered_domains, discovered_ips
+
+    def _query_providers_for_ip(self, ip: str) -> None:
+        """
+        Query all enabled providers for information about an IP address.
+
+        Args:
+            ip: IP address to investigate
+        """
+        print(f"Querying {len(self.providers)} providers for IP: {ip}")
+
+        if not self.providers:
+            print("No providers available")
+            return
+
+        # Query providers concurrently
+        with ThreadPoolExecutor(max_workers=len(self.providers)) as executor:
+            # Submit queries for all providers
+            future_to_provider = {
+                executor.submit(self._safe_provider_query_ip, provider, ip): provider 
+                for provider in self.providers
+            }
+            
+            # Collect results as they complete
+            for future in as_completed(future_to_provider):
+                if self.stop_requested:
+                    break
+                    
+                provider = future_to_provider[future]
+                
+                try:
+                    relationships = future.result()
+                    print(f"Provider {provider.get_name()} returned {len(relationships)} relationships for IP {ip}")
+                    
+                    for source, target, rel_type, confidence, raw_data in relationships:
+                        # Determine node type based on target
+                        if self._is_valid_domain(target):
+                            target_node_type = NodeType.DOMAIN
+                        elif target.startswith('AS'):
+                            target_node_type = NodeType.ASN
+                        else:
+                            target_node_type = NodeType.IP
+                        
+                        # Add nodes and relationship to graph
+                        self.graph.add_node(source, NodeType.IP)
+                        self.graph.add_node(target, target_node_type)
+                        
+                        success = self.graph.add_edge(
+                            source, target, rel_type, confidence,
+                            provider.get_name(), raw_data
+                        )
+                        
+                        if success:
+                            print(f"Added IP relationship: {source} -> {target} ({rel_type.relationship_name})")
+                
+                except Exception as e:
+                    print(f"Provider {provider.get_name()} failed for IP {ip}: {e}")
+
+    def _safe_provider_query_domain(self, provider, domain: str):
+        """Safely query provider for domain with error handling."""
+        try:
+            return provider.query_domain(domain)
+        except Exception as e:
+            print(f"Provider {provider.get_name()} query_domain failed: {e}")
+            return []
+
+    def _safe_provider_query_ip(self, provider, ip: str):
+        """Safely query provider for IP with error handling."""
+        try:
+            return provider.query_ip(ip)
+        except Exception as e:
+            print(f"Provider {provider.get_name()} query_ip failed: {e}")
+            return []
+
     def stop_scan(self) -> bool:
         """
         Request scan termination.
@@ -218,159 +555,6 @@ class Scanner:
             return 0.0
         return min(100.0, (self.indicators_processed / self.total_indicators_found) * 100)
 
-    def _execute_scan_sync(self, target_domain: str, max_depth: int) -> None:
-        """
-        Execute the reconnaissance scan synchronously (for debugging).
-
-        Args:
-            target_domain: Target domain to investigate
-            max_depth: Maximum recursion depth
-        """
-        print(f"_execute_scan_sync started for {target_domain} with depth {max_depth}")
-
-        try:
-            print("Setting status to RUNNING")
-            self.status = ScanStatus.RUNNING
-
-            # Log scan start
-            enabled_providers = [provider.get_name() for provider in self.providers]
-            self.logger.log_scan_start(target_domain, max_depth, enabled_providers)
-            print(f"Logged scan start with providers: {enabled_providers}")
-
-            # Initialize with target domain
-            print(f"Adding target domain '{target_domain}' as initial node")
-            self.graph.add_node(target_domain, NodeType.DOMAIN)
-
-            # BFS-style exploration with depth limiting
-            current_level_domains = {target_domain}
-            processed_domains = set()
-
-            print(f"Starting BFS exploration...")
-
-            for depth in range(max_depth + 1):
-                if self.stop_requested:
-                    print(f"Stop requested at depth {depth}")
-                    break
-
-                self.current_depth = depth
-                print(f"Processing depth level {depth} with {len(current_level_domains)} domains")
-
-                if not current_level_domains:
-                    print("No domains to process at this level")
-                    break
-
-                # Update progress tracking
-                self.total_indicators_found += len(current_level_domains)
-                next_level_domains = set()
-
-                # Process domains at current depth level
-                for domain in current_level_domains:
-                    if self.stop_requested:
-                        print(f"Stop requested while processing domain {domain}")
-                        break
-
-                    if domain in processed_domains:
-                        print(f"Domain {domain} already processed, skipping")
-                        continue
-
-                    print(f"Processing domain: {domain}")
-                    self.current_indicator = domain
-                    self.indicators_processed += 1
-
-                    # Query all providers for this domain
-                    discovered_domains = self._query_providers_for_domain(domain)
-                    print(f"Discovered {len(discovered_domains)} new domains from {domain}")
-
-                    # Add discovered domains to next level if not at max depth
-                    if depth < max_depth:
-                        for discovered_domain in discovered_domains:
-                            if discovered_domain not in processed_domains:
-                                next_level_domains.add(discovered_domain)
-                                print(f"Adding {discovered_domain} to next level")
-
-                    processed_domains.add(domain)
-
-                current_level_domains = next_level_domains
-                print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level")
-
-            # Finalize scan
-            if self.stop_requested:
-                self.status = ScanStatus.STOPPED
-                print("Scan completed with STOPPED status")
-            else:
-                self.status = ScanStatus.COMPLETED
-                print("Scan completed with COMPLETED status")
-
-            self.logger.log_scan_complete()
-
-            # Print final statistics
-            stats = self.graph.get_statistics()
-            print(f"Final scan statistics:")
-            print(f"  - Total nodes: {stats['basic_metrics']['total_nodes']}")
-            print(f"  - Total edges: {stats['basic_metrics']['total_edges']}")
-            print(f"  - Domains processed: {len(processed_domains)}")
-
-        except Exception as e:
-            print(f"ERROR: Scan execution failed with error: {e}")
-            traceback.print_exc()
-            self.status = ScanStatus.FAILED
-            self.logger.logger.error(f"Scan failed: {e}")
-
-    def _query_providers_for_domain(self, domain: str) -> Set[str]:
-        """
-        Query all enabled providers for information about a domain.
-
-        Args:
-            domain: Domain to investigate
-
-        Returns:
-            Set of newly discovered domains
-        """
-        print(f"Querying {len(self.providers)} providers for domain: {domain}")
-        discovered_domains = set()
-
-        if not self.providers:
-            print("No providers available")
-            return discovered_domains
-
-        # Query providers sequentially for debugging
-        for provider in self.providers:
-            if self.stop_requested:
-                print("Stop requested, cancelling provider queries")
-                break
-
-            try:
-                print(f"Querying provider: {provider.get_name()}")
-                relationships = provider.query_domain(domain)
-                print(f"Provider {provider.get_name()} returned {len(relationships)} relationships")
-
-                for source, target, rel_type, confidence, raw_data in relationships:
-                    print(f"Processing relationship: {source} -> {target} ({rel_type.relationship_name})")
-
-                    # Add target node to graph if it doesn't exist
-                    self.graph.add_node(target, NodeType.DOMAIN)
-
-                    # Add relationship
-                    success = self.graph.add_edge(
-                        source, target, rel_type, confidence,
-                        provider.get_name(), raw_data
-                    )
-
-                    if success:
-                        print(f"Added new relationship: {source} -> {target}")
-                    else:
-                        print(f"Relationship already exists or failed to add: {source} -> {target}")
-
-                    discovered_domains.add(target)
-
-            except Exception as e:
-                print(f"Provider {provider.get_name()} failed for {domain}: {e}")
-                traceback.print_exc()
-                self.logger.logger.error(f"Provider {provider.get_name()} failed for {domain}: {e}")
-
-        print(f"Total unique domains discovered: {len(discovered_domains)}")
-        return discovered_domains
-
     def get_graph_data(self) -> Dict[str, Any]:
         """
         Get current graph data for visualization.

providers/__init__.py

@@ -5,11 +5,17 @@ Contains implementations for various reconnaissance data sources.
 
 from .base_provider import BaseProvider, RateLimiter
 from .crtsh_provider import CrtShProvider
+from .dns_provider import DNSProvider
+from .shodan_provider import ShodanProvider
+from .virustotal_provider import VirusTotalProvider
 
 __all__ = [
     'BaseProvider',
     'RateLimiter', 
-    'CrtShProvider'
+    'CrtShProvider',
+    'DNSProvider',
+    'ShodanProvider',
+    'VirusTotalProvider'
 ]
 
-__version__ = "1.0.0-phase1"
+__version__ = "1.0.0-phase2"

providers/base_provider.py

@@ -115,9 +115,10 @@ class BaseProvider(ABC):
     def make_request(self, url: str, method: str = "GET",
                         params: Optional[Dict[str, Any]] = None,
                         headers: Optional[Dict[str, str]] = None,
-                    target_indicator: str = "") -> Optional[requests.Response]:
+                        target_indicator: str = "",
+                        max_retries: int = 3) -> Optional[requests.Response]:
             """
-        Make a rate-limited HTTP request with forensic logging.
+            Make a rate-limited HTTP request with forensic logging and retry logic.
 
             Args:
                 url: Request URL
@@ -125,10 +126,12 @@ class BaseProvider(ABC):
                 params: Query parameters
                 headers: Additional headers
                 target_indicator: The indicator being investigated
+                max_retries: Maximum number of retry attempts
 
             Returns:
                 Response object or None if request failed
             """
+            for attempt in range(max_retries + 1):
                 # Apply rate limiting
                 self.rate_limiter.wait_if_needed()
 
@@ -144,7 +147,7 @@ class BaseProvider(ABC):
                     if headers:
                         request_headers.update(headers)
 
-            print(f"Making {method} request to: {url}")
+                    print(f"Making {method} request to: {url} (attempt {attempt + 1})")
 
                     # Make request
                     if method.upper() == "GET":
@@ -168,19 +171,42 @@ class BaseProvider(ABC):
                     response.raise_for_status()
                     self.successful_requests += 1
                     
+                    # Success - log and return
+                    duration_ms = (time.time() - start_time) * 1000
+                    self.logger.log_api_request(
+                        provider=self.name,
+                        url=url,
+                        method=method.upper(),
+                        status_code=response.status_code,
+                        response_size=len(response.content),
+                        duration_ms=duration_ms,
+                        error=None,
+                        target_indicator=target_indicator
+                    )
+                    return response
+
                 except requests.exceptions.RequestException as e:
                     error = str(e)
                     self.failed_requests += 1
-            print(f"Request failed: {error}")
+                    print(f"Request failed (attempt {attempt + 1}): {error}")
+                    
+                    # Check if we should retry
+                    if attempt < max_retries and self._should_retry(e):
+                        backoff_time = (2 ** attempt) * 1  # Exponential backoff: 1s, 2s, 4s
+                        print(f"Retrying in {backoff_time} seconds...")
+                        time.sleep(backoff_time)
+                        continue
+                    else:
+                        break
 
                 except Exception as e:
                     error = f"Unexpected error: {str(e)}"
                     self.failed_requests += 1
                     print(f"Unexpected error: {error}")
+                    break
 
-        # Calculate duration and log request
+            # All attempts failed - log and return None
             duration_ms = (time.time() - start_time) * 1000
-
             self.logger.log_api_request(
                 provider=self.name,
                 url=url,
@@ -192,7 +218,29 @@ class BaseProvider(ABC):
                 target_indicator=target_indicator
             )
             
-        return response if error is None else None
+            return None
+
+    def _should_retry(self, exception: requests.exceptions.RequestException) -> bool:
+        """
+        Determine if a request should be retried based on the exception.
+        
+        Args:
+            exception: The request exception that occurred
+            
+        Returns:
+            True if the request should be retried
+        """
+        # Retry on connection errors, timeouts, and 5xx server errors
+        if isinstance(exception, (requests.exceptions.ConnectionError, 
+                                requests.exceptions.Timeout)):
+            return True
+        
+        if isinstance(exception, requests.exceptions.HTTPError):
+            if hasattr(exception, 'response') and exception.response:
+                # Retry on server errors (5xx) but not client errors (4xx)
+                return exception.response.status_code >= 500
+        
+        return False
 
     def log_relationship_discovery(self, source_node: str, target_node: str,
                                  relationship_type: RelationshipType,

providers/dns_provider.py

@@ -0,0 +1,338 @@
+"""
+DNS resolution provider for DNSRecon.
+Discovers domain relationships through DNS record analysis.
+"""
+
+import socket
+import dns.resolver
+import dns.reversename
+from typing import List, Dict, Any, Tuple, Optional
+from .base_provider import BaseProvider
+from core.graph_manager import RelationshipType, NodeType
+
+
+class DNSProvider(BaseProvider):
+    """
+    Provider for standard DNS resolution and reverse DNS lookups.
+    Discovers domain-to-IP and IP-to-domain relationships through DNS records.
+    """
+    
+    def __init__(self):
+        """Initialize DNS provider with appropriate rate limiting."""
+        super().__init__(
+            name="dns",
+            rate_limit=100,  # DNS queries can be faster
+            timeout=10
+        )
+        
+        # Configure DNS resolver
+        self.resolver = dns.resolver.Resolver()
+        self.resolver.timeout = 5
+        self.resolver.lifetime = 10
+    
+    def get_name(self) -> str:
+        """Return the provider name."""
+        return "dns"
+    
+    def is_available(self) -> bool:
+        """DNS is always available - no API key required."""
+        return True
+    
+    def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """
+        Query DNS records for the domain to discover relationships.
+        
+        Args:
+            domain: Domain to investigate
+            
+        Returns:
+            List of relationships discovered from DNS analysis
+        """
+        if not self._is_valid_domain(domain):
+            return []
+        
+        relationships = []
+        
+        # Query A records
+        relationships.extend(self._query_a_records(domain))
+        
+        # Query AAAA records (IPv6)
+        relationships.extend(self._query_aaaa_records(domain))
+        
+        # Query CNAME records
+        relationships.extend(self._query_cname_records(domain))
+        
+        # Query MX records
+        relationships.extend(self._query_mx_records(domain))
+        
+        # Query NS records
+        relationships.extend(self._query_ns_records(domain))
+        
+        return relationships
+    
+    def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """
+        Query reverse DNS for the IP address.
+        
+        Args:
+            ip: IP address to investigate
+            
+        Returns:
+            List of relationships discovered from reverse DNS
+        """
+        if not self._is_valid_ip(ip):
+            return []
+        
+        relationships = []
+        
+        try:
+            # Perform reverse DNS lookup
+            reverse_name = dns.reversename.from_address(ip)
+            response = self.resolver.resolve(reverse_name, 'PTR')
+            
+            for ptr_record in response:
+                hostname = str(ptr_record).rstrip('.')
+                
+                if self._is_valid_domain(hostname):
+                    raw_data = {
+                        'query_type': 'PTR',
+                        'ip_address': ip,
+                        'hostname': hostname,
+                        'ttl': response.ttl
+                    }
+                    
+                    relationships.append((
+                        ip,
+                        hostname,
+                        RelationshipType.A_RECORD,  # Reverse relationship
+                        RelationshipType.A_RECORD.default_confidence,
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=ip,
+                        target_node=hostname,
+                        relationship_type=RelationshipType.A_RECORD,
+                        confidence_score=RelationshipType.A_RECORD.default_confidence,
+                        raw_data=raw_data,
+                        discovery_method="reverse_dns_lookup"
+                    )
+        
+        except Exception as e:
+            self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}")
+        
+        return relationships
+    
+    def _query_a_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query A records for the domain."""
+        relationships = []
+        
+        #if not DNS_AVAILABLE:
+        #    return relationships
+        
+        try:
+            response = self.resolver.resolve(domain, 'A')
+            
+            for a_record in response:
+                ip_address = str(a_record)
+                
+                raw_data = {
+                    'query_type': 'A',
+                    'domain': domain,
+                    'ip_address': ip_address,
+                    'ttl': response.ttl
+                }
+                
+                relationships.append((
+                    domain,
+                    ip_address,
+                    RelationshipType.A_RECORD,
+                    RelationshipType.A_RECORD.default_confidence,
+                    raw_data
+                ))
+                
+                self.log_relationship_discovery(
+                    source_node=domain,
+                    target_node=ip_address,
+                    relationship_type=RelationshipType.A_RECORD,
+                    confidence_score=RelationshipType.A_RECORD.default_confidence,
+                    raw_data=raw_data,
+                    discovery_method="dns_a_record"
+                )
+        
+        except Exception as e:
+            self.logger.logger.debug(f"A record query failed for {domain}: {e}")
+        
+        return relationships
+    
+    def _query_aaaa_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query AAAA records (IPv6) for the domain."""
+        relationships = []
+        
+        #if not DNS_AVAILABLE:
+        #    return relationships
+        
+        try:
+            response = self.resolver.resolve(domain, 'AAAA')
+            
+            for aaaa_record in response:
+                ip_address = str(aaaa_record)
+                
+                raw_data = {
+                    'query_type': 'AAAA',
+                    'domain': domain,
+                    'ip_address': ip_address,
+                    'ttl': response.ttl
+                }
+                
+                relationships.append((
+                    domain,
+                    ip_address,
+                    RelationshipType.A_RECORD,  # Using same type for IPv6
+                    RelationshipType.A_RECORD.default_confidence,
+                    raw_data
+                ))
+                
+                self.log_relationship_discovery(
+                    source_node=domain,
+                    target_node=ip_address,
+                    relationship_type=RelationshipType.A_RECORD,
+                    confidence_score=RelationshipType.A_RECORD.default_confidence,
+                    raw_data=raw_data,
+                    discovery_method="dns_aaaa_record"
+                )
+        
+        except Exception as e:
+            self.logger.logger.debug(f"AAAA record query failed for {domain}: {e}")
+        
+        return relationships
+    
+    def _query_cname_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query CNAME records for the domain."""
+        relationships = []
+        
+        #if not DNS_AVAILABLE:
+        #    return relationships
+        
+        try:
+            response = self.resolver.resolve(domain, 'CNAME')
+            
+            for cname_record in response:
+                target_domain = str(cname_record).rstrip('.')
+                
+                if self._is_valid_domain(target_domain):
+                    raw_data = {
+                        'query_type': 'CNAME',
+                        'source_domain': domain,
+                        'target_domain': target_domain,
+                        'ttl': response.ttl
+                    }
+                    
+                    relationships.append((
+                        domain,
+                        target_domain,
+                        RelationshipType.CNAME_RECORD,
+                        RelationshipType.CNAME_RECORD.default_confidence,
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=domain,
+                        target_node=target_domain,
+                        relationship_type=RelationshipType.CNAME_RECORD,
+                        confidence_score=RelationshipType.CNAME_RECORD.default_confidence,
+                        raw_data=raw_data,
+                        discovery_method="dns_cname_record"
+                    )
+        
+        except Exception as e:
+            self.logger.logger.debug(f"CNAME record query failed for {domain}: {e}")
+        
+        return relationships
+    
+    def _query_mx_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query MX records for the domain."""
+        relationships = []
+        
+        #if not DNS_AVAILABLE:
+        #    return relationships
+        
+        try:
+            response = self.resolver.resolve(domain, 'MX')
+            
+            for mx_record in response:
+                mx_host = str(mx_record.exchange).rstrip('.')
+                
+                if self._is_valid_domain(mx_host):
+                    raw_data = {
+                        'query_type': 'MX',
+                        'domain': domain,
+                        'mx_host': mx_host,
+                        'priority': mx_record.preference,
+                        'ttl': response.ttl
+                    }
+                    
+                    relationships.append((
+                        domain,
+                        mx_host,
+                        RelationshipType.MX_RECORD,
+                        RelationshipType.MX_RECORD.default_confidence,
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=domain,
+                        target_node=mx_host,
+                        relationship_type=RelationshipType.MX_RECORD,
+                        confidence_score=RelationshipType.MX_RECORD.default_confidence,
+                        raw_data=raw_data,
+                        discovery_method="dns_mx_record"
+                    )
+        
+        except Exception as e:
+            self.logger.logger.debug(f"MX record query failed for {domain}: {e}")
+        
+        return relationships
+    
+    def _query_ns_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query NS records for the domain."""
+        relationships = []
+        
+        #if not DNS_AVAILABLE:
+        #    return relationships
+        
+        try:
+            response = self.resolver.resolve(domain, 'NS')
+            
+            for ns_record in response:
+                ns_host = str(ns_record).rstrip('.')
+                
+                if self._is_valid_domain(ns_host):
+                    raw_data = {
+                        'query_type': 'NS',
+                        'domain': domain,
+                        'ns_host': ns_host,
+                        'ttl': response.ttl
+                    }
+                    
+                    relationships.append((
+                        domain,
+                        ns_host,
+                        RelationshipType.NS_RECORD,
+                        RelationshipType.NS_RECORD.default_confidence,
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=domain,
+                        target_node=ns_host,
+                        relationship_type=RelationshipType.NS_RECORD,
+                        confidence_score=RelationshipType.NS_RECORD.default_confidence,
+                        raw_data=raw_data,
+                        discovery_method="dns_ns_record"
+                    )
+        
+        except Exception as e:
+            self.logger.logger.debug(f"NS record query failed for {domain}: {e}")
+        
+        return relationships

providers/shodan_provider.py

@@ -0,0 +1,299 @@
+"""
+Shodan provider for DNSRecon.
+Discovers IP relationships and infrastructure context through Shodan API.
+"""
+
+import json
+from typing import List, Dict, Any, Tuple, Optional
+from urllib.parse import quote
+from .base_provider import BaseProvider
+from core.graph_manager import RelationshipType
+from config import config
+
+
+class ShodanProvider(BaseProvider):
+    """
+    Provider for querying Shodan API for IP address and hostname information.
+    Requires valid API key and respects Shodan's rate limits.
+    """
+    
+    def __init__(self):
+        """Initialize Shodan provider with appropriate rate limiting."""
+        super().__init__(
+            name="shodan",
+            rate_limit=60,  # Shodan API has various rate limits depending on plan
+            timeout=30
+        )
+        self.base_url = "https://api.shodan.io"
+        self.api_key = config.get_api_key('shodan')
+    
+    def get_name(self) -> str:
+        """Return the provider name."""
+        return "shodan"
+    
+    def is_available(self) -> bool:
+        """
+        Check if Shodan provider is available (has valid API key).
+        """
+        return self.api_key is not None and len(self.api_key.strip()) > 0
+    
+    def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """
+        Query Shodan for information about a domain.
+        Uses Shodan's hostname search to find associated IPs.
+        
+        Args:
+            domain: Domain to investigate
+            
+        Returns:
+            List of relationships discovered from Shodan data
+        """
+        if not self._is_valid_domain(domain) or not self.is_available():
+            return []
+        
+        relationships = []
+        
+        try:
+            # Search for hostname in Shodan
+            search_query = f"hostname:{domain}"
+            url = f"{self.base_url}/shodan/host/search"
+            params = {
+                'key': self.api_key,
+                'query': search_query,
+                'minify': True  # Get minimal data to reduce bandwidth
+            }
+            
+            response = self.make_request(url, method="GET", params=params, target_indicator=domain)
+            
+            if not response or response.status_code != 200:
+                return []
+            
+            data = response.json()
+            
+            if 'matches' not in data:
+                return []
+            
+            # Process search results
+            for match in data['matches']:
+                ip_address = match.get('ip_str')
+                hostnames = match.get('hostnames', [])
+                
+                if ip_address and domain in hostnames:
+                    raw_data = {
+                        'ip_address': ip_address,
+                        'hostnames': hostnames,
+                        'country': match.get('location', {}).get('country_name', ''),
+                        'city': match.get('location', {}).get('city', ''),
+                        'isp': match.get('isp', ''),
+                        'org': match.get('org', ''),
+                        'ports': match.get('ports', []),
+                        'last_update': match.get('last_update', '')
+                    }
+                    
+                    relationships.append((
+                        domain,
+                        ip_address,
+                        RelationshipType.A_RECORD,  # Domain resolves to IP
+                        RelationshipType.A_RECORD.default_confidence,
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=domain,
+                        target_node=ip_address,
+                        relationship_type=RelationshipType.A_RECORD,
+                        confidence_score=RelationshipType.A_RECORD.default_confidence,
+                        raw_data=raw_data,
+                        discovery_method="shodan_hostname_search"
+                    )
+                    
+                    # Also create relationships to other hostnames on the same IP
+                    for hostname in hostnames:
+                        if hostname != domain and self._is_valid_domain(hostname):
+                            hostname_raw_data = {
+                                'shared_ip': ip_address,
+                                'all_hostnames': hostnames,
+                                'discovery_context': 'shared_hosting'
+                            }
+                            
+                            relationships.append((
+                                domain,
+                                hostname,
+                                RelationshipType.PASSIVE_DNS,  # Shared hosting relationship
+                                0.6,  # Lower confidence for shared hosting
+                                hostname_raw_data
+                            ))
+                            
+                            self.log_relationship_discovery(
+                                source_node=domain,
+                                target_node=hostname,
+                                relationship_type=RelationshipType.PASSIVE_DNS,
+                                confidence_score=0.6,
+                                raw_data=hostname_raw_data,
+                                discovery_method="shodan_shared_hosting"
+                            )
+        
+        except json.JSONDecodeError as e:
+            self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
+        except Exception as e:
+            self.logger.logger.error(f"Error querying Shodan for domain {domain}: {e}")
+        
+        return relationships
+    
+    def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """
+        Query Shodan for information about an IP address.
+        
+        Args:
+            ip: IP address to investigate
+            
+        Returns:
+            List of relationships discovered from Shodan IP data
+        """
+        if not self._is_valid_ip(ip) or not self.is_available():
+            return []
+        
+        relationships = []
+        
+        try:
+            # Query Shodan host information
+            url = f"{self.base_url}/shodan/host/{ip}"
+            params = {'key': self.api_key}
+            
+            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
+            
+            if not response or response.status_code != 200:
+                return []
+            
+            data = response.json()
+            
+            # Extract hostname relationships
+            hostnames = data.get('hostnames', [])
+            for hostname in hostnames:
+                if self._is_valid_domain(hostname):
+                    raw_data = {
+                        'ip_address': ip,
+                        'hostname': hostname,
+                        'country': data.get('country_name', ''),
+                        'city': data.get('city', ''),
+                        'isp': data.get('isp', ''),
+                        'org': data.get('org', ''),
+                        'asn': data.get('asn', ''),
+                        'ports': data.get('ports', []),
+                        'last_update': data.get('last_update', ''),
+                        'os': data.get('os', '')
+                    }
+                    
+                    relationships.append((
+                        ip,
+                        hostname,
+                        RelationshipType.A_RECORD,  # IP resolves to hostname
+                        RelationshipType.A_RECORD.default_confidence,
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=ip,
+                        target_node=hostname,
+                        relationship_type=RelationshipType.A_RECORD,
+                        confidence_score=RelationshipType.A_RECORD.default_confidence,
+                        raw_data=raw_data,
+                        discovery_method="shodan_host_lookup"
+                    )
+            
+            # Extract ASN relationship if available
+            asn = data.get('asn')
+            if asn:
+                asn_name = f"AS{asn}"
+                
+                asn_raw_data = {
+                    'ip_address': ip,
+                    'asn': asn,
+                    'isp': data.get('isp', ''),
+                    'org': data.get('org', '')
+                }
+                
+                relationships.append((
+                    ip,
+                    asn_name,
+                    RelationshipType.ASN_MEMBERSHIP,
+                    RelationshipType.ASN_MEMBERSHIP.default_confidence,
+                    asn_raw_data
+                ))
+                
+                self.log_relationship_discovery(
+                    source_node=ip,
+                    target_node=asn_name,
+                    relationship_type=RelationshipType.ASN_MEMBERSHIP,
+                    confidence_score=RelationshipType.ASN_MEMBERSHIP.default_confidence,
+                    raw_data=asn_raw_data,
+                    discovery_method="shodan_asn_lookup"
+                )
+        
+        except json.JSONDecodeError as e:
+            self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
+        except Exception as e:
+            self.logger.logger.error(f"Error querying Shodan for IP {ip}: {e}")
+        
+        return relationships
+    
+    def search_by_organization(self, org_name: str) -> List[Dict[str, Any]]:
+        """
+        Search Shodan for hosts belonging to a specific organization.
+        
+        Args:
+            org_name: Organization name to search for
+            
+        Returns:
+            List of host information dictionaries
+        """
+        if not self.is_available():
+            return []
+        
+        try:
+            search_query = f"org:\"{org_name}\""
+            url = f"{self.base_url}/shodan/host/search"
+            params = {
+                'key': self.api_key,
+                'query': search_query,
+                'minify': True
+            }
+            
+            response = self.make_request(url, method="GET", params=params, target_indicator=org_name)
+            
+            if response and response.status_code == 200:
+                data = response.json()
+                return data.get('matches', [])
+        
+        except Exception as e:
+            self.logger.logger.error(f"Error searching Shodan by organization {org_name}: {e}")
+        
+        return []
+    
+    def get_host_services(self, ip: str) -> List[Dict[str, Any]]:
+        """
+        Get service information for a specific IP address.
+        
+        Args:
+            ip: IP address to query
+            
+        Returns:
+            List of service information dictionaries
+        """
+        if not self._is_valid_ip(ip) or not self.is_available():
+            return []
+        
+        try:
+            url = f"{self.base_url}/shodan/host/{ip}"
+            params = {'key': self.api_key}
+            
+            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
+            
+            if response and response.status_code == 200:
+                data = response.json()
+                return data.get('data', [])  # Service banners
+        
+        except Exception as e:
+            self.logger.logger.error(f"Error getting Shodan services for IP {ip}: {e}")
+        
+        return []

providers/virustotal_provider.py

@@ -0,0 +1,334 @@
+"""
+VirusTotal provider for DNSRecon.
+Discovers domain relationships through passive DNS and URL analysis.
+"""
+
+import json
+from typing import List, Dict, Any, Tuple, Optional
+from .base_provider import BaseProvider
+from core.graph_manager import RelationshipType
+from config import config
+
+
+class VirusTotalProvider(BaseProvider):
+    """
+    Provider for querying VirusTotal API for passive DNS and domain reputation data.
+    Requires valid API key and strictly respects free tier rate limits.
+    """
+    
+    def __init__(self):
+        """Initialize VirusTotal provider with strict rate limiting for free tier."""
+        super().__init__(
+            name="virustotal",
+            rate_limit=4,  # Free tier: 4 requests per minute
+            timeout=30
+        )
+        self.base_url = "https://www.virustotal.com/vtapi/v2"
+        self.api_key = config.get_api_key('virustotal')
+    
+    def get_name(self) -> str:
+        """Return the provider name."""
+        return "virustotal"
+    
+    def is_available(self) -> bool:
+        """
+        Check if VirusTotal provider is available (has valid API key).
+        """
+        return self.api_key is not None and len(self.api_key.strip()) > 0
+    
+    def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """
+        Query VirusTotal for domain information including passive DNS.
+        
+        Args:
+            domain: Domain to investigate
+            
+        Returns:
+            List of relationships discovered from VirusTotal data
+        """
+        if not self._is_valid_domain(domain) or not self.is_available():
+            return []
+        
+        relationships = []
+        
+        # Query domain report
+        domain_relationships = self._query_domain_report(domain)
+        relationships.extend(domain_relationships)
+        
+        # Query passive DNS for the domain
+        passive_dns_relationships = self._query_passive_dns_domain(domain)
+        relationships.extend(passive_dns_relationships)
+        
+        return relationships
+    
+    def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """
+        Query VirusTotal for IP address information including passive DNS.
+        
+        Args:
+            ip: IP address to investigate
+            
+        Returns:
+            List of relationships discovered from VirusTotal IP data
+        """
+        if not self._is_valid_ip(ip) or not self.is_available():
+            return []
+        
+        relationships = []
+        
+        # Query IP report
+        ip_relationships = self._query_ip_report(ip)
+        relationships.extend(ip_relationships)
+        
+        # Query passive DNS for the IP
+        passive_dns_relationships = self._query_passive_dns_ip(ip)
+        relationships.extend(passive_dns_relationships)
+        
+        return relationships
+    
+    def _query_domain_report(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query VirusTotal domain report."""
+        relationships = []
+        
+        try:
+            url = f"{self.base_url}/domain/report"
+            params = {
+                'apikey': self.api_key,
+                'domain': domain,
+                'allinfo': 1  # Get comprehensive information
+            }
+            
+            response = self.make_request(url, method="GET", params=params, target_indicator=domain)
+            
+            if not response or response.status_code != 200:
+                return []
+            
+            data = response.json()
+            
+            if data.get('response_code') != 1:
+                return []
+            
+            # Extract resolved IPs
+            resolutions = data.get('resolutions', [])
+            for resolution in resolutions:
+                ip_address = resolution.get('ip_address')
+                last_resolved = resolution.get('last_resolved')
+                
+                if ip_address and self._is_valid_ip(ip_address):
+                    raw_data = {
+                        'domain': domain,
+                        'ip_address': ip_address,
+                        'last_resolved': last_resolved,
+                        'source': 'virustotal_domain_report'
+                    }
+                    
+                    relationships.append((
+                        domain,
+                        ip_address,
+                        RelationshipType.PASSIVE_DNS,
+                        RelationshipType.PASSIVE_DNS.default_confidence,
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=domain,
+                        target_node=ip_address,
+                        relationship_type=RelationshipType.PASSIVE_DNS,
+                        confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
+                        raw_data=raw_data,
+                        discovery_method="virustotal_domain_resolution"
+                    )
+            
+            # Extract subdomains
+            subdomains = data.get('subdomains', [])
+            for subdomain in subdomains:
+                if subdomain != domain and self._is_valid_domain(subdomain):
+                    raw_data = {
+                        'parent_domain': domain,
+                        'subdomain': subdomain,
+                        'source': 'virustotal_subdomain_discovery'
+                    }
+                    
+                    relationships.append((
+                        domain,
+                        subdomain,
+                        RelationshipType.PASSIVE_DNS,
+                        0.7,  # Medium-high confidence for subdomains
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=domain,
+                        target_node=subdomain,
+                        relationship_type=RelationshipType.PASSIVE_DNS,
+                        confidence_score=0.7,
+                        raw_data=raw_data,
+                        discovery_method="virustotal_subdomain_discovery"
+                    )
+        
+        except json.JSONDecodeError as e:
+            self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
+        except Exception as e:
+            self.logger.logger.error(f"Error querying VirusTotal domain report for {domain}: {e}")
+        
+        return relationships
+    
+    def _query_ip_report(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query VirusTotal IP report."""
+        relationships = []
+        
+        try:
+            url = f"{self.base_url}/ip-address/report"
+            params = {
+                'apikey': self.api_key,
+                'ip': ip
+            }
+            
+            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
+            
+            if not response or response.status_code != 200:
+                return []
+            
+            data = response.json()
+            
+            if data.get('response_code') != 1:
+                return []
+            
+            # Extract resolved domains
+            resolutions = data.get('resolutions', [])
+            for resolution in resolutions:
+                hostname = resolution.get('hostname')
+                last_resolved = resolution.get('last_resolved')
+                
+                if hostname and self._is_valid_domain(hostname):
+                    raw_data = {
+                        'ip_address': ip,
+                        'hostname': hostname,
+                        'last_resolved': last_resolved,
+                        'source': 'virustotal_ip_report'
+                    }
+                    
+                    relationships.append((
+                        ip,
+                        hostname,
+                        RelationshipType.PASSIVE_DNS,
+                        RelationshipType.PASSIVE_DNS.default_confidence,
+                        raw_data
+                    ))
+                    
+                    self.log_relationship_discovery(
+                        source_node=ip,
+                        target_node=hostname,
+                        relationship_type=RelationshipType.PASSIVE_DNS,
+                        confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
+                        raw_data=raw_data,
+                        discovery_method="virustotal_ip_resolution"
+                    )
+        
+        except json.JSONDecodeError as e:
+            self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
+        except Exception as e:
+            self.logger.logger.error(f"Error querying VirusTotal IP report for {ip}: {e}")
+        
+        return relationships
+    
+    def _query_passive_dns_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query VirusTotal passive DNS for domain."""
+        # Note: VirusTotal's passive DNS API might require a premium subscription
+        # This is a placeholder for the endpoint structure
+        return []
+    
+    def _query_passive_dns_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+        """Query VirusTotal passive DNS for IP."""
+        # Note: VirusTotal's passive DNS API might require a premium subscription
+        # This is a placeholder for the endpoint structure
+        return []
+    
+    def get_domain_reputation(self, domain: str) -> Dict[str, Any]:
+        """
+        Get domain reputation information from VirusTotal.
+        
+        Args:
+            domain: Domain to check reputation for
+            
+        Returns:
+            Dictionary containing reputation data
+        """
+        if not self._is_valid_domain(domain) or not self.is_available():
+            return {}
+        
+        try:
+            url = f"{self.base_url}/domain/report"
+            params = {
+                'apikey': self.api_key,
+                'domain': domain
+            }
+            
+            response = self.make_request(url, method="GET", params=params, target_indicator=domain)
+            
+            if response and response.status_code == 200:
+                data = response.json()
+                
+                if data.get('response_code') == 1:
+                    return {
+                        'positives': data.get('positives', 0),
+                        'total': data.get('total', 0),
+                        'scan_date': data.get('scan_date', ''),
+                        'permalink': data.get('permalink', ''),
+                        'reputation_score': self._calculate_reputation_score(data)
+                    }
+        
+        except Exception as e:
+            self.logger.logger.error(f"Error getting VirusTotal reputation for domain {domain}: {e}")
+        
+        return {}
+    
+    def get_ip_reputation(self, ip: str) -> Dict[str, Any]:
+        """
+        Get IP reputation information from VirusTotal.
+        
+        Args:
+            ip: IP address to check reputation for
+            
+        Returns:
+            Dictionary containing reputation data
+        """
+        if not self._is_valid_ip(ip) or not self.is_available():
+            return {}
+        
+        try:
+            url = f"{self.base_url}/ip-address/report"
+            params = {
+                'apikey': self.api_key,
+                'ip': ip
+            }
+            
+            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
+            
+            if response and response.status_code == 200:
+                data = response.json()
+                
+                if data.get('response_code') == 1:
+                    return {
+                        'positives': data.get('positives', 0),
+                        'total': data.get('total', 0),
+                        'scan_date': data.get('scan_date', ''),
+                        'permalink': data.get('permalink', ''),
+                        'reputation_score': self._calculate_reputation_score(data)
+                    }
+        
+        except Exception as e:
+            self.logger.logger.error(f"Error getting VirusTotal reputation for IP {ip}: {e}")
+        
+        return {}
+    
+    def _calculate_reputation_score(self, data: Dict[str, Any]) -> float:
+        """Calculate a normalized reputation score (0.0 to 1.0)."""
+        positives = data.get('positives', 0)
+        total = data.get('total', 1)  # Avoid division by zero
+        
+        if total == 0:
+            return 1.0  # No data means neutral
+        
+        # Score is inverse of detection ratio (lower detection = higher reputation)
+        return max(0.0, 1.0 - (positives / total))

requirements.txt

@@ -4,3 +4,4 @@ requests>=2.31.0
 python-dateutil>=2.8.2
 Werkzeug>=2.3.7
 urllib3>=2.0.0
+dnspython>=2.4.2

static/css/main.css

@@ -64,6 +64,18 @@ body {
     gap: 0.5rem;
 }
 
+.status-indicator.scanning {
+    animation: pulse 1.5s infinite;
+}
+
+.status-indicator.completed {
+    background-color: #00ff41;
+}
+
+.status-indicator.error {
+    background-color: #ff6b6b;
+}
+
 .status-dot {
     width: 8px;
     height: 8px;
@@ -266,6 +278,7 @@ input[type="text"]:focus, select:focus {
     background-color: #1a1a1a;
     border: 1px solid #444;
     overflow: hidden;
+    position: relative;
 }
 
 .progress-fill {
@@ -274,6 +287,23 @@ input[type="text"]:focus, select:focus {
     width: 0%;
     transition: width 0.3s ease;
     box-shadow: 0 0 5px rgba(0, 255, 65, 0.5);
+    position: relative;
+}
+
+.progress-fill::after {
+    content: '';
+    position: absolute;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background: linear-gradient(90deg, transparent, rgba(255,255,255,0.2), transparent);
+    animation: shimmer 2s infinite;
+}
+
+@keyframes shimmer {
+    0% { transform: translateX(-100%); }
+    100% { transform: translateX(100%); }
 }
 
 /* Visualization Panel */
@@ -292,6 +322,37 @@ input[type="text"]:focus, select:focus {
     position: relative;
     background-color: #1a1a1a;
     border-top: 1px solid #444;
+    transition: height 0.3s ease;
+}
+
+.graph-container.expanded {
+    height: 700px;
+}
+
+.graph-controls {
+    position: absolute;
+    top: 10px;
+    right: 10px;
+    z-index: 10;
+    display: flex;
+    gap: 0.5rem;
+}
+
+.graph-control-btn {
+    background: rgba(42, 42, 42, 0.9);
+    border: 1px solid #555;
+    color: #c7c7c7;
+    padding: 0.5rem;
+    font-family: 'Roboto Mono', monospace;
+    font-size: 0.8rem;
+    cursor: pointer;
+    transition: all 0.3s ease;
+}
+
+.graph-control-btn:hover {
+    border-color: #00ff41;
+    color: #00ff41;
+    background: rgba(42, 42, 42, 1);
 }
 
 .graph-placeholder {
@@ -333,6 +394,20 @@ input[type="text"]:focus, select:focus {
     border-top: 1px solid #444;
 }
 
+.legend-section {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.legend-title {
+    font-size: 0.7rem;
+    color: #00ff41;
+    text-transform: uppercase;
+    font-weight: 500;
+    margin-bottom: 0.25rem;
+}
+
 .legend-item {
     display: flex;
     align-items: center;
@@ -344,6 +419,7 @@ input[type="text"]:focus, select:focus {
     width: 12px;
     height: 12px;
     border-radius: 50%;
+    border: 1px solid #444;
 }
 
 .legend-edge {
@@ -353,10 +429,16 @@ input[type="text"]:focus, select:focus {
 
 .legend-edge.high-confidence {
     background-color: #00ff41;
+    box-shadow: 0 0 3px rgba(0, 255, 65, 0.5);
 }
 
 .legend-edge.medium-confidence {
     background-color: #ff9900;
+    box-shadow: 0 0 3px rgba(255, 153, 0, 0.5);
+}
+
+.legend-edge.low-confidence {
+    background-color: #666666;
 }
 
 /* Provider Panel */
@@ -375,9 +457,11 @@ input[type="text"]:focus, select:focus {
     background-color: #1a1a1a;
     border: 1px solid #444;
     padding: 1rem;
-    display: flex;
+    transition: border-color 0.3s ease;
-    justify-content: space-between;
+}
-    align-items: center;
+
+.provider-item:hover {
+    border-color: #555;
 }
 
 .provider-name {
@@ -389,6 +473,7 @@ input[type="text"]:focus, select:focus {
     font-size: 0.8rem;
     padding: 0.25rem 0.5rem;
     border-radius: 3px;
+    font-weight: 500;
 }
 
 .provider-status.enabled {
@@ -401,12 +486,78 @@ input[type="text"]:focus, select:focus {
     color: #e0e0e0;
 }
 
+.provider-status.api-key-required {
+    background-color: #5c4c2c;
+    color: #e0e0e0;
+}
+
 .provider-stats {
     font-size: 0.8rem;
     color: #999;
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 0.5rem;
     margin-top: 0.5rem;
 }
 
+.provider-stat {
+    display: flex;
+    justify-content: space-between;
+}
+
+.provider-stat-label {
+    color: #666;
+}
+
+.provider-stat-value {
+    color: #00ff41;
+    font-weight: 500;
+}
+
+.provider-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 0.5rem;
+}
+
+.node-info-popup {
+    position: fixed;
+    background: rgba(42, 42, 42, 0.95);
+    border: 1px solid #555;
+    padding: 1rem;
+    border-radius: 4px;
+    color: #c7c7c7;
+    font-family: 'Roboto Mono', monospace;
+    font-size: 0.8rem;
+    max-width: 300px;
+    z-index: 1001;
+    box-shadow: 0 4px 6px rgba(0,0,0,0.3);
+}
+
+.node-info-title {
+    color: #00ff41;
+    font-weight: bold;
+    margin-bottom: 0.5rem;
+    border-bottom: 1px solid #444;
+    padding-bottom: 0.25rem;
+}
+
+.node-info-detail {
+    margin-bottom: 0.25rem;
+    display: flex;
+    justify-content: space-between;
+}
+
+.node-info-label {
+    color: #999;
+}
+
+.node-info-value {
+    color: #c7c7c7;
+    font-weight: 500;
+}
+
 /* Footer */
 .footer {
     background-color: #0a0a0a;
@@ -437,6 +588,7 @@ input[type="text"]:focus, select:focus {
     width: 100%;
     height: 100%;
     background-color: rgba(0, 0, 0, 0.8);
+    animation: fadeIn 0.3s ease-out;
 }
 
 .modal-content {
@@ -447,6 +599,18 @@ input[type="text"]:focus, select:focus {
     max-width: 600px;
     max-height: 80vh;
     overflow-y: auto;
+    animation: slideInDown 0.3s ease-out;
+}
+
+@keyframes slideInDown {
+    from {
+        opacity: 0;
+        transform: translateY(-50px);
+    }
+    to {
+        opacity: 1;
+        transform: translateY(0);
+    }
 }
 
 .modal-header {
@@ -480,6 +644,12 @@ input[type="text"]:focus, select:focus {
     padding: 1.5rem;
 }
 
+.modal-description {
+    color: #999;
+    margin-bottom: 1.5rem;
+    line-height: 1.6;
+}
+
 .detail-row {
     display: flex;
     justify-content: space-between;
@@ -495,6 +665,7 @@ input[type="text"]:focus, select:focus {
 
 .detail-value {
     color: #c7c7c7;
+    word-break: break-word;
 }
 
 /* Responsive Design */
@@ -552,6 +723,40 @@ input[type="text"]:focus, select:focus {
     pointer-events: none;
 }
 
+.loading-overlay {
+    position: absolute;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background: rgba(26, 26, 26, 0.8);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    z-index: 100;
+}
+
+.loading-spinner {
+    width: 40px;
+    height: 40px;
+    border: 3px solid #444;
+    border-top: 3px solid #00ff41;
+    border-radius: 50%;
+    animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+    0% { transform: rotate(0deg); }
+    100% { transform: rotate(360deg); }
+}
+
+.loading-text {
+    margin-top: 1rem;
+    color: #999;
+    font-family: 'Roboto Mono', monospace;
+    font-size: 0.9rem;
+}
+
 .error {
     color: #ff6b6b !important;
     border-color: #ff6b6b !important;
@@ -599,3 +804,100 @@ input[type="text"]:focus, select:focus {
 .amber {
     color: #ff9900;
 }
+
+
+.apikey-section {
+    margin-bottom: 1.5rem;
+}
+
+.apikey-section label {
+    display: block;
+    margin-bottom: 0.5rem;
+    color: #c7c7c7;
+    font-size: 0.9rem;
+    font-weight: 500;
+}
+
+.apikey-section input[type="password"] {
+    width: 100%;
+    padding: 0.75rem;
+    background-color: #1a1a1a;
+    border: 1px solid #555;
+    color: #c7c7c7;
+    font-family: 'Roboto Mono', monospace;
+    font-size: 0.9rem;
+    transition: border-color 0.3s ease, box-shadow 0.3s ease;
+}
+
+.apikey-section input[type="password"]:focus {
+    outline: none;
+    border-color: #00ff41;
+    box-shadow: 0 0 5px rgba(0, 255, 65, 0.5);
+}
+
+.apikey-help {
+    font-size: 0.8rem;
+    color: #666;
+    margin-top: 0.25rem;
+    font-style: italic;
+}
+
+
+.message-container {
+    position: fixed;
+    top: 20px;
+    right: 20px;
+    z-index: 1002;
+    max-width: 400px;
+}
+
+.message-toast {
+    margin-bottom: 10px;
+    border-radius: 4px;
+    font-family: 'Roboto Mono', monospace;
+    font-size: 0.9rem;
+    box-shadow: 0 4px 6px rgba(0,0,0,0.3);
+    animation: slideInRight 0.3s ease-out;
+}
+
+.message-toast.success {
+    background: #2c5c34;
+    border-left: 4px solid #00ff41;
+}
+
+.message-toast.error {
+    background: #5c2c2c;
+    border-left: 4px solid #ff6b6b;
+}
+
+.message-toast.warning {
+    background: #5c4c2c;
+    border-left: 4px solid #ff9900;
+}
+
+.message-toast.info {
+    background: #2c3e5c;
+    border-left: 4px solid #00aaff;
+}
+
+@keyframes slideInRight {
+    from {
+        transform: translateX(100%);
+        opacity: 0;
+    }
+    to {
+        transform: translateX(0);
+        opacity: 1;
+    }
+}
+
+@keyframes slideOutRight {
+    from {
+        transform: translateX(0);
+        opacity: 1;
+    }
+    to {
+        transform: translateX(100%);
+        opacity: 0;
+    }
+}

static/js/graph.js

@@ -1,6 +1,6 @@
 /**
  * Graph visualization module for DNSRecon
- * Handles network graph rendering using vis.js
+ * Handles network graph rendering using vis.js with enhanced Phase 2 features
  */
 
 class GraphManager {
@@ -10,40 +10,57 @@ class GraphManager {
         this.nodes = new vis.DataSet();
         this.edges = new vis.DataSet();
         this.isInitialized = false;
+        this.currentLayout = 'physics';
+        this.nodeInfoPopup = null;
 
-        // Graph options for cybersecurity theme
+        // Enhanced graph options for Phase 2
         this.options = {
             nodes: {
                 shape: 'dot',
-                size: 12,
+                size: 15,
                 font: {
-                    size: 11,
+                    size: 12,
                     color: '#c7c7c7',
                     face: 'Roboto Mono, monospace',
-                    background: 'rgba(26, 26, 26, 0.8)',
+                    background: 'rgba(26, 26, 26, 0.9)',
-                    strokeWidth: 1,
+                    strokeWidth: 2,
                     strokeColor: '#000000'
                 },
                 borderWidth: 2,
                 borderColor: '#444',
                 shadow: {
                     enabled: true,
-                    color: 'rgba(0, 0, 0, 0.3)',
+                    color: 'rgba(0, 0, 0, 0.5)',
-                    size: 3,
+                    size: 5,
-                    x: 1,
+                    x: 2,
-                    y: 1
+                    y: 2
                 },
                 scaling: {
+                    min: 10,
+                    max: 30,
+                    label: {
+                        enabled: true,
                         min: 8,
-                    max: 20
+                        max: 16
+                    }
+                },
+                chosen: {
+                    node: (values, id, selected, hovering) => {
+                        values.borderColor = '#00ff41';
+                        values.borderWidth = 3;
+                        values.shadow = true;
+                        values.shadowColor = 'rgba(0, 255, 65, 0.6)';
+                        values.shadowSize = 10;
+                    }
                 }
             },
             edges: {
                 width: 2,
                 color: {
-                    color: '#444',
+                    color: '#555',
                     highlight: '#00ff41',
-                    hover: '#ff9900'
+                    hover: '#ff9900',
+                    inherit: false
                 },
                 font: {
                     size: 10,
@@ -56,62 +73,84 @@ class GraphManager {
                 arrows: {
                     to: {
                         enabled: true,
-                        scaleFactor: 0.8,
+                        scaleFactor: 1,
                         type: 'arrow'
                     }
                 },
                 smooth: {
                     enabled: true,
                     type: 'dynamic',
-                    roundness: 0.5
+                    roundness: 0.6
                 },
                 shadow: {
                     enabled: true,
-                    color: 'rgba(0, 0, 0, 0.2)',
+                    color: 'rgba(0, 0, 0, 0.3)',
-                    size: 2,
+                    size: 3,
                     x: 1,
                     y: 1
+                },
+                chosen: {
+                    edge: (values, id, selected, hovering) => {
+                        values.color = '#00ff41';
+                        values.width = 4;
+                        values.shadow = true;
+                        values.shadowColor = 'rgba(0, 255, 65, 0.4)';
+                    }
                 }
             },
             physics: {
                 enabled: true,
                 stabilization: {
                     enabled: true,
-                    iterations: 100,
+                    iterations: 150,
-                    updateInterval: 25
+                    updateInterval: 50
                 },
                 barnesHut: {
-                    gravitationalConstant: -2000,
+                    gravitationalConstant: -3000,
-                    centralGravity: 0.3,
+                    centralGravity: 0.4,
-                    springLength: 95,
+                    springLength: 120,
-                    springConstant: 0.04,
+                    springConstant: 0.05,
-                    damping: 0.09,
+                    damping: 0.1,
-                    avoidOverlap: 0.1
+                    avoidOverlap: 0.2
                 },
-                maxVelocity: 50,
+                maxVelocity: 30,
                 minVelocity: 0.1,
                 solver: 'barnesHut',
-                timestep: 0.35,
+                timestep: 0.4,
                 adaptiveTimestep: true
             },
             interaction: {
                 hover: true,
                 hoverConnectedEdges: true,
                 selectConnectedEdges: true,
-                tooltipDelay: 200,
+                tooltipDelay: 300,
                 hideEdgesOnDrag: false,
-                hideNodesOnDrag: false
+                hideNodesOnDrag: false,
+                zoomView: true,
+                dragView: true,
+                multiselect: true
             },
             layout: {
-                improvedLayout: true
+                improvedLayout: true,
+                randomSeed: 2
             }
         };
 
-        this.setupEventHandlers();
+        this.createNodeInfoPopup();
     }
 
     /**
-     * Initialize the network graph
+     * Create floating node info popup
+     */
+    createNodeInfoPopup() {
+        this.nodeInfoPopup = document.createElement('div');
+        this.nodeInfoPopup.className = 'node-info-popup';
+        this.nodeInfoPopup.style.display = 'none';
+        document.body.appendChild(this.nodeInfoPopup);
+    }
+
+    /**
+     * Initialize the network graph with enhanced features
      */
     initialize() {
         if (this.isInitialized) {
@@ -134,7 +173,10 @@ class GraphManager {
                 placeholder.style.display = 'none';
             }
 
-            console.log('Graph initialized successfully');
+            // Add graph controls
+            this.addGraphControls();
+
+            console.log('Enhanced graph initialized successfully');
         } catch (error) {
             console.error('Failed to initialize graph:', error);
             this.showError('Failed to initialize visualization');
@@ -142,33 +184,89 @@ class GraphManager {
     }
 
     /**
-     * Setup network event handlers
+     * Add interactive graph controls
+     */
+    addGraphControls() {
+        const controlsContainer = document.createElement('div');
+        controlsContainer.className = 'graph-controls';
+        controlsContainer.innerHTML = `
+            <button class="graph-control-btn" id="graph-fit" title="Fit to Screen">[FIT]</button>
+            <button class="graph-control-btn" id="graph-reset" title="Reset View">[RESET]</button>
+            <button class="graph-control-btn" id="graph-physics" title="Toggle Physics">[PHYSICS]</button>
+            <button class="graph-control-btn" id="graph-cluster" title="Cluster Nodes">[CLUSTER]</button>
+        `;
+
+        this.container.appendChild(controlsContainer);
+
+        // Add control event listeners
+        document.getElementById('graph-fit').addEventListener('click', () => this.fitView());
+        document.getElementById('graph-reset').addEventListener('click', () => this.resetView());
+        document.getElementById('graph-physics').addEventListener('click', () => this.togglePhysics());
+        document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering());
+    }
+
+    /**
+     * Setup enhanced network event handlers
      */
     setupNetworkEvents() {
         if (!this.network) return;
 
-        // Node click event
+        // Node click event with enhanced details
         this.network.on('click', (params) => {
             if (params.nodes.length > 0) {
                 const nodeId = params.nodes[0];
                 this.showNodeDetails(nodeId);
+                this.highlightNodeConnections(nodeId);
+            } else {
+                this.clearHighlights();
             }
         });
 
-        // Hover events for tooltips
+        // Enhanced hover events
         this.network.on('hoverNode', (params) => {
             const nodeId = params.node;
             const node = this.nodes.get(nodeId);
             if (node) {
-                this.showTooltip(params.pointer.DOM, node);
+                this.showNodeInfoPopup(params.pointer.DOM, node);
+                this.highlightConnectedNodes(nodeId, true);
             }
         });
 
-        this.network.on('blurNode', () => {
+        this.network.on('blurNode', (params) => {
-            this.hideTooltip();
+            this.hideNodeInfoPopup();
+            this.clearHoverHighlights();
         });
 
-        // Stabilization events
+        // Edge hover events
+        this.network.on('hoverEdge', (params) => {
+            const edgeId = params.edge;
+            const edge = this.edges.get(edgeId);
+            if (edge) {
+                this.showEdgeInfo(params.pointer.DOM, edge);
+            }
+        });
+
+        this.network.on('blurEdge', () => {
+            this.hideNodeInfoPopup();
+        });
+
+        // Double-click to focus on node
+        this.network.on('doubleClick', (params) => {
+            if (params.nodes.length > 0) {
+                const nodeId = params.nodes[0];
+                this.focusOnNode(nodeId);
+            }
+        });
+
+        // Context menu (right-click)
+        this.network.on('oncontext', (params) => {
+            params.event.preventDefault();
+            if (params.nodes.length > 0) {
+                this.showNodeContextMenu(params.pointer.DOM, params.nodes[0]);
+            }
+        });
+
+        // Stabilization events with progress
         this.network.on('stabilizationProgress', (params) => {
             const progress = params.iterations / params.total;
             this.updateStabilizationProgress(progress);
@@ -177,10 +275,16 @@ class GraphManager {
         this.network.on('stabilizationIterationsDone', () => {
             this.onStabilizationComplete();
         });
+
+        // Selection events
+        this.network.on('select', (params) => {
+            console.log('Selected nodes:', params.nodes);
+            console.log('Selected edges:', params.edges);
+        });
     }
 
     /**
-     * Update graph with new data
+     * Update graph with new data and enhanced processing
      * @param {Object} graphData - Graph data from backend
      */
     updateGraph(graphData) {
@@ -195,30 +299,41 @@ class GraphManager {
                 this.initialize();
             }
 
-            // Process nodes
+            // Process nodes with enhanced attributes
             const processedNodes = graphData.nodes.map(node => this.processNode(node));
             const processedEdges = graphData.edges.map(edge => this.processEdge(edge));
 
-            // Update datasets
+            // Update datasets with animation
-            this.nodes.clear();
+            const existingNodeIds = this.nodes.getIds();
-            this.edges.clear();
+            const existingEdgeIds = this.edges.getIds();
-            this.nodes.add(processedNodes);
-            this.edges.add(processedEdges);
 
-            // Fit the view if this is the first update or graph is small
+            // Add new nodes with fade-in animation
-            if (processedNodes.length <= 10) {
+            const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id));
-                setTimeout(() => this.fitView(), 500);
+            const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id));
+
+            // Update existing data
+            this.nodes.update(processedNodes);
+            this.edges.update(processedEdges);
+
+            // Highlight new additions briefly
+            if (newNodes.length > 0 || newEdges.length > 0) {
+                setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100);
             }
 
-            console.log(`Graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges`);
+            // Auto-fit view for small graphs or first update
+            if (processedNodes.length <= 10 || existingNodeIds.length === 0) {
+                setTimeout(() => this.fitView(), 800);
+            }
+
+            console.log(`Enhanced graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
         } catch (error) {
-            console.error('Failed to update graph:', error);
+            console.error('Failed to update enhanced graph:', error);
             this.showError('Failed to update visualization');
         }
     }
 
     /**
-     * Process node data for visualization
+     * Process node data with enhanced styling and metadata
      * @param {Object} node - Raw node data
      * @returns {Object} Processed node data
      */
@@ -230,25 +345,32 @@ class GraphManager {
             color: this.getNodeColor(node.type),
             size: this.getNodeSize(node.type),
             borderColor: this.getNodeBorderColor(node.type),
-            metadata: node.metadata || {}
+            shape: this.getNodeShape(node.type),
+            metadata: node.metadata || {},
+            type: node.type
         };
 
-        // Add type-specific styling
+        // Add confidence-based styling
-        if (node.type === 'domain') {
+        if (node.confidence) {
-            processedNode.shape = 'dot';
+            processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5));
-        } else if (node.type === 'ip') {
+        }
-            processedNode.shape = 'square';
+
-        } else if (node.type === 'certificate') {
+        // Add special styling for important nodes
-            processedNode.shape = 'diamond';
+        if (this.isImportantNode(node)) {
-        } else if (node.type === 'asn') {
+            processedNode.shadow = {
-            processedNode.shape = 'triangle';
+                enabled: true,
+                color: 'rgba(0, 255, 65, 0.6)',
+                size: 10,
+                x: 2,
+                y: 2
+            };
         }
 
         return processedNode;
     }
 
     /**
-     * Process edge data for visualization
+     * Process edge data with enhanced styling and metadata
      * @param {Object} edge - Raw edge data
      * @returns {Object} Processed edge data
      */
@@ -262,9 +384,26 @@ class GraphManager {
             title: this.createEdgeTooltip(edge),
             width: this.getEdgeWidth(confidence),
             color: this.getEdgeColor(confidence),
-            dashes: confidence < 0.6 ? [5, 5] : false
+            dashes: confidence < 0.6 ? [5, 5] : false,
+            metadata: {
+                relationship_type: edge.label,
+                confidence_score: confidence,
+                source_provider: edge.source_provider,
+                discovery_timestamp: edge.discovery_timestamp
+            }
         };
 
+        // Add animation for high-confidence edges
+        if (confidence >= 0.8) {
+            processedEdge.shadow = {
+                enabled: true,
+                color: 'rgba(0, 255, 65, 0.3)',
+                size: 5,
+                x: 1,
+                y: 1
+            };
+        }
+
         return processedEdge;
     }
 
@@ -340,6 +479,21 @@ class GraphManager {
         return sizes[nodeType] || 12;
     }
     
+    /**
+     * Get enhanced node shape based on type
+     * @param {string} nodeType - Node type
+     * @returns {string} Shape name
+     */
+    getNodeShape(nodeType) {
+        const shapes = {
+            'domain': 'dot',
+            'ip': 'square',
+            'certificate': 'diamond',
+            'asn': 'triangle'
+        };
+        return shapes[nodeType] || 'dot';
+    }
+    
     /**
      * Get edge color based on confidence
      * @param {number} confidence - Confidence score
@@ -412,6 +566,19 @@ class GraphManager {
         return tooltip;
     }
 
+    /**
+     * Determine if node is important based on connections or metadata
+     * @param {Object} node - Node data
+     * @returns {boolean} True if node is important
+     */
+    isImportantNode(node) {
+        // Mark nodes as important based on criteria
+        if (node.type === 'domain' && node.id.includes('www.')) return false;
+        if (node.metadata && node.metadata.connection_count > 3) return true;
+        if (node.type === 'asn') return true;
+        return false;
+    }
+
     /**
      * Show node details in modal
      * @param {string} nodeId - Node identifier
@@ -428,20 +595,247 @@ class GraphManager {
     }
     
     /**
-     * Show tooltip
+     * Show enhanced node info popup
      * @param {Object} position - Mouse position
      * @param {Object} node - Node data
      */
-    showTooltip(position, node) {
+    showNodeInfoPopup(position, node) {
-        // Tooltip is handled by vis.js automatically
+        if (!this.nodeInfoPopup) return;
-        // This method is for custom tooltip implementation if needed
+
+        const html = `
+            <div class="node-info-title">${node.id}</div>
+            <div class="node-info-detail">
+                <span class="node-info-label">Type:</span>
+                <span class="node-info-value">${node.type || 'Unknown'}</span>
+            </div>
+            ${node.metadata && Object.keys(node.metadata).length > 0 ? 
+                '<div class="node-info-detail"><span class="node-info-label">Details:</span><span class="node-info-value">Click for more</span></div>' : 
+                ''}
+        `;
+
+        this.nodeInfoPopup.innerHTML = html;
+        this.nodeInfoPopup.style.display = 'block';
+        this.nodeInfoPopup.style.left = position.x + 15 + 'px';
+        this.nodeInfoPopup.style.top = position.y - 10 + 'px';
+
+        // Ensure popup stays in viewport
+        const rect = this.nodeInfoPopup.getBoundingClientRect();
+        if (rect.right > window.innerWidth) {
+            this.nodeInfoPopup.style.left = position.x - rect.width - 15 + 'px';
+        }
+        if (rect.bottom > window.innerHeight) {
+            this.nodeInfoPopup.style.top = position.y - rect.height + 10 + 'px';
+        }
     }
 
     /**
-     * Hide tooltip
+     * Show edge information tooltip
+     * @param {Object} position - Mouse position
+     * @param {Object} edge - Edge data
      */
-    hideTooltip() {
+    showEdgeInfo(position, edge) {
-        // Tooltip hiding is handled by vis.js automatically
+        if (!this.nodeInfoPopup) return;
+        
+        const confidence = edge.metadata ? edge.metadata.confidence_score : 0;
+        const provider = edge.metadata ? edge.metadata.source_provider : 'Unknown';
+        
+        const html = `
+            <div class="node-info-title">${edge.metadata ? edge.metadata.relationship_type : 'Relationship'}</div>
+            <div class="node-info-detail">
+                <span class="node-info-label">Confidence:</span>
+                <span class="node-info-value">${(confidence * 100).toFixed(1)}%</span>
+            </div>
+            <div class="node-info-detail">
+                <span class="node-info-label">Provider:</span>
+                <span class="node-info-value">${provider}</span>
+            </div>
+        `;
+        
+        this.nodeInfoPopup.innerHTML = html;
+        this.nodeInfoPopup.style.display = 'block';
+        this.nodeInfoPopup.style.left = position.x + 15 + 'px';
+        this.nodeInfoPopup.style.top = position.y - 10 + 'px';
+    }
+    
+    /**
+     * Hide node info popup
+     */
+    hideNodeInfoPopup() {
+        if (this.nodeInfoPopup) {
+            this.nodeInfoPopup.style.display = 'none';
+        }
+    }
+    
+    /**
+     * Highlight node connections
+     * @param {string} nodeId - Node to highlight
+     */
+    highlightNodeConnections(nodeId) {
+        const connectedNodes = this.network.getConnectedNodes(nodeId);
+        const connectedEdges = this.network.getConnectedEdges(nodeId);
+        
+        // Update node colors
+        const nodeUpdates = connectedNodes.map(id => ({
+            id: id,
+            borderColor: '#ff9900',
+            borderWidth: 3
+        }));
+        
+        nodeUpdates.push({
+            id: nodeId,
+            borderColor: '#00ff41',
+            borderWidth: 4
+        });
+        
+        // Update edge colors
+        const edgeUpdates = connectedEdges.map(id => ({
+            id: id,
+            color: { color: '#ff9900' },
+            width: 3
+        }));
+        
+        this.nodes.update(nodeUpdates);
+        this.edges.update(edgeUpdates);
+        
+        // Store for cleanup
+        this.highlightedElements = {
+            nodes: connectedNodes.concat([nodeId]),
+            edges: connectedEdges
+        };
+    }
+
+    /**
+     * Highlight connected nodes on hover
+     * @param {string} nodeId - Node ID
+     * @param {boolean} highlight - Whether to highlight or unhighlight
+     */
+    highlightConnectedNodes(nodeId, highlight) {
+        const connectedNodes = this.network.getConnectedNodes(nodeId);
+        const connectedEdges = this.network.getConnectedEdges(nodeId);
+        
+        if (highlight) {
+            // Dim all other elements
+            this.dimUnconnectedElements([nodeId, ...connectedNodes], connectedEdges);
+        }
+    }
+
+    /**
+     * Dim elements not connected to the specified nodes
+     * @param {Array} nodeIds - Node IDs to keep highlighted
+     * @param {Array} edgeIds - Edge IDs to keep highlighted
+     */
+    dimUnconnectedElements(nodeIds, edgeIds) {
+        const allNodes = this.nodes.get();
+        const allEdges = this.edges.get();
+        
+        const nodeUpdates = allNodes.map(node => ({
+            id: node.id,
+            opacity: nodeIds.includes(node.id) ? 1 : 0.3
+        }));
+        
+        const edgeUpdates = allEdges.map(edge => ({
+            id: edge.id,
+            opacity: edgeIds.includes(edge.id) ? 1 : 0.1
+        }));
+        
+        this.nodes.update(nodeUpdates);
+        this.edges.update(edgeUpdates);
+    }
+    
+    /**
+     * Clear all highlights
+     */
+    clearHighlights() {
+        if (this.highlightedElements) {
+            // Reset highlighted nodes
+            const nodeUpdates = this.highlightedElements.nodes.map(id => {
+                const originalNode = this.nodes.get(id);
+                return {
+                    id: id,
+                    borderColor: this.getNodeBorderColor(originalNode.type),
+                    borderWidth: 2
+                };
+            });
+            
+            // Reset highlighted edges
+            const edgeUpdates = this.highlightedElements.edges.map(id => {
+                const originalEdge = this.edges.get(id);
+                return {
+                    id: id,
+                    color: this.getEdgeColor(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5),
+                    width: this.getEdgeWidth(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5)
+                };
+            });
+            
+            this.nodes.update(nodeUpdates);
+            this.edges.update(edgeUpdates);
+            
+            this.highlightedElements = null;
+        }
+    }
+
+    /**
+     * Clear hover highlights
+     */
+    clearHoverHighlights() {
+        const allNodes = this.nodes.get();
+        const allEdges = this.edges.get();
+        
+        const nodeUpdates = allNodes.map(node => ({ id: node.id, opacity: 1 }));
+        const edgeUpdates = allEdges.map(edge => ({ id: edge.id, opacity: 1 }));
+        
+        this.nodes.update(nodeUpdates);
+        this.edges.update(edgeUpdates);
+    }
+
+    /**
+     * Highlight newly added elements
+     * @param {Array} newNodes - New nodes
+     * @param {Array} newEdges - New edges
+     */
+    highlightNewElements(newNodes, newEdges) {
+        // Briefly highlight new nodes
+        const nodeHighlights = newNodes.map(node => ({
+            id: node.id,
+            borderColor: '#00ff41',
+            borderWidth: 4,
+            shadow: {
+                enabled: true,
+                color: 'rgba(0, 255, 65, 0.8)',
+                size: 15,
+                x: 2,
+                y: 2
+            }
+        }));
+        
+        // Briefly highlight new edges
+        const edgeHighlights = newEdges.map(edge => ({
+            id: edge.id,
+            color: '#00ff41',
+            width: 4
+        }));
+        
+        this.nodes.update(nodeHighlights);
+        this.edges.update(edgeHighlights);
+        
+        // Reset after animation
+        setTimeout(() => {
+            const nodeResets = newNodes.map(node => ({
+                id: node.id,
+                borderColor: this.getNodeBorderColor(node.type),
+                borderWidth: 2,
+                shadow: node.shadow || { enabled: false }
+            }));
+            
+            const edgeResets = newEdges.map(edge => ({
+                id: edge.id,
+                color: this.getEdgeColor(edge.metadata ? edge.metadata.confidence_score : 0.5),
+                width: this.getEdgeWidth(edge.metadata ? edge.metadata.confidence_score : 0.5)
+            }));
+            
+            this.nodes.update(nodeResets);
+            this.edges.update(edgeResets);
+        }, 2000);
     }
 
     /**
@@ -460,6 +854,63 @@ class GraphManager {
         console.log('Graph stabilization complete');
     }
 
+    /**
+     * Focus view on specific node
+     * @param {string} nodeId - Node to focus on
+     */
+    focusOnNode(nodeId) {
+        const nodePosition = this.network.getPositions([nodeId]);
+        if (nodePosition[nodeId]) {
+            this.network.moveTo({
+                position: nodePosition[nodeId],
+                scale: 1.5,
+                animation: {
+                    duration: 1000,
+                    easingFunction: 'easeInOutQuart'
+                }
+            });
+        }
+    }
+    
+    /**
+     * Toggle physics simulation
+     */
+    togglePhysics() {
+        const currentPhysics = this.network.physics.physicsEnabled;
+        this.network.setOptions({ physics: !currentPhysics });
+        
+        const button = document.getElementById('graph-physics');
+        if (button) {
+            button.textContent = currentPhysics ? '[PHYSICS OFF]' : '[PHYSICS ON]';
+            button.style.color = currentPhysics ? '#ff9900' : '#00ff41';
+        }
+    }
+
+    /**
+     * Toggle node clustering
+     */
+    toggleClustering() {
+        // Simple clustering by node type
+        const clusterOptionsByType = {
+            joinCondition: (childOptions) => {
+                return childOptions.type === 'domain';
+            },
+            clusterNodeProperties: {
+                id: 'domain-cluster',
+                borderWidth: 3,
+                shape: 'database',
+                label: 'Domains',
+                color: '#00ff41'
+            }
+        };
+        
+        if (this.network.clustering.isCluster('domain-cluster')) {
+            this.network.clustering.openCluster('domain-cluster');
+        } else {
+            this.network.clustering.cluster(clusterOptionsByType);
+        }
+    }
+
     /**
      * Fit the view to show all nodes
      */
@@ -516,24 +967,6 @@ class GraphManager {
         }
     }
 
-    /**
-     * Setup control event handlers
-     */
-    setupEventHandlers() {
-        // Reset view button
-        document.addEventListener('DOMContentLoaded', () => {
-            const resetBtn = document.getElementById('reset-view');
-            if (resetBtn) {
-                resetBtn.addEventListener('click', () => this.resetView());
-            }
-            
-            const fitBtn = document.getElementById('fit-view');
-            if (fitBtn) {
-                fitBtn.addEventListener('click', () => this.fitView());
-            }
-        });
-    }
-    
     /**
      * Get network statistics
      * @returns {Object} Statistics object

static/js/main.js

@@ -447,11 +447,19 @@ class DNSReconApp {
         try {
             console.log('Updating status display...');
             
-            // Update status text
+            // Update status text with animation
             if (this.elements.scanStatus) {
-                this.elements.scanStatus.textContent = this.formatStatus(status.status);
+                const formattedStatus = this.formatStatus(status.status);
-                console.log('Updated status display:', status.status);
+                if (this.elements.scanStatus.textContent !== formattedStatus) {
+                    this.elements.scanStatus.textContent = formattedStatus;
+                    this.elements.scanStatus.classList.add('fade-in');
+                    setTimeout(() => this.elements.scanStatus.classList.remove('fade-in'), 300);
                 }
+                
+                // Add status-specific classes for styling
+                this.elements.scanStatus.className = `status-value status-${status.status}`;
+            }
+            
             if (this.elements.targetDisplay) {
                 this.elements.targetDisplay.textContent = status.target_domain || 'None';
             }
@@ -465,9 +473,16 @@ class DNSReconApp {
                 this.elements.indicatorsDisplay.textContent = status.indicators_processed || 0;
             }
             
-            // Update progress bar
+            // Update progress bar with smooth animation
             if (this.elements.progressFill) {
                 this.elements.progressFill.style.width = `${status.progress_percentage}%`;
+                
+                // Add pulsing animation for active scans
+                if (status.status === 'running') {
+                    this.elements.progressFill.parentElement.classList.add('scanning');
+                } else {
+                    this.elements.progressFill.parentElement.classList.remove('scanning');
+                }
             }
             
             // Update session ID
@@ -492,12 +507,16 @@ class DNSReconApp {
             case 'running':
                 this.setUIState('scanning');
                 this.showSuccess('Scan is running');
+                // Reset polling frequency for active scans
+                this.pollFrequency = 2000;
+                this.updateConnectionStatus('active');
                 break;
                 
             case 'completed':
                 this.setUIState('completed');
                 this.stopPolling();
                 this.showSuccess('Scan completed successfully');
+                this.updateConnectionStatus('completed');
                 // Force a final graph update
                 console.log('Scan completed - forcing final graph update');
                 setTimeout(() => this.updateGraph(), 100);
@@ -507,21 +526,55 @@ class DNSReconApp {
                 this.setUIState('failed');
                 this.stopPolling();
                 this.showError('Scan failed');
+                this.updateConnectionStatus('error');
                 break;
                 
             case 'stopped':
                 this.setUIState('stopped');
                 this.stopPolling();
                 this.showSuccess('Scan stopped');
+                this.updateConnectionStatus('stopped');
                 break;
                 
             case 'idle':
                 this.setUIState('idle');
                 this.stopPolling();
+                this.updateConnectionStatus('idle');
                 break;
         }
     }
 
+    /**
+     * Update connection status indicator
+     * @param {string} status - Connection status
+     */
+    updateConnectionStatus(status) {
+        if (!this.elements.connectionStatus) return;
+        
+        const statusColors = {
+            'idle': '#c7c7c7',
+            'active': '#00ff41',
+            'completed': '#00aa2e',
+            'stopped': '#ff9900',
+            'error': '#ff6b6b'
+        };
+        
+        this.elements.connectionStatus.style.backgroundColor = statusColors[status] || '#c7c7c7';
+        
+        const statusText = this.elements.connectionStatus.parentElement?.querySelector('.status-text');
+        if (statusText) {
+            const statusTexts = {
+                'idle': 'System Ready',
+                'active': 'Scanning Active',
+                'completed': 'Scan Complete',
+                'stopped': 'Scan Stopped',
+                'error': 'Connection Error'
+            };
+            
+            statusText.textContent = statusTexts[status] || 'System Online';
+        }
+    }
+    
     /**
      * Set UI state based on scan status
      * @param {string} state - UI state
@@ -532,10 +585,17 @@ class DNSReconApp {
         switch (state) {
             case 'scanning':
                 this.isScanning = true;
-                if (this.elements.startScan) this.elements.startScan.disabled = true;
+                if (this.elements.startScan) {
-                if (this.elements.stopScan) this.elements.stopScan.disabled = false;
+                    this.elements.startScan.disabled = true;
+                    this.elements.startScan.classList.add('loading');
+                }
+                if (this.elements.stopScan) {
+                    this.elements.stopScan.disabled = false;
+                    this.elements.stopScan.classList.remove('loading');
+                }
                 if (this.elements.targetDomain) this.elements.targetDomain.disabled = true;
                 if (this.elements.maxDepth) this.elements.maxDepth.disabled = true;
+                if (this.elements.configureApiKeys) this.elements.configureApiKeys.disabled = true;
                 break;
                 
             case 'idle':
@@ -543,10 +603,17 @@ class DNSReconApp {
             case 'failed':
             case 'stopped':
                 this.isScanning = false;
-                if (this.elements.startScan) this.elements.startScan.disabled = false;
+                if (this.elements.startScan) {
-                if (this.elements.stopScan) this.elements.stopScan.disabled = true;
+                    this.elements.startScan.disabled = false;
+                    this.elements.startScan.classList.remove('loading');
+                }
+                if (this.elements.stopScan) {
+                    this.elements.stopScan.disabled = true;
+                    this.elements.stopScan.classList.add('loading');
+                }
                 if (this.elements.targetDomain) this.elements.targetDomain.disabled = false;
                 if (this.elements.maxDepth) this.elements.maxDepth.disabled = false;
+                if (this.elements.configureApiKeys) this.elements.configureApiKeys.disabled = false;
                 break;
         }
     }
@@ -580,20 +647,42 @@ class DNSReconApp {
         
         for (const [name, info] of Object.entries(providers)) {
             const providerItem = document.createElement('div');
-            providerItem.className = 'provider-item';
+            providerItem.className = 'provider-item fade-in';
             
-            const status = info.enabled ? 'enabled' : 'disabled';
+            let statusClass = 'disabled';
-            const statusClass = info.enabled ? 'enabled' : 'disabled';
+            let statusText = 'Disabled';
+            
+            if (info.enabled) {
+                statusClass = 'enabled';
+                statusText = 'Enabled';
+            } else if (info.requires_api_key) {
+                statusClass = 'api-key-required';
+                statusText = 'API Key Required';
+            }
             
             providerItem.innerHTML = `
-                <div>
+                <div class="provider-header">
                     <div class="provider-name">${name.toUpperCase()}</div>
+                    <div class="provider-status ${statusClass}">${statusText}</div>
+                </div>
                 <div class="provider-stats">
-                        Requests: ${info.statistics.total_requests || 0} | 
+                    <div class="provider-stat">
-                        Success Rate: ${(info.statistics.success_rate || 0).toFixed(1)}%
+                        <span class="provider-stat-label">Requests:</span>
+                        <span class="provider-stat-value">${info.statistics.total_requests || 0}</span>
+                    </div>
+                    <div class="provider-stat">
+                        <span class="provider-stat-label">Success Rate:</span>
+                        <span class="provider-stat-value">${(info.statistics.success_rate || 0).toFixed(1)}%</span>
+                    </div>
+                    <div class="provider-stat">
+                        <span class="provider-stat-label">Relationships:</span>
+                        <span class="provider-stat-value">${info.statistics.relationships_found || 0}</span>
+                    </div>
+                    <div class="provider-stat">
+                        <span class="provider-stat-label">Rate Limit:</span>
+                        <span class="provider-stat-value">${info.rate_limit}/min</span>
                     </div>
                 </div>
-                <div class="provider-status ${statusClass}">${status}</div>
             `;
             
             this.elements.providerList.appendChild(providerItem);
@@ -614,7 +703,7 @@ class DNSReconApp {
         
         let detailsHtml = '';
         detailsHtml += `<div class="detail-row"><span class="detail-label">Identifier:</span><span class="detail-value">${nodeId}</span></div>`;
-        detailsHtml += `<div class="detail-row"><span class="detail-label">Type:</span><span class="detail-value">${node.metadata.type || 'Unknown'}</span></div>`;
+        detailsHtml += `<div class="detail-row"><span class="detail-label">Type:</span><span class="detail-value">${node.metadata.type || node.type || 'Unknown'}</span></div>`;
         
         if (node.metadata) {
             for (const [key, value] of Object.entries(node.metadata)) {
@@ -624,6 +713,12 @@ class DNSReconApp {
             }
         }
         
+        // Add timestamps if available
+        if (node.added_timestamp) {
+            const addedDate = new Date(node.added_timestamp);
+            detailsHtml += `<div class="detail-row"><span class="detail-label">Added:</span><span class="detail-value">${addedDate.toLocaleString()}</span></div>`;
+        }
+        
         if (this.elements.modalDetails) {
             this.elements.modalDetails.innerHTML = detailsHtml;
         }
@@ -645,12 +740,24 @@ class DNSReconApp {
      * @returns {boolean} True if data has changed
      */
     hasGraphChanged(graphData) {
-        // Simple check based on node and edge counts
+        // Simple check based on node and edge counts and timestamps
         const currentStats = this.graphManager.getStatistics();
         const newNodeCount = graphData.nodes ? graphData.nodes.length : 0;
         const newEdgeCount = graphData.edges ? graphData.edges.length : 0;
         
-        const changed = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount;
+        // Check if counts changed
+        const countsChanged = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount;
+        
+        // Also check if we have new timestamp data
+        const hasNewTimestamp = graphData.statistics && 
+                               graphData.statistics.last_modified && 
+                               graphData.statistics.last_modified !== this.lastGraphTimestamp;
+        
+        if (hasNewTimestamp) {
+            this.lastGraphTimestamp = graphData.statistics.last_modified;
+        }
+        
+        const changed = countsChanged || hasNewTimestamp;
         
         console.log(`Graph change check: Current(${currentStats.nodeCount}n, ${currentStats.edgeCount}e) vs New(${newNodeCount}n, ${newEdgeCount}e) = ${changed}`);
         
@@ -816,6 +923,10 @@ class DNSReconApp {
         this.showMessage(message, 'info');
     }
     
+    showWarning(message) {
+        this.showMessage(message, 'warning');
+    }
+
     /**
      * Show error message
      * @param {string} message - Error message
@@ -828,13 +939,7 @@ class DNSReconApp {
      * Show connection error
      */
     showConnectionError() {
-        if (this.elements.connectionStatus) {
+        this.updateConnectionStatus('error');
-            this.elements.connectionStatus.style.backgroundColor = '#ff6b6b';
-        }
-        const statusText = this.elements.connectionStatus?.parentElement?.querySelector('.status-text');
-        if (statusText) {
-            statusText.textContent = 'Connection Error';
-        }
     }
     
     /**
@@ -848,24 +953,12 @@ class DNSReconApp {
         // Create message element
         const messageElement = document.createElement('div');
         messageElement.className = `message-toast message-${type}`;
-        messageElement.style.cssText = `
-            background: ${this.getMessageColor(type)};
-            color: #fff;
-            padding: 12px 20px;
-            margin-bottom: 10px;
-            border-radius: 4px;
-            font-family: 'Roboto Mono', monospace;
-            font-size: 0.9rem;
-            box-shadow: 0 4px 6px rgba(0,0,0,0.3);
-            border-left: 4px solid ${this.getMessageBorderColor(type)};
-            animation: slideInRight 0.3s ease-out;
-        `;
         
         messageElement.innerHTML = `
-            <div style="display: flex; justify-content: space-between; align-items: center;">
+            <div style="display: flex; justify-content: space-between; align-items: center; padding: 12px 20px;">
-                <span>${message}</span>
+                <span style="flex: 1;">${message}</span>
                 <button onclick="this.parentElement.parentElement.remove()" 
-                        style="background: none; border: none; color: #fff; cursor: pointer; font-size: 16px; margin-left: 10px;">×</button>
+                        style="background: none; border: none; color: #fff; cursor: pointer; font-size: 16px; margin-left: 10px; opacity: 0.7;">×</button>
             </div>
         `;
         
@@ -888,13 +981,8 @@ class DNSReconApp {
         }
         
         // Update connection status to show activity
-        if (type === 'success' && this.elements.connectionStatus) {
+        if (type === 'success' && this.consecutiveErrors === 0) {
-            this.elements.connectionStatus.style.backgroundColor = '#00ff41';
+            this.updateConnectionStatus(this.isScanning ? 'active' : 'idle');
-            setTimeout(() => {
-                if (this.elements.connectionStatus) {
-                    this.elements.connectionStatus.style.backgroundColor = '#00ff41';
-                }
-            }, 2000);
         }
     }
 

templates/index.html

@@ -63,6 +63,10 @@
                             <span class="btn-icon">[EXPORT]</span>
                             <span>Download Results</span>
                         </button>
+                        <button id="configure-api-keys" class="btn btn-secondary">
+                            <span class="btn-icon">[API]</span>
+                            <span>Configure API Keys</span>
+                        </button>
                     </div>
                 </div>
             </section>