From ce0e11cf0b70a652e75b2ce8d5b026df438cb684 Mon Sep 17 00:00:00 2001 From: overcuriousity Date: Wed, 10 Sep 2025 15:17:17 +0200 Subject: [PATCH] progress --- app.py | 18 +- config.py | 2 + core/__init__.py | 3 +- core/graph_manager.py | 86 +++- core/logger.py | 15 +- core/scanner.py | 562 ++++++++++++++++--------- providers/__init__.py | 10 +- providers/base_provider.py | 200 +++++---- providers/dns_provider.py | 338 +++++++++++++++ providers/shodan_provider.py | 299 +++++++++++++ providers/virustotal_provider.py | 334 +++++++++++++++ requirements.txt | 3 +- static/css/main.css | 308 +++++++++++++- static/js/graph.js | 695 +++++++++++++++++++++++++------ static/js/main.js | 184 +++++--- templates/index.html | 4 + 16 files changed, 2577 insertions(+), 484 deletions(-) create mode 100644 providers/shodan_provider.py create mode 100644 providers/virustotal_provider.py diff --git a/app.py b/app.py index 932f140..b987244 100644 --- a/app.py +++ b/app.py @@ -6,7 +6,7 @@ Provides REST API endpoints and serves the web interface. import json import traceback from flask import Flask, render_template, request, jsonify, send_file -from datetime import datetime +from datetime import datetime, timezone import io from core.scanner import scanner @@ -173,7 +173,7 @@ def export_results(): results = scanner.export_results() # Create filename with timestamp - timestamp = datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S') + timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S') target = scanner.current_target or 'unknown' filename = f"dnsrecon_{target}_{timestamp}.json" @@ -284,12 +284,22 @@ def set_api_keys(): @app.route('/api/health', methods=['GET']) def health_check(): - """Health check endpoint.""" + """Health check endpoint with enhanced Phase 2 information.""" return jsonify({ 'success': True, 'status': 'healthy', 'timestamp': datetime.now(datetime.UTC).isoformat(), - 'version': '1.0.0-phase1' + 'version': '1.0.0-phase2', + 'phase': 2, + 'features': { + 'multi_provider': True, + 'concurrent_processing': True, + 'real_time_updates': True, + 'api_key_management': True, + 'enhanced_visualization': True, + 'retry_logic': True + }, + 'providers_available': len(scanner.providers) if hasattr(scanner, 'providers') else 0 }) diff --git a/config.py b/config.py index b799f9e..1f55542 100644 --- a/config.py +++ b/config.py @@ -111,6 +111,8 @@ class Config: # Override default settings from environment self.default_recursion_depth = int(os.getenv('DEFAULT_RECURSION_DEPTH', '2')) self.flask_debug = os.getenv('FLASK_DEBUG', 'True').lower() == 'true' + self.default_timeout = 30 + self.max_concurrent_requests = 5 # Global configuration instance diff --git a/core/__init__.py b/core/__init__.py index cb68feb..8d3e34d 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,6 +1,7 @@ """ Core modules for DNSRecon passive reconnaissance tool. Contains graph management, scanning orchestration, and forensic logging. +Phase 2: Enhanced with concurrent processing and real-time capabilities. """ from .graph_manager import GraphManager, NodeType, RelationshipType @@ -19,4 +20,4 @@ __all__ = [ 'new_session' ] -__version__ = "1.0.0-phase1" \ No newline at end of file +__version__ = "1.0.0-phase2" \ No newline at end of file diff --git a/core/graph_manager.py b/core/graph_manager.py index d0768ad..968e4aa 100644 --- a/core/graph_manager.py +++ b/core/graph_manager.py @@ -8,6 +8,7 @@ import threading from datetime import datetime from typing import Dict, List, Any, Optional, Tuple, Set from enum import Enum +from datetime import timezone import networkx as nx @@ -44,8 +45,8 @@ class GraphManager: def __init__(self): """Initialize empty directed graph.""" self.graph = nx.DiGraph() - #self.lock = threading.Lock() - self.creation_time = datetime.now(datetime.UTC).isoformat() + # self.lock = threading.Lock() + self.creation_time = datetime.now(timezone.utc).isoformat() self.last_modified = self.creation_time def add_node(self, node_id: str, node_type: NodeType, @@ -71,12 +72,12 @@ class GraphManager: node_attributes = { 'type': node_type.value, - 'added_timestamp': datetime.now(datetime.UTC).isoformat(), + 'added_timestamp': datetime.now(timezone.utc).isoformat(), 'metadata': metadata or {} } self.graph.add_node(node_id, **node_attributes) - self.last_modified = datetime.now(datetime.UTC).isoformat() + self.last_modified = datetime.now(timezone.utc).isoformat() return True def add_edge(self, source_id: str, target_id: str, @@ -111,7 +112,7 @@ class GraphManager: if new_confidence > existing_confidence: self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence - self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(datetime.UTC).isoformat() + self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat() self.graph.edges[source_id, target_id]['updated_by'] = source_provider return False @@ -120,12 +121,12 @@ class GraphManager: 'relationship_type': relationship_type.relationship_name, 'confidence_score': confidence_score or relationship_type.default_confidence, 'source_provider': source_provider, - 'discovery_timestamp': datetime.now(datetime.UTC).isoformat(), + 'discovery_timestamp': datetime.now(timezone.utc).isoformat(), 'raw_data': raw_data or {} } self.graph.add_edge(source_id, target_id, **edge_attributes) - self.last_modified = datetime.now(datetime.UTC).isoformat() + self.last_modified = datetime.now(timezone.utc).isoformat() return True def get_node_count(self) -> int: @@ -210,14 +211,36 @@ class GraphManager: 'added_timestamp': attributes.get('added_timestamp') } - # Color coding by type + # Color coding by type - now returns color objects for enhanced visualization type_colors = { - 'domain': '#00ff41', # Green for domains - 'ip': '#ff9900', # Amber for IPs - 'certificate': '#c7c7c7', # Gray for certificates - 'asn': '#00aaff' # Blue for ASNs + 'domain': { + 'background': '#00ff41', + 'border': '#00aa2e', + 'highlight': {'background': '#44ff75', 'border': '#00ff41'}, + 'hover': {'background': '#22ff63', 'border': '#00cc35'} + }, + 'ip': { + 'background': '#ff9900', + 'border': '#cc7700', + 'highlight': {'background': '#ffbb44', 'border': '#ff9900'}, + 'hover': {'background': '#ffaa22', 'border': '#dd8800'} + }, + 'certificate': { + 'background': '#c7c7c7', + 'border': '#999999', + 'highlight': {'background': '#e0e0e0', 'border': '#c7c7c7'}, + 'hover': {'background': '#d4d4d4', 'border': '#aaaaaa'} + }, + 'asn': { + 'background': '#00aaff', + 'border': '#0088cc', + 'highlight': {'background': '#44ccff', 'border': '#00aaff'}, + 'hover': {'background': '#22bbff', 'border': '#0099dd'} + } } - node_data['color'] = type_colors.get(attributes.get('type'), '#ffffff') + + node_color_config = type_colors.get(attributes.get('type', 'unknown'), type_colors['domain']) + node_data['color'] = node_color_config nodes.append(node_data) # Format edges for visualization @@ -231,17 +254,36 @@ class GraphManager: 'discovery_timestamp': attributes.get('discovery_timestamp') } - # Edge styling based on confidence + # Enhanced edge styling based on confidence confidence = attributes.get('confidence_score', 0) if confidence >= 0.8: - edge_data['color'] = '#00ff41' # Green for high confidence - edge_data['width'] = 3 + edge_data['color'] = { + 'color': '#00ff41', + 'highlight': '#44ff75', + 'hover': '#22ff63', + 'inherit': False + } + edge_data['width'] = 4 elif confidence >= 0.6: - edge_data['color'] = '#ff9900' # Amber for medium confidence - edge_data['width'] = 2 + edge_data['color'] = { + 'color': '#ff9900', + 'highlight': '#ffbb44', + 'hover': '#ffaa22', + 'inherit': False + } + edge_data['width'] = 3 else: - edge_data['color'] = '#444444' # Dark gray for low confidence - edge_data['width'] = 1 + edge_data['color'] = { + 'color': '#666666', + 'highlight': '#888888', + 'hover': '#777777', + 'inherit': False + } + edge_data['width'] = 2 + + # Add dashed line for low confidence + if confidence < 0.6: + edge_data['dashes'] = [5, 5] edges.append(edge_data) @@ -270,7 +312,7 @@ class GraphManager: # Add comprehensive metadata export_data = { 'export_metadata': { - 'export_timestamp': datetime.now(datetime.UTC).isoformat(), + 'export_timestamp': datetime.now(timezone.utc).isoformat(), 'graph_creation_time': self.creation_time, 'last_modified': self.last_modified, 'total_nodes': self.graph.number_of_nodes(), @@ -351,5 +393,5 @@ class GraphManager: """Clear all nodes and edges from the graph.""" #with self.lock: self.graph.clear() - self.creation_time = datetime.now(datetime.UTC).isoformat() + self.creation_time = datetime.now(timezone.utc).isoformat() self.last_modified = self.creation_time \ No newline at end of file diff --git a/core/logger.py b/core/logger.py index a01495b..c9f6964 100644 --- a/core/logger.py +++ b/core/logger.py @@ -9,6 +9,7 @@ import threading from datetime import datetime from typing import Dict, Any, Optional, List from dataclasses import dataclass, asdict +from datetime import timezone @dataclass @@ -60,7 +61,7 @@ class ForensicLogger: self.relationships: List[RelationshipDiscovery] = [] self.session_metadata = { 'session_id': self.session_id, - 'start_time': datetime.now(datetime.UTC).isoformat(), + 'start_time': datetime.now(timezone.utc).isoformat(), 'end_time': None, 'total_requests': 0, 'total_relationships': 0, @@ -85,7 +86,7 @@ class ForensicLogger: def _generate_session_id(self) -> str: """Generate unique session identifier.""" - return f"dnsrecon_{datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S')}" + return f"dnsrecon_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}" def log_api_request(self, provider: str, url: str, method: str = "GET", status_code: Optional[int] = None, @@ -110,7 +111,7 @@ class ForensicLogger: """ #with self.lock: api_request = APIRequest( - timestamp=datetime.now(datetime.UTC).isoformat(), + timestamp=datetime.now(timezone.utc).isoformat(), provider=provider, url=url, method=method, @@ -153,7 +154,7 @@ class ForensicLogger: """ #with self.lock: relationship = RelationshipDiscovery( - timestamp=datetime.now(datetime.UTC).isoformat(), + timestamp=datetime.now(timezone.utc).isoformat(), source_node=source_node, target_node=target_node, relationship_type=relationship_type, @@ -183,7 +184,7 @@ class ForensicLogger: def log_scan_complete(self) -> None: """Log the completion of a reconnaissance scan.""" #with self.lock: - self.session_metadata['end_time'] = datetime.now(datetime.UTC).isoformat() + self.session_metadata['end_time'] = datetime.now(timezone.utc).isoformat() self.session_metadata['providers_used'] = list(self.session_metadata['providers_used']) self.session_metadata['target_domains'] = list(self.session_metadata['target_domains']) @@ -203,7 +204,7 @@ class ForensicLogger: 'session_metadata': self.session_metadata.copy(), 'api_requests': [asdict(req) for req in self.api_requests], 'relationships': [asdict(rel) for rel in self.relationships], - 'export_timestamp': datetime.now(datetime.UTC).isoformat() + 'export_timestamp': datetime.now(timezone.utc).isoformat() } def get_forensic_summary(self) -> Dict[str, Any]: @@ -239,7 +240,7 @@ class ForensicLogger: def _calculate_session_duration(self) -> float: """Calculate session duration in minutes.""" if not self.session_metadata['end_time']: - end_time = datetime.now(datetime.UTC) + end_time = datetime.now(timezone.utc) else: end_time = datetime.fromisoformat(self.session_metadata['end_time']) diff --git a/core/scanner.py b/core/scanner.py index 222d942..40c8a56 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -6,12 +6,15 @@ Coordinates data gathering from multiple providers and builds the infrastructure import threading import time import traceback -from typing import List, Set, Dict, Any, Optional +from typing import List, Set, Dict, Any, Optional, Tuple from concurrent.futures import ThreadPoolExecutor, as_completed from core.graph_manager import GraphManager, NodeType, RelationshipType from core.logger import get_forensic_logger, new_session from providers.crtsh_provider import CrtShProvider +from providers.dns_provider import DNSProvider +from providers.shodan_provider import ShodanProvider +from providers.virustotal_provider import VirusTotalProvider from config import config @@ -27,17 +30,16 @@ class ScanStatus: class Scanner: """ Main scanning orchestrator for DNSRecon passive reconnaissance. - Manages multi-provider data gathering and graph construction. + Manages multi-provider data gathering and graph construction with concurrent processing. """ def __init__(self): - """Initialize scanner with default providers and empty graph.""" + """Initialize scanner with all available providers and empty graph.""" print("Initializing Scanner instance...") try: - from providers.base_provider import BaseProvider self.graph = GraphManager() - self.providers: List[BaseProvider] = [] + self.providers = [] self.status = ScanStatus.IDLE self.current_target = None self.current_depth = 0 @@ -49,6 +51,9 @@ class Scanner: self.total_indicators_found = 0 self.indicators_processed = 0 self.current_indicator = "" + + # Concurrent processing configuration + self.max_workers = config.max_concurrent_requests # Initialize providers print("Calling _initialize_providers...") @@ -66,36 +71,54 @@ class Scanner: raise def _initialize_providers(self) -> None: - """Initialize available providers based on configuration.""" + """Initialize all available providers based on configuration.""" self.providers = [] print("Initializing providers...") # Always add free providers - if config.is_provider_enabled('crtsh'): - try: - crtsh_provider = CrtShProvider() - if crtsh_provider.is_available(): - self.providers.append(crtsh_provider) - print("✓ CrtSh provider initialized successfully") - else: - print("✗ CrtSh provider is not available") - except Exception as e: - print(f"✗ Failed to initialize CrtSh provider: {e}") - traceback.print_exc() + free_providers = [ + ('crtsh', CrtShProvider), + ('dns', DNSProvider) + ] + + for provider_name, provider_class in free_providers: + if config.is_provider_enabled(provider_name): + try: + provider = provider_class() + if provider.is_available(): + self.providers.append(provider) + print(f"✓ {provider_name.title()} provider initialized successfully") + else: + print(f"✗ {provider_name.title()} provider is not available") + except Exception as e: + print(f"✗ Failed to initialize {provider_name.title()} provider: {e}") + traceback.print_exc() + + # Add API key-dependent providers + api_providers = [ + ('shodan', ShodanProvider), + ('virustotal', VirusTotalProvider) + ] + + for provider_name, provider_class in api_providers: + if config.is_provider_enabled(provider_name): + try: + provider = provider_class() + if provider.is_available(): + self.providers.append(provider) + print(f"✓ {provider_name.title()} provider initialized successfully") + else: + print(f"✗ {provider_name.title()} provider is not available (API key required)") + except Exception as e: + print(f"✗ Failed to initialize {provider_name.title()} provider: {e}") + traceback.print_exc() print(f"Initialized {len(self.providers)} providers") - def _debug_threads(self): - """Debug function to show current threads.""" - print("=== THREAD DEBUG INFO ===") - for t in threading.enumerate(): - print(f"Thread: {t.name} | Alive: {t.is_alive()} | Daemon: {t.daemon}") - print("=== END THREAD DEBUG ===") - def start_scan(self, target_domain: str, max_depth: int = 2) -> bool: """ - Start a new reconnaissance scan. + Start a new reconnaissance scan with concurrent processing. Args: target_domain: Initial domain to investigate @@ -107,9 +130,6 @@ class Scanner: print(f"Scanner.start_scan called with target='{target_domain}', depth={max_depth}") try: - print("Checking current status...") - self._debug_threads() - if self.status == ScanStatus.RUNNING: print("Scan already running, rejecting new scan") return False @@ -119,8 +139,6 @@ class Scanner: print("No providers available, cannot start scan") return False - print(f"Current status: {self.status}, Providers: {len(self.providers)}") - # Stop any existing scan thread if self.scan_thread and self.scan_thread.is_alive(): print("Stopping existing scan thread...") @@ -132,9 +150,7 @@ class Scanner: # Reset state print("Resetting scanner state...") - #print("Running graph.clear()") - #self.graph.clear() - print("running self.current_target = target_domain.lower().strip()") + self.graph.clear() self.current_target = target_domain.lower().strip() self.max_depth = max_depth self.current_depth = 0 @@ -147,9 +163,15 @@ class Scanner: print("Starting new forensic session...") self.logger = new_session() - # FOR DEBUGGING: Run scan synchronously instead of in thread - print("Running scan synchronously for debugging...") - self._execute_scan_sync(self.current_target, max_depth) + # Start scan in separate thread for Phase 2 + print("Starting scan thread...") + self.scan_thread = threading.Thread( + target=self._execute_scan_async, + args=(self.current_target, max_depth), + daemon=True + ) + self.scan_thread.start() + return True except Exception as e: @@ -157,6 +179,321 @@ class Scanner: traceback.print_exc() return False + def _execute_scan_async(self, target_domain: str, max_depth: int) -> None: + """ + Execute the reconnaissance scan asynchronously with concurrent provider queries. + + Args: + target_domain: Target domain to investigate + max_depth: Maximum recursion depth + """ + print(f"_execute_scan_async started for {target_domain} with depth {max_depth}") + + try: + print("Setting status to RUNNING") + self.status = ScanStatus.RUNNING + + # Log scan start + enabled_providers = [provider.get_name() for provider in self.providers] + self.logger.log_scan_start(target_domain, max_depth, enabled_providers) + print(f"Logged scan start with providers: {enabled_providers}") + + # Initialize with target domain + print(f"Adding target domain '{target_domain}' as initial node") + self.graph.add_node(target_domain, NodeType.DOMAIN) + + # BFS-style exploration with depth limiting and concurrent processing + current_level_domains = {target_domain} + processed_domains = set() + all_discovered_ips = set() + + print(f"Starting BFS exploration...") + + for depth in range(max_depth + 1): + if self.stop_requested: + print(f"Stop requested at depth {depth}") + break + + self.current_depth = depth + print(f"Processing depth level {depth} with {len(current_level_domains)} domains") + + if not current_level_domains: + print("No domains to process at this level") + break + + # Update progress tracking + self.total_indicators_found += len(current_level_domains) + next_level_domains = set() + + # Process domains at current depth level with concurrent queries + domain_results = self._process_domains_concurrent(current_level_domains, processed_domains) + + for domain, discovered_domains, discovered_ips in domain_results: + if self.stop_requested: + break + + processed_domains.add(domain) + all_discovered_ips.update(discovered_ips) + + # Add discovered domains to next level if not at max depth + if depth < max_depth: + for discovered_domain in discovered_domains: + if discovered_domain not in processed_domains: + next_level_domains.add(discovered_domain) + print(f"Adding {discovered_domain} to next level") + + # Process discovered IPs concurrently + if all_discovered_ips: + print(f"Processing {len(all_discovered_ips)} discovered IP addresses") + self._process_ips_concurrent(all_discovered_ips) + + current_level_domains = next_level_domains + print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level") + + # Finalize scan + if self.stop_requested: + self.status = ScanStatus.STOPPED + print("Scan completed with STOPPED status") + else: + self.status = ScanStatus.COMPLETED + print("Scan completed with COMPLETED status") + + self.logger.log_scan_complete() + + # Print final statistics + stats = self.graph.get_statistics() + print(f"Final scan statistics:") + print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}") + print(f" - Total edges: {stats['basic_metrics']['total_edges']}") + print(f" - Domains processed: {len(processed_domains)}") + print(f" - IPs discovered: {len(all_discovered_ips)}") + + except Exception as e: + print(f"ERROR: Scan execution failed with error: {e}") + traceback.print_exc() + self.status = ScanStatus.FAILED + self.logger.logger.error(f"Scan failed: {e}") + + def _process_domains_concurrent(self, domains: Set[str], processed_domains: Set[str]) -> List[Tuple[str, Set[str], Set[str]]]: + """ + Process multiple domains concurrently using thread pool. + + Args: + domains: Set of domains to process + processed_domains: Set of already processed domains + + Returns: + List of tuples (domain, discovered_domains, discovered_ips) + """ + results = [] + + # Filter out already processed domains + domains_to_process = domains - processed_domains + + if not domains_to_process: + return results + + print(f"Processing {len(domains_to_process)} domains concurrently with {self.max_workers} workers") + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + # Submit all domain processing tasks + future_to_domain = { + executor.submit(self._query_providers_for_domain, domain): domain + for domain in domains_to_process + } + + # Collect results as they complete + for future in as_completed(future_to_domain): + if self.stop_requested: + break + + domain = future_to_domain[future] + + try: + discovered_domains, discovered_ips = future.result() + results.append((domain, discovered_domains, discovered_ips)) + self.indicators_processed += 1 + print(f"Completed processing domain: {domain} ({len(discovered_domains)} domains, {len(discovered_ips)} IPs)") + + except Exception as e: + print(f"Error processing domain {domain}: {e}") + traceback.print_exc() + + return results + + def _process_ips_concurrent(self, ips: Set[str]) -> None: + """ + Process multiple IP addresses concurrently. + + Args: + ips: Set of IP addresses to process + """ + if not ips: + return + + print(f"Processing {len(ips)} IP addresses concurrently") + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + # Submit all IP processing tasks + future_to_ip = { + executor.submit(self._query_providers_for_ip, ip): ip + for ip in ips + } + + # Collect results as they complete + for future in as_completed(future_to_ip): + if self.stop_requested: + break + + ip = future_to_ip[future] + + try: + future.result() # Just wait for completion + print(f"Completed processing IP: {ip}") + + except Exception as e: + print(f"Error processing IP {ip}: {e}") + traceback.print_exc() + + def _query_providers_for_domain(self, domain: str) -> Tuple[Set[str], Set[str]]: + """ + Query all enabled providers for information about a domain. + + Args: + domain: Domain to investigate + + Returns: + Tuple of (discovered_domains, discovered_ips) + """ + print(f"Querying {len(self.providers)} providers for domain: {domain}") + discovered_domains = set() + discovered_ips = set() + + if not self.providers: + print("No providers available") + return discovered_domains, discovered_ips + + # Query providers concurrently for better performance + with ThreadPoolExecutor(max_workers=len(self.providers)) as executor: + # Submit queries for all providers + future_to_provider = { + executor.submit(self._safe_provider_query_domain, provider, domain): provider + for provider in self.providers + } + + # Collect results as they complete + for future in as_completed(future_to_provider): + if self.stop_requested: + break + + provider = future_to_provider[future] + + try: + relationships = future.result() + print(f"Provider {provider.get_name()} returned {len(relationships)} relationships") + + for source, target, rel_type, confidence, raw_data in relationships: + # Determine node type based on target + if self._is_valid_ip(target): + target_node_type = NodeType.IP + discovered_ips.add(target) + elif self._is_valid_domain(target): + target_node_type = NodeType.DOMAIN + discovered_domains.add(target) + else: + # Could be ASN or certificate + target_node_type = NodeType.ASN if target.startswith('AS') else NodeType.CERTIFICATE + + # Add nodes and relationship to graph + self.graph.add_node(source, NodeType.DOMAIN) + self.graph.add_node(target, target_node_type) + + success = self.graph.add_edge( + source, target, rel_type, confidence, + provider.get_name(), raw_data + ) + + if success: + print(f"Added relationship: {source} -> {target} ({rel_type.relationship_name})") + + except Exception as e: + print(f"Provider {provider.get_name()} failed for {domain}: {e}") + + print(f"Domain {domain}: discovered {len(discovered_domains)} domains, {len(discovered_ips)} IPs") + return discovered_domains, discovered_ips + + def _query_providers_for_ip(self, ip: str) -> None: + """ + Query all enabled providers for information about an IP address. + + Args: + ip: IP address to investigate + """ + print(f"Querying {len(self.providers)} providers for IP: {ip}") + + if not self.providers: + print("No providers available") + return + + # Query providers concurrently + with ThreadPoolExecutor(max_workers=len(self.providers)) as executor: + # Submit queries for all providers + future_to_provider = { + executor.submit(self._safe_provider_query_ip, provider, ip): provider + for provider in self.providers + } + + # Collect results as they complete + for future in as_completed(future_to_provider): + if self.stop_requested: + break + + provider = future_to_provider[future] + + try: + relationships = future.result() + print(f"Provider {provider.get_name()} returned {len(relationships)} relationships for IP {ip}") + + for source, target, rel_type, confidence, raw_data in relationships: + # Determine node type based on target + if self._is_valid_domain(target): + target_node_type = NodeType.DOMAIN + elif target.startswith('AS'): + target_node_type = NodeType.ASN + else: + target_node_type = NodeType.IP + + # Add nodes and relationship to graph + self.graph.add_node(source, NodeType.IP) + self.graph.add_node(target, target_node_type) + + success = self.graph.add_edge( + source, target, rel_type, confidence, + provider.get_name(), raw_data + ) + + if success: + print(f"Added IP relationship: {source} -> {target} ({rel_type.relationship_name})") + + except Exception as e: + print(f"Provider {provider.get_name()} failed for IP {ip}: {e}") + + def _safe_provider_query_domain(self, provider, domain: str): + """Safely query provider for domain with error handling.""" + try: + return provider.query_domain(domain) + except Exception as e: + print(f"Provider {provider.get_name()} query_domain failed: {e}") + return [] + + def _safe_provider_query_ip(self, provider, ip: str): + """Safely query provider for IP with error handling.""" + try: + return provider.query_ip(ip) + except Exception as e: + print(f"Provider {provider.get_name()} query_ip failed: {e}") + return [] + def stop_scan(self) -> bool: """ Request scan termination. @@ -218,159 +555,6 @@ class Scanner: return 0.0 return min(100.0, (self.indicators_processed / self.total_indicators_found) * 100) - def _execute_scan_sync(self, target_domain: str, max_depth: int) -> None: - """ - Execute the reconnaissance scan synchronously (for debugging). - - Args: - target_domain: Target domain to investigate - max_depth: Maximum recursion depth - """ - print(f"_execute_scan_sync started for {target_domain} with depth {max_depth}") - - try: - print("Setting status to RUNNING") - self.status = ScanStatus.RUNNING - - # Log scan start - enabled_providers = [provider.get_name() for provider in self.providers] - self.logger.log_scan_start(target_domain, max_depth, enabled_providers) - print(f"Logged scan start with providers: {enabled_providers}") - - # Initialize with target domain - print(f"Adding target domain '{target_domain}' as initial node") - self.graph.add_node(target_domain, NodeType.DOMAIN) - - # BFS-style exploration with depth limiting - current_level_domains = {target_domain} - processed_domains = set() - - print(f"Starting BFS exploration...") - - for depth in range(max_depth + 1): - if self.stop_requested: - print(f"Stop requested at depth {depth}") - break - - self.current_depth = depth - print(f"Processing depth level {depth} with {len(current_level_domains)} domains") - - if not current_level_domains: - print("No domains to process at this level") - break - - # Update progress tracking - self.total_indicators_found += len(current_level_domains) - next_level_domains = set() - - # Process domains at current depth level - for domain in current_level_domains: - if self.stop_requested: - print(f"Stop requested while processing domain {domain}") - break - - if domain in processed_domains: - print(f"Domain {domain} already processed, skipping") - continue - - print(f"Processing domain: {domain}") - self.current_indicator = domain - self.indicators_processed += 1 - - # Query all providers for this domain - discovered_domains = self._query_providers_for_domain(domain) - print(f"Discovered {len(discovered_domains)} new domains from {domain}") - - # Add discovered domains to next level if not at max depth - if depth < max_depth: - for discovered_domain in discovered_domains: - if discovered_domain not in processed_domains: - next_level_domains.add(discovered_domain) - print(f"Adding {discovered_domain} to next level") - - processed_domains.add(domain) - - current_level_domains = next_level_domains - print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level") - - # Finalize scan - if self.stop_requested: - self.status = ScanStatus.STOPPED - print("Scan completed with STOPPED status") - else: - self.status = ScanStatus.COMPLETED - print("Scan completed with COMPLETED status") - - self.logger.log_scan_complete() - - # Print final statistics - stats = self.graph.get_statistics() - print(f"Final scan statistics:") - print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}") - print(f" - Total edges: {stats['basic_metrics']['total_edges']}") - print(f" - Domains processed: {len(processed_domains)}") - - except Exception as e: - print(f"ERROR: Scan execution failed with error: {e}") - traceback.print_exc() - self.status = ScanStatus.FAILED - self.logger.logger.error(f"Scan failed: {e}") - - def _query_providers_for_domain(self, domain: str) -> Set[str]: - """ - Query all enabled providers for information about a domain. - - Args: - domain: Domain to investigate - - Returns: - Set of newly discovered domains - """ - print(f"Querying {len(self.providers)} providers for domain: {domain}") - discovered_domains = set() - - if not self.providers: - print("No providers available") - return discovered_domains - - # Query providers sequentially for debugging - for provider in self.providers: - if self.stop_requested: - print("Stop requested, cancelling provider queries") - break - - try: - print(f"Querying provider: {provider.get_name()}") - relationships = provider.query_domain(domain) - print(f"Provider {provider.get_name()} returned {len(relationships)} relationships") - - for source, target, rel_type, confidence, raw_data in relationships: - print(f"Processing relationship: {source} -> {target} ({rel_type.relationship_name})") - - # Add target node to graph if it doesn't exist - self.graph.add_node(target, NodeType.DOMAIN) - - # Add relationship - success = self.graph.add_edge( - source, target, rel_type, confidence, - provider.get_name(), raw_data - ) - - if success: - print(f"Added new relationship: {source} -> {target}") - else: - print(f"Relationship already exists or failed to add: {source} -> {target}") - - discovered_domains.add(target) - - except Exception as e: - print(f"Provider {provider.get_name()} failed for {domain}: {e}") - traceback.print_exc() - self.logger.logger.error(f"Provider {provider.get_name()} failed for {domain}: {e}") - - print(f"Total unique domains discovered: {len(discovered_domains)}") - return discovered_domains - def get_graph_data(self) -> Dict[str, Any]: """ Get current graph data for visualization. diff --git a/providers/__init__.py b/providers/__init__.py index bc3275f..a071381 100644 --- a/providers/__init__.py +++ b/providers/__init__.py @@ -5,11 +5,17 @@ Contains implementations for various reconnaissance data sources. from .base_provider import BaseProvider, RateLimiter from .crtsh_provider import CrtShProvider +from .dns_provider import DNSProvider +from .shodan_provider import ShodanProvider +from .virustotal_provider import VirusTotalProvider __all__ = [ 'BaseProvider', 'RateLimiter', - 'CrtShProvider' + 'CrtShProvider', + 'DNSProvider', + 'ShodanProvider', + 'VirusTotalProvider' ] -__version__ = "1.0.0-phase1" \ No newline at end of file +__version__ = "1.0.0-phase2" \ No newline at end of file diff --git a/providers/base_provider.py b/providers/base_provider.py index 551b811..7733731 100644 --- a/providers/base_provider.py +++ b/providers/base_provider.py @@ -113,86 +113,134 @@ class BaseProvider(ABC): pass def make_request(self, url: str, method: str = "GET", - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, - target_indicator: str = "") -> Optional[requests.Response]: - """ - Make a rate-limited HTTP request with forensic logging. + params: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + target_indicator: str = "", + max_retries: int = 3) -> Optional[requests.Response]: + """ + Make a rate-limited HTTP request with forensic logging and retry logic. + Args: + url: Request URL + method: HTTP method + params: Query parameters + headers: Additional headers + target_indicator: The indicator being investigated + max_retries: Maximum number of retry attempts + + Returns: + Response object or None if request failed + """ + for attempt in range(max_retries + 1): + # Apply rate limiting + self.rate_limiter.wait_if_needed() + + start_time = time.time() + response = None + error = None + + try: + self.total_requests += 1 + + # Prepare request + request_headers = self.session.headers.copy() + if headers: + request_headers.update(headers) + + print(f"Making {method} request to: {url} (attempt {attempt + 1})") + + # Make request + if method.upper() == "GET": + response = self.session.get( + url, + params=params, + headers=request_headers, + timeout=self.timeout + ) + elif method.upper() == "POST": + response = self.session.post( + url, + json=params, + headers=request_headers, + timeout=self.timeout + ) + else: + raise ValueError(f"Unsupported HTTP method: {method}") + + print(f"Response status: {response.status_code}") + response.raise_for_status() + self.successful_requests += 1 + + # Success - log and return + duration_ms = (time.time() - start_time) * 1000 + self.logger.log_api_request( + provider=self.name, + url=url, + method=method.upper(), + status_code=response.status_code, + response_size=len(response.content), + duration_ms=duration_ms, + error=None, + target_indicator=target_indicator + ) + return response + + except requests.exceptions.RequestException as e: + error = str(e) + self.failed_requests += 1 + print(f"Request failed (attempt {attempt + 1}): {error}") + + # Check if we should retry + if attempt < max_retries and self._should_retry(e): + backoff_time = (2 ** attempt) * 1 # Exponential backoff: 1s, 2s, 4s + print(f"Retrying in {backoff_time} seconds...") + time.sleep(backoff_time) + continue + else: + break + + except Exception as e: + error = f"Unexpected error: {str(e)}" + self.failed_requests += 1 + print(f"Unexpected error: {error}") + break + + # All attempts failed - log and return None + duration_ms = (time.time() - start_time) * 1000 + self.logger.log_api_request( + provider=self.name, + url=url, + method=method.upper(), + status_code=response.status_code if response else None, + response_size=len(response.content) if response else None, + duration_ms=duration_ms, + error=error, + target_indicator=target_indicator + ) + + return None + + def _should_retry(self, exception: requests.exceptions.RequestException) -> bool: + """ + Determine if a request should be retried based on the exception. + Args: - url: Request URL - method: HTTP method - params: Query parameters - headers: Additional headers - target_indicator: The indicator being investigated - + exception: The request exception that occurred + Returns: - Response object or None if request failed + True if the request should be retried """ - # Apply rate limiting - self.rate_limiter.wait_if_needed() - - start_time = time.time() - response = None - error = None - - try: - self.total_requests += 1 - - # Prepare request - request_headers = self.session.headers.copy() - if headers: - request_headers.update(headers) - - print(f"Making {method} request to: {url}") - - # Make request - if method.upper() == "GET": - response = self.session.get( - url, - params=params, - headers=request_headers, - timeout=self.timeout - ) - elif method.upper() == "POST": - response = self.session.post( - url, - json=params, - headers=request_headers, - timeout=self.timeout - ) - else: - raise ValueError(f"Unsupported HTTP method: {method}") - - print(f"Response status: {response.status_code}") - response.raise_for_status() - self.successful_requests += 1 - - except requests.exceptions.RequestException as e: - error = str(e) - self.failed_requests += 1 - print(f"Request failed: {error}") - - except Exception as e: - error = f"Unexpected error: {str(e)}" - self.failed_requests += 1 - print(f"Unexpected error: {error}") - - # Calculate duration and log request - duration_ms = (time.time() - start_time) * 1000 - - self.logger.log_api_request( - provider=self.name, - url=url, - method=method.upper(), - status_code=response.status_code if response else None, - response_size=len(response.content) if response else None, - duration_ms=duration_ms, - error=error, - target_indicator=target_indicator - ) - - return response if error is None else None + # Retry on connection errors, timeouts, and 5xx server errors + if isinstance(exception, (requests.exceptions.ConnectionError, + requests.exceptions.Timeout)): + return True + + if isinstance(exception, requests.exceptions.HTTPError): + if hasattr(exception, 'response') and exception.response: + # Retry on server errors (5xx) but not client errors (4xx) + return exception.response.status_code >= 500 + + return False def log_relationship_discovery(self, source_node: str, target_node: str, relationship_type: RelationshipType, diff --git a/providers/dns_provider.py b/providers/dns_provider.py index e69de29..d3c52e8 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -0,0 +1,338 @@ +""" +DNS resolution provider for DNSRecon. +Discovers domain relationships through DNS record analysis. +""" + +import socket +import dns.resolver +import dns.reversename +from typing import List, Dict, Any, Tuple, Optional +from .base_provider import BaseProvider +from core.graph_manager import RelationshipType, NodeType + + +class DNSProvider(BaseProvider): + """ + Provider for standard DNS resolution and reverse DNS lookups. + Discovers domain-to-IP and IP-to-domain relationships through DNS records. + """ + + def __init__(self): + """Initialize DNS provider with appropriate rate limiting.""" + super().__init__( + name="dns", + rate_limit=100, # DNS queries can be faster + timeout=10 + ) + + # Configure DNS resolver + self.resolver = dns.resolver.Resolver() + self.resolver.timeout = 5 + self.resolver.lifetime = 10 + + def get_name(self) -> str: + """Return the provider name.""" + return "dns" + + def is_available(self) -> bool: + """DNS is always available - no API key required.""" + return True + + def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """ + Query DNS records for the domain to discover relationships. + + Args: + domain: Domain to investigate + + Returns: + List of relationships discovered from DNS analysis + """ + if not self._is_valid_domain(domain): + return [] + + relationships = [] + + # Query A records + relationships.extend(self._query_a_records(domain)) + + # Query AAAA records (IPv6) + relationships.extend(self._query_aaaa_records(domain)) + + # Query CNAME records + relationships.extend(self._query_cname_records(domain)) + + # Query MX records + relationships.extend(self._query_mx_records(domain)) + + # Query NS records + relationships.extend(self._query_ns_records(domain)) + + return relationships + + def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """ + Query reverse DNS for the IP address. + + Args: + ip: IP address to investigate + + Returns: + List of relationships discovered from reverse DNS + """ + if not self._is_valid_ip(ip): + return [] + + relationships = [] + + try: + # Perform reverse DNS lookup + reverse_name = dns.reversename.from_address(ip) + response = self.resolver.resolve(reverse_name, 'PTR') + + for ptr_record in response: + hostname = str(ptr_record).rstrip('.') + + if self._is_valid_domain(hostname): + raw_data = { + 'query_type': 'PTR', + 'ip_address': ip, + 'hostname': hostname, + 'ttl': response.ttl + } + + relationships.append(( + ip, + hostname, + RelationshipType.A_RECORD, # Reverse relationship + RelationshipType.A_RECORD.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=ip, + target_node=hostname, + relationship_type=RelationshipType.A_RECORD, + confidence_score=RelationshipType.A_RECORD.default_confidence, + raw_data=raw_data, + discovery_method="reverse_dns_lookup" + ) + + except Exception as e: + self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}") + + return relationships + + def _query_a_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query A records for the domain.""" + relationships = [] + + #if not DNS_AVAILABLE: + # return relationships + + try: + response = self.resolver.resolve(domain, 'A') + + for a_record in response: + ip_address = str(a_record) + + raw_data = { + 'query_type': 'A', + 'domain': domain, + 'ip_address': ip_address, + 'ttl': response.ttl + } + + relationships.append(( + domain, + ip_address, + RelationshipType.A_RECORD, + RelationshipType.A_RECORD.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=ip_address, + relationship_type=RelationshipType.A_RECORD, + confidence_score=RelationshipType.A_RECORD.default_confidence, + raw_data=raw_data, + discovery_method="dns_a_record" + ) + + except Exception as e: + self.logger.logger.debug(f"A record query failed for {domain}: {e}") + + return relationships + + def _query_aaaa_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query AAAA records (IPv6) for the domain.""" + relationships = [] + + #if not DNS_AVAILABLE: + # return relationships + + try: + response = self.resolver.resolve(domain, 'AAAA') + + for aaaa_record in response: + ip_address = str(aaaa_record) + + raw_data = { + 'query_type': 'AAAA', + 'domain': domain, + 'ip_address': ip_address, + 'ttl': response.ttl + } + + relationships.append(( + domain, + ip_address, + RelationshipType.A_RECORD, # Using same type for IPv6 + RelationshipType.A_RECORD.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=ip_address, + relationship_type=RelationshipType.A_RECORD, + confidence_score=RelationshipType.A_RECORD.default_confidence, + raw_data=raw_data, + discovery_method="dns_aaaa_record" + ) + + except Exception as e: + self.logger.logger.debug(f"AAAA record query failed for {domain}: {e}") + + return relationships + + def _query_cname_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query CNAME records for the domain.""" + relationships = [] + + #if not DNS_AVAILABLE: + # return relationships + + try: + response = self.resolver.resolve(domain, 'CNAME') + + for cname_record in response: + target_domain = str(cname_record).rstrip('.') + + if self._is_valid_domain(target_domain): + raw_data = { + 'query_type': 'CNAME', + 'source_domain': domain, + 'target_domain': target_domain, + 'ttl': response.ttl + } + + relationships.append(( + domain, + target_domain, + RelationshipType.CNAME_RECORD, + RelationshipType.CNAME_RECORD.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=target_domain, + relationship_type=RelationshipType.CNAME_RECORD, + confidence_score=RelationshipType.CNAME_RECORD.default_confidence, + raw_data=raw_data, + discovery_method="dns_cname_record" + ) + + except Exception as e: + self.logger.logger.debug(f"CNAME record query failed for {domain}: {e}") + + return relationships + + def _query_mx_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query MX records for the domain.""" + relationships = [] + + #if not DNS_AVAILABLE: + # return relationships + + try: + response = self.resolver.resolve(domain, 'MX') + + for mx_record in response: + mx_host = str(mx_record.exchange).rstrip('.') + + if self._is_valid_domain(mx_host): + raw_data = { + 'query_type': 'MX', + 'domain': domain, + 'mx_host': mx_host, + 'priority': mx_record.preference, + 'ttl': response.ttl + } + + relationships.append(( + domain, + mx_host, + RelationshipType.MX_RECORD, + RelationshipType.MX_RECORD.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=mx_host, + relationship_type=RelationshipType.MX_RECORD, + confidence_score=RelationshipType.MX_RECORD.default_confidence, + raw_data=raw_data, + discovery_method="dns_mx_record" + ) + + except Exception as e: + self.logger.logger.debug(f"MX record query failed for {domain}: {e}") + + return relationships + + def _query_ns_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query NS records for the domain.""" + relationships = [] + + #if not DNS_AVAILABLE: + # return relationships + + try: + response = self.resolver.resolve(domain, 'NS') + + for ns_record in response: + ns_host = str(ns_record).rstrip('.') + + if self._is_valid_domain(ns_host): + raw_data = { + 'query_type': 'NS', + 'domain': domain, + 'ns_host': ns_host, + 'ttl': response.ttl + } + + relationships.append(( + domain, + ns_host, + RelationshipType.NS_RECORD, + RelationshipType.NS_RECORD.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=ns_host, + relationship_type=RelationshipType.NS_RECORD, + confidence_score=RelationshipType.NS_RECORD.default_confidence, + raw_data=raw_data, + discovery_method="dns_ns_record" + ) + + except Exception as e: + self.logger.logger.debug(f"NS record query failed for {domain}: {e}") + + return relationships \ No newline at end of file diff --git a/providers/shodan_provider.py b/providers/shodan_provider.py new file mode 100644 index 0000000..3dd2c09 --- /dev/null +++ b/providers/shodan_provider.py @@ -0,0 +1,299 @@ +""" +Shodan provider for DNSRecon. +Discovers IP relationships and infrastructure context through Shodan API. +""" + +import json +from typing import List, Dict, Any, Tuple, Optional +from urllib.parse import quote +from .base_provider import BaseProvider +from core.graph_manager import RelationshipType +from config import config + + +class ShodanProvider(BaseProvider): + """ + Provider for querying Shodan API for IP address and hostname information. + Requires valid API key and respects Shodan's rate limits. + """ + + def __init__(self): + """Initialize Shodan provider with appropriate rate limiting.""" + super().__init__( + name="shodan", + rate_limit=60, # Shodan API has various rate limits depending on plan + timeout=30 + ) + self.base_url = "https://api.shodan.io" + self.api_key = config.get_api_key('shodan') + + def get_name(self) -> str: + """Return the provider name.""" + return "shodan" + + def is_available(self) -> bool: + """ + Check if Shodan provider is available (has valid API key). + """ + return self.api_key is not None and len(self.api_key.strip()) > 0 + + def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """ + Query Shodan for information about a domain. + Uses Shodan's hostname search to find associated IPs. + + Args: + domain: Domain to investigate + + Returns: + List of relationships discovered from Shodan data + """ + if not self._is_valid_domain(domain) or not self.is_available(): + return [] + + relationships = [] + + try: + # Search for hostname in Shodan + search_query = f"hostname:{domain}" + url = f"{self.base_url}/shodan/host/search" + params = { + 'key': self.api_key, + 'query': search_query, + 'minify': True # Get minimal data to reduce bandwidth + } + + response = self.make_request(url, method="GET", params=params, target_indicator=domain) + + if not response or response.status_code != 200: + return [] + + data = response.json() + + if 'matches' not in data: + return [] + + # Process search results + for match in data['matches']: + ip_address = match.get('ip_str') + hostnames = match.get('hostnames', []) + + if ip_address and domain in hostnames: + raw_data = { + 'ip_address': ip_address, + 'hostnames': hostnames, + 'country': match.get('location', {}).get('country_name', ''), + 'city': match.get('location', {}).get('city', ''), + 'isp': match.get('isp', ''), + 'org': match.get('org', ''), + 'ports': match.get('ports', []), + 'last_update': match.get('last_update', '') + } + + relationships.append(( + domain, + ip_address, + RelationshipType.A_RECORD, # Domain resolves to IP + RelationshipType.A_RECORD.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=ip_address, + relationship_type=RelationshipType.A_RECORD, + confidence_score=RelationshipType.A_RECORD.default_confidence, + raw_data=raw_data, + discovery_method="shodan_hostname_search" + ) + + # Also create relationships to other hostnames on the same IP + for hostname in hostnames: + if hostname != domain and self._is_valid_domain(hostname): + hostname_raw_data = { + 'shared_ip': ip_address, + 'all_hostnames': hostnames, + 'discovery_context': 'shared_hosting' + } + + relationships.append(( + domain, + hostname, + RelationshipType.PASSIVE_DNS, # Shared hosting relationship + 0.6, # Lower confidence for shared hosting + hostname_raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=hostname, + relationship_type=RelationshipType.PASSIVE_DNS, + confidence_score=0.6, + raw_data=hostname_raw_data, + discovery_method="shodan_shared_hosting" + ) + + except json.JSONDecodeError as e: + self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}") + except Exception as e: + self.logger.logger.error(f"Error querying Shodan for domain {domain}: {e}") + + return relationships + + def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """ + Query Shodan for information about an IP address. + + Args: + ip: IP address to investigate + + Returns: + List of relationships discovered from Shodan IP data + """ + if not self._is_valid_ip(ip) or not self.is_available(): + return [] + + relationships = [] + + try: + # Query Shodan host information + url = f"{self.base_url}/shodan/host/{ip}" + params = {'key': self.api_key} + + response = self.make_request(url, method="GET", params=params, target_indicator=ip) + + if not response or response.status_code != 200: + return [] + + data = response.json() + + # Extract hostname relationships + hostnames = data.get('hostnames', []) + for hostname in hostnames: + if self._is_valid_domain(hostname): + raw_data = { + 'ip_address': ip, + 'hostname': hostname, + 'country': data.get('country_name', ''), + 'city': data.get('city', ''), + 'isp': data.get('isp', ''), + 'org': data.get('org', ''), + 'asn': data.get('asn', ''), + 'ports': data.get('ports', []), + 'last_update': data.get('last_update', ''), + 'os': data.get('os', '') + } + + relationships.append(( + ip, + hostname, + RelationshipType.A_RECORD, # IP resolves to hostname + RelationshipType.A_RECORD.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=ip, + target_node=hostname, + relationship_type=RelationshipType.A_RECORD, + confidence_score=RelationshipType.A_RECORD.default_confidence, + raw_data=raw_data, + discovery_method="shodan_host_lookup" + ) + + # Extract ASN relationship if available + asn = data.get('asn') + if asn: + asn_name = f"AS{asn}" + + asn_raw_data = { + 'ip_address': ip, + 'asn': asn, + 'isp': data.get('isp', ''), + 'org': data.get('org', '') + } + + relationships.append(( + ip, + asn_name, + RelationshipType.ASN_MEMBERSHIP, + RelationshipType.ASN_MEMBERSHIP.default_confidence, + asn_raw_data + )) + + self.log_relationship_discovery( + source_node=ip, + target_node=asn_name, + relationship_type=RelationshipType.ASN_MEMBERSHIP, + confidence_score=RelationshipType.ASN_MEMBERSHIP.default_confidence, + raw_data=asn_raw_data, + discovery_method="shodan_asn_lookup" + ) + + except json.JSONDecodeError as e: + self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}") + except Exception as e: + self.logger.logger.error(f"Error querying Shodan for IP {ip}: {e}") + + return relationships + + def search_by_organization(self, org_name: str) -> List[Dict[str, Any]]: + """ + Search Shodan for hosts belonging to a specific organization. + + Args: + org_name: Organization name to search for + + Returns: + List of host information dictionaries + """ + if not self.is_available(): + return [] + + try: + search_query = f"org:\"{org_name}\"" + url = f"{self.base_url}/shodan/host/search" + params = { + 'key': self.api_key, + 'query': search_query, + 'minify': True + } + + response = self.make_request(url, method="GET", params=params, target_indicator=org_name) + + if response and response.status_code == 200: + data = response.json() + return data.get('matches', []) + + except Exception as e: + self.logger.logger.error(f"Error searching Shodan by organization {org_name}: {e}") + + return [] + + def get_host_services(self, ip: str) -> List[Dict[str, Any]]: + """ + Get service information for a specific IP address. + + Args: + ip: IP address to query + + Returns: + List of service information dictionaries + """ + if not self._is_valid_ip(ip) or not self.is_available(): + return [] + + try: + url = f"{self.base_url}/shodan/host/{ip}" + params = {'key': self.api_key} + + response = self.make_request(url, method="GET", params=params, target_indicator=ip) + + if response and response.status_code == 200: + data = response.json() + return data.get('data', []) # Service banners + + except Exception as e: + self.logger.logger.error(f"Error getting Shodan services for IP {ip}: {e}") + + return [] \ No newline at end of file diff --git a/providers/virustotal_provider.py b/providers/virustotal_provider.py new file mode 100644 index 0000000..9949810 --- /dev/null +++ b/providers/virustotal_provider.py @@ -0,0 +1,334 @@ +""" +VirusTotal provider for DNSRecon. +Discovers domain relationships through passive DNS and URL analysis. +""" + +import json +from typing import List, Dict, Any, Tuple, Optional +from .base_provider import BaseProvider +from core.graph_manager import RelationshipType +from config import config + + +class VirusTotalProvider(BaseProvider): + """ + Provider for querying VirusTotal API for passive DNS and domain reputation data. + Requires valid API key and strictly respects free tier rate limits. + """ + + def __init__(self): + """Initialize VirusTotal provider with strict rate limiting for free tier.""" + super().__init__( + name="virustotal", + rate_limit=4, # Free tier: 4 requests per minute + timeout=30 + ) + self.base_url = "https://www.virustotal.com/vtapi/v2" + self.api_key = config.get_api_key('virustotal') + + def get_name(self) -> str: + """Return the provider name.""" + return "virustotal" + + def is_available(self) -> bool: + """ + Check if VirusTotal provider is available (has valid API key). + """ + return self.api_key is not None and len(self.api_key.strip()) > 0 + + def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """ + Query VirusTotal for domain information including passive DNS. + + Args: + domain: Domain to investigate + + Returns: + List of relationships discovered from VirusTotal data + """ + if not self._is_valid_domain(domain) or not self.is_available(): + return [] + + relationships = [] + + # Query domain report + domain_relationships = self._query_domain_report(domain) + relationships.extend(domain_relationships) + + # Query passive DNS for the domain + passive_dns_relationships = self._query_passive_dns_domain(domain) + relationships.extend(passive_dns_relationships) + + return relationships + + def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """ + Query VirusTotal for IP address information including passive DNS. + + Args: + ip: IP address to investigate + + Returns: + List of relationships discovered from VirusTotal IP data + """ + if not self._is_valid_ip(ip) or not self.is_available(): + return [] + + relationships = [] + + # Query IP report + ip_relationships = self._query_ip_report(ip) + relationships.extend(ip_relationships) + + # Query passive DNS for the IP + passive_dns_relationships = self._query_passive_dns_ip(ip) + relationships.extend(passive_dns_relationships) + + return relationships + + def _query_domain_report(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query VirusTotal domain report.""" + relationships = [] + + try: + url = f"{self.base_url}/domain/report" + params = { + 'apikey': self.api_key, + 'domain': domain, + 'allinfo': 1 # Get comprehensive information + } + + response = self.make_request(url, method="GET", params=params, target_indicator=domain) + + if not response or response.status_code != 200: + return [] + + data = response.json() + + if data.get('response_code') != 1: + return [] + + # Extract resolved IPs + resolutions = data.get('resolutions', []) + for resolution in resolutions: + ip_address = resolution.get('ip_address') + last_resolved = resolution.get('last_resolved') + + if ip_address and self._is_valid_ip(ip_address): + raw_data = { + 'domain': domain, + 'ip_address': ip_address, + 'last_resolved': last_resolved, + 'source': 'virustotal_domain_report' + } + + relationships.append(( + domain, + ip_address, + RelationshipType.PASSIVE_DNS, + RelationshipType.PASSIVE_DNS.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=ip_address, + relationship_type=RelationshipType.PASSIVE_DNS, + confidence_score=RelationshipType.PASSIVE_DNS.default_confidence, + raw_data=raw_data, + discovery_method="virustotal_domain_resolution" + ) + + # Extract subdomains + subdomains = data.get('subdomains', []) + for subdomain in subdomains: + if subdomain != domain and self._is_valid_domain(subdomain): + raw_data = { + 'parent_domain': domain, + 'subdomain': subdomain, + 'source': 'virustotal_subdomain_discovery' + } + + relationships.append(( + domain, + subdomain, + RelationshipType.PASSIVE_DNS, + 0.7, # Medium-high confidence for subdomains + raw_data + )) + + self.log_relationship_discovery( + source_node=domain, + target_node=subdomain, + relationship_type=RelationshipType.PASSIVE_DNS, + confidence_score=0.7, + raw_data=raw_data, + discovery_method="virustotal_subdomain_discovery" + ) + + except json.JSONDecodeError as e: + self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}") + except Exception as e: + self.logger.logger.error(f"Error querying VirusTotal domain report for {domain}: {e}") + + return relationships + + def _query_ip_report(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query VirusTotal IP report.""" + relationships = [] + + try: + url = f"{self.base_url}/ip-address/report" + params = { + 'apikey': self.api_key, + 'ip': ip + } + + response = self.make_request(url, method="GET", params=params, target_indicator=ip) + + if not response or response.status_code != 200: + return [] + + data = response.json() + + if data.get('response_code') != 1: + return [] + + # Extract resolved domains + resolutions = data.get('resolutions', []) + for resolution in resolutions: + hostname = resolution.get('hostname') + last_resolved = resolution.get('last_resolved') + + if hostname and self._is_valid_domain(hostname): + raw_data = { + 'ip_address': ip, + 'hostname': hostname, + 'last_resolved': last_resolved, + 'source': 'virustotal_ip_report' + } + + relationships.append(( + ip, + hostname, + RelationshipType.PASSIVE_DNS, + RelationshipType.PASSIVE_DNS.default_confidence, + raw_data + )) + + self.log_relationship_discovery( + source_node=ip, + target_node=hostname, + relationship_type=RelationshipType.PASSIVE_DNS, + confidence_score=RelationshipType.PASSIVE_DNS.default_confidence, + raw_data=raw_data, + discovery_method="virustotal_ip_resolution" + ) + + except json.JSONDecodeError as e: + self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}") + except Exception as e: + self.logger.logger.error(f"Error querying VirusTotal IP report for {ip}: {e}") + + return relationships + + def _query_passive_dns_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query VirusTotal passive DNS for domain.""" + # Note: VirusTotal's passive DNS API might require a premium subscription + # This is a placeholder for the endpoint structure + return [] + + def _query_passive_dns_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: + """Query VirusTotal passive DNS for IP.""" + # Note: VirusTotal's passive DNS API might require a premium subscription + # This is a placeholder for the endpoint structure + return [] + + def get_domain_reputation(self, domain: str) -> Dict[str, Any]: + """ + Get domain reputation information from VirusTotal. + + Args: + domain: Domain to check reputation for + + Returns: + Dictionary containing reputation data + """ + if not self._is_valid_domain(domain) or not self.is_available(): + return {} + + try: + url = f"{self.base_url}/domain/report" + params = { + 'apikey': self.api_key, + 'domain': domain + } + + response = self.make_request(url, method="GET", params=params, target_indicator=domain) + + if response and response.status_code == 200: + data = response.json() + + if data.get('response_code') == 1: + return { + 'positives': data.get('positives', 0), + 'total': data.get('total', 0), + 'scan_date': data.get('scan_date', ''), + 'permalink': data.get('permalink', ''), + 'reputation_score': self._calculate_reputation_score(data) + } + + except Exception as e: + self.logger.logger.error(f"Error getting VirusTotal reputation for domain {domain}: {e}") + + return {} + + def get_ip_reputation(self, ip: str) -> Dict[str, Any]: + """ + Get IP reputation information from VirusTotal. + + Args: + ip: IP address to check reputation for + + Returns: + Dictionary containing reputation data + """ + if not self._is_valid_ip(ip) or not self.is_available(): + return {} + + try: + url = f"{self.base_url}/ip-address/report" + params = { + 'apikey': self.api_key, + 'ip': ip + } + + response = self.make_request(url, method="GET", params=params, target_indicator=ip) + + if response and response.status_code == 200: + data = response.json() + + if data.get('response_code') == 1: + return { + 'positives': data.get('positives', 0), + 'total': data.get('total', 0), + 'scan_date': data.get('scan_date', ''), + 'permalink': data.get('permalink', ''), + 'reputation_score': self._calculate_reputation_score(data) + } + + except Exception as e: + self.logger.logger.error(f"Error getting VirusTotal reputation for IP {ip}: {e}") + + return {} + + def _calculate_reputation_score(self, data: Dict[str, Any]) -> float: + """Calculate a normalized reputation score (0.0 to 1.0).""" + positives = data.get('positives', 0) + total = data.get('total', 1) # Avoid division by zero + + if total == 0: + return 1.0 # No data means neutral + + # Score is inverse of detection ratio (lower detection = higher reputation) + return max(0.0, 1.0 - (positives / total)) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f6e6d13..bab2a1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ networkx>=3.1 requests>=2.31.0 python-dateutil>=2.8.2 Werkzeug>=2.3.7 -urllib3>=2.0.0 \ No newline at end of file +urllib3>=2.0.0 +dnspython>=2.4.2 \ No newline at end of file diff --git a/static/css/main.css b/static/css/main.css index 21766ec..eff10da 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -64,6 +64,18 @@ body { gap: 0.5rem; } +.status-indicator.scanning { + animation: pulse 1.5s infinite; +} + +.status-indicator.completed { + background-color: #00ff41; +} + +.status-indicator.error { + background-color: #ff6b6b; +} + .status-dot { width: 8px; height: 8px; @@ -266,6 +278,7 @@ input[type="text"]:focus, select:focus { background-color: #1a1a1a; border: 1px solid #444; overflow: hidden; + position: relative; } .progress-fill { @@ -274,6 +287,23 @@ input[type="text"]:focus, select:focus { width: 0%; transition: width 0.3s ease; box-shadow: 0 0 5px rgba(0, 255, 65, 0.5); + position: relative; +} + +.progress-fill::after { + content: ''; + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: linear-gradient(90deg, transparent, rgba(255,255,255,0.2), transparent); + animation: shimmer 2s infinite; +} + +@keyframes shimmer { + 0% { transform: translateX(-100%); } + 100% { transform: translateX(100%); } } /* Visualization Panel */ @@ -292,6 +322,37 @@ input[type="text"]:focus, select:focus { position: relative; background-color: #1a1a1a; border-top: 1px solid #444; + transition: height 0.3s ease; +} + +.graph-container.expanded { + height: 700px; +} + +.graph-controls { + position: absolute; + top: 10px; + right: 10px; + z-index: 10; + display: flex; + gap: 0.5rem; +} + +.graph-control-btn { + background: rgba(42, 42, 42, 0.9); + border: 1px solid #555; + color: #c7c7c7; + padding: 0.5rem; + font-family: 'Roboto Mono', monospace; + font-size: 0.8rem; + cursor: pointer; + transition: all 0.3s ease; +} + +.graph-control-btn:hover { + border-color: #00ff41; + color: #00ff41; + background: rgba(42, 42, 42, 1); } .graph-placeholder { @@ -333,6 +394,20 @@ input[type="text"]:focus, select:focus { border-top: 1px solid #444; } +.legend-section { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.legend-title { + font-size: 0.7rem; + color: #00ff41; + text-transform: uppercase; + font-weight: 500; + margin-bottom: 0.25rem; +} + .legend-item { display: flex; align-items: center; @@ -344,6 +419,7 @@ input[type="text"]:focus, select:focus { width: 12px; height: 12px; border-radius: 50%; + border: 1px solid #444; } .legend-edge { @@ -353,10 +429,16 @@ input[type="text"]:focus, select:focus { .legend-edge.high-confidence { background-color: #00ff41; + box-shadow: 0 0 3px rgba(0, 255, 65, 0.5); } .legend-edge.medium-confidence { background-color: #ff9900; + box-shadow: 0 0 3px rgba(255, 153, 0, 0.5); +} + +.legend-edge.low-confidence { + background-color: #666666; } /* Provider Panel */ @@ -375,9 +457,11 @@ input[type="text"]:focus, select:focus { background-color: #1a1a1a; border: 1px solid #444; padding: 1rem; - display: flex; - justify-content: space-between; - align-items: center; + transition: border-color 0.3s ease; +} + +.provider-item:hover { + border-color: #555; } .provider-name { @@ -389,6 +473,7 @@ input[type="text"]:focus, select:focus { font-size: 0.8rem; padding: 0.25rem 0.5rem; border-radius: 3px; + font-weight: 500; } .provider-status.enabled { @@ -401,12 +486,78 @@ input[type="text"]:focus, select:focus { color: #e0e0e0; } +.provider-status.api-key-required { + background-color: #5c4c2c; + color: #e0e0e0; +} + .provider-stats { font-size: 0.8rem; color: #999; + display: grid; + grid-template-columns: 1fr 1fr; + gap: 0.5rem; margin-top: 0.5rem; } +.provider-stat { + display: flex; + justify-content: space-between; +} + +.provider-stat-label { + color: #666; +} + +.provider-stat-value { + color: #00ff41; + font-weight: 500; +} + +.provider-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.5rem; +} + +.node-info-popup { + position: fixed; + background: rgba(42, 42, 42, 0.95); + border: 1px solid #555; + padding: 1rem; + border-radius: 4px; + color: #c7c7c7; + font-family: 'Roboto Mono', monospace; + font-size: 0.8rem; + max-width: 300px; + z-index: 1001; + box-shadow: 0 4px 6px rgba(0,0,0,0.3); +} + +.node-info-title { + color: #00ff41; + font-weight: bold; + margin-bottom: 0.5rem; + border-bottom: 1px solid #444; + padding-bottom: 0.25rem; +} + +.node-info-detail { + margin-bottom: 0.25rem; + display: flex; + justify-content: space-between; +} + +.node-info-label { + color: #999; +} + +.node-info-value { + color: #c7c7c7; + font-weight: 500; +} + /* Footer */ .footer { background-color: #0a0a0a; @@ -437,6 +588,7 @@ input[type="text"]:focus, select:focus { width: 100%; height: 100%; background-color: rgba(0, 0, 0, 0.8); + animation: fadeIn 0.3s ease-out; } .modal-content { @@ -447,6 +599,18 @@ input[type="text"]:focus, select:focus { max-width: 600px; max-height: 80vh; overflow-y: auto; + animation: slideInDown 0.3s ease-out; +} + +@keyframes slideInDown { + from { + opacity: 0; + transform: translateY(-50px); + } + to { + opacity: 1; + transform: translateY(0); + } } .modal-header { @@ -480,6 +644,12 @@ input[type="text"]:focus, select:focus { padding: 1.5rem; } +.modal-description { + color: #999; + margin-bottom: 1.5rem; + line-height: 1.6; +} + .detail-row { display: flex; justify-content: space-between; @@ -495,6 +665,7 @@ input[type="text"]:focus, select:focus { .detail-value { color: #c7c7c7; + word-break: break-word; } /* Responsive Design */ @@ -552,6 +723,40 @@ input[type="text"]:focus, select:focus { pointer-events: none; } +.loading-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(26, 26, 26, 0.8); + display: flex; + align-items: center; + justify-content: center; + z-index: 100; +} + +.loading-spinner { + width: 40px; + height: 40px; + border: 3px solid #444; + border-top: 3px solid #00ff41; + border-radius: 50%; + animation: spin 1s linear infinite; +} + +@keyframes spin { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } +} + +.loading-text { + margin-top: 1rem; + color: #999; + font-family: 'Roboto Mono', monospace; + font-size: 0.9rem; +} + .error { color: #ff6b6b !important; border-color: #ff6b6b !important; @@ -598,4 +803,101 @@ input[type="text"]:focus, select:focus { .amber { color: #ff9900; +} + + +.apikey-section { + margin-bottom: 1.5rem; +} + +.apikey-section label { + display: block; + margin-bottom: 0.5rem; + color: #c7c7c7; + font-size: 0.9rem; + font-weight: 500; +} + +.apikey-section input[type="password"] { + width: 100%; + padding: 0.75rem; + background-color: #1a1a1a; + border: 1px solid #555; + color: #c7c7c7; + font-family: 'Roboto Mono', monospace; + font-size: 0.9rem; + transition: border-color 0.3s ease, box-shadow 0.3s ease; +} + +.apikey-section input[type="password"]:focus { + outline: none; + border-color: #00ff41; + box-shadow: 0 0 5px rgba(0, 255, 65, 0.5); +} + +.apikey-help { + font-size: 0.8rem; + color: #666; + margin-top: 0.25rem; + font-style: italic; +} + + +.message-container { + position: fixed; + top: 20px; + right: 20px; + z-index: 1002; + max-width: 400px; +} + +.message-toast { + margin-bottom: 10px; + border-radius: 4px; + font-family: 'Roboto Mono', monospace; + font-size: 0.9rem; + box-shadow: 0 4px 6px rgba(0,0,0,0.3); + animation: slideInRight 0.3s ease-out; +} + +.message-toast.success { + background: #2c5c34; + border-left: 4px solid #00ff41; +} + +.message-toast.error { + background: #5c2c2c; + border-left: 4px solid #ff6b6b; +} + +.message-toast.warning { + background: #5c4c2c; + border-left: 4px solid #ff9900; +} + +.message-toast.info { + background: #2c3e5c; + border-left: 4px solid #00aaff; +} + +@keyframes slideInRight { + from { + transform: translateX(100%); + opacity: 0; + } + to { + transform: translateX(0); + opacity: 1; + } +} + +@keyframes slideOutRight { + from { + transform: translateX(0); + opacity: 1; + } + to { + transform: translateX(100%); + opacity: 0; + } } \ No newline at end of file diff --git a/static/js/graph.js b/static/js/graph.js index 016880e..90c9703 100644 --- a/static/js/graph.js +++ b/static/js/graph.js @@ -1,6 +1,6 @@ /** * Graph visualization module for DNSRecon - * Handles network graph rendering using vis.js + * Handles network graph rendering using vis.js with enhanced Phase 2 features */ class GraphManager { @@ -10,40 +10,57 @@ class GraphManager { this.nodes = new vis.DataSet(); this.edges = new vis.DataSet(); this.isInitialized = false; - - // Graph options for cybersecurity theme + this.currentLayout = 'physics'; + this.nodeInfoPopup = null; + + // Enhanced graph options for Phase 2 this.options = { nodes: { shape: 'dot', - size: 12, + size: 15, font: { - size: 11, + size: 12, color: '#c7c7c7', face: 'Roboto Mono, monospace', - background: 'rgba(26, 26, 26, 0.8)', - strokeWidth: 1, + background: 'rgba(26, 26, 26, 0.9)', + strokeWidth: 2, strokeColor: '#000000' }, borderWidth: 2, borderColor: '#444', shadow: { enabled: true, - color: 'rgba(0, 0, 0, 0.3)', - size: 3, - x: 1, - y: 1 + color: 'rgba(0, 0, 0, 0.5)', + size: 5, + x: 2, + y: 2 }, scaling: { - min: 8, - max: 20 + min: 10, + max: 30, + label: { + enabled: true, + min: 8, + max: 16 + } + }, + chosen: { + node: (values, id, selected, hovering) => { + values.borderColor = '#00ff41'; + values.borderWidth = 3; + values.shadow = true; + values.shadowColor = 'rgba(0, 255, 65, 0.6)'; + values.shadowSize = 10; + } } }, edges: { width: 2, color: { - color: '#444', + color: '#555', highlight: '#00ff41', - hover: '#ff9900' + hover: '#ff9900', + inherit: false }, font: { size: 10, @@ -56,131 +73,218 @@ class GraphManager { arrows: { to: { enabled: true, - scaleFactor: 0.8, + scaleFactor: 1, type: 'arrow' } }, smooth: { enabled: true, type: 'dynamic', - roundness: 0.5 + roundness: 0.6 }, shadow: { enabled: true, - color: 'rgba(0, 0, 0, 0.2)', - size: 2, + color: 'rgba(0, 0, 0, 0.3)', + size: 3, x: 1, y: 1 + }, + chosen: { + edge: (values, id, selected, hovering) => { + values.color = '#00ff41'; + values.width = 4; + values.shadow = true; + values.shadowColor = 'rgba(0, 255, 65, 0.4)'; + } } }, physics: { enabled: true, stabilization: { enabled: true, - iterations: 100, - updateInterval: 25 + iterations: 150, + updateInterval: 50 }, barnesHut: { - gravitationalConstant: -2000, - centralGravity: 0.3, - springLength: 95, - springConstant: 0.04, - damping: 0.09, - avoidOverlap: 0.1 + gravitationalConstant: -3000, + centralGravity: 0.4, + springLength: 120, + springConstant: 0.05, + damping: 0.1, + avoidOverlap: 0.2 }, - maxVelocity: 50, + maxVelocity: 30, minVelocity: 0.1, solver: 'barnesHut', - timestep: 0.35, + timestep: 0.4, adaptiveTimestep: true }, interaction: { hover: true, hoverConnectedEdges: true, selectConnectedEdges: true, - tooltipDelay: 200, + tooltipDelay: 300, hideEdgesOnDrag: false, - hideNodesOnDrag: false + hideNodesOnDrag: false, + zoomView: true, + dragView: true, + multiselect: true }, layout: { - improvedLayout: true + improvedLayout: true, + randomSeed: 2 } }; - - this.setupEventHandlers(); + + this.createNodeInfoPopup(); } - + /** - * Initialize the network graph + * Create floating node info popup + */ + createNodeInfoPopup() { + this.nodeInfoPopup = document.createElement('div'); + this.nodeInfoPopup.className = 'node-info-popup'; + this.nodeInfoPopup.style.display = 'none'; + document.body.appendChild(this.nodeInfoPopup); + } + + /** + * Initialize the network graph with enhanced features */ initialize() { if (this.isInitialized) { return; } - + try { const data = { nodes: this.nodes, edges: this.edges }; - + this.network = new vis.Network(this.container, data, this.options); this.setupNetworkEvents(); this.isInitialized = true; - + // Hide placeholder const placeholder = this.container.querySelector('.graph-placeholder'); if (placeholder) { placeholder.style.display = 'none'; } - - console.log('Graph initialized successfully'); + + // Add graph controls + this.addGraphControls(); + + console.log('Enhanced graph initialized successfully'); } catch (error) { console.error('Failed to initialize graph:', error); this.showError('Failed to initialize visualization'); } } - + /** - * Setup network event handlers + * Add interactive graph controls + */ + addGraphControls() { + const controlsContainer = document.createElement('div'); + controlsContainer.className = 'graph-controls'; + controlsContainer.innerHTML = ` + + + + + `; + + this.container.appendChild(controlsContainer); + + // Add control event listeners + document.getElementById('graph-fit').addEventListener('click', () => this.fitView()); + document.getElementById('graph-reset').addEventListener('click', () => this.resetView()); + document.getElementById('graph-physics').addEventListener('click', () => this.togglePhysics()); + document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering()); + } + + /** + * Setup enhanced network event handlers */ setupNetworkEvents() { if (!this.network) return; - - // Node click event + + // Node click event with enhanced details this.network.on('click', (params) => { if (params.nodes.length > 0) { const nodeId = params.nodes[0]; this.showNodeDetails(nodeId); + this.highlightNodeConnections(nodeId); + } else { + this.clearHighlights(); } }); - - // Hover events for tooltips + + // Enhanced hover events this.network.on('hoverNode', (params) => { const nodeId = params.node; const node = this.nodes.get(nodeId); if (node) { - this.showTooltip(params.pointer.DOM, node); + this.showNodeInfoPopup(params.pointer.DOM, node); + this.highlightConnectedNodes(nodeId, true); } }); - - this.network.on('blurNode', () => { - this.hideTooltip(); + + this.network.on('blurNode', (params) => { + this.hideNodeInfoPopup(); + this.clearHoverHighlights(); }); - - // Stabilization events + + // Edge hover events + this.network.on('hoverEdge', (params) => { + const edgeId = params.edge; + const edge = this.edges.get(edgeId); + if (edge) { + this.showEdgeInfo(params.pointer.DOM, edge); + } + }); + + this.network.on('blurEdge', () => { + this.hideNodeInfoPopup(); + }); + + // Double-click to focus on node + this.network.on('doubleClick', (params) => { + if (params.nodes.length > 0) { + const nodeId = params.nodes[0]; + this.focusOnNode(nodeId); + } + }); + + // Context menu (right-click) + this.network.on('oncontext', (params) => { + params.event.preventDefault(); + if (params.nodes.length > 0) { + this.showNodeContextMenu(params.pointer.DOM, params.nodes[0]); + } + }); + + // Stabilization events with progress this.network.on('stabilizationProgress', (params) => { const progress = params.iterations / params.total; this.updateStabilizationProgress(progress); }); - + this.network.on('stabilizationIterationsDone', () => { this.onStabilizationComplete(); }); + + // Selection events + this.network.on('select', (params) => { + console.log('Selected nodes:', params.nodes); + console.log('Selected edges:', params.edges); + }); } - + /** - * Update graph with new data + * Update graph with new data and enhanced processing * @param {Object} graphData - Graph data from backend */ updateGraph(graphData) { @@ -188,37 +292,48 @@ class GraphManager { console.warn('Invalid graph data received'); return; } - + try { // Initialize if not already done if (!this.isInitialized) { this.initialize(); } - - // Process nodes + + // Process nodes with enhanced attributes const processedNodes = graphData.nodes.map(node => this.processNode(node)); const processedEdges = graphData.edges.map(edge => this.processEdge(edge)); - - // Update datasets - this.nodes.clear(); - this.edges.clear(); - this.nodes.add(processedNodes); - this.edges.add(processedEdges); - - // Fit the view if this is the first update or graph is small - if (processedNodes.length <= 10) { - setTimeout(() => this.fitView(), 500); + + // Update datasets with animation + const existingNodeIds = this.nodes.getIds(); + const existingEdgeIds = this.edges.getIds(); + + // Add new nodes with fade-in animation + const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id)); + const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id)); + + // Update existing data + this.nodes.update(processedNodes); + this.edges.update(processedEdges); + + // Highlight new additions briefly + if (newNodes.length > 0 || newEdges.length > 0) { + setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100); } - - console.log(`Graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges`); + + // Auto-fit view for small graphs or first update + if (processedNodes.length <= 10 || existingNodeIds.length === 0) { + setTimeout(() => this.fitView(), 800); + } + + console.log(`Enhanced graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`); } catch (error) { - console.error('Failed to update graph:', error); + console.error('Failed to update enhanced graph:', error); this.showError('Failed to update visualization'); } } - + /** - * Process node data for visualization + * Process node data with enhanced styling and metadata * @param {Object} node - Raw node data * @returns {Object} Processed node data */ @@ -230,25 +345,32 @@ class GraphManager { color: this.getNodeColor(node.type), size: this.getNodeSize(node.type), borderColor: this.getNodeBorderColor(node.type), - metadata: node.metadata || {} + shape: this.getNodeShape(node.type), + metadata: node.metadata || {}, + type: node.type }; - - // Add type-specific styling - if (node.type === 'domain') { - processedNode.shape = 'dot'; - } else if (node.type === 'ip') { - processedNode.shape = 'square'; - } else if (node.type === 'certificate') { - processedNode.shape = 'diamond'; - } else if (node.type === 'asn') { - processedNode.shape = 'triangle'; + + // Add confidence-based styling + if (node.confidence) { + processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5)); } - + + // Add special styling for important nodes + if (this.isImportantNode(node)) { + processedNode.shadow = { + enabled: true, + color: 'rgba(0, 255, 65, 0.6)', + size: 10, + x: 2, + y: 2 + }; + } + return processedNode; } - + /** - * Process edge data for visualization + * Process edge data with enhanced styling and metadata * @param {Object} edge - Raw edge data * @returns {Object} Processed edge data */ @@ -262,12 +384,29 @@ class GraphManager { title: this.createEdgeTooltip(edge), width: this.getEdgeWidth(confidence), color: this.getEdgeColor(confidence), - dashes: confidence < 0.6 ? [5, 5] : false + dashes: confidence < 0.6 ? [5, 5] : false, + metadata: { + relationship_type: edge.label, + confidence_score: confidence, + source_provider: edge.source_provider, + discovery_timestamp: edge.discovery_timestamp + } }; - + + // Add animation for high-confidence edges + if (confidence >= 0.8) { + processedEdge.shadow = { + enabled: true, + color: 'rgba(0, 255, 65, 0.3)', + size: 5, + x: 1, + y: 1 + }; + } + return processedEdge; } - + /** * Format node label for display * @param {string} nodeId - Node identifier @@ -281,7 +420,7 @@ class GraphManager { } return nodeId; } - + /** * Format edge label for display * @param {string} relationshipType - Type of relationship @@ -290,7 +429,7 @@ class GraphManager { */ formatEdgeLabel(relationshipType, confidence) { if (!relationshipType) return ''; - + const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○'; return `${relationshipType} ${confidenceText}`; } @@ -309,7 +448,7 @@ class GraphManager { }; return colors[nodeType] || '#ffffff'; } - + /** * Get node border color based on type * @param {string} nodeType - Node type @@ -324,7 +463,7 @@ class GraphManager { }; return borderColors[nodeType] || '#666666'; } - + /** * Get node size based on type * @param {string} nodeType - Node type @@ -340,6 +479,21 @@ class GraphManager { return sizes[nodeType] || 12; } + /** + * Get enhanced node shape based on type + * @param {string} nodeType - Node type + * @returns {string} Shape name + */ + getNodeShape(nodeType) { + const shapes = { + 'domain': 'dot', + 'ip': 'square', + 'certificate': 'diamond', + 'asn': 'triangle' + }; + return shapes[nodeType] || 'dot'; + } + /** * Get edge color based on confidence * @param {number} confidence - Confidence score @@ -354,7 +508,7 @@ class GraphManager { return '#666666'; // Low confidence - gray } } - + /** * Get edge width based on confidence * @param {number} confidence - Confidence score @@ -411,7 +565,20 @@ class GraphManager { tooltip += ``; return tooltip; } - + + /** + * Determine if node is important based on connections or metadata + * @param {Object} node - Node data + * @returns {boolean} True if node is important + */ + isImportantNode(node) { + // Mark nodes as important based on criteria + if (node.type === 'domain' && node.id.includes('www.')) return false; + if (node.metadata && node.metadata.connection_count > 3) return true; + if (node.type === 'asn') return true; + return false; + } + /** * Show node details in modal * @param {string} nodeId - Node identifier @@ -419,7 +586,7 @@ class GraphManager { showNodeDetails(nodeId) { const node = this.nodes.get(nodeId); if (!node) return; - + // Trigger custom event for main application to handle const event = new CustomEvent('nodeSelected', { detail: { nodeId, node } @@ -428,22 +595,249 @@ class GraphManager { } /** - * Show tooltip + * Show enhanced node info popup * @param {Object} position - Mouse position * @param {Object} node - Node data */ - showTooltip(position, node) { - // Tooltip is handled by vis.js automatically - // This method is for custom tooltip implementation if needed + showNodeInfoPopup(position, node) { + if (!this.nodeInfoPopup) return; + + const html = ` +
${node.id}
+
+ Type: + ${node.type || 'Unknown'} +
+ ${node.metadata && Object.keys(node.metadata).length > 0 ? + '
Details:Click for more
' : + ''} + `; + + this.nodeInfoPopup.innerHTML = html; + this.nodeInfoPopup.style.display = 'block'; + this.nodeInfoPopup.style.left = position.x + 15 + 'px'; + this.nodeInfoPopup.style.top = position.y - 10 + 'px'; + + // Ensure popup stays in viewport + const rect = this.nodeInfoPopup.getBoundingClientRect(); + if (rect.right > window.innerWidth) { + this.nodeInfoPopup.style.left = position.x - rect.width - 15 + 'px'; + } + if (rect.bottom > window.innerHeight) { + this.nodeInfoPopup.style.top = position.y - rect.height + 10 + 'px'; + } + } + + /** + * Show edge information tooltip + * @param {Object} position - Mouse position + * @param {Object} edge - Edge data + */ + showEdgeInfo(position, edge) { + if (!this.nodeInfoPopup) return; + + const confidence = edge.metadata ? edge.metadata.confidence_score : 0; + const provider = edge.metadata ? edge.metadata.source_provider : 'Unknown'; + + const html = ` +
${edge.metadata ? edge.metadata.relationship_type : 'Relationship'}
+
+ Confidence: + ${(confidence * 100).toFixed(1)}% +
+
+ Provider: + ${provider} +
+ `; + + this.nodeInfoPopup.innerHTML = html; + this.nodeInfoPopup.style.display = 'block'; + this.nodeInfoPopup.style.left = position.x + 15 + 'px'; + this.nodeInfoPopup.style.top = position.y - 10 + 'px'; } /** - * Hide tooltip + * Hide node info popup */ - hideTooltip() { - // Tooltip hiding is handled by vis.js automatically + hideNodeInfoPopup() { + if (this.nodeInfoPopup) { + this.nodeInfoPopup.style.display = 'none'; + } } + /** + * Highlight node connections + * @param {string} nodeId - Node to highlight + */ + highlightNodeConnections(nodeId) { + const connectedNodes = this.network.getConnectedNodes(nodeId); + const connectedEdges = this.network.getConnectedEdges(nodeId); + + // Update node colors + const nodeUpdates = connectedNodes.map(id => ({ + id: id, + borderColor: '#ff9900', + borderWidth: 3 + })); + + nodeUpdates.push({ + id: nodeId, + borderColor: '#00ff41', + borderWidth: 4 + }); + + // Update edge colors + const edgeUpdates = connectedEdges.map(id => ({ + id: id, + color: { color: '#ff9900' }, + width: 3 + })); + + this.nodes.update(nodeUpdates); + this.edges.update(edgeUpdates); + + // Store for cleanup + this.highlightedElements = { + nodes: connectedNodes.concat([nodeId]), + edges: connectedEdges + }; + } + + /** + * Highlight connected nodes on hover + * @param {string} nodeId - Node ID + * @param {boolean} highlight - Whether to highlight or unhighlight + */ + highlightConnectedNodes(nodeId, highlight) { + const connectedNodes = this.network.getConnectedNodes(nodeId); + const connectedEdges = this.network.getConnectedEdges(nodeId); + + if (highlight) { + // Dim all other elements + this.dimUnconnectedElements([nodeId, ...connectedNodes], connectedEdges); + } + } + + /** + * Dim elements not connected to the specified nodes + * @param {Array} nodeIds - Node IDs to keep highlighted + * @param {Array} edgeIds - Edge IDs to keep highlighted + */ + dimUnconnectedElements(nodeIds, edgeIds) { + const allNodes = this.nodes.get(); + const allEdges = this.edges.get(); + + const nodeUpdates = allNodes.map(node => ({ + id: node.id, + opacity: nodeIds.includes(node.id) ? 1 : 0.3 + })); + + const edgeUpdates = allEdges.map(edge => ({ + id: edge.id, + opacity: edgeIds.includes(edge.id) ? 1 : 0.1 + })); + + this.nodes.update(nodeUpdates); + this.edges.update(edgeUpdates); + } + + /** + * Clear all highlights + */ + clearHighlights() { + if (this.highlightedElements) { + // Reset highlighted nodes + const nodeUpdates = this.highlightedElements.nodes.map(id => { + const originalNode = this.nodes.get(id); + return { + id: id, + borderColor: this.getNodeBorderColor(originalNode.type), + borderWidth: 2 + }; + }); + + // Reset highlighted edges + const edgeUpdates = this.highlightedElements.edges.map(id => { + const originalEdge = this.edges.get(id); + return { + id: id, + color: this.getEdgeColor(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5), + width: this.getEdgeWidth(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5) + }; + }); + + this.nodes.update(nodeUpdates); + this.edges.update(edgeUpdates); + + this.highlightedElements = null; + } + } + + /** + * Clear hover highlights + */ + clearHoverHighlights() { + const allNodes = this.nodes.get(); + const allEdges = this.edges.get(); + + const nodeUpdates = allNodes.map(node => ({ id: node.id, opacity: 1 })); + const edgeUpdates = allEdges.map(edge => ({ id: edge.id, opacity: 1 })); + + this.nodes.update(nodeUpdates); + this.edges.update(edgeUpdates); + } + + /** + * Highlight newly added elements + * @param {Array} newNodes - New nodes + * @param {Array} newEdges - New edges + */ + highlightNewElements(newNodes, newEdges) { + // Briefly highlight new nodes + const nodeHighlights = newNodes.map(node => ({ + id: node.id, + borderColor: '#00ff41', + borderWidth: 4, + shadow: { + enabled: true, + color: 'rgba(0, 255, 65, 0.8)', + size: 15, + x: 2, + y: 2 + } + })); + + // Briefly highlight new edges + const edgeHighlights = newEdges.map(edge => ({ + id: edge.id, + color: '#00ff41', + width: 4 + })); + + this.nodes.update(nodeHighlights); + this.edges.update(edgeHighlights); + + // Reset after animation + setTimeout(() => { + const nodeResets = newNodes.map(node => ({ + id: node.id, + borderColor: this.getNodeBorderColor(node.type), + borderWidth: 2, + shadow: node.shadow || { enabled: false } + })); + + const edgeResets = newEdges.map(edge => ({ + id: edge.id, + color: this.getEdgeColor(edge.metadata ? edge.metadata.confidence_score : 0.5), + width: this.getEdgeWidth(edge.metadata ? edge.metadata.confidence_score : 0.5) + })); + + this.nodes.update(nodeResets); + this.edges.update(edgeResets); + }, 2000); + } + /** * Update stabilization progress * @param {number} progress - Progress value (0-1) @@ -452,14 +846,71 @@ class GraphManager { // Could show a progress indicator if needed console.log(`Graph stabilization: ${(progress * 100).toFixed(1)}%`); } - + /** * Handle stabilization completion */ onStabilizationComplete() { console.log('Graph stabilization complete'); } + + /** + * Focus view on specific node + * @param {string} nodeId - Node to focus on + */ + focusOnNode(nodeId) { + const nodePosition = this.network.getPositions([nodeId]); + if (nodePosition[nodeId]) { + this.network.moveTo({ + position: nodePosition[nodeId], + scale: 1.5, + animation: { + duration: 1000, + easingFunction: 'easeInOutQuart' + } + }); + } + } + /** + * Toggle physics simulation + */ + togglePhysics() { + const currentPhysics = this.network.physics.physicsEnabled; + this.network.setOptions({ physics: !currentPhysics }); + + const button = document.getElementById('graph-physics'); + if (button) { + button.textContent = currentPhysics ? '[PHYSICS OFF]' : '[PHYSICS ON]'; + button.style.color = currentPhysics ? '#ff9900' : '#00ff41'; + } + } + + /** + * Toggle node clustering + */ + toggleClustering() { + // Simple clustering by node type + const clusterOptionsByType = { + joinCondition: (childOptions) => { + return childOptions.type === 'domain'; + }, + clusterNodeProperties: { + id: 'domain-cluster', + borderWidth: 3, + shape: 'database', + label: 'Domains', + color: '#00ff41' + } + }; + + if (this.network.clustering.isCluster('domain-cluster')) { + this.network.clustering.openCluster('domain-cluster'); + } else { + this.network.clustering.cluster(clusterOptionsByType); + } + } + /** * Fit the view to show all nodes */ @@ -473,7 +924,7 @@ class GraphManager { }); } } - + /** * Reset the view to initial state */ @@ -489,21 +940,21 @@ class GraphManager { }); } } - + /** * Clear the graph */ clear() { this.nodes.clear(); this.edges.clear(); - + // Show placeholder const placeholder = this.container.querySelector('.graph-placeholder'); if (placeholder) { placeholder.style.display = 'flex'; } } - + /** * Show error message * @param {string} message - Error message @@ -515,25 +966,7 @@ class GraphManager { placeholder.style.color = '#ff6b6b'; } } - - /** - * Setup control event handlers - */ - setupEventHandlers() { - // Reset view button - document.addEventListener('DOMContentLoaded', () => { - const resetBtn = document.getElementById('reset-view'); - if (resetBtn) { - resetBtn.addEventListener('click', () => this.resetView()); - } - - const fitBtn = document.getElementById('fit-view'); - if (fitBtn) { - fitBtn.addEventListener('click', () => this.fitView()); - } - }); - } - + /** * Get network statistics * @returns {Object} Statistics object @@ -545,7 +978,7 @@ class GraphManager { //isStabilized: this.network ? this.network.isStabilized() : false }; } - + /** * Export graph as image (if needed for future implementation) * @param {string} format - Image format ('png', 'jpeg') @@ -553,7 +986,7 @@ class GraphManager { */ exportAsImage(format = 'png') { if (!this.network) return null; - + // This would require additional vis.js functionality // Placeholder for future implementation console.log('Image export not yet implemented'); diff --git a/static/js/main.js b/static/js/main.js index dd4d840..3cdc34a 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -447,11 +447,19 @@ class DNSReconApp { try { console.log('Updating status display...'); - // Update status text + // Update status text with animation if (this.elements.scanStatus) { - this.elements.scanStatus.textContent = this.formatStatus(status.status); - console.log('Updated status display:', status.status); + const formattedStatus = this.formatStatus(status.status); + if (this.elements.scanStatus.textContent !== formattedStatus) { + this.elements.scanStatus.textContent = formattedStatus; + this.elements.scanStatus.classList.add('fade-in'); + setTimeout(() => this.elements.scanStatus.classList.remove('fade-in'), 300); + } + + // Add status-specific classes for styling + this.elements.scanStatus.className = `status-value status-${status.status}`; } + if (this.elements.targetDisplay) { this.elements.targetDisplay.textContent = status.target_domain || 'None'; } @@ -465,9 +473,16 @@ class DNSReconApp { this.elements.indicatorsDisplay.textContent = status.indicators_processed || 0; } - // Update progress bar + // Update progress bar with smooth animation if (this.elements.progressFill) { this.elements.progressFill.style.width = `${status.progress_percentage}%`; + + // Add pulsing animation for active scans + if (status.status === 'running') { + this.elements.progressFill.parentElement.classList.add('scanning'); + } else { + this.elements.progressFill.parentElement.classList.remove('scanning'); + } } // Update session ID @@ -492,12 +507,16 @@ class DNSReconApp { case 'running': this.setUIState('scanning'); this.showSuccess('Scan is running'); + // Reset polling frequency for active scans + this.pollFrequency = 2000; + this.updateConnectionStatus('active'); break; case 'completed': this.setUIState('completed'); this.stopPolling(); this.showSuccess('Scan completed successfully'); + this.updateConnectionStatus('completed'); // Force a final graph update console.log('Scan completed - forcing final graph update'); setTimeout(() => this.updateGraph(), 100); @@ -507,20 +526,54 @@ class DNSReconApp { this.setUIState('failed'); this.stopPolling(); this.showError('Scan failed'); + this.updateConnectionStatus('error'); break; case 'stopped': this.setUIState('stopped'); this.stopPolling(); this.showSuccess('Scan stopped'); + this.updateConnectionStatus('stopped'); break; case 'idle': this.setUIState('idle'); this.stopPolling(); + this.updateConnectionStatus('idle'); break; } } + + /** + * Update connection status indicator + * @param {string} status - Connection status + */ + updateConnectionStatus(status) { + if (!this.elements.connectionStatus) return; + + const statusColors = { + 'idle': '#c7c7c7', + 'active': '#00ff41', + 'completed': '#00aa2e', + 'stopped': '#ff9900', + 'error': '#ff6b6b' + }; + + this.elements.connectionStatus.style.backgroundColor = statusColors[status] || '#c7c7c7'; + + const statusText = this.elements.connectionStatus.parentElement?.querySelector('.status-text'); + if (statusText) { + const statusTexts = { + 'idle': 'System Ready', + 'active': 'Scanning Active', + 'completed': 'Scan Complete', + 'stopped': 'Scan Stopped', + 'error': 'Connection Error' + }; + + statusText.textContent = statusTexts[status] || 'System Online'; + } + } /** * Set UI state based on scan status @@ -532,10 +585,17 @@ class DNSReconApp { switch (state) { case 'scanning': this.isScanning = true; - if (this.elements.startScan) this.elements.startScan.disabled = true; - if (this.elements.stopScan) this.elements.stopScan.disabled = false; + if (this.elements.startScan) { + this.elements.startScan.disabled = true; + this.elements.startScan.classList.add('loading'); + } + if (this.elements.stopScan) { + this.elements.stopScan.disabled = false; + this.elements.stopScan.classList.remove('loading'); + } if (this.elements.targetDomain) this.elements.targetDomain.disabled = true; if (this.elements.maxDepth) this.elements.maxDepth.disabled = true; + if (this.elements.configureApiKeys) this.elements.configureApiKeys.disabled = true; break; case 'idle': @@ -543,10 +603,17 @@ class DNSReconApp { case 'failed': case 'stopped': this.isScanning = false; - if (this.elements.startScan) this.elements.startScan.disabled = false; - if (this.elements.stopScan) this.elements.stopScan.disabled = true; + if (this.elements.startScan) { + this.elements.startScan.disabled = false; + this.elements.startScan.classList.remove('loading'); + } + if (this.elements.stopScan) { + this.elements.stopScan.disabled = true; + this.elements.stopScan.classList.add('loading'); + } if (this.elements.targetDomain) this.elements.targetDomain.disabled = false; if (this.elements.maxDepth) this.elements.maxDepth.disabled = false; + if (this.elements.configureApiKeys) this.elements.configureApiKeys.disabled = false; break; } } @@ -580,20 +647,42 @@ class DNSReconApp { for (const [name, info] of Object.entries(providers)) { const providerItem = document.createElement('div'); - providerItem.className = 'provider-item'; + providerItem.className = 'provider-item fade-in'; - const status = info.enabled ? 'enabled' : 'disabled'; - const statusClass = info.enabled ? 'enabled' : 'disabled'; + let statusClass = 'disabled'; + let statusText = 'Disabled'; + + if (info.enabled) { + statusClass = 'enabled'; + statusText = 'Enabled'; + } else if (info.requires_api_key) { + statusClass = 'api-key-required'; + statusText = 'API Key Required'; + } providerItem.innerHTML = ` -
+
${name.toUpperCase()}
-
- Requests: ${info.statistics.total_requests || 0} | - Success Rate: ${(info.statistics.success_rate || 0).toFixed(1)}% +
${statusText}
+
+
+
+ Requests: + ${info.statistics.total_requests || 0} +
+
+ Success Rate: + ${(info.statistics.success_rate || 0).toFixed(1)}% +
+
+ Relationships: + ${info.statistics.relationships_found || 0} +
+
+ Rate Limit: + ${info.rate_limit}/min
-
${status}
`; this.elements.providerList.appendChild(providerItem); @@ -614,7 +703,7 @@ class DNSReconApp { let detailsHtml = ''; detailsHtml += `
Identifier:${nodeId}
`; - detailsHtml += `
Type:${node.metadata.type || 'Unknown'}
`; + detailsHtml += `
Type:${node.metadata.type || node.type || 'Unknown'}
`; if (node.metadata) { for (const [key, value] of Object.entries(node.metadata)) { @@ -624,6 +713,12 @@ class DNSReconApp { } } + // Add timestamps if available + if (node.added_timestamp) { + const addedDate = new Date(node.added_timestamp); + detailsHtml += `
Added:${addedDate.toLocaleString()}
`; + } + if (this.elements.modalDetails) { this.elements.modalDetails.innerHTML = detailsHtml; } @@ -645,12 +740,24 @@ class DNSReconApp { * @returns {boolean} True if data has changed */ hasGraphChanged(graphData) { - // Simple check based on node and edge counts + // Simple check based on node and edge counts and timestamps const currentStats = this.graphManager.getStatistics(); const newNodeCount = graphData.nodes ? graphData.nodes.length : 0; const newEdgeCount = graphData.edges ? graphData.edges.length : 0; - const changed = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount; + // Check if counts changed + const countsChanged = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount; + + // Also check if we have new timestamp data + const hasNewTimestamp = graphData.statistics && + graphData.statistics.last_modified && + graphData.statistics.last_modified !== this.lastGraphTimestamp; + + if (hasNewTimestamp) { + this.lastGraphTimestamp = graphData.statistics.last_modified; + } + + const changed = countsChanged || hasNewTimestamp; console.log(`Graph change check: Current(${currentStats.nodeCount}n, ${currentStats.edgeCount}e) vs New(${newNodeCount}n, ${newEdgeCount}e) = ${changed}`); @@ -816,6 +923,10 @@ class DNSReconApp { this.showMessage(message, 'info'); } + showWarning(message) { + this.showMessage(message, 'warning'); + } + /** * Show error message * @param {string} message - Error message @@ -828,13 +939,7 @@ class DNSReconApp { * Show connection error */ showConnectionError() { - if (this.elements.connectionStatus) { - this.elements.connectionStatus.style.backgroundColor = '#ff6b6b'; - } - const statusText = this.elements.connectionStatus?.parentElement?.querySelector('.status-text'); - if (statusText) { - statusText.textContent = 'Connection Error'; - } + this.updateConnectionStatus('error'); } /** @@ -848,24 +953,12 @@ class DNSReconApp { // Create message element const messageElement = document.createElement('div'); messageElement.className = `message-toast message-${type}`; - messageElement.style.cssText = ` - background: ${this.getMessageColor(type)}; - color: #fff; - padding: 12px 20px; - margin-bottom: 10px; - border-radius: 4px; - font-family: 'Roboto Mono', monospace; - font-size: 0.9rem; - box-shadow: 0 4px 6px rgba(0,0,0,0.3); - border-left: 4px solid ${this.getMessageBorderColor(type)}; - animation: slideInRight 0.3s ease-out; - `; messageElement.innerHTML = ` -
- ${message} +
+ ${message} + style="background: none; border: none; color: #fff; cursor: pointer; font-size: 16px; margin-left: 10px; opacity: 0.7;">×
`; @@ -888,13 +981,8 @@ class DNSReconApp { } // Update connection status to show activity - if (type === 'success' && this.elements.connectionStatus) { - this.elements.connectionStatus.style.backgroundColor = '#00ff41'; - setTimeout(() => { - if (this.elements.connectionStatus) { - this.elements.connectionStatus.style.backgroundColor = '#00ff41'; - } - }, 2000); + if (type === 'success' && this.consecutiveErrors === 0) { + this.updateConnectionStatus(this.isScanning ? 'active' : 'idle'); } } diff --git a/templates/index.html b/templates/index.html index 16010a6..aca67df 100644 --- a/templates/index.html +++ b/templates/index.html @@ -63,6 +63,10 @@ [EXPORT] Download Results +