mstoeck3/dnscope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
overcuriousity 2025-09-10 15:17:17 +02:00
parent 696cec0723
commit ce0e11cf0b
16 changed files with 2577 additions and 484 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app.py
View File

@ -6,7 +6,7 @@ Provides REST API endpoints and serves the web interface.
import json
import traceback
from flask import Flask, render_template, request, jsonify, send_file
from datetime import datetime
from datetime import datetime, timezone
import io
from core.scanner import scanner
@ -173,7 +173,7 @@ def export_results():
results = scanner.export_results()
# Create filename with timestamp
timestamp = datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S')
timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')
target = scanner.current_target or 'unknown'
filename = f"dnsrecon_{target}_{timestamp}.json"
@ -284,12 +284,22 @@ def set_api_keys():
@app.route('/api/health', methods=['GET'])
def health_check():
"""Health check endpoint."""
"""Health check endpoint with enhanced Phase 2 information."""
return jsonify({
'success': True,
'status': 'healthy',
'timestamp': datetime.now(datetime.UTC).isoformat(),
'version': '1.0.0-phase1'
'version': '1.0.0-phase2',
'phase': 2,
'features': {
'multi_provider': True,
'concurrent_processing': True,
'real_time_updates': True,
'api_key_management': True,
'enhanced_visualization': True,
'retry_logic': True
},
'providers_available': len(scanner.providers) if hasattr(scanner, 'providers') else 0
})

2
config.py
View File

@ -111,6 +111,8 @@ class Config:
# Override default settings from environment
self.default_recursion_depth = int(os.getenv('DEFAULT_RECURSION_DEPTH', '2'))
self.flask_debug = os.getenv('FLASK_DEBUG', 'True').lower() == 'true'
self.default_timeout = 30
self.max_concurrent_requests = 5
# Global configuration instance

3
core/__init__.py
View File

@ -1,6 +1,7 @@
"""
Core modules for DNSRecon passive reconnaissance tool.
Contains graph management, scanning orchestration, and forensic logging.
Phase 2: Enhanced with concurrent processing and real-time capabilities.
"""
from .graph_manager import GraphManager, NodeType, RelationshipType
@ -19,4 +20,4 @@ __all__ = [
'new_session'
]
__version__ = "1.0.0-phase1"
__version__ = "1.0.0-phase2"

86
core/graph_manager.py
View File

@ -8,6 +8,7 @@ import threading
from datetime import datetime
from typing import Dict, List, Any, Optional, Tuple, Set
from enum import Enum
from datetime import timezone
import networkx as nx
@ -44,8 +45,8 @@ class GraphManager:
def __init__(self):
"""Initialize empty directed graph."""
self.graph = nx.DiGraph()
#self.lock = threading.Lock()
self.creation_time = datetime.now(datetime.UTC).isoformat()
# self.lock = threading.Lock()
self.creation_time = datetime.now(timezone.utc).isoformat()
self.last_modified = self.creation_time
def add_node(self, node_id: str, node_type: NodeType,
@ -71,12 +72,12 @@ class GraphManager:
node_attributes = {
'type': node_type.value,
'added_timestamp': datetime.now(datetime.UTC).isoformat(),
'added_timestamp': datetime.now(timezone.utc).isoformat(),
'metadata': metadata or {}
}
self.graph.add_node(node_id, **node_attributes)
self.last_modified = datetime.now(datetime.UTC).isoformat()
self.last_modified = datetime.now(timezone.utc).isoformat()
return True
def add_edge(self, source_id: str, target_id: str,
@ -111,7 +112,7 @@ class GraphManager:
if new_confidence > existing_confidence:
self.graph.edges[source_id, target_id]['confidence_score'] = new_confidence
self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(datetime.UTC).isoformat()
self.graph.edges[source_id, target_id]['updated_timestamp'] = datetime.now(timezone.utc).isoformat()
self.graph.edges[source_id, target_id]['updated_by'] = source_provider
return False
@ -120,12 +121,12 @@ class GraphManager:
'relationship_type': relationship_type.relationship_name,
'confidence_score': confidence_score or relationship_type.default_confidence,
'source_provider': source_provider,
'discovery_timestamp': datetime.now(datetime.UTC).isoformat(),
'discovery_timestamp': datetime.now(timezone.utc).isoformat(),
'raw_data': raw_data or {}
}
self.graph.add_edge(source_id, target_id, **edge_attributes)
self.last_modified = datetime.now(datetime.UTC).isoformat()
self.last_modified = datetime.now(timezone.utc).isoformat()
return True
def get_node_count(self) -> int:
@ -210,14 +211,36 @@ class GraphManager:
'added_timestamp': attributes.get('added_timestamp')
}
# Color coding by type
# Color coding by type - now returns color objects for enhanced visualization
type_colors = {
'domain': '#00ff41', # Green for domains
'ip': '#ff9900', # Amber for IPs
'certificate': '#c7c7c7', # Gray for certificates
'asn': '#00aaff' # Blue for ASNs
'domain': {
'background': '#00ff41',
'border': '#00aa2e',
'highlight': {'background': '#44ff75', 'border': '#00ff41'},
'hover': {'background': '#22ff63', 'border': '#00cc35'}
},
'ip': {
'background': '#ff9900',
'border': '#cc7700',
'highlight': {'background': '#ffbb44', 'border': '#ff9900'},
'hover': {'background': '#ffaa22', 'border': '#dd8800'}
},
'certificate': {
'background': '#c7c7c7',
'border': '#999999',
'highlight': {'background': '#e0e0e0', 'border': '#c7c7c7'},
'hover': {'background': '#d4d4d4', 'border': '#aaaaaa'}
},
'asn': {
'background': '#00aaff',
'border': '#0088cc',
'highlight': {'background': '#44ccff', 'border': '#00aaff'},
'hover': {'background': '#22bbff', 'border': '#0099dd'}
}
}
node_data['color'] = type_colors.get(attributes.get('type'), '#ffffff')
node_color_config = type_colors.get(attributes.get('type', 'unknown'), type_colors['domain'])
node_data['color'] = node_color_config
nodes.append(node_data)
# Format edges for visualization
@ -231,17 +254,36 @@ class GraphManager:
'discovery_timestamp': attributes.get('discovery_timestamp')
}
# Edge styling based on confidence
# Enhanced edge styling based on confidence
confidence = attributes.get('confidence_score', 0)
if confidence >= 0.8:
edge_data['color'] = '#00ff41' # Green for high confidence
edge_data['width'] = 3
edge_data['color'] = {
'color': '#00ff41',
'highlight': '#44ff75',
'hover': '#22ff63',
'inherit': False
}
edge_data['width'] = 4
elif confidence >= 0.6:
edge_data['color'] = '#ff9900' # Amber for medium confidence
edge_data['width'] = 2
edge_data['color'] = {
'color': '#ff9900',
'highlight': '#ffbb44',
'hover': '#ffaa22',
'inherit': False
}
edge_data['width'] = 3
else:
edge_data['color'] = '#444444' # Dark gray for low confidence
edge_data['width'] = 1
edge_data['color'] = {
'color': '#666666',
'highlight': '#888888',
'hover': '#777777',
'inherit': False
}
edge_data['width'] = 2
# Add dashed line for low confidence
if confidence < 0.6:
edge_data['dashes'] = [5, 5]
edges.append(edge_data)
@ -270,7 +312,7 @@ class GraphManager:
# Add comprehensive metadata
export_data = {
'export_metadata': {
'export_timestamp': datetime.now(datetime.UTC).isoformat(),
'export_timestamp': datetime.now(timezone.utc).isoformat(),
'graph_creation_time': self.creation_time,
'last_modified': self.last_modified,
'total_nodes': self.graph.number_of_nodes(),
@ -351,5 +393,5 @@ class GraphManager:
"""Clear all nodes and edges from the graph."""
#with self.lock:
self.graph.clear()
self.creation_time = datetime.now(datetime.UTC).isoformat()
self.creation_time = datetime.now(timezone.utc).isoformat()
self.last_modified = self.creation_time

15
core/logger.py
View File

@ -9,6 +9,7 @@ import threading
from datetime import datetime
from typing import Dict, Any, Optional, List
from dataclasses import dataclass, asdict
from datetime import timezone
@dataclass
@ -60,7 +61,7 @@ class ForensicLogger:
self.relationships: List[RelationshipDiscovery] = []
self.session_metadata = {
'session_id': self.session_id,
'start_time': datetime.now(datetime.UTC).isoformat(),
'start_time': datetime.now(timezone.utc).isoformat(),
'end_time': None,
'total_requests': 0,
'total_relationships': 0,
@ -85,7 +86,7 @@ class ForensicLogger:
def _generate_session_id(self) -> str:
"""Generate unique session identifier."""
return f"dnsrecon_{datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S')}"
return f"dnsrecon_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
def log_api_request(self, provider: str, url: str, method: str = "GET",
status_code: Optional[int] = None,
@ -110,7 +111,7 @@ class ForensicLogger:
"""
#with self.lock:
api_request = APIRequest(
timestamp=datetime.now(datetime.UTC).isoformat(),
timestamp=datetime.now(timezone.utc).isoformat(),
provider=provider,
url=url,
method=method,
@ -153,7 +154,7 @@ class ForensicLogger:
"""
#with self.lock:
relationship = RelationshipDiscovery(
timestamp=datetime.now(datetime.UTC).isoformat(),
timestamp=datetime.now(timezone.utc).isoformat(),
source_node=source_node,
target_node=target_node,
relationship_type=relationship_type,
@ -183,7 +184,7 @@ class ForensicLogger:
def log_scan_complete(self) -> None:
"""Log the completion of a reconnaissance scan."""
#with self.lock:
self.session_metadata['end_time'] = datetime.now(datetime.UTC).isoformat()
self.session_metadata['end_time'] = datetime.now(timezone.utc).isoformat()
self.session_metadata['providers_used'] = list(self.session_metadata['providers_used'])
self.session_metadata['target_domains'] = list(self.session_metadata['target_domains'])
@ -203,7 +204,7 @@ class ForensicLogger:
'session_metadata': self.session_metadata.copy(),
'api_requests': [asdict(req) for req in self.api_requests],
'relationships': [asdict(rel) for rel in self.relationships],
'export_timestamp': datetime.now(datetime.UTC).isoformat()
'export_timestamp': datetime.now(timezone.utc).isoformat()
}
def get_forensic_summary(self) -> Dict[str, Any]:
@ -239,7 +240,7 @@ class ForensicLogger:
def _calculate_session_duration(self) -> float:
"""Calculate session duration in minutes."""
if not self.session_metadata['end_time']:
end_time = datetime.now(datetime.UTC)
end_time = datetime.now(timezone.utc)
else:
end_time = datetime.fromisoformat(self.session_metadata['end_time'])

562
core/scanner.py
View File

@ -6,12 +6,15 @@ Coordinates data gathering from multiple providers and builds the infrastructure
import threading
import time
import traceback
from typing import List, Set, Dict, Any, Optional
from typing import List, Set, Dict, Any, Optional, Tuple
from concurrent.futures import ThreadPoolExecutor, as_completed
from core.graph_manager import GraphManager, NodeType, RelationshipType
from core.logger import get_forensic_logger, new_session
from providers.crtsh_provider import CrtShProvider
from providers.dns_provider import DNSProvider
from providers.shodan_provider import ShodanProvider
from providers.virustotal_provider import VirusTotalProvider
from config import config
@ -27,17 +30,16 @@ class ScanStatus:
class Scanner:
"""
Main scanning orchestrator for DNSRecon passive reconnaissance.
Manages multi-provider data gathering and graph construction.
Manages multi-provider data gathering and graph construction with concurrent processing.
"""
def __init__(self):
"""Initialize scanner with default providers and empty graph."""
"""Initialize scanner with all available providers and empty graph."""
print("Initializing Scanner instance...")
try:
from providers.base_provider import BaseProvider
self.graph = GraphManager()
self.providers: List[BaseProvider] = []
self.providers = []
self.status = ScanStatus.IDLE
self.current_target = None
self.current_depth = 0
@ -49,6 +51,9 @@ class Scanner:
self.total_indicators_found = 0
self.indicators_processed = 0
self.current_indicator = ""
# Concurrent processing configuration
self.max_workers = config.max_concurrent_requests
# Initialize providers
print("Calling _initialize_providers...")
@ -66,36 +71,54 @@ class Scanner:
raise
def _initialize_providers(self) -> None:
"""Initialize available providers based on configuration."""
"""Initialize all available providers based on configuration."""
self.providers = []
print("Initializing providers...")
# Always add free providers
if config.is_provider_enabled('crtsh'):
try:
crtsh_provider = CrtShProvider()
if crtsh_provider.is_available():
self.providers.append(crtsh_provider)
print("✓ CrtSh provider initialized successfully")
else:
print("✗ CrtSh provider is not available")
except Exception as e:
print(f"✗ Failed to initialize CrtSh provider: {e}")
traceback.print_exc()
free_providers = [
('crtsh', CrtShProvider),
('dns', DNSProvider)
]
for provider_name, provider_class in free_providers:
if config.is_provider_enabled(provider_name):
try:
provider = provider_class()
if provider.is_available():
self.providers.append(provider)
print(f"✓ {provider_name.title()} provider initialized successfully")
else:
print(f"✗ {provider_name.title()} provider is not available")
except Exception as e:
print(f"✗ Failed to initialize {provider_name.title()} provider: {e}")
traceback.print_exc()
# Add API key-dependent providers
api_providers = [
('shodan', ShodanProvider),
('virustotal', VirusTotalProvider)
]
for provider_name, provider_class in api_providers:
if config.is_provider_enabled(provider_name):
try:
provider = provider_class()
if provider.is_available():
self.providers.append(provider)
print(f"✓ {provider_name.title()} provider initialized successfully")
else:
print(f"✗ {provider_name.title()} provider is not available (API key required)")
except Exception as e:
print(f"✗ Failed to initialize {provider_name.title()} provider: {e}")
traceback.print_exc()
print(f"Initialized {len(self.providers)} providers")
def _debug_threads(self):
"""Debug function to show current threads."""
print("=== THREAD DEBUG INFO ===")
for t in threading.enumerate():
print(f"Thread: {t.name} | Alive: {t.is_alive()} | Daemon: {t.daemon}")
print("=== END THREAD DEBUG ===")
def start_scan(self, target_domain: str, max_depth: int = 2) -> bool:
"""
Start a new reconnaissance scan.
Start a new reconnaissance scan with concurrent processing.
Args:
target_domain: Initial domain to investigate
@ -107,9 +130,6 @@ class Scanner:
print(f"Scanner.start_scan called with target='{target_domain}', depth={max_depth}")
try:
print("Checking current status...")
self._debug_threads()
if self.status == ScanStatus.RUNNING:
print("Scan already running, rejecting new scan")
return False
@ -119,8 +139,6 @@ class Scanner:
print("No providers available, cannot start scan")
return False
print(f"Current status: {self.status}, Providers: {len(self.providers)}")
# Stop any existing scan thread
if self.scan_thread and self.scan_thread.is_alive():
print("Stopping existing scan thread...")
@ -132,9 +150,7 @@ class Scanner:
# Reset state
print("Resetting scanner state...")
#print("Running graph.clear()")
#self.graph.clear()
print("running self.current_target = target_domain.lower().strip()")
self.graph.clear()
self.current_target = target_domain.lower().strip()
self.max_depth = max_depth
self.current_depth = 0
@ -147,9 +163,15 @@ class Scanner:
print("Starting new forensic session...")
self.logger = new_session()
# FOR DEBUGGING: Run scan synchronously instead of in thread
print("Running scan synchronously for debugging...")
self._execute_scan_sync(self.current_target, max_depth)
# Start scan in separate thread for Phase 2
print("Starting scan thread...")
self.scan_thread = threading.Thread(
target=self._execute_scan_async,
args=(self.current_target, max_depth),
daemon=True
)
self.scan_thread.start()
return True
except Exception as e:
@ -157,6 +179,321 @@ class Scanner:
traceback.print_exc()
return False
def _execute_scan_async(self, target_domain: str, max_depth: int) -> None:
"""
Execute the reconnaissance scan asynchronously with concurrent provider queries.
Args:
target_domain: Target domain to investigate
max_depth: Maximum recursion depth
"""
print(f"_execute_scan_async started for {target_domain} with depth {max_depth}")
try:
print("Setting status to RUNNING")
self.status = ScanStatus.RUNNING
# Log scan start
enabled_providers = [provider.get_name() for provider in self.providers]
self.logger.log_scan_start(target_domain, max_depth, enabled_providers)
print(f"Logged scan start with providers: {enabled_providers}")
# Initialize with target domain
print(f"Adding target domain '{target_domain}' as initial node")
self.graph.add_node(target_domain, NodeType.DOMAIN)
# BFS-style exploration with depth limiting and concurrent processing
current_level_domains = {target_domain}
processed_domains = set()
all_discovered_ips = set()
print(f"Starting BFS exploration...")
for depth in range(max_depth + 1):
if self.stop_requested:
print(f"Stop requested at depth {depth}")
break
self.current_depth = depth
print(f"Processing depth level {depth} with {len(current_level_domains)} domains")
if not current_level_domains:
print("No domains to process at this level")
break
# Update progress tracking
self.total_indicators_found += len(current_level_domains)
next_level_domains = set()
# Process domains at current depth level with concurrent queries
domain_results = self._process_domains_concurrent(current_level_domains, processed_domains)
for domain, discovered_domains, discovered_ips in domain_results:
if self.stop_requested:
break
processed_domains.add(domain)
all_discovered_ips.update(discovered_ips)
# Add discovered domains to next level if not at max depth
if depth < max_depth:
for discovered_domain in discovered_domains:
if discovered_domain not in processed_domains:
next_level_domains.add(discovered_domain)
print(f"Adding {discovered_domain} to next level")
# Process discovered IPs concurrently
if all_discovered_ips:
print(f"Processing {len(all_discovered_ips)} discovered IP addresses")
self._process_ips_concurrent(all_discovered_ips)
current_level_domains = next_level_domains
print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level")
# Finalize scan
if self.stop_requested:
self.status = ScanStatus.STOPPED
print("Scan completed with STOPPED status")
else:
self.status = ScanStatus.COMPLETED
print("Scan completed with COMPLETED status")
self.logger.log_scan_complete()
# Print final statistics
stats = self.graph.get_statistics()
print(f"Final scan statistics:")
print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}")
print(f" - Total edges: {stats['basic_metrics']['total_edges']}")
print(f" - Domains processed: {len(processed_domains)}")
print(f" - IPs discovered: {len(all_discovered_ips)}")
except Exception as e:
print(f"ERROR: Scan execution failed with error: {e}")
traceback.print_exc()
self.status = ScanStatus.FAILED
self.logger.logger.error(f"Scan failed: {e}")
def _process_domains_concurrent(self, domains: Set[str], processed_domains: Set[str]) -> List[Tuple[str, Set[str], Set[str]]]:
"""
Process multiple domains concurrently using thread pool.
Args:
domains: Set of domains to process
processed_domains: Set of already processed domains
Returns:
List of tuples (domain, discovered_domains, discovered_ips)
"""
results = []
# Filter out already processed domains
domains_to_process = domains - processed_domains
if not domains_to_process:
return results
print(f"Processing {len(domains_to_process)} domains concurrently with {self.max_workers} workers")
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
# Submit all domain processing tasks
future_to_domain = {
executor.submit(self._query_providers_for_domain, domain): domain
for domain in domains_to_process
}
# Collect results as they complete
for future in as_completed(future_to_domain):
if self.stop_requested:
break
domain = future_to_domain[future]
try:
discovered_domains, discovered_ips = future.result()
results.append((domain, discovered_domains, discovered_ips))
self.indicators_processed += 1
print(f"Completed processing domain: {domain} ({len(discovered_domains)} domains, {len(discovered_ips)} IPs)")
except Exception as e:
print(f"Error processing domain {domain}: {e}")
traceback.print_exc()
return results
def _process_ips_concurrent(self, ips: Set[str]) -> None:
"""
Process multiple IP addresses concurrently.
Args:
ips: Set of IP addresses to process
"""
if not ips:
return
print(f"Processing {len(ips)} IP addresses concurrently")
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
# Submit all IP processing tasks
future_to_ip = {
executor.submit(self._query_providers_for_ip, ip): ip
for ip in ips
}
# Collect results as they complete
for future in as_completed(future_to_ip):
if self.stop_requested:
break
ip = future_to_ip[future]
try:
future.result() # Just wait for completion
print(f"Completed processing IP: {ip}")
except Exception as e:
print(f"Error processing IP {ip}: {e}")
traceback.print_exc()
def _query_providers_for_domain(self, domain: str) -> Tuple[Set[str], Set[str]]:
"""
Query all enabled providers for information about a domain.
Args:
domain: Domain to investigate
Returns:
Tuple of (discovered_domains, discovered_ips)
"""
print(f"Querying {len(self.providers)} providers for domain: {domain}")
discovered_domains = set()
discovered_ips = set()
if not self.providers:
print("No providers available")
return discovered_domains, discovered_ips
# Query providers concurrently for better performance
with ThreadPoolExecutor(max_workers=len(self.providers)) as executor:
# Submit queries for all providers
future_to_provider = {
executor.submit(self._safe_provider_query_domain, provider, domain): provider
for provider in self.providers
}
# Collect results as they complete
for future in as_completed(future_to_provider):
if self.stop_requested:
break
provider = future_to_provider[future]
try:
relationships = future.result()
print(f"Provider {provider.get_name()} returned {len(relationships)} relationships")
for source, target, rel_type, confidence, raw_data in relationships:
# Determine node type based on target
if self._is_valid_ip(target):
target_node_type = NodeType.IP
discovered_ips.add(target)
elif self._is_valid_domain(target):
target_node_type = NodeType.DOMAIN
discovered_domains.add(target)
else:
# Could be ASN or certificate
target_node_type = NodeType.ASN if target.startswith('AS') else NodeType.CERTIFICATE
# Add nodes and relationship to graph
self.graph.add_node(source, NodeType.DOMAIN)
self.graph.add_node(target, target_node_type)
success = self.graph.add_edge(
source, target, rel_type, confidence,
provider.get_name(), raw_data
)
if success:
print(f"Added relationship: {source} -> {target} ({rel_type.relationship_name})")
except Exception as e:
print(f"Provider {provider.get_name()} failed for {domain}: {e}")
print(f"Domain {domain}: discovered {len(discovered_domains)} domains, {len(discovered_ips)} IPs")
return discovered_domains, discovered_ips
def _query_providers_for_ip(self, ip: str) -> None:
"""
Query all enabled providers for information about an IP address.
Args:
ip: IP address to investigate
"""
print(f"Querying {len(self.providers)} providers for IP: {ip}")
if not self.providers:
print("No providers available")
return
# Query providers concurrently
with ThreadPoolExecutor(max_workers=len(self.providers)) as executor:
# Submit queries for all providers
future_to_provider = {
executor.submit(self._safe_provider_query_ip, provider, ip): provider
for provider in self.providers
}
# Collect results as they complete
for future in as_completed(future_to_provider):
if self.stop_requested:
break
provider = future_to_provider[future]
try:
relationships = future.result()
print(f"Provider {provider.get_name()} returned {len(relationships)} relationships for IP {ip}")
for source, target, rel_type, confidence, raw_data in relationships:
# Determine node type based on target
if self._is_valid_domain(target):
target_node_type = NodeType.DOMAIN
elif target.startswith('AS'):
target_node_type = NodeType.ASN
else:
target_node_type = NodeType.IP
# Add nodes and relationship to graph
self.graph.add_node(source, NodeType.IP)
self.graph.add_node(target, target_node_type)
success = self.graph.add_edge(
source, target, rel_type, confidence,
provider.get_name(), raw_data
)
if success:
print(f"Added IP relationship: {source} -> {target} ({rel_type.relationship_name})")
except Exception as e:
print(f"Provider {provider.get_name()} failed for IP {ip}: {e}")
def _safe_provider_query_domain(self, provider, domain: str):
"""Safely query provider for domain with error handling."""
try:
return provider.query_domain(domain)
except Exception as e:
print(f"Provider {provider.get_name()} query_domain failed: {e}")
return []
def _safe_provider_query_ip(self, provider, ip: str):
"""Safely query provider for IP with error handling."""
try:
return provider.query_ip(ip)
except Exception as e:
print(f"Provider {provider.get_name()} query_ip failed: {e}")
return []
def stop_scan(self) -> bool:
"""
Request scan termination.
@ -218,159 +555,6 @@ class Scanner:
return 0.0
return min(100.0, (self.indicators_processed / self.total_indicators_found) * 100)
def _execute_scan_sync(self, target_domain: str, max_depth: int) -> None:
"""
Execute the reconnaissance scan synchronously (for debugging).
Args:
target_domain: Target domain to investigate
max_depth: Maximum recursion depth
"""
print(f"_execute_scan_sync started for {target_domain} with depth {max_depth}")
try:
print("Setting status to RUNNING")
self.status = ScanStatus.RUNNING
# Log scan start
enabled_providers = [provider.get_name() for provider in self.providers]
self.logger.log_scan_start(target_domain, max_depth, enabled_providers)
print(f"Logged scan start with providers: {enabled_providers}")
# Initialize with target domain
print(f"Adding target domain '{target_domain}' as initial node")
self.graph.add_node(target_domain, NodeType.DOMAIN)
# BFS-style exploration with depth limiting
current_level_domains = {target_domain}
processed_domains = set()
print(f"Starting BFS exploration...")
for depth in range(max_depth + 1):
if self.stop_requested:
print(f"Stop requested at depth {depth}")
break
self.current_depth = depth
print(f"Processing depth level {depth} with {len(current_level_domains)} domains")
if not current_level_domains:
print("No domains to process at this level")
break
# Update progress tracking
self.total_indicators_found += len(current_level_domains)
next_level_domains = set()
# Process domains at current depth level
for domain in current_level_domains:
if self.stop_requested:
print(f"Stop requested while processing domain {domain}")
break
if domain in processed_domains:
print(f"Domain {domain} already processed, skipping")
continue
print(f"Processing domain: {domain}")
self.current_indicator = domain
self.indicators_processed += 1
# Query all providers for this domain
discovered_domains = self._query_providers_for_domain(domain)
print(f"Discovered {len(discovered_domains)} new domains from {domain}")
# Add discovered domains to next level if not at max depth
if depth < max_depth:
for discovered_domain in discovered_domains:
if discovered_domain not in processed_domains:
next_level_domains.add(discovered_domain)
print(f"Adding {discovered_domain} to next level")
processed_domains.add(domain)
current_level_domains = next_level_domains
print(f"Completed depth {depth}, {len(next_level_domains)} domains for next level")
# Finalize scan
if self.stop_requested:
self.status = ScanStatus.STOPPED
print("Scan completed with STOPPED status")
else:
self.status = ScanStatus.COMPLETED
print("Scan completed with COMPLETED status")
self.logger.log_scan_complete()
# Print final statistics
stats = self.graph.get_statistics()
print(f"Final scan statistics:")
print(f" - Total nodes: {stats['basic_metrics']['total_nodes']}")
print(f" - Total edges: {stats['basic_metrics']['total_edges']}")
print(f" - Domains processed: {len(processed_domains)}")
except Exception as e:
print(f"ERROR: Scan execution failed with error: {e}")
traceback.print_exc()
self.status = ScanStatus.FAILED
self.logger.logger.error(f"Scan failed: {e}")
def _query_providers_for_domain(self, domain: str) -> Set[str]:
"""
Query all enabled providers for information about a domain.
Args:
domain: Domain to investigate
Returns:
Set of newly discovered domains
"""
print(f"Querying {len(self.providers)} providers for domain: {domain}")
discovered_domains = set()
if not self.providers:
print("No providers available")
return discovered_domains
# Query providers sequentially for debugging
for provider in self.providers:
if self.stop_requested:
print("Stop requested, cancelling provider queries")
break
try:
print(f"Querying provider: {provider.get_name()}")
relationships = provider.query_domain(domain)
print(f"Provider {provider.get_name()} returned {len(relationships)} relationships")
for source, target, rel_type, confidence, raw_data in relationships:
print(f"Processing relationship: {source} -> {target} ({rel_type.relationship_name})")
# Add target node to graph if it doesn't exist
self.graph.add_node(target, NodeType.DOMAIN)
# Add relationship
success = self.graph.add_edge(
source, target, rel_type, confidence,
provider.get_name(), raw_data
)
if success:
print(f"Added new relationship: {source} -> {target}")
else:
print(f"Relationship already exists or failed to add: {source} -> {target}")
discovered_domains.add(target)
except Exception as e:
print(f"Provider {provider.get_name()} failed for {domain}: {e}")
traceback.print_exc()
self.logger.logger.error(f"Provider {provider.get_name()} failed for {domain}: {e}")
print(f"Total unique domains discovered: {len(discovered_domains)}")
return discovered_domains
def get_graph_data(self) -> Dict[str, Any]:
"""
Get current graph data for visualization.

10
providers/__init__.py
View File

@ -5,11 +5,17 @@ Contains implementations for various reconnaissance data sources.
from .base_provider import BaseProvider, RateLimiter
from .crtsh_provider import CrtShProvider
from .dns_provider import DNSProvider
from .shodan_provider import ShodanProvider
from .virustotal_provider import VirusTotalProvider
__all__ = [
'BaseProvider',
'RateLimiter',
'CrtShProvider'
'CrtShProvider',
'DNSProvider',
'ShodanProvider',
'VirusTotalProvider'
]
__version__ = "1.0.0-phase1"
__version__ = "1.0.0-phase2"

200
providers/base_provider.py
View File

@ -113,86 +113,134 @@ class BaseProvider(ABC):
pass
def make_request(self, url: str, method: str = "GET",
params: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
target_indicator: str = "") -> Optional[requests.Response]:
"""
Make a rate-limited HTTP request with forensic logging.
params: Optional[Dict[str, Any]] = None,
headers: Optional[Dict[str, str]] = None,
target_indicator: str = "",
max_retries: int = 3) -> Optional[requests.Response]:
"""
Make a rate-limited HTTP request with forensic logging and retry logic.
Args:
url: Request URL
method: HTTP method
params: Query parameters
headers: Additional headers
target_indicator: The indicator being investigated
max_retries: Maximum number of retry attempts
Returns:
Response object or None if request failed
"""
for attempt in range(max_retries + 1):
# Apply rate limiting
self.rate_limiter.wait_if_needed()
start_time = time.time()
response = None
error = None
try:
self.total_requests += 1
# Prepare request
request_headers = self.session.headers.copy()
if headers:
request_headers.update(headers)
print(f"Making {method} request to: {url} (attempt {attempt + 1})")
# Make request
if method.upper() == "GET":
response = self.session.get(
url,
params=params,
headers=request_headers,
timeout=self.timeout
)
elif method.upper() == "POST":
response = self.session.post(
url,
json=params,
headers=request_headers,
timeout=self.timeout
)
else:
raise ValueError(f"Unsupported HTTP method: {method}")
print(f"Response status: {response.status_code}")
response.raise_for_status()
self.successful_requests += 1
# Success - log and return
duration_ms = (time.time() - start_time) * 1000
self.logger.log_api_request(
provider=self.name,
url=url,
method=method.upper(),
status_code=response.status_code,
response_size=len(response.content),
duration_ms=duration_ms,
error=None,
target_indicator=target_indicator
)
return response
except requests.exceptions.RequestException as e:
error = str(e)
self.failed_requests += 1
print(f"Request failed (attempt {attempt + 1}): {error}")
# Check if we should retry
if attempt < max_retries and self._should_retry(e):
backoff_time = (2 ** attempt) * 1 # Exponential backoff: 1s, 2s, 4s
print(f"Retrying in {backoff_time} seconds...")
time.sleep(backoff_time)
continue
else:
break
except Exception as e:
error = f"Unexpected error: {str(e)}"
self.failed_requests += 1
print(f"Unexpected error: {error}")
break
# All attempts failed - log and return None
duration_ms = (time.time() - start_time) * 1000
self.logger.log_api_request(
provider=self.name,
url=url,
method=method.upper(),
status_code=response.status_code if response else None,
response_size=len(response.content) if response else None,
duration_ms=duration_ms,
error=error,
target_indicator=target_indicator
)
return None
def _should_retry(self, exception: requests.exceptions.RequestException) -> bool:
"""
Determine if a request should be retried based on the exception.
Args:
url: Request URL
method: HTTP method
params: Query parameters
headers: Additional headers
target_indicator: The indicator being investigated
exception: The request exception that occurred
Returns:
Response object or None if request failed
True if the request should be retried
"""
# Apply rate limiting
self.rate_limiter.wait_if_needed()
start_time = time.time()
response = None
error = None
try:
self.total_requests += 1
# Prepare request
request_headers = self.session.headers.copy()
if headers:
request_headers.update(headers)
print(f"Making {method} request to: {url}")
# Make request
if method.upper() == "GET":
response = self.session.get(
url,
params=params,
headers=request_headers,
timeout=self.timeout
)
elif method.upper() == "POST":
response = self.session.post(
url,
json=params,
headers=request_headers,
timeout=self.timeout
)
else:
raise ValueError(f"Unsupported HTTP method: {method}")
print(f"Response status: {response.status_code}")
response.raise_for_status()
self.successful_requests += 1
except requests.exceptions.RequestException as e:
error = str(e)
self.failed_requests += 1
print(f"Request failed: {error}")
except Exception as e:
error = f"Unexpected error: {str(e)}"
self.failed_requests += 1
print(f"Unexpected error: {error}")
# Calculate duration and log request
duration_ms = (time.time() - start_time) * 1000
self.logger.log_api_request(
provider=self.name,
url=url,
method=method.upper(),
status_code=response.status_code if response else None,
response_size=len(response.content) if response else None,
duration_ms=duration_ms,
error=error,
target_indicator=target_indicator
)
return response if error is None else None
# Retry on connection errors, timeouts, and 5xx server errors
if isinstance(exception, (requests.exceptions.ConnectionError,
requests.exceptions.Timeout)):
return True
if isinstance(exception, requests.exceptions.HTTPError):
if hasattr(exception, 'response') and exception.response:
# Retry on server errors (5xx) but not client errors (4xx)
return exception.response.status_code >= 500
return False
def log_relationship_discovery(self, source_node: str, target_node: str,
relationship_type: RelationshipType,

338
providers/dns_provider.py
View File

@ -0,0 +1,338 @@
"""
DNS resolution provider for DNSRecon.
Discovers domain relationships through DNS record analysis.
"""
import socket
import dns.resolver
import dns.reversename
from typing import List, Dict, Any, Tuple, Optional
from .base_provider import BaseProvider
from core.graph_manager import RelationshipType, NodeType
class DNSProvider(BaseProvider):
"""
Provider for standard DNS resolution and reverse DNS lookups.
Discovers domain-to-IP and IP-to-domain relationships through DNS records.
"""
def __init__(self):
"""Initialize DNS provider with appropriate rate limiting."""
super().__init__(
name="dns",
rate_limit=100, # DNS queries can be faster
timeout=10
)
# Configure DNS resolver
self.resolver = dns.resolver.Resolver()
self.resolver.timeout = 5
self.resolver.lifetime = 10
def get_name(self) -> str:
"""Return the provider name."""
return "dns"
def is_available(self) -> bool:
"""DNS is always available - no API key required."""
return True
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query DNS records for the domain to discover relationships.
Args:
domain: Domain to investigate
Returns:
List of relationships discovered from DNS analysis
"""
if not self._is_valid_domain(domain):
return []
relationships = []
# Query A records
relationships.extend(self._query_a_records(domain))
# Query AAAA records (IPv6)
relationships.extend(self._query_aaaa_records(domain))
# Query CNAME records
relationships.extend(self._query_cname_records(domain))
# Query MX records
relationships.extend(self._query_mx_records(domain))
# Query NS records
relationships.extend(self._query_ns_records(domain))
return relationships
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query reverse DNS for the IP address.
Args:
ip: IP address to investigate
Returns:
List of relationships discovered from reverse DNS
"""
if not self._is_valid_ip(ip):
return []
relationships = []
try:
# Perform reverse DNS lookup
reverse_name = dns.reversename.from_address(ip)
response = self.resolver.resolve(reverse_name, 'PTR')
for ptr_record in response:
hostname = str(ptr_record).rstrip('.')
if self._is_valid_domain(hostname):
raw_data = {
'query_type': 'PTR',
'ip_address': ip,
'hostname': hostname,
'ttl': response.ttl
}
relationships.append((
ip,
hostname,
RelationshipType.A_RECORD, # Reverse relationship
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="reverse_dns_lookup"
)
except Exception as e:
self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}")
return relationships
def _query_a_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query A records for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
response = self.resolver.resolve(domain, 'A')
for a_record in response:
ip_address = str(a_record)
raw_data = {
'query_type': 'A',
'domain': domain,
'ip_address': ip_address,
'ttl': response.ttl
}
relationships.append((
domain,
ip_address,
RelationshipType.A_RECORD,
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ip_address,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_a_record"
)
except Exception as e:
self.logger.logger.debug(f"A record query failed for {domain}: {e}")
return relationships
def _query_aaaa_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query AAAA records (IPv6) for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
response = self.resolver.resolve(domain, 'AAAA')
for aaaa_record in response:
ip_address = str(aaaa_record)
raw_data = {
'query_type': 'AAAA',
'domain': domain,
'ip_address': ip_address,
'ttl': response.ttl
}
relationships.append((
domain,
ip_address,
RelationshipType.A_RECORD, # Using same type for IPv6
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ip_address,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_aaaa_record"
)
except Exception as e:
self.logger.logger.debug(f"AAAA record query failed for {domain}: {e}")
return relationships
def _query_cname_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query CNAME records for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
response = self.resolver.resolve(domain, 'CNAME')
for cname_record in response:
target_domain = str(cname_record).rstrip('.')
if self._is_valid_domain(target_domain):
raw_data = {
'query_type': 'CNAME',
'source_domain': domain,
'target_domain': target_domain,
'ttl': response.ttl
}
relationships.append((
domain,
target_domain,
RelationshipType.CNAME_RECORD,
RelationshipType.CNAME_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=target_domain,
relationship_type=RelationshipType.CNAME_RECORD,
confidence_score=RelationshipType.CNAME_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_cname_record"
)
except Exception as e:
self.logger.logger.debug(f"CNAME record query failed for {domain}: {e}")
return relationships
def _query_mx_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query MX records for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
response = self.resolver.resolve(domain, 'MX')
for mx_record in response:
mx_host = str(mx_record.exchange).rstrip('.')
if self._is_valid_domain(mx_host):
raw_data = {
'query_type': 'MX',
'domain': domain,
'mx_host': mx_host,
'priority': mx_record.preference,
'ttl': response.ttl
}
relationships.append((
domain,
mx_host,
RelationshipType.MX_RECORD,
RelationshipType.MX_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=mx_host,
relationship_type=RelationshipType.MX_RECORD,
confidence_score=RelationshipType.MX_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_mx_record"
)
except Exception as e:
self.logger.logger.debug(f"MX record query failed for {domain}: {e}")
return relationships
def _query_ns_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query NS records for the domain."""
relationships = []
#if not DNS_AVAILABLE:
# return relationships
try:
response = self.resolver.resolve(domain, 'NS')
for ns_record in response:
ns_host = str(ns_record).rstrip('.')
if self._is_valid_domain(ns_host):
raw_data = {
'query_type': 'NS',
'domain': domain,
'ns_host': ns_host,
'ttl': response.ttl
}
relationships.append((
domain,
ns_host,
RelationshipType.NS_RECORD,
RelationshipType.NS_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ns_host,
relationship_type=RelationshipType.NS_RECORD,
confidence_score=RelationshipType.NS_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="dns_ns_record"
)
except Exception as e:
self.logger.logger.debug(f"NS record query failed for {domain}: {e}")
return relationships

299
providers/shodan_provider.py Normal file
View File

@ -0,0 +1,299 @@
"""
Shodan provider for DNSRecon.
Discovers IP relationships and infrastructure context through Shodan API.
"""
import json
from typing import List, Dict, Any, Tuple, Optional
from urllib.parse import quote
from .base_provider import BaseProvider
from core.graph_manager import RelationshipType
from config import config
class ShodanProvider(BaseProvider):
"""
Provider for querying Shodan API for IP address and hostname information.
Requires valid API key and respects Shodan's rate limits.
"""
def __init__(self):
"""Initialize Shodan provider with appropriate rate limiting."""
super().__init__(
name="shodan",
rate_limit=60, # Shodan API has various rate limits depending on plan
timeout=30
)
self.base_url = "https://api.shodan.io"
self.api_key = config.get_api_key('shodan')
def get_name(self) -> str:
"""Return the provider name."""
return "shodan"
def is_available(self) -> bool:
"""
Check if Shodan provider is available (has valid API key).
"""
return self.api_key is not None and len(self.api_key.strip()) > 0
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query Shodan for information about a domain.
Uses Shodan's hostname search to find associated IPs.
Args:
domain: Domain to investigate
Returns:
List of relationships discovered from Shodan data
"""
if not self._is_valid_domain(domain) or not self.is_available():
return []
relationships = []
try:
# Search for hostname in Shodan
search_query = f"hostname:{domain}"
url = f"{self.base_url}/shodan/host/search"
params = {
'key': self.api_key,
'query': search_query,
'minify': True # Get minimal data to reduce bandwidth
}
response = self.make_request(url, method="GET", params=params, target_indicator=domain)
if not response or response.status_code != 200:
return []
data = response.json()
if 'matches' not in data:
return []
# Process search results
for match in data['matches']:
ip_address = match.get('ip_str')
hostnames = match.get('hostnames', [])
if ip_address and domain in hostnames:
raw_data = {
'ip_address': ip_address,
'hostnames': hostnames,
'country': match.get('location', {}).get('country_name', ''),
'city': match.get('location', {}).get('city', ''),
'isp': match.get('isp', ''),
'org': match.get('org', ''),
'ports': match.get('ports', []),
'last_update': match.get('last_update', '')
}
relationships.append((
domain,
ip_address,
RelationshipType.A_RECORD, # Domain resolves to IP
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ip_address,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="shodan_hostname_search"
)
# Also create relationships to other hostnames on the same IP
for hostname in hostnames:
if hostname != domain and self._is_valid_domain(hostname):
hostname_raw_data = {
'shared_ip': ip_address,
'all_hostnames': hostnames,
'discovery_context': 'shared_hosting'
}
relationships.append((
domain,
hostname,
RelationshipType.PASSIVE_DNS, # Shared hosting relationship
0.6, # Lower confidence for shared hosting
hostname_raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=hostname,
relationship_type=RelationshipType.PASSIVE_DNS,
confidence_score=0.6,
raw_data=hostname_raw_data,
discovery_method="shodan_shared_hosting"
)
except json.JSONDecodeError as e:
self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
except Exception as e:
self.logger.logger.error(f"Error querying Shodan for domain {domain}: {e}")
return relationships
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query Shodan for information about an IP address.
Args:
ip: IP address to investigate
Returns:
List of relationships discovered from Shodan IP data
"""
if not self._is_valid_ip(ip) or not self.is_available():
return []
relationships = []
try:
# Query Shodan host information
url = f"{self.base_url}/shodan/host/{ip}"
params = {'key': self.api_key}
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
if not response or response.status_code != 200:
return []
data = response.json()
# Extract hostname relationships
hostnames = data.get('hostnames', [])
for hostname in hostnames:
if self._is_valid_domain(hostname):
raw_data = {
'ip_address': ip,
'hostname': hostname,
'country': data.get('country_name', ''),
'city': data.get('city', ''),
'isp': data.get('isp', ''),
'org': data.get('org', ''),
'asn': data.get('asn', ''),
'ports': data.get('ports', []),
'last_update': data.get('last_update', ''),
'os': data.get('os', '')
}
relationships.append((
ip,
hostname,
RelationshipType.A_RECORD, # IP resolves to hostname
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="shodan_host_lookup"
)
# Extract ASN relationship if available
asn = data.get('asn')
if asn:
asn_name = f"AS{asn}"
asn_raw_data = {
'ip_address': ip,
'asn': asn,
'isp': data.get('isp', ''),
'org': data.get('org', '')
}
relationships.append((
ip,
asn_name,
RelationshipType.ASN_MEMBERSHIP,
RelationshipType.ASN_MEMBERSHIP.default_confidence,
asn_raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=asn_name,
relationship_type=RelationshipType.ASN_MEMBERSHIP,
confidence_score=RelationshipType.ASN_MEMBERSHIP.default_confidence,
raw_data=asn_raw_data,
discovery_method="shodan_asn_lookup"
)
except json.JSONDecodeError as e:
self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
except Exception as e:
self.logger.logger.error(f"Error querying Shodan for IP {ip}: {e}")
return relationships
def search_by_organization(self, org_name: str) -> List[Dict[str, Any]]:
"""
Search Shodan for hosts belonging to a specific organization.
Args:
org_name: Organization name to search for
Returns:
List of host information dictionaries
"""
if not self.is_available():
return []
try:
search_query = f"org:\"{org_name}\""
url = f"{self.base_url}/shodan/host/search"
params = {
'key': self.api_key,
'query': search_query,
'minify': True
}
response = self.make_request(url, method="GET", params=params, target_indicator=org_name)
if response and response.status_code == 200:
data = response.json()
return data.get('matches', [])
except Exception as e:
self.logger.logger.error(f"Error searching Shodan by organization {org_name}: {e}")
return []
def get_host_services(self, ip: str) -> List[Dict[str, Any]]:
"""
Get service information for a specific IP address.
Args:
ip: IP address to query
Returns:
List of service information dictionaries
"""
if not self._is_valid_ip(ip) or not self.is_available():
return []
try:
url = f"{self.base_url}/shodan/host/{ip}"
params = {'key': self.api_key}
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
if response and response.status_code == 200:
data = response.json()
return data.get('data', []) # Service banners
except Exception as e:
self.logger.logger.error(f"Error getting Shodan services for IP {ip}: {e}")
return []

334
providers/virustotal_provider.py Normal file
View File

@ -0,0 +1,334 @@
"""
VirusTotal provider for DNSRecon.
Discovers domain relationships through passive DNS and URL analysis.
"""
import json
from typing import List, Dict, Any, Tuple, Optional
from .base_provider import BaseProvider
from core.graph_manager import RelationshipType
from config import config
class VirusTotalProvider(BaseProvider):
"""
Provider for querying VirusTotal API for passive DNS and domain reputation data.
Requires valid API key and strictly respects free tier rate limits.
"""
def __init__(self):
"""Initialize VirusTotal provider with strict rate limiting for free tier."""
super().__init__(
name="virustotal",
rate_limit=4, # Free tier: 4 requests per minute
timeout=30
)
self.base_url = "https://www.virustotal.com/vtapi/v2"
self.api_key = config.get_api_key('virustotal')
def get_name(self) -> str:
"""Return the provider name."""
return "virustotal"
def is_available(self) -> bool:
"""
Check if VirusTotal provider is available (has valid API key).
"""
return self.api_key is not None and len(self.api_key.strip()) > 0
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query VirusTotal for domain information including passive DNS.
Args:
domain: Domain to investigate
Returns:
List of relationships discovered from VirusTotal data
"""
if not self._is_valid_domain(domain) or not self.is_available():
return []
relationships = []
# Query domain report
domain_relationships = self._query_domain_report(domain)
relationships.extend(domain_relationships)
# Query passive DNS for the domain
passive_dns_relationships = self._query_passive_dns_domain(domain)
relationships.extend(passive_dns_relationships)
return relationships
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query VirusTotal for IP address information including passive DNS.
Args:
ip: IP address to investigate
Returns:
List of relationships discovered from VirusTotal IP data
"""
if not self._is_valid_ip(ip) or not self.is_available():
return []
relationships = []
# Query IP report
ip_relationships = self._query_ip_report(ip)
relationships.extend(ip_relationships)
# Query passive DNS for the IP
passive_dns_relationships = self._query_passive_dns_ip(ip)
relationships.extend(passive_dns_relationships)
return relationships
def _query_domain_report(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query VirusTotal domain report."""
relationships = []
try:
url = f"{self.base_url}/domain/report"
params = {
'apikey': self.api_key,
'domain': domain,
'allinfo': 1 # Get comprehensive information
}
response = self.make_request(url, method="GET", params=params, target_indicator=domain)
if not response or response.status_code != 200:
return []
data = response.json()
if data.get('response_code') != 1:
return []
# Extract resolved IPs
resolutions = data.get('resolutions', [])
for resolution in resolutions:
ip_address = resolution.get('ip_address')
last_resolved = resolution.get('last_resolved')
if ip_address and self._is_valid_ip(ip_address):
raw_data = {
'domain': domain,
'ip_address': ip_address,
'last_resolved': last_resolved,
'source': 'virustotal_domain_report'
}
relationships.append((
domain,
ip_address,
RelationshipType.PASSIVE_DNS,
RelationshipType.PASSIVE_DNS.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ip_address,
relationship_type=RelationshipType.PASSIVE_DNS,
confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
raw_data=raw_data,
discovery_method="virustotal_domain_resolution"
)
# Extract subdomains
subdomains = data.get('subdomains', [])
for subdomain in subdomains:
if subdomain != domain and self._is_valid_domain(subdomain):
raw_data = {
'parent_domain': domain,
'subdomain': subdomain,
'source': 'virustotal_subdomain_discovery'
}
relationships.append((
domain,
subdomain,
RelationshipType.PASSIVE_DNS,
0.7, # Medium-high confidence for subdomains
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=subdomain,
relationship_type=RelationshipType.PASSIVE_DNS,
confidence_score=0.7,
raw_data=raw_data,
discovery_method="virustotal_subdomain_discovery"
)
except json.JSONDecodeError as e:
self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
except Exception as e:
self.logger.logger.error(f"Error querying VirusTotal domain report for {domain}: {e}")
return relationships
def _query_ip_report(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query VirusTotal IP report."""
relationships = []
try:
url = f"{self.base_url}/ip-address/report"
params = {
'apikey': self.api_key,
'ip': ip
}
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
if not response or response.status_code != 200:
return []
data = response.json()
if data.get('response_code') != 1:
return []
# Extract resolved domains
resolutions = data.get('resolutions', [])
for resolution in resolutions:
hostname = resolution.get('hostname')
last_resolved = resolution.get('last_resolved')
if hostname and self._is_valid_domain(hostname):
raw_data = {
'ip_address': ip,
'hostname': hostname,
'last_resolved': last_resolved,
'source': 'virustotal_ip_report'
}
relationships.append((
ip,
hostname,
RelationshipType.PASSIVE_DNS,
RelationshipType.PASSIVE_DNS.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type=RelationshipType.PASSIVE_DNS,
confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
raw_data=raw_data,
discovery_method="virustotal_ip_resolution"
)
except json.JSONDecodeError as e:
self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
except Exception as e:
self.logger.logger.error(f"Error querying VirusTotal IP report for {ip}: {e}")
return relationships
def _query_passive_dns_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query VirusTotal passive DNS for domain."""
# Note: VirusTotal's passive DNS API might require a premium subscription
# This is a placeholder for the endpoint structure
return []
def _query_passive_dns_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query VirusTotal passive DNS for IP."""
# Note: VirusTotal's passive DNS API might require a premium subscription
# This is a placeholder for the endpoint structure
return []
def get_domain_reputation(self, domain: str) -> Dict[str, Any]:
"""
Get domain reputation information from VirusTotal.
Args:
domain: Domain to check reputation for
Returns:
Dictionary containing reputation data
"""
if not self._is_valid_domain(domain) or not self.is_available():
return {}
try:
url = f"{self.base_url}/domain/report"
params = {
'apikey': self.api_key,
'domain': domain
}
response = self.make_request(url, method="GET", params=params, target_indicator=domain)
if response and response.status_code == 200:
data = response.json()
if data.get('response_code') == 1:
return {
'positives': data.get('positives', 0),
'total': data.get('total', 0),
'scan_date': data.get('scan_date', ''),
'permalink': data.get('permalink', ''),
'reputation_score': self._calculate_reputation_score(data)
}
except Exception as e:
self.logger.logger.error(f"Error getting VirusTotal reputation for domain {domain}: {e}")
return {}
def get_ip_reputation(self, ip: str) -> Dict[str, Any]:
"""
Get IP reputation information from VirusTotal.
Args:
ip: IP address to check reputation for
Returns:
Dictionary containing reputation data
"""
if not self._is_valid_ip(ip) or not self.is_available():
return {}
try:
url = f"{self.base_url}/ip-address/report"
params = {
'apikey': self.api_key,
'ip': ip
}
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
if response and response.status_code == 200:
data = response.json()
if data.get('response_code') == 1:
return {
'positives': data.get('positives', 0),
'total': data.get('total', 0),
'scan_date': data.get('scan_date', ''),
'permalink': data.get('permalink', ''),
'reputation_score': self._calculate_reputation_score(data)
}
except Exception as e:
self.logger.logger.error(f"Error getting VirusTotal reputation for IP {ip}: {e}")
return {}
def _calculate_reputation_score(self, data: Dict[str, Any]) -> float:
"""Calculate a normalized reputation score (0.0 to 1.0)."""
positives = data.get('positives', 0)
total = data.get('total', 1) # Avoid division by zero
if total == 0:
return 1.0 # No data means neutral
# Score is inverse of detection ratio (lower detection = higher reputation)
return max(0.0, 1.0 - (positives / total))

3
requirements.txt
View File

@ -3,4 +3,5 @@ networkx>=3.1
requests>=2.31.0
python-dateutil>=2.8.2
Werkzeug>=2.3.7
urllib3>=2.0.0
urllib3>=2.0.0
dnspython>=2.4.2

308
static/css/main.css
View File

@ -64,6 +64,18 @@ body {
gap: 0.5rem;
}
.status-indicator.scanning {
animation: pulse 1.5s infinite;
}
.status-indicator.completed {
background-color: #00ff41;
}
.status-indicator.error {
background-color: #ff6b6b;
}
.status-dot {
width: 8px;
height: 8px;
@ -266,6 +278,7 @@ input[type="text"]:focus, select:focus {
background-color: #1a1a1a;
border: 1px solid #444;
overflow: hidden;
position: relative;
}
.progress-fill {
@ -274,6 +287,23 @@ input[type="text"]:focus, select:focus {
width: 0%;
transition: width 0.3s ease;
box-shadow: 0 0 5px rgba(0, 255, 65, 0.5);
position: relative;
}
.progress-fill::after {
content: '';
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: linear-gradient(90deg, transparent, rgba(255,255,255,0.2), transparent);
animation: shimmer 2s infinite;
}
@keyframes shimmer {
0% { transform: translateX(-100%); }
100% { transform: translateX(100%); }
}
/* Visualization Panel */
@ -292,6 +322,37 @@ input[type="text"]:focus, select:focus {
position: relative;
background-color: #1a1a1a;
border-top: 1px solid #444;
transition: height 0.3s ease;
}
.graph-container.expanded {
height: 700px;
}
.graph-controls {
position: absolute;
top: 10px;
right: 10px;
z-index: 10;
display: flex;
gap: 0.5rem;
}
.graph-control-btn {
background: rgba(42, 42, 42, 0.9);
border: 1px solid #555;
color: #c7c7c7;
padding: 0.5rem;
font-family: 'Roboto Mono', monospace;
font-size: 0.8rem;
cursor: pointer;
transition: all 0.3s ease;
}
.graph-control-btn:hover {
border-color: #00ff41;
color: #00ff41;
background: rgba(42, 42, 42, 1);
}
.graph-placeholder {
@ -333,6 +394,20 @@ input[type="text"]:focus, select:focus {
border-top: 1px solid #444;
}
.legend-section {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.legend-title {
font-size: 0.7rem;
color: #00ff41;
text-transform: uppercase;
font-weight: 500;
margin-bottom: 0.25rem;
}
.legend-item {
display: flex;
align-items: center;
@ -344,6 +419,7 @@ input[type="text"]:focus, select:focus {
width: 12px;
height: 12px;
border-radius: 50%;
border: 1px solid #444;
}
.legend-edge {
@ -353,10 +429,16 @@ input[type="text"]:focus, select:focus {
.legend-edge.high-confidence {
background-color: #00ff41;
box-shadow: 0 0 3px rgba(0, 255, 65, 0.5);
}
.legend-edge.medium-confidence {
background-color: #ff9900;
box-shadow: 0 0 3px rgba(255, 153, 0, 0.5);
}
.legend-edge.low-confidence {
background-color: #666666;
}
/* Provider Panel */
@ -375,9 +457,11 @@ input[type="text"]:focus, select:focus {
background-color: #1a1a1a;
border: 1px solid #444;
padding: 1rem;
display: flex;
justify-content: space-between;
align-items: center;
transition: border-color 0.3s ease;
}
.provider-item:hover {
border-color: #555;
}
.provider-name {
@ -389,6 +473,7 @@ input[type="text"]:focus, select:focus {
font-size: 0.8rem;
padding: 0.25rem 0.5rem;
border-radius: 3px;
font-weight: 500;
}
.provider-status.enabled {
@ -401,12 +486,78 @@ input[type="text"]:focus, select:focus {
color: #e0e0e0;
}
.provider-status.api-key-required {
background-color: #5c4c2c;
color: #e0e0e0;
}
.provider-stats {
font-size: 0.8rem;
color: #999;
display: grid;
grid-template-columns: 1fr 1fr;
gap: 0.5rem;
margin-top: 0.5rem;
}
.provider-stat {
display: flex;
justify-content: space-between;
}
.provider-stat-label {
color: #666;
}
.provider-stat-value {
color: #00ff41;
font-weight: 500;
}
.provider-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.node-info-popup {
position: fixed;
background: rgba(42, 42, 42, 0.95);
border: 1px solid #555;
padding: 1rem;
border-radius: 4px;
color: #c7c7c7;
font-family: 'Roboto Mono', monospace;
font-size: 0.8rem;
max-width: 300px;
z-index: 1001;
box-shadow: 0 4px 6px rgba(0,0,0,0.3);
}
.node-info-title {
color: #00ff41;
font-weight: bold;
margin-bottom: 0.5rem;
border-bottom: 1px solid #444;
padding-bottom: 0.25rem;
}
.node-info-detail {
margin-bottom: 0.25rem;
display: flex;
justify-content: space-between;
}
.node-info-label {
color: #999;
}
.node-info-value {
color: #c7c7c7;
font-weight: 500;
}
/* Footer */
.footer {
background-color: #0a0a0a;
@ -437,6 +588,7 @@ input[type="text"]:focus, select:focus {
width: 100%;
height: 100%;
background-color: rgba(0, 0, 0, 0.8);
animation: fadeIn 0.3s ease-out;
}
.modal-content {
@ -447,6 +599,18 @@ input[type="text"]:focus, select:focus {
max-width: 600px;
max-height: 80vh;
overflow-y: auto;
animation: slideInDown 0.3s ease-out;
}
@keyframes slideInDown {
from {
opacity: 0;
transform: translateY(-50px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
.modal-header {
@ -480,6 +644,12 @@ input[type="text"]:focus, select:focus {
padding: 1.5rem;
}
.modal-description {
color: #999;
margin-bottom: 1.5rem;
line-height: 1.6;
}
.detail-row {
display: flex;
justify-content: space-between;
@ -495,6 +665,7 @@ input[type="text"]:focus, select:focus {
.detail-value {
color: #c7c7c7;
word-break: break-word;
}
/* Responsive Design */
@ -552,6 +723,40 @@ input[type="text"]:focus, select:focus {
pointer-events: none;
}
.loading-overlay {
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(26, 26, 26, 0.8);
display: flex;
align-items: center;
justify-content: center;
z-index: 100;
}
.loading-spinner {
width: 40px;
height: 40px;
border: 3px solid #444;
border-top: 3px solid #00ff41;
border-radius: 50%;
animation: spin 1s linear infinite;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
.loading-text {
margin-top: 1rem;
color: #999;
font-family: 'Roboto Mono', monospace;
font-size: 0.9rem;
}
.error {
color: #ff6b6b !important;
border-color: #ff6b6b !important;
@ -598,4 +803,101 @@ input[type="text"]:focus, select:focus {
.amber {
color: #ff9900;
}
.apikey-section {
margin-bottom: 1.5rem;
}
.apikey-section label {
display: block;
margin-bottom: 0.5rem;
color: #c7c7c7;
font-size: 0.9rem;
font-weight: 500;
}
.apikey-section input[type="password"] {
width: 100%;
padding: 0.75rem;
background-color: #1a1a1a;
border: 1px solid #555;
color: #c7c7c7;
font-family: 'Roboto Mono', monospace;
font-size: 0.9rem;
transition: border-color 0.3s ease, box-shadow 0.3s ease;
}
.apikey-section input[type="password"]:focus {
outline: none;
border-color: #00ff41;
box-shadow: 0 0 5px rgba(0, 255, 65, 0.5);
}
.apikey-help {
font-size: 0.8rem;
color: #666;
margin-top: 0.25rem;
font-style: italic;
}
.message-container {
position: fixed;
top: 20px;
right: 20px;
z-index: 1002;
max-width: 400px;
}
.message-toast {
margin-bottom: 10px;
border-radius: 4px;
font-family: 'Roboto Mono', monospace;
font-size: 0.9rem;
box-shadow: 0 4px 6px rgba(0,0,0,0.3);
animation: slideInRight 0.3s ease-out;
}
.message-toast.success {
background: #2c5c34;
border-left: 4px solid #00ff41;
}
.message-toast.error {
background: #5c2c2c;
border-left: 4px solid #ff6b6b;
}
.message-toast.warning {
background: #5c4c2c;
border-left: 4px solid #ff9900;
}
.message-toast.info {
background: #2c3e5c;
border-left: 4px solid #00aaff;
}
@keyframes slideInRight {
from {
transform: translateX(100%);
opacity: 0;
}
to {
transform: translateX(0);
opacity: 1;
}
}
@keyframes slideOutRight {
from {
transform: translateX(0);
opacity: 1;
}
to {
transform: translateX(100%);
opacity: 0;
}
}

695
static/js/graph.js
View File

@ -1,6 +1,6 @@
/**
* Graph visualization module for DNSRecon
* Handles network graph rendering using vis.js
* Handles network graph rendering using vis.js with enhanced Phase 2 features
*/
class GraphManager {
@ -10,40 +10,57 @@ class GraphManager {
this.nodes = new vis.DataSet();
this.edges = new vis.DataSet();
this.isInitialized = false;
// Graph options for cybersecurity theme
this.currentLayout = 'physics';
this.nodeInfoPopup = null;
// Enhanced graph options for Phase 2
this.options = {
nodes: {
shape: 'dot',
size: 12,
size: 15,
font: {
size: 11,
size: 12,
color: '#c7c7c7',
face: 'Roboto Mono, monospace',
background: 'rgba(26, 26, 26, 0.8)',
strokeWidth: 1,
background: 'rgba(26, 26, 26, 0.9)',
strokeWidth: 2,
strokeColor: '#000000'
},
borderWidth: 2,
borderColor: '#444',
shadow: {
enabled: true,
color: 'rgba(0, 0, 0, 0.3)',
size: 3,
x: 1,
y: 1
color: 'rgba(0, 0, 0, 0.5)',
size: 5,
x: 2,
y: 2
},
scaling: {
min: 8,
max: 20
min: 10,
max: 30,
label: {
enabled: true,
min: 8,
max: 16
}
},
chosen: {
node: (values, id, selected, hovering) => {
values.borderColor = '#00ff41';
values.borderWidth = 3;
values.shadow = true;
values.shadowColor = 'rgba(0, 255, 65, 0.6)';
values.shadowSize = 10;
}
}
},
edges: {
width: 2,
color: {
color: '#444',
color: '#555',
highlight: '#00ff41',
hover: '#ff9900'
hover: '#ff9900',
inherit: false
},
font: {
size: 10,
@ -56,131 +73,218 @@ class GraphManager {
arrows: {
to: {
enabled: true,
scaleFactor: 0.8,
scaleFactor: 1,
type: 'arrow'
}
},
smooth: {
enabled: true,
type: 'dynamic',
roundness: 0.5
roundness: 0.6
},
shadow: {
enabled: true,
color: 'rgba(0, 0, 0, 0.2)',
size: 2,
color: 'rgba(0, 0, 0, 0.3)',
size: 3,
x: 1,
y: 1
},
chosen: {
edge: (values, id, selected, hovering) => {
values.color = '#00ff41';
values.width = 4;
values.shadow = true;
values.shadowColor = 'rgba(0, 255, 65, 0.4)';
}
}
},
physics: {
enabled: true,
stabilization: {
enabled: true,
iterations: 100,
updateInterval: 25
iterations: 150,
updateInterval: 50
},
barnesHut: {
gravitationalConstant: -2000,
centralGravity: 0.3,
springLength: 95,
springConstant: 0.04,
damping: 0.09,
avoidOverlap: 0.1
gravitationalConstant: -3000,
centralGravity: 0.4,
springLength: 120,
springConstant: 0.05,
damping: 0.1,
avoidOverlap: 0.2
},
maxVelocity: 50,
maxVelocity: 30,
minVelocity: 0.1,
solver: 'barnesHut',
timestep: 0.35,
timestep: 0.4,
adaptiveTimestep: true
},
interaction: {
hover: true,
hoverConnectedEdges: true,
selectConnectedEdges: true,
tooltipDelay: 200,
tooltipDelay: 300,
hideEdgesOnDrag: false,
hideNodesOnDrag: false
hideNodesOnDrag: false,
zoomView: true,
dragView: true,
multiselect: true
},
layout: {
improvedLayout: true
improvedLayout: true,
randomSeed: 2
}
};
this.setupEventHandlers();
this.createNodeInfoPopup();
}
/**
* Initialize the network graph
* Create floating node info popup
*/
createNodeInfoPopup() {
this.nodeInfoPopup = document.createElement('div');
this.nodeInfoPopup.className = 'node-info-popup';
this.nodeInfoPopup.style.display = 'none';
document.body.appendChild(this.nodeInfoPopup);
}
/**
* Initialize the network graph with enhanced features
*/
initialize() {
if (this.isInitialized) {
return;
}
try {
const data = {
nodes: this.nodes,
edges: this.edges
};
this.network = new vis.Network(this.container, data, this.options);
this.setupNetworkEvents();
this.isInitialized = true;
// Hide placeholder
const placeholder = this.container.querySelector('.graph-placeholder');
if (placeholder) {
placeholder.style.display = 'none';
}
console.log('Graph initialized successfully');
// Add graph controls
this.addGraphControls();
console.log('Enhanced graph initialized successfully');
} catch (error) {
console.error('Failed to initialize graph:', error);
this.showError('Failed to initialize visualization');
}
}
/**
* Setup network event handlers
* Add interactive graph controls
*/
addGraphControls() {
const controlsContainer = document.createElement('div');
controlsContainer.className = 'graph-controls';
controlsContainer.innerHTML = `
<button class="graph-control-btn" id="graph-fit" title="Fit to Screen">[FIT]</button>
<button class="graph-control-btn" id="graph-reset" title="Reset View">[RESET]</button>
<button class="graph-control-btn" id="graph-physics" title="Toggle Physics">[PHYSICS]</button>
<button class="graph-control-btn" id="graph-cluster" title="Cluster Nodes">[CLUSTER]</button>
`;
this.container.appendChild(controlsContainer);
// Add control event listeners
document.getElementById('graph-fit').addEventListener('click', () => this.fitView());
document.getElementById('graph-reset').addEventListener('click', () => this.resetView());
document.getElementById('graph-physics').addEventListener('click', () => this.togglePhysics());
document.getElementById('graph-cluster').addEventListener('click', () => this.toggleClustering());
}
/**
* Setup enhanced network event handlers
*/
setupNetworkEvents() {
if (!this.network) return;
// Node click event
// Node click event with enhanced details
this.network.on('click', (params) => {
if (params.nodes.length > 0) {
const nodeId = params.nodes[0];
this.showNodeDetails(nodeId);
this.highlightNodeConnections(nodeId);
} else {
this.clearHighlights();
}
});
// Hover events for tooltips
// Enhanced hover events
this.network.on('hoverNode', (params) => {
const nodeId = params.node;
const node = this.nodes.get(nodeId);
if (node) {
this.showTooltip(params.pointer.DOM, node);
this.showNodeInfoPopup(params.pointer.DOM, node);
this.highlightConnectedNodes(nodeId, true);
}
});
this.network.on('blurNode', () => {
this.hideTooltip();
this.network.on('blurNode', (params) => {
this.hideNodeInfoPopup();
this.clearHoverHighlights();
});
// Stabilization events
// Edge hover events
this.network.on('hoverEdge', (params) => {
const edgeId = params.edge;
const edge = this.edges.get(edgeId);
if (edge) {
this.showEdgeInfo(params.pointer.DOM, edge);
}
});
this.network.on('blurEdge', () => {
this.hideNodeInfoPopup();
});
// Double-click to focus on node
this.network.on('doubleClick', (params) => {
if (params.nodes.length > 0) {
const nodeId = params.nodes[0];
this.focusOnNode(nodeId);
}
});
// Context menu (right-click)
this.network.on('oncontext', (params) => {
params.event.preventDefault();
if (params.nodes.length > 0) {
this.showNodeContextMenu(params.pointer.DOM, params.nodes[0]);
}
});
// Stabilization events with progress
this.network.on('stabilizationProgress', (params) => {
const progress = params.iterations / params.total;
this.updateStabilizationProgress(progress);
});
this.network.on('stabilizationIterationsDone', () => {
this.onStabilizationComplete();
});
// Selection events
this.network.on('select', (params) => {
console.log('Selected nodes:', params.nodes);
console.log('Selected edges:', params.edges);
});
}
/**
* Update graph with new data
* Update graph with new data and enhanced processing
* @param {Object} graphData - Graph data from backend
*/
updateGraph(graphData) {
@ -188,37 +292,48 @@ class GraphManager {
console.warn('Invalid graph data received');
return;
}
try {
// Initialize if not already done
if (!this.isInitialized) {
this.initialize();
}
// Process nodes
// Process nodes with enhanced attributes
const processedNodes = graphData.nodes.map(node => this.processNode(node));
const processedEdges = graphData.edges.map(edge => this.processEdge(edge));
// Update datasets
this.nodes.clear();
this.edges.clear();
this.nodes.add(processedNodes);
this.edges.add(processedEdges);
// Fit the view if this is the first update or graph is small
if (processedNodes.length <= 10) {
setTimeout(() => this.fitView(), 500);
// Update datasets with animation
const existingNodeIds = this.nodes.getIds();
const existingEdgeIds = this.edges.getIds();
// Add new nodes with fade-in animation
const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id));
const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id));
// Update existing data
this.nodes.update(processedNodes);
this.edges.update(processedEdges);
// Highlight new additions briefly
if (newNodes.length > 0 || newEdges.length > 0) {
setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100);
}
console.log(`Graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges`);
// Auto-fit view for small graphs or first update
if (processedNodes.length <= 10 || existingNodeIds.length === 0) {
setTimeout(() => this.fitView(), 800);
}
console.log(`Enhanced graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
} catch (error) {
console.error('Failed to update graph:', error);
console.error('Failed to update enhanced graph:', error);
this.showError('Failed to update visualization');
}
}
/**
* Process node data for visualization
* Process node data with enhanced styling and metadata
* @param {Object} node - Raw node data
* @returns {Object} Processed node data
*/
@ -230,25 +345,32 @@ class GraphManager {
color: this.getNodeColor(node.type),
size: this.getNodeSize(node.type),
borderColor: this.getNodeBorderColor(node.type),
metadata: node.metadata || {}
shape: this.getNodeShape(node.type),
metadata: node.metadata || {},
type: node.type
};
// Add type-specific styling
if (node.type === 'domain') {
processedNode.shape = 'dot';
} else if (node.type === 'ip') {
processedNode.shape = 'square';
} else if (node.type === 'certificate') {
processedNode.shape = 'diamond';
} else if (node.type === 'asn') {
processedNode.shape = 'triangle';
// Add confidence-based styling
if (node.confidence) {
processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5));
}
// Add special styling for important nodes
if (this.isImportantNode(node)) {
processedNode.shadow = {
enabled: true,
color: 'rgba(0, 255, 65, 0.6)',
size: 10,
x: 2,
y: 2
};
}
return processedNode;
}
/**
* Process edge data for visualization
* Process edge data with enhanced styling and metadata
* @param {Object} edge - Raw edge data
* @returns {Object} Processed edge data
*/
@ -262,12 +384,29 @@ class GraphManager {
title: this.createEdgeTooltip(edge),
width: this.getEdgeWidth(confidence),
color: this.getEdgeColor(confidence),
dashes: confidence < 0.6 ? [5, 5] : false
dashes: confidence < 0.6 ? [5, 5] : false,
metadata: {
relationship_type: edge.label,
confidence_score: confidence,
source_provider: edge.source_provider,
discovery_timestamp: edge.discovery_timestamp
}
};
// Add animation for high-confidence edges
if (confidence >= 0.8) {
processedEdge.shadow = {
enabled: true,
color: 'rgba(0, 255, 65, 0.3)',
size: 5,
x: 1,
y: 1
};
}
return processedEdge;
}
/**
* Format node label for display
* @param {string} nodeId - Node identifier
@ -281,7 +420,7 @@ class GraphManager {
}
return nodeId;
}
/**
* Format edge label for display
* @param {string} relationshipType - Type of relationship
@ -290,7 +429,7 @@ class GraphManager {
*/
formatEdgeLabel(relationshipType, confidence) {
if (!relationshipType) return '';
const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○';
return `${relationshipType} ${confidenceText}`;
}
@ -309,7 +448,7 @@ class GraphManager {
};
return colors[nodeType] || '#ffffff';
}
/**
* Get node border color based on type
* @param {string} nodeType - Node type
@ -324,7 +463,7 @@ class GraphManager {
};
return borderColors[nodeType] || '#666666';
}
/**
* Get node size based on type
* @param {string} nodeType - Node type
@ -340,6 +479,21 @@ class GraphManager {
return sizes[nodeType] || 12;
}
/**
* Get enhanced node shape based on type
* @param {string} nodeType - Node type
* @returns {string} Shape name
*/
getNodeShape(nodeType) {
const shapes = {
'domain': 'dot',
'ip': 'square',
'certificate': 'diamond',
'asn': 'triangle'
};
return shapes[nodeType] || 'dot';
}
/**
* Get edge color based on confidence
* @param {number} confidence - Confidence score
@ -354,7 +508,7 @@ class GraphManager {
return '#666666'; // Low confidence - gray
}
}
/**
* Get edge width based on confidence
* @param {number} confidence - Confidence score
@ -411,7 +565,20 @@ class GraphManager {
tooltip += `</div>`;
return tooltip;
}
/**
* Determine if node is important based on connections or metadata
* @param {Object} node - Node data
* @returns {boolean} True if node is important
*/
isImportantNode(node) {
// Mark nodes as important based on criteria
if (node.type === 'domain' && node.id.includes('www.')) return false;
if (node.metadata && node.metadata.connection_count > 3) return true;
if (node.type === 'asn') return true;
return false;
}
/**
* Show node details in modal
* @param {string} nodeId - Node identifier
@ -419,7 +586,7 @@ class GraphManager {
showNodeDetails(nodeId) {
const node = this.nodes.get(nodeId);
if (!node) return;
// Trigger custom event for main application to handle
const event = new CustomEvent('nodeSelected', {
detail: { nodeId, node }
@ -428,22 +595,249 @@ class GraphManager {
}
/**
* Show tooltip
* Show enhanced node info popup
* @param {Object} position - Mouse position
* @param {Object} node - Node data
*/
showTooltip(position, node) {
// Tooltip is handled by vis.js automatically
// This method is for custom tooltip implementation if needed
showNodeInfoPopup(position, node) {
if (!this.nodeInfoPopup) return;
const html = `
<div class="node-info-title">${node.id}</div>
<div class="node-info-detail">
<span class="node-info-label">Type:</span>
<span class="node-info-value">${node.type || 'Unknown'}</span>
</div>
${node.metadata && Object.keys(node.metadata).length > 0 ?
'<div class="node-info-detail"><span class="node-info-label">Details:</span><span class="node-info-value">Click for more</span></div>' :
''}
`;
this.nodeInfoPopup.innerHTML = html;
this.nodeInfoPopup.style.display = 'block';
this.nodeInfoPopup.style.left = position.x + 15 + 'px';
this.nodeInfoPopup.style.top = position.y - 10 + 'px';
// Ensure popup stays in viewport
const rect = this.nodeInfoPopup.getBoundingClientRect();
if (rect.right > window.innerWidth) {
this.nodeInfoPopup.style.left = position.x - rect.width - 15 + 'px';
}
if (rect.bottom > window.innerHeight) {
this.nodeInfoPopup.style.top = position.y - rect.height + 10 + 'px';
}
}
/**
* Show edge information tooltip
* @param {Object} position - Mouse position
* @param {Object} edge - Edge data
*/
showEdgeInfo(position, edge) {
if (!this.nodeInfoPopup) return;
const confidence = edge.metadata ? edge.metadata.confidence_score : 0;
const provider = edge.metadata ? edge.metadata.source_provider : 'Unknown';
const html = `
<div class="node-info-title">${edge.metadata ? edge.metadata.relationship_type : 'Relationship'}</div>
<div class="node-info-detail">
<span class="node-info-label">Confidence:</span>
<span class="node-info-value">${(confidence * 100).toFixed(1)}%</span>
</div>
<div class="node-info-detail">
<span class="node-info-label">Provider:</span>
<span class="node-info-value">${provider}</span>
</div>
`;
this.nodeInfoPopup.innerHTML = html;
this.nodeInfoPopup.style.display = 'block';
this.nodeInfoPopup.style.left = position.x + 15 + 'px';
this.nodeInfoPopup.style.top = position.y - 10 + 'px';
}
/**
* Hide tooltip
* Hide node info popup
*/
hideTooltip() {
// Tooltip hiding is handled by vis.js automatically
hideNodeInfoPopup() {
if (this.nodeInfoPopup) {
this.nodeInfoPopup.style.display = 'none';
}
}
/**
* Highlight node connections
* @param {string} nodeId - Node to highlight
*/
highlightNodeConnections(nodeId) {
const connectedNodes = this.network.getConnectedNodes(nodeId);
const connectedEdges = this.network.getConnectedEdges(nodeId);
// Update node colors
const nodeUpdates = connectedNodes.map(id => ({
id: id,
borderColor: '#ff9900',
borderWidth: 3
}));
nodeUpdates.push({
id: nodeId,
borderColor: '#00ff41',
borderWidth: 4
});
// Update edge colors
const edgeUpdates = connectedEdges.map(id => ({
id: id,
color: { color: '#ff9900' },
width: 3
}));
this.nodes.update(nodeUpdates);
this.edges.update(edgeUpdates);
// Store for cleanup
this.highlightedElements = {
nodes: connectedNodes.concat([nodeId]),
edges: connectedEdges
};
}
/**
* Highlight connected nodes on hover
* @param {string} nodeId - Node ID
* @param {boolean} highlight - Whether to highlight or unhighlight
*/
highlightConnectedNodes(nodeId, highlight) {
const connectedNodes = this.network.getConnectedNodes(nodeId);
const connectedEdges = this.network.getConnectedEdges(nodeId);
if (highlight) {
// Dim all other elements
this.dimUnconnectedElements([nodeId, ...connectedNodes], connectedEdges);
}
}
/**
* Dim elements not connected to the specified nodes
* @param {Array} nodeIds - Node IDs to keep highlighted
* @param {Array} edgeIds - Edge IDs to keep highlighted
*/
dimUnconnectedElements(nodeIds, edgeIds) {
const allNodes = this.nodes.get();
const allEdges = this.edges.get();
const nodeUpdates = allNodes.map(node => ({
id: node.id,
opacity: nodeIds.includes(node.id) ? 1 : 0.3
}));
const edgeUpdates = allEdges.map(edge => ({
id: edge.id,
opacity: edgeIds.includes(edge.id) ? 1 : 0.1
}));
this.nodes.update(nodeUpdates);
this.edges.update(edgeUpdates);
}
/**
* Clear all highlights
*/
clearHighlights() {
if (this.highlightedElements) {
// Reset highlighted nodes
const nodeUpdates = this.highlightedElements.nodes.map(id => {
const originalNode = this.nodes.get(id);
return {
id: id,
borderColor: this.getNodeBorderColor(originalNode.type),
borderWidth: 2
};
});
// Reset highlighted edges
const edgeUpdates = this.highlightedElements.edges.map(id => {
const originalEdge = this.edges.get(id);
return {
id: id,
color: this.getEdgeColor(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5),
width: this.getEdgeWidth(originalEdge.metadata ? originalEdge.metadata.confidence_score : 0.5)
};
});
this.nodes.update(nodeUpdates);
this.edges.update(edgeUpdates);
this.highlightedElements = null;
}
}
/**
* Clear hover highlights
*/
clearHoverHighlights() {
const allNodes = this.nodes.get();
const allEdges = this.edges.get();
const nodeUpdates = allNodes.map(node => ({ id: node.id, opacity: 1 }));
const edgeUpdates = allEdges.map(edge => ({ id: edge.id, opacity: 1 }));
this.nodes.update(nodeUpdates);
this.edges.update(edgeUpdates);
}
/**
* Highlight newly added elements
* @param {Array} newNodes - New nodes
* @param {Array} newEdges - New edges
*/
highlightNewElements(newNodes, newEdges) {
// Briefly highlight new nodes
const nodeHighlights = newNodes.map(node => ({
id: node.id,
borderColor: '#00ff41',
borderWidth: 4,
shadow: {
enabled: true,
color: 'rgba(0, 255, 65, 0.8)',
size: 15,
x: 2,
y: 2
}
}));
// Briefly highlight new edges
const edgeHighlights = newEdges.map(edge => ({
id: edge.id,
color: '#00ff41',
width: 4
}));
this.nodes.update(nodeHighlights);
this.edges.update(edgeHighlights);
// Reset after animation
setTimeout(() => {
const nodeResets = newNodes.map(node => ({
id: node.id,
borderColor: this.getNodeBorderColor(node.type),
borderWidth: 2,
shadow: node.shadow || { enabled: false }
}));
const edgeResets = newEdges.map(edge => ({
id: edge.id,
color: this.getEdgeColor(edge.metadata ? edge.metadata.confidence_score : 0.5),
width: this.getEdgeWidth(edge.metadata ? edge.metadata.confidence_score : 0.5)
}));
this.nodes.update(nodeResets);
this.edges.update(edgeResets);
}, 2000);
}
/**
* Update stabilization progress
* @param {number} progress - Progress value (0-1)
@ -452,14 +846,71 @@ class GraphManager {
// Could show a progress indicator if needed
console.log(`Graph stabilization: ${(progress * 100).toFixed(1)}%`);
}
/**
* Handle stabilization completion
*/
onStabilizationComplete() {
console.log('Graph stabilization complete');
}
/**
* Focus view on specific node
* @param {string} nodeId - Node to focus on
*/
focusOnNode(nodeId) {
const nodePosition = this.network.getPositions([nodeId]);
if (nodePosition[nodeId]) {
this.network.moveTo({
position: nodePosition[nodeId],
scale: 1.5,
animation: {
duration: 1000,
easingFunction: 'easeInOutQuart'
}
});
}
}
/**
* Toggle physics simulation
*/
togglePhysics() {
const currentPhysics = this.network.physics.physicsEnabled;
this.network.setOptions({ physics: !currentPhysics });
const button = document.getElementById('graph-physics');
if (button) {
button.textContent = currentPhysics ? '[PHYSICS OFF]' : '[PHYSICS ON]';
button.style.color = currentPhysics ? '#ff9900' : '#00ff41';
}
}
/**
* Toggle node clustering
*/
toggleClustering() {
// Simple clustering by node type
const clusterOptionsByType = {
joinCondition: (childOptions) => {
return childOptions.type === 'domain';
},
clusterNodeProperties: {
id: 'domain-cluster',
borderWidth: 3,
shape: 'database',
label: 'Domains',
color: '#00ff41'
}
};
if (this.network.clustering.isCluster('domain-cluster')) {
this.network.clustering.openCluster('domain-cluster');
} else {
this.network.clustering.cluster(clusterOptionsByType);
}
}
/**
* Fit the view to show all nodes
*/
@ -473,7 +924,7 @@ class GraphManager {
});
}
}
/**
* Reset the view to initial state
*/
@ -489,21 +940,21 @@ class GraphManager {
});
}
}
/**
* Clear the graph
*/
clear() {
this.nodes.clear();
this.edges.clear();
// Show placeholder
const placeholder = this.container.querySelector('.graph-placeholder');
if (placeholder) {
placeholder.style.display = 'flex';
}
}
/**
* Show error message
* @param {string} message - Error message
@ -515,25 +966,7 @@ class GraphManager {
placeholder.style.color = '#ff6b6b';
}
}
/**
* Setup control event handlers
*/
setupEventHandlers() {
// Reset view button
document.addEventListener('DOMContentLoaded', () => {
const resetBtn = document.getElementById('reset-view');
if (resetBtn) {
resetBtn.addEventListener('click', () => this.resetView());
}
const fitBtn = document.getElementById('fit-view');
if (fitBtn) {
fitBtn.addEventListener('click', () => this.fitView());
}
});
}
/**
* Get network statistics
* @returns {Object} Statistics object
@ -545,7 +978,7 @@ class GraphManager {
//isStabilized: this.network ? this.network.isStabilized() : false
};
}
/**
* Export graph as image (if needed for future implementation)
* @param {string} format - Image format ('png', 'jpeg')
@ -553,7 +986,7 @@ class GraphManager {
*/
exportAsImage(format = 'png') {
if (!this.network) return null;
// This would require additional vis.js functionality
// Placeholder for future implementation
console.log('Image export not yet implemented');

184
static/js/main.js
View File

@ -447,11 +447,19 @@ class DNSReconApp {
try {
console.log('Updating status display...');
// Update status text
// Update status text with animation
if (this.elements.scanStatus) {
this.elements.scanStatus.textContent = this.formatStatus(status.status);
console.log('Updated status display:', status.status);
const formattedStatus = this.formatStatus(status.status);
if (this.elements.scanStatus.textContent !== formattedStatus) {
this.elements.scanStatus.textContent = formattedStatus;
this.elements.scanStatus.classList.add('fade-in');
setTimeout(() => this.elements.scanStatus.classList.remove('fade-in'), 300);
}
// Add status-specific classes for styling
this.elements.scanStatus.className = `status-value status-${status.status}`;
}
if (this.elements.targetDisplay) {
this.elements.targetDisplay.textContent = status.target_domain || 'None';
}
@ -465,9 +473,16 @@ class DNSReconApp {
this.elements.indicatorsDisplay.textContent = status.indicators_processed || 0;
}
// Update progress bar
// Update progress bar with smooth animation
if (this.elements.progressFill) {
this.elements.progressFill.style.width = `${status.progress_percentage}%`;
// Add pulsing animation for active scans
if (status.status === 'running') {
this.elements.progressFill.parentElement.classList.add('scanning');
} else {
this.elements.progressFill.parentElement.classList.remove('scanning');
}
}
// Update session ID
@ -492,12 +507,16 @@ class DNSReconApp {
case 'running':
this.setUIState('scanning');
this.showSuccess('Scan is running');
// Reset polling frequency for active scans
this.pollFrequency = 2000;
this.updateConnectionStatus('active');
break;
case 'completed':
this.setUIState('completed');
this.stopPolling();
this.showSuccess('Scan completed successfully');
this.updateConnectionStatus('completed');
// Force a final graph update
console.log('Scan completed - forcing final graph update');
setTimeout(() => this.updateGraph(), 100);
@ -507,20 +526,54 @@ class DNSReconApp {
this.setUIState('failed');
this.stopPolling();
this.showError('Scan failed');
this.updateConnectionStatus('error');
break;
case 'stopped':
this.setUIState('stopped');
this.stopPolling();
this.showSuccess('Scan stopped');
this.updateConnectionStatus('stopped');
break;
case 'idle':
this.setUIState('idle');
this.stopPolling();
this.updateConnectionStatus('idle');
break;
}
}
/**
* Update connection status indicator
* @param {string} status - Connection status
*/
updateConnectionStatus(status) {
if (!this.elements.connectionStatus) return;
const statusColors = {
'idle': '#c7c7c7',
'active': '#00ff41',
'completed': '#00aa2e',
'stopped': '#ff9900',
'error': '#ff6b6b'
};
this.elements.connectionStatus.style.backgroundColor = statusColors[status] || '#c7c7c7';
const statusText = this.elements.connectionStatus.parentElement?.querySelector('.status-text');
if (statusText) {
const statusTexts = {
'idle': 'System Ready',
'active': 'Scanning Active',
'completed': 'Scan Complete',
'stopped': 'Scan Stopped',
'error': 'Connection Error'
};
statusText.textContent = statusTexts[status] || 'System Online';
}
}
/**
* Set UI state based on scan status
@ -532,10 +585,17 @@ class DNSReconApp {
switch (state) {
case 'scanning':
this.isScanning = true;
if (this.elements.startScan) this.elements.startScan.disabled = true;
if (this.elements.stopScan) this.elements.stopScan.disabled = false;
if (this.elements.startScan) {
this.elements.startScan.disabled = true;
this.elements.startScan.classList.add('loading');
}
if (this.elements.stopScan) {
this.elements.stopScan.disabled = false;
this.elements.stopScan.classList.remove('loading');
}
if (this.elements.targetDomain) this.elements.targetDomain.disabled = true;
if (this.elements.maxDepth) this.elements.maxDepth.disabled = true;
if (this.elements.configureApiKeys) this.elements.configureApiKeys.disabled = true;
break;
case 'idle':
@ -543,10 +603,17 @@ class DNSReconApp {
case 'failed':
case 'stopped':
this.isScanning = false;
if (this.elements.startScan) this.elements.startScan.disabled = false;
if (this.elements.stopScan) this.elements.stopScan.disabled = true;
if (this.elements.startScan) {
this.elements.startScan.disabled = false;
this.elements.startScan.classList.remove('loading');
}
if (this.elements.stopScan) {
this.elements.stopScan.disabled = true;
this.elements.stopScan.classList.add('loading');
}
if (this.elements.targetDomain) this.elements.targetDomain.disabled = false;
if (this.elements.maxDepth) this.elements.maxDepth.disabled = false;
if (this.elements.configureApiKeys) this.elements.configureApiKeys.disabled = false;
break;
}
}
@ -580,20 +647,42 @@ class DNSReconApp {
for (const [name, info] of Object.entries(providers)) {
const providerItem = document.createElement('div');
providerItem.className = 'provider-item';
providerItem.className = 'provider-item fade-in';
const status = info.enabled ? 'enabled' : 'disabled';
const statusClass = info.enabled ? 'enabled' : 'disabled';
let statusClass = 'disabled';
let statusText = 'Disabled';
if (info.enabled) {
statusClass = 'enabled';
statusText = 'Enabled';
} else if (info.requires_api_key) {
statusClass = 'api-key-required';
statusText = 'API Key Required';
}
providerItem.innerHTML = `
<div>
<div class="provider-header">
<div class="provider-name">${name.toUpperCase()}</div>
<div class="provider-stats">
Requests: ${info.statistics.total_requests || 0} |
Success Rate: ${(info.statistics.success_rate || 0).toFixed(1)}%
<div class="provider-status ${statusClass}">${statusText}</div>
</div>
<div class="provider-stats">
<div class="provider-stat">
<span class="provider-stat-label">Requests:</span>
<span class="provider-stat-value">${info.statistics.total_requests || 0}</span>
</div>
<div class="provider-stat">
<span class="provider-stat-label">Success Rate:</span>
<span class="provider-stat-value">${(info.statistics.success_rate || 0).toFixed(1)}%</span>
</div>
<div class="provider-stat">
<span class="provider-stat-label">Relationships:</span>
<span class="provider-stat-value">${info.statistics.relationships_found || 0}</span>
</div>
<div class="provider-stat">
<span class="provider-stat-label">Rate Limit:</span>
<span class="provider-stat-value">${info.rate_limit}/min</span>
</div>
</div>
<div class="provider-status ${statusClass}">${status}</div>
`;
this.elements.providerList.appendChild(providerItem);
@ -614,7 +703,7 @@ class DNSReconApp {
let detailsHtml = '';
detailsHtml += `<div class="detail-row"><span class="detail-label">Identifier:</span><span class="detail-value">${nodeId}</span></div>`;
detailsHtml += `<div class="detail-row"><span class="detail-label">Type:</span><span class="detail-value">${node.metadata.type || 'Unknown'}</span></div>`;
detailsHtml += `<div class="detail-row"><span class="detail-label">Type:</span><span class="detail-value">${node.metadata.type || node.type || 'Unknown'}</span></div>`;
if (node.metadata) {
for (const [key, value] of Object.entries(node.metadata)) {
@ -624,6 +713,12 @@ class DNSReconApp {
}
}
// Add timestamps if available
if (node.added_timestamp) {
const addedDate = new Date(node.added_timestamp);
detailsHtml += `<div class="detail-row"><span class="detail-label">Added:</span><span class="detail-value">${addedDate.toLocaleString()}</span></div>`;
}
if (this.elements.modalDetails) {
this.elements.modalDetails.innerHTML = detailsHtml;
}
@ -645,12 +740,24 @@ class DNSReconApp {
* @returns {boolean} True if data has changed
*/
hasGraphChanged(graphData) {
// Simple check based on node and edge counts
// Simple check based on node and edge counts and timestamps
const currentStats = this.graphManager.getStatistics();
const newNodeCount = graphData.nodes ? graphData.nodes.length : 0;
const newEdgeCount = graphData.edges ? graphData.edges.length : 0;
const changed = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount;
// Check if counts changed
const countsChanged = currentStats.nodeCount !== newNodeCount || currentStats.edgeCount !== newEdgeCount;
// Also check if we have new timestamp data
const hasNewTimestamp = graphData.statistics &&
graphData.statistics.last_modified &&
graphData.statistics.last_modified !== this.lastGraphTimestamp;
if (hasNewTimestamp) {
this.lastGraphTimestamp = graphData.statistics.last_modified;
}
const changed = countsChanged || hasNewTimestamp;
console.log(`Graph change check: Current(${currentStats.nodeCount}n, ${currentStats.edgeCount}e) vs New(${newNodeCount}n, ${newEdgeCount}e) = ${changed}`);
@ -816,6 +923,10 @@ class DNSReconApp {
this.showMessage(message, 'info');
}
showWarning(message) {
this.showMessage(message, 'warning');
}
/**
* Show error message
* @param {string} message - Error message
@ -828,13 +939,7 @@ class DNSReconApp {
* Show connection error
*/
showConnectionError() {
if (this.elements.connectionStatus) {
this.elements.connectionStatus.style.backgroundColor = '#ff6b6b';
}
const statusText = this.elements.connectionStatus?.parentElement?.querySelector('.status-text');
if (statusText) {
statusText.textContent = 'Connection Error';
}
this.updateConnectionStatus('error');
}
/**
@ -848,24 +953,12 @@ class DNSReconApp {
// Create message element
const messageElement = document.createElement('div');
messageElement.className = `message-toast message-${type}`;
messageElement.style.cssText = `
background: ${this.getMessageColor(type)};
color: #fff;
padding: 12px 20px;
margin-bottom: 10px;
border-radius: 4px;
font-family: 'Roboto Mono', monospace;
font-size: 0.9rem;
box-shadow: 0 4px 6px rgba(0,0,0,0.3);
border-left: 4px solid ${this.getMessageBorderColor(type)};
animation: slideInRight 0.3s ease-out;
`;
messageElement.innerHTML = `
<div style="display: flex; justify-content: space-between; align-items: center;">
<span>${message}</span>
<div style="display: flex; justify-content: space-between; align-items: center; padding: 12px 20px;">
<span style="flex: 1;">${message}</span>
<button onclick="this.parentElement.parentElement.remove()"
style="background: none; border: none; color: #fff; cursor: pointer; font-size: 16px; margin-left: 10px;">×</button>
style="background: none; border: none; color: #fff; cursor: pointer; font-size: 16px; margin-left: 10px; opacity: 0.7;">×</button>
</div>
`;
@ -888,13 +981,8 @@ class DNSReconApp {
}
// Update connection status to show activity
if (type === 'success' && this.elements.connectionStatus) {
this.elements.connectionStatus.style.backgroundColor = '#00ff41';
setTimeout(() => {
if (this.elements.connectionStatus) {
this.elements.connectionStatus.style.backgroundColor = '#00ff41';
}
}, 2000);
if (type === 'success' && this.consecutiveErrors === 0) {
this.updateConnectionStatus(this.isScanning ? 'active' : 'idle');
}
}

4
templates/index.html
View File

@ -63,6 +63,10 @@
<span class="btn-icon">[EXPORT]</span>
<span>Download Results</span>
</button>
<button id="configure-api-keys" class="btn btn-secondary">
<span class="btn-icon">[API]</span>
<span>Configure API Keys</span>
</button>
</div>
</div>
</section>