mstoeck3/dnsrecon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

it

Browse Source
This commit is contained in:
overcuriousity 2025-09-13 23:45:36 +02:00
parent 41d556e2ce
commit b7a57f1552
11 changed files with 125 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
app.py
View File

@ -1,7 +1,6 @@
""" """
Flask application entry point for DNSRecon web interface. Flask application entry point for DNSRecon web interface.
Provides REST API endpoints and serves the web interface with user session support. Provides REST API endpoints and serves the web interface with user session support.
Enhanced with better session debugging and isolation.
""" """
import json import json
@ -20,7 +19,7 @@ app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=2) # 2 hour session
def get_user_scanner(): def get_user_scanner():
""" """
Enhanced user scanner retrieval with better error handling and debugging. User scanner retrieval with better error handling and debugging.
""" """
# Get current Flask session info for debugging # Get current Flask session info for debugging
current_flask_session_id = session.get('dnsrecon_session_id') current_flask_session_id = session.get('dnsrecon_session_id')
@ -184,7 +183,7 @@ def stop_scan():
if not scanner.session_id: if not scanner.session_id:
scanner.session_id = user_session_id scanner.session_id = user_session_id
# Use the enhanced stop mechanism # Use the stop mechanism
success = scanner.stop_scan() success = scanner.stop_scan()
# Also set the Redis stop signal directly for extra reliability # Also set the Redis stop signal directly for extra reliability
@ -203,7 +202,7 @@ def stop_scan():
'message': 'Scan stop requested - termination initiated', 'message': 'Scan stop requested - termination initiated',
'user_session_id': user_session_id, 'user_session_id': user_session_id,
'scanner_status': scanner.status, 'scanner_status': scanner.status,
'stop_method': 'enhanced_cross_process' 'stop_method': 'cross_process'
}) })
except Exception as e: except Exception as e:
@ -217,7 +216,7 @@ def stop_scan():
@app.route('/api/scan/status', methods=['GET']) @app.route('/api/scan/status', methods=['GET'])
def get_scan_status(): def get_scan_status():
"""Get current scan status with enhanced error handling.""" """Get current scan status with error handling."""
try: try:
# Get user-specific scanner # Get user-specific scanner
user_session_id, scanner = get_user_scanner() user_session_id, scanner = get_user_scanner()
@ -279,7 +278,7 @@ def get_scan_status():
@app.route('/api/graph', methods=['GET']) @app.route('/api/graph', methods=['GET'])
def get_graph_data(): def get_graph_data():
"""Get current graph data with enhanced error handling.""" """Get current graph data with error handling."""
try: try:
# Get user-specific scanner # Get user-specific scanner
user_session_id, scanner = get_user_scanner() user_session_id, scanner = get_user_scanner()
@ -524,7 +523,7 @@ def list_sessions():
@app.route('/api/health', methods=['GET']) @app.route('/api/health', methods=['GET'])
def health_check(): def health_check():
"""Health check endpoint with enhanced Phase 2 information.""" """Health check endpoint."""
try: try:
# Get session stats # Get session stats
session_stats = session_manager.get_statistics() session_stats = session_manager.get_statistics()
@ -540,7 +539,7 @@ def health_check():
'concurrent_processing': True, 'concurrent_processing': True,
'real_time_updates': True, 'real_time_updates': True,
'api_key_management': True, 'api_key_management': True,
'enhanced_visualization': True, 'visualization': True,
'retry_logic': True, 'retry_logic': True,
'user_sessions': True, 'user_sessions': True,
'session_isolation': True 'session_isolation': True

17
core/__init__.py
View File

@ -1,28 +1,25 @@
""" """
Core modules for DNSRecon passive reconnaissance tool. Core modules for DNSRecon passive reconnaissance tool.
Contains graph management, scanning orchestration, and forensic logging. Contains graph management, scanning orchestration, and forensic logging.
Phase 2: Enhanced with concurrent processing and real-time capabilities.
""" """
from .graph_manager import GraphManager, NodeType, RelationshipType from .graph_manager import GraphManager, NodeType
from .scanner import Scanner, ScanStatus # Remove 'scanner' global instance from .scanner import Scanner, ScanStatus
from .logger import ForensicLogger, get_forensic_logger, new_session from .logger import ForensicLogger, get_forensic_logger, new_session
from .session_manager import session_manager # Add session manager from .session_manager import session_manager
from .session_config import SessionConfig, create_session_config # Add session config from .session_config import SessionConfig, create_session_config
__all__ = [ __all__ = [
'GraphManager', 'GraphManager',
'NodeType', 'NodeType',
'RelationshipType',
'Scanner', 'Scanner',
'ScanStatus', 'ScanStatus',
# 'scanner', # Remove this - no more global scanner
'ForensicLogger', 'ForensicLogger',
'get_forensic_logger', 'get_forensic_logger',
'new_session', 'new_session',
'session_manager', # Add this 'session_manager',
'SessionConfig', # Add this 'SessionConfig',
'create_session_config' # Add this 'create_session_config'
] ]
__version__ = "1.0.0-phase2" __version__ = "1.0.0-phase2"

53
core/graph_manager.py
View File

@ -22,28 +22,6 @@ class NodeType(Enum):
return self.value return self.value
class RelationshipType(Enum):
"""Enumeration of supported relationship types with confidence scores."""
SAN_CERTIFICATE = ("san", 0.9)
A_RECORD = ("a_record", 0.8)
AAAA_RECORD = ("aaaa_record", 0.8)
CNAME_RECORD = ("cname", 0.8)
MX_RECORD = ("mx_record", 0.7)
NS_RECORD = ("ns_record", 0.7)
PTR_RECORD = ("ptr_record", 0.8)
SOA_RECORD = ("soa_record", 0.7)
PASSIVE_DNS = ("passive_dns", 0.6)
ASN_MEMBERSHIP = ("asn", 0.7)
CORRELATED_TO = ("correlated_to", 0.9)
def __init__(self, relationship_name: str, default_confidence: float):
self.relationship_name = relationship_name
self.default_confidence = default_confidence
def __repr__(self):
return self.relationship_name
class GraphManager: class GraphManager:
""" """
Thread-safe graph manager for DNSRecon infrastructure mapping. Thread-safe graph manager for DNSRecon infrastructure mapping.
@ -180,19 +158,23 @@ class GraphManager:
for corr in correlations: for corr in correlations:
value = corr['value'] value = corr['value']
# FIXED: Check if the correlation value contains an existing node ID.
found_major_node_id = None found_major_node_id = None
if isinstance(value, str): if isinstance(value, str):
# Check if the value contains ANY existing major node ID from the entire graph
for existing_node in self.graph.nodes(): for existing_node in self.graph.nodes():
if existing_node in value: # Ensure the existing_node is a major type (domain/ip/asn) and is a substring of the correlation value
if (self.graph.nodes[existing_node].get('type') in [NodeType.DOMAIN.value, NodeType.IP.value, NodeType.ASN.value] and
existing_node in value):
found_major_node_id = existing_node found_major_node_id = existing_node
break break # Found a major node, no need to check further
if found_major_node_id: if found_major_node_id:
# An existing major node is part of the value; link to it directly. # An existing major node is part of the value; link to it directly.
for c_node_id in set(corr['nodes']): for c_node_id in set(corr['nodes']):
if self.graph.has_node(c_node_id) and c_node_id != found_major_node_id: if self.graph.has_node(c_node_id) and c_node_id != found_major_node_id:
self.add_edge(c_node_id, found_major_node_id, RelationshipType.CORRELATED_TO) attribute = corr['sources'][0]['path'].split('.')[-1]
relationship_type = f"c_{attribute}"
self.add_edge(c_node_id, found_major_node_id, relationship_type, confidence_score=0.9)
continue # Skip creating a redundant correlation node continue # Skip creating a redundant correlation node
# Proceed to create a new correlation node if no major node was found. # Proceed to create a new correlation node if no major node was found.
@ -211,22 +193,29 @@ class GraphManager:
existing_meta['sources'] = [{'node_id': nid, 'path': p} for nid, p in existing_sources] existing_meta['sources'] = [{'node_id': nid, 'path': p} for nid, p in existing_sources]
for c_node_id in set(corr['nodes']): for c_node_id in set(corr['nodes']):
self.add_edge(c_node_id, correlation_node_id, RelationshipType.CORRELATED_TO) attribute = corr['sources'][0]['path'].split('.')[-1]
relationship_type = f"c_{attribute}"
self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9)
self._update_correlation_index(node_id, attributes) self._update_correlation_index(node_id, attributes)
self.last_modified = datetime.now(timezone.utc).isoformat() self.last_modified = datetime.now(timezone.utc).isoformat()
return is_new_node return is_new_node
def add_edge(self, source_id: str, target_id: str, relationship_type: RelationshipType, def add_edge(self, source_id: str, target_id: str, relationship_type: str,
confidence_score: Optional[float] = None, source_provider: str = "unknown", confidence_score: float = 0.5, source_provider: str = "unknown",
raw_data: Optional[Dict[str, Any]] = None) -> bool: raw_data: Optional[Dict[str, Any]] = None) -> bool:
"""Add or update an edge between two nodes, ensuring nodes exist.""" """Add or update an edge between two nodes, ensuring nodes exist."""
# LOGIC FIX: Ensure both source and target nodes exist before adding an edge.
if not self.graph.has_node(source_id) or not self.graph.has_node(target_id): if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
return False return False
new_confidence = confidence_score or relationship_type.default_confidence new_confidence = confidence_score
if relationship_type.startswith("c_"):
edge_label = relationship_type
else:
edge_label = f"{source_provider}_{relationship_type}"
if self.graph.has_edge(source_id, target_id): if self.graph.has_edge(source_id, target_id):
# If edge exists, update confidence if the new score is higher. # If edge exists, update confidence if the new score is higher.
if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0): if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0):
@ -237,7 +226,7 @@ class GraphManager:
# Add a new edge with all attributes. # Add a new edge with all attributes.
self.graph.add_edge(source_id, target_id, self.graph.add_edge(source_id, target_id,
relationship_type=relationship_type.relationship_name, relationship_type=edge_label,
confidence_score=new_confidence, confidence_score=new_confidence,
source_provider=source_provider, source_provider=source_provider,
discovery_timestamp=datetime.now(timezone.utc).isoformat(), discovery_timestamp=datetime.now(timezone.utc).isoformat(),

24
core/scanner.py
View File

@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError,
from collections import defaultdict, deque from collections import defaultdict, deque
from datetime import datetime, timezone from datetime import datetime, timezone
from core.graph_manager import GraphManager, NodeType, RelationshipType from core.graph_manager import GraphManager, NodeType
from core.logger import get_forensic_logger, new_session from core.logger import get_forensic_logger, new_session
from utils.helpers import _is_valid_ip, _is_valid_domain from utils.helpers import _is_valid_ip, _is_valid_domain
from providers.base_provider import BaseProvider from providers.base_provider import BaseProvider
@ -28,7 +28,6 @@ class ScanStatus:
class Scanner: class Scanner:
""" """
Main scanning orchestrator for DNSRecon passive reconnaissance. Main scanning orchestrator for DNSRecon passive reconnaissance.
Enhanced with reliable cross-process termination capabilities.
""" """
def __init__(self, session_config=None): def __init__(self, session_config=None):
@ -507,7 +506,7 @@ class Scanner:
self.logger.log_relationship_discovery( self.logger.log_relationship_discovery(
source_node=source, source_node=source,
target_node=rel_target, target_node=rel_target,
relationship_type=rel_type.relationship_name, relationship_type=rel_type,
confidence_score=confidence, confidence_score=confidence,
provider=provider_name, provider=provider_name,
raw_data=raw_data, raw_data=raw_data,
@ -519,18 +518,18 @@ class Scanner:
if _is_valid_ip(rel_target): if _is_valid_ip(rel_target):
self.graph.add_node(rel_target, NodeType.IP) self.graph.add_node(rel_target, NodeType.IP)
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data): if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
print(f"Added IP relationship: {source} -> {rel_target} ({rel_type.relationship_name})") print(f"Added IP relationship: {source} -> {rel_target} ({rel_type})")
discovered_targets.add(rel_target) discovered_targets.add(rel_target)
elif rel_target.startswith('AS') and rel_target[2:].isdigit(): elif rel_target.startswith('AS') and rel_target[2:].isdigit():
self.graph.add_node(rel_target, NodeType.ASN) self.graph.add_node(rel_target, NodeType.ASN)
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data): if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type.relationship_name})") print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type})")
elif _is_valid_domain(rel_target): elif _is_valid_domain(rel_target):
self.graph.add_node(rel_target, NodeType.DOMAIN) self.graph.add_node(rel_target, NodeType.DOMAIN)
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data): if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
print(f"Added domain relationship: {source} -> {rel_target} ({rel_type.relationship_name})") print(f"Added domain relationship: {source} -> {rel_target} ({rel_type})")
discovered_targets.add(rel_target) discovered_targets.add(rel_target)
self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target]) self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target])
@ -577,10 +576,10 @@ class Scanner:
return set(targets) return set(targets)
def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: RelationshipType, def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: str,
target: str, raw_data: Dict[str, Any], attributes: Dict[str, Any]) -> None: target: str, raw_data: Dict[str, Any], attributes: Dict[str, Any]) -> None:
"""Collect and organize attributes for a node.""" """Collect and organize attributes for a node."""
self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type.relationship_name}") self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type}")
if provider_name == 'dns': if provider_name == 'dns':
record_type = raw_data.get('query_type', 'UNKNOWN') record_type = raw_data.get('query_type', 'UNKNOWN')
@ -590,7 +589,7 @@ class Scanner:
attributes.setdefault('dns_records', []).append(dns_entry) attributes.setdefault('dns_records', []).append(dns_entry)
elif provider_name == 'crtsh': elif provider_name == 'crtsh':
if rel_type == RelationshipType.SAN_CERTIFICATE: if rel_type == "san_certificate":
domain_certs = raw_data.get('domain_certificates', {}) domain_certs = raw_data.get('domain_certificates', {})
if node_id in domain_certs: if node_id in domain_certs:
cert_summary = domain_certs[node_id] cert_summary = domain_certs[node_id]
@ -604,7 +603,7 @@ class Scanner:
if key not in shodan_attributes or not shodan_attributes.get(key): if key not in shodan_attributes or not shodan_attributes.get(key):
shodan_attributes[key] = value shodan_attributes[key] = value
if rel_type == RelationshipType.ASN_MEMBERSHIP: if rel_type == "asn_membership":
attributes['asn'] = { attributes['asn'] = {
'id': target, 'id': target,
'description': raw_data.get('org', ''), 'description': raw_data.get('org', ''),
@ -612,7 +611,7 @@ class Scanner:
'country': raw_data.get('country', '') 'country': raw_data.get('country', '')
} }
record_type_name = rel_type.relationship_name record_type_name = rel_type
if record_type_name not in attributes: if record_type_name not in attributes:
attributes[record_type_name] = [] attributes[record_type_name] = []
@ -719,8 +718,7 @@ class Scanner:
'final_status': self.status, 'final_status': self.status,
'total_indicators_processed': self.indicators_processed, 'total_indicators_processed': self.indicators_processed,
'enabled_providers': list(provider_stats.keys()), 'enabled_providers': list(provider_stats.keys()),
'session_id': self.session_id, 'session_id': self.session_id
'forensic_note': 'Enhanced scanner with reliable cross-process termination'
}, },
'graph_data': graph_data, 'graph_data': graph_data,
'forensic_audit': audit_trail, 'forensic_audit': audit_trail,

3
core/session_manager.py
View File

@ -16,7 +16,6 @@ from core.scanner import Scanner
class SessionManager: class SessionManager:
""" """
Manages multiple scanner instances for concurrent user sessions using Redis. Manages multiple scanner instances for concurrent user sessions using Redis.
Enhanced with reliable cross-process stop signal management and immediate state updates.
""" """
def __init__(self, session_timeout_minutes: int = 60): def __init__(self, session_timeout_minutes: int = 60):
@ -250,7 +249,7 @@ class SessionManager:
def get_session(self, session_id: str) -> Optional[Scanner]: def get_session(self, session_id: str) -> Optional[Scanner]:
""" """
Get scanner instance for a session from Redis with enhanced session ID management. Get scanner instance for a session from Redis with session ID management.
""" """
if not session_id: if not session_id:
return None return None

9
providers/base_provider.py
View File

@ -9,7 +9,6 @@ from abc import ABC, abstractmethod
from typing import List, Dict, Any, Optional, Tuple from typing import List, Dict, Any, Optional, Tuple
from core.logger import get_forensic_logger from core.logger import get_forensic_logger
from core.graph_manager import RelationshipType
class RateLimiter: class RateLimiter:
@ -147,7 +146,7 @@ class BaseProvider(ABC):
pass pass
@abstractmethod @abstractmethod
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query the provider for information about a domain. Query the provider for information about a domain.
@ -160,7 +159,7 @@ class BaseProvider(ABC):
pass pass
@abstractmethod @abstractmethod
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query the provider for information about an IP address. Query the provider for information about an IP address.
@ -419,7 +418,7 @@ class BaseProvider(ABC):
return False return False
def log_relationship_discovery(self, source_node: str, target_node: str, def log_relationship_discovery(self, source_node: str, target_node: str,
relationship_type: RelationshipType, relationship_type: str,
confidence_score: float, confidence_score: float,
raw_data: Dict[str, Any], raw_data: Dict[str, Any],
discovery_method: str) -> None: discovery_method: str) -> None:
@ -439,7 +438,7 @@ class BaseProvider(ABC):
self.logger.log_relationship_discovery( self.logger.log_relationship_discovery(
source_node=source_node, source_node=source_node,
target_node=target_node, target_node=target_node,
relationship_type=relationship_type.relationship_name, relationship_type=relationship_type,
confidence_score=confidence_score, confidence_score=confidence_score,
provider=self.name, provider=self.name,
raw_data=raw_data, raw_data=raw_data,

21
providers/crtsh_provider.py
View File

@ -9,10 +9,10 @@ import re
from typing import List, Dict, Any, Tuple, Set from typing import List, Dict, Any, Tuple, Set
from urllib.parse import quote from urllib.parse import quote
from datetime import datetime, timezone from datetime import datetime, timezone
import requests
from .base_provider import BaseProvider from .base_provider import BaseProvider
from utils.helpers import _is_valid_domain from utils.helpers import _is_valid_domain
from core.graph_manager import RelationshipType
class CrtShProvider(BaseProvider): class CrtShProvider(BaseProvider):
@ -145,7 +145,6 @@ class CrtShProvider(BaseProvider):
'source': 'crt.sh' 'source': 'crt.sh'
} }
# Add computed fields
try: try:
if metadata['not_before'] and metadata['not_after']: if metadata['not_before'] and metadata['not_after']:
not_before = self._parse_certificate_date(metadata['not_before']) not_before = self._parse_certificate_date(metadata['not_before'])
@ -166,10 +165,9 @@ class CrtShProvider(BaseProvider):
return metadata return metadata
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query crt.sh for certificates containing the domain. Query crt.sh for certificates containing the domain.
Enhanced with more frequent stop signal checking for reliable termination.
""" """
if not _is_valid_domain(domain): if not _is_valid_domain(domain):
return [] return []
@ -184,7 +182,7 @@ class CrtShProvider(BaseProvider):
try: try:
# Query crt.sh for certificates # Query crt.sh for certificates
url = f"{self.base_url}?q={quote(domain)}&output=json" url = f"{self.base_url}?q={quote(domain)}&output=json"
response = self.make_request(url, target_indicator=domain, max_retries=1) # Reduce retries for faster cancellation response = self.make_request(url, target_indicator=domain, max_retries=3)
if not response or response.status_code != 200: if not response or response.status_code != 200:
return [] return []
@ -208,7 +206,7 @@ class CrtShProvider(BaseProvider):
domain_certificates = {} domain_certificates = {}
all_discovered_domains = set() all_discovered_domains = set()
# Process certificates with enhanced cancellation checking # Process certificates with cancellation checking
for i, cert_data in enumerate(certificates): for i, cert_data in enumerate(certificates):
# Check for cancellation every 5 certificates instead of 10 for faster response # Check for cancellation every 5 certificates instead of 10 for faster response
if i % 5 == 0 and self._stop_event and self._stop_event.is_set(): if i % 5 == 0 and self._stop_event and self._stop_event.is_set():
@ -283,7 +281,7 @@ class CrtShProvider(BaseProvider):
relationships.append(( relationships.append((
domain, domain,
discovered_domain, discovered_domain,
RelationshipType.SAN_CERTIFICATE, 'san_certificate',
confidence, confidence,
relationship_raw_data relationship_raw_data
)) ))
@ -292,7 +290,7 @@ class CrtShProvider(BaseProvider):
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=domain, source_node=domain,
target_node=discovered_domain, target_node=discovered_domain,
relationship_type=RelationshipType.SAN_CERTIFICATE, relationship_type='san_certificate',
confidence_score=confidence, confidence_score=confidence,
raw_data=relationship_raw_data, raw_data=relationship_raw_data,
discovery_method="certificate_transparency_analysis" discovery_method="certificate_transparency_analysis"
@ -300,6 +298,9 @@ class CrtShProvider(BaseProvider):
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
self.logger.logger.error(f"Failed to parse JSON response from crt.sh: {e}") self.logger.logger.error(f"Failed to parse JSON response from crt.sh: {e}")
except requests.exceptions.RequestException as e:
self.logger.logger.error(f"HTTP request to crt.sh failed: {e}")
return relationships return relationships
@ -394,7 +395,7 @@ class CrtShProvider(BaseProvider):
Returns: Returns:
Confidence score between 0.0 and 1.0 Confidence score between 0.0 and 1.0
""" """
base_confidence = RelationshipType.SAN_CERTIFICATE.default_confidence base_confidence = 0.9
# Adjust confidence based on domain relationship context # Adjust confidence based on domain relationship context
relationship_context = self._determine_relationship_context(domain2, domain1) relationship_context = self._determine_relationship_context(domain2, domain1)
@ -462,7 +463,7 @@ class CrtShProvider(BaseProvider):
else: else:
return 'related_domain' return 'related_domain'
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query crt.sh for certificates containing the IP address. Query crt.sh for certificates containing the IP address.
Note: crt.sh doesn't typically index by IP, so this returns empty results. Note: crt.sh doesn't typically index by IP, so this returns empty results.

38
providers/dns_provider.py
View File

@ -5,7 +5,6 @@ import dns.reversename
from typing import List, Dict, Any, Tuple from typing import List, Dict, Any, Tuple
from .base_provider import BaseProvider from .base_provider import BaseProvider
from utils.helpers import _is_valid_ip, _is_valid_domain from utils.helpers import _is_valid_ip, _is_valid_domain
from core.graph_manager import RelationshipType
class DNSProvider(BaseProvider): class DNSProvider(BaseProvider):
@ -49,7 +48,7 @@ class DNSProvider(BaseProvider):
"""DNS is always available - no API key required.""" """DNS is always available - no API key required."""
return True return True
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query DNS records for the domain to discover relationships. Query DNS records for the domain to discover relationships.
@ -70,7 +69,7 @@ class DNSProvider(BaseProvider):
return relationships return relationships
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query reverse DNS for the IP address. Query reverse DNS for the IP address.
@ -106,16 +105,16 @@ class DNSProvider(BaseProvider):
relationships.append(( relationships.append((
ip, ip,
hostname, hostname,
RelationshipType.PTR_RECORD, 'ptr_record',
RelationshipType.PTR_RECORD.default_confidence, 0.8,
raw_data raw_data
)) ))
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=ip, source_node=ip,
target_node=hostname, target_node=hostname,
relationship_type=RelationshipType.PTR_RECORD, relationship_type='ptr_record',
confidence_score=RelationshipType.PTR_RECORD.default_confidence, confidence_score=0.8,
raw_data=raw_data, raw_data=raw_data,
discovery_method="reverse_dns_lookup" discovery_method="reverse_dns_lookup"
) )
@ -126,7 +125,7 @@ class DNSProvider(BaseProvider):
return relationships return relationships
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query a specific type of DNS record for the domain. Query a specific type of DNS record for the domain.
""" """
@ -147,7 +146,8 @@ class DNSProvider(BaseProvider):
elif record_type == 'SOA': elif record_type == 'SOA':
target = str(record.mname).rstrip('.') target = str(record.mname).rstrip('.')
elif record_type in ['TXT']: elif record_type in ['TXT']:
target = b' '.join(record.strings).decode('utf-8', 'ignore') # TXT records are treated as metadata, not relationships.
continue
elif record_type == 'SRV': elif record_type == 'SRV':
target = str(record.target).rstrip('.') target = str(record.target).rstrip('.')
elif record_type == 'CAA': elif record_type == 'CAA':
@ -155,7 +155,6 @@ class DNSProvider(BaseProvider):
else: else:
target = str(record) target = str(record)
if target: if target:
raw_data = { raw_data = {
'query_type': record_type, 'query_type': record_type,
@ -163,32 +162,25 @@ class DNSProvider(BaseProvider):
'value': target, 'value': target,
'ttl': response.ttl 'ttl': response.ttl
} }
try: relationship_type = f"{record_type.lower()}_record"
relationship_type_enum_name = f"{record_type}_RECORD" confidence = 0.8 # Default confidence for DNS records
# Handle TXT records as metadata, not relationships
if record_type == 'TXT':
relationship_type_enum = RelationshipType.A_RECORD # Dummy value, won't be used
else:
relationship_type_enum = getattr(RelationshipType, relationship_type_enum_name)
relationships.append(( relationships.append((
domain, domain,
target, target,
relationship_type_enum, relationship_type,
relationship_type_enum.default_confidence, confidence,
raw_data raw_data
)) ))
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=domain, source_node=domain,
target_node=target, target_node=target,
relationship_type=relationship_type_enum, relationship_type=relationship_type,
confidence_score=relationship_type_enum.default_confidence, confidence_score=confidence,
raw_data=raw_data, raw_data=raw_data,
discovery_method=f"dns_{record_type.lower()}_record" discovery_method=f"dns_{record_type.lower()}_record"
) )
except AttributeError:
self.logger.logger.error(f"Unsupported record type '{record_type}' encountered for domain {domain}")
except Exception as e: except Exception as e:
self.failed_requests += 1 self.failed_requests += 1

33
providers/shodan_provider.py
View File

@ -7,7 +7,6 @@ import json
from typing import List, Dict, Any, Tuple from typing import List, Dict, Any, Tuple
from .base_provider import BaseProvider from .base_provider import BaseProvider
from utils.helpers import _is_valid_ip, _is_valid_domain from utils.helpers import _is_valid_ip, _is_valid_domain
from core.graph_manager import RelationshipType
class ShodanProvider(BaseProvider): class ShodanProvider(BaseProvider):
@ -47,7 +46,7 @@ class ShodanProvider(BaseProvider):
"""Return a dictionary indicating if the provider can query domains and/or IPs.""" """Return a dictionary indicating if the provider can query domains and/or IPs."""
return {'domains': True, 'ips': True} return {'domains': True, 'ips': True}
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query Shodan for information about a domain. Query Shodan for information about a domain.
Uses Shodan's hostname search to find associated IPs. Uses Shodan's hostname search to find associated IPs.
@ -103,16 +102,16 @@ class ShodanProvider(BaseProvider):
relationships.append(( relationships.append((
domain, domain,
ip_address, ip_address,
RelationshipType.A_RECORD, # Domain resolves to IP 'a_record', # Domain resolves to IP
RelationshipType.A_RECORD.default_confidence, 0.8,
raw_data raw_data
)) ))
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=domain, source_node=domain,
target_node=ip_address, target_node=ip_address,
relationship_type=RelationshipType.A_RECORD, relationship_type='a_record',
confidence_score=RelationshipType.A_RECORD.default_confidence, confidence_score=0.8,
raw_data=raw_data, raw_data=raw_data,
discovery_method="shodan_hostname_search" discovery_method="shodan_hostname_search"
) )
@ -129,7 +128,7 @@ class ShodanProvider(BaseProvider):
relationships.append(( relationships.append((
domain, domain,
hostname, hostname,
RelationshipType.PASSIVE_DNS, # Shared hosting relationship 'passive_dns', # Shared hosting relationship
0.6, # Lower confidence for shared hosting 0.6, # Lower confidence for shared hosting
hostname_raw_data hostname_raw_data
)) ))
@ -137,7 +136,7 @@ class ShodanProvider(BaseProvider):
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=domain, source_node=domain,
target_node=hostname, target_node=hostname,
relationship_type=RelationshipType.PASSIVE_DNS, relationship_type='passive_dns',
confidence_score=0.6, confidence_score=0.6,
raw_data=hostname_raw_data, raw_data=hostname_raw_data,
discovery_method="shodan_shared_hosting" discovery_method="shodan_shared_hosting"
@ -148,7 +147,7 @@ class ShodanProvider(BaseProvider):
return relationships return relationships
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]: def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
""" """
Query Shodan for information about an IP address. Query Shodan for information about an IP address.
@ -195,16 +194,16 @@ class ShodanProvider(BaseProvider):
relationships.append(( relationships.append((
ip, ip,
hostname, hostname,
RelationshipType.A_RECORD, # IP resolves to hostname 'a_record', # IP resolves to hostname
RelationshipType.A_RECORD.default_confidence, 0.8,
raw_data raw_data
)) ))
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=ip, source_node=ip,
target_node=hostname, target_node=hostname,
relationship_type=RelationshipType.A_RECORD, relationship_type='a_record',
confidence_score=RelationshipType.A_RECORD.default_confidence, confidence_score=0.8,
raw_data=raw_data, raw_data=raw_data,
discovery_method="shodan_host_lookup" discovery_method="shodan_host_lookup"
) )
@ -230,16 +229,16 @@ class ShodanProvider(BaseProvider):
relationships.append(( relationships.append((
ip, ip,
asn_name, asn_name,
RelationshipType.ASN_MEMBERSHIP, 'asn_membership',
RelationshipType.ASN_MEMBERSHIP.default_confidence, 0.7,
asn_raw_data asn_raw_data
)) ))
self.log_relationship_discovery( self.log_relationship_discovery(
source_node=ip, source_node=ip,
target_node=asn_name, target_node=asn_name,
relationship_type=RelationshipType.ASN_MEMBERSHIP, relationship_type='asn_membership',
confidence_score=RelationshipType.ASN_MEMBERSHIP.default_confidence, confidence_score=0.7,
raw_data=asn_raw_data, raw_data=asn_raw_data,
discovery_method="shodan_asn_lookup" discovery_method="shodan_asn_lookup"
) )

28
static/js/graph.js
View File

@ -1,6 +1,6 @@
/** /**
* Graph visualization module for DNSRecon * Graph visualization module for DNSRecon
* Handles network graph rendering using vis.js with enhanced Phase 2 features * Handles network graph rendering using vis.js
*/ */
class GraphManager { class GraphManager {
@ -130,7 +130,7 @@ class GraphManager {
} }
/** /**
* Initialize the network graph with enhanced features * Initialize the network graph
*/ */
initialize() { initialize() {
if (this.isInitialized) { if (this.isInitialized) {
@ -156,7 +156,7 @@ class GraphManager {
// Add graph controls // Add graph controls
this.addGraphControls(); this.addGraphControls();
console.log('Enhanced graph initialized successfully'); console.log('Graph initialized successfully');
} catch (error) { } catch (error) {
console.error('Failed to initialize graph:', error); console.error('Failed to initialize graph:', error);
this.showError('Failed to initialize visualization'); this.showError('Failed to initialize visualization');
@ -184,12 +184,12 @@ class GraphManager {
} }
/** /**
* Setup enhanced network event handlers * Setup network event handlers
*/ */
setupNetworkEvents() { setupNetworkEvents() {
if (!this.network) return; if (!this.network) return;
// Node click event with enhanced details // Node click event with details
this.network.on('click', (params) => { this.network.on('click', (params) => {
if (params.nodes.length > 0) { if (params.nodes.length > 0) {
const nodeId = params.nodes[0]; const nodeId = params.nodes[0];
@ -207,7 +207,7 @@ class GraphManager {
} }
}); });
// Enhanced hover events // Hover events
this.network.on('hoverNode', (params) => { this.network.on('hoverNode', (params) => {
const nodeId = params.node; const nodeId = params.node;
const node = this.nodes.get(nodeId); const node = this.nodes.get(nodeId);
@ -242,7 +242,6 @@ class GraphManager {
} }
/** /**
* Update graph with new data and enhanced processing
* @param {Object} graphData - Graph data from backend * @param {Object} graphData - Graph data from backend
*/ */
updateGraph(graphData) { updateGraph(graphData) {
@ -326,15 +325,15 @@ class GraphManager {
setTimeout(() => this.fitView(), 800); setTimeout(() => this.fitView(), 800);
} }
console.log(`Enhanced graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`); console.log(`Graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
} catch (error) { } catch (error) {
console.error('Failed to update enhanced graph:', error); console.error('Failed to update graph:', error);
this.showError('Failed to update visualization'); this.showError('Failed to update visualization');
} }
} }
/** /**
* Process node data with enhanced styling and metadata * Process node data with styling and metadata
* @param {Object} node - Raw node data * @param {Object} node - Raw node data
* @returns {Object} Processed node data * @returns {Object} Processed node data
*/ */
@ -367,14 +366,17 @@ class GraphManager {
} }
if (node.type === 'correlation_object') { if (node.type === 'correlation_object') {
processedNode.label = this.formatNodeLabel(node.metadata.value, node.type); const value = node.metadata.value;
const label = Array.isArray(value) ? `Correlated (${value.length})` : String(value);
processedNode.label = this.formatNodeLabel(label, node.type);
processedNode.title = Array.isArray(value) ? value.join(', ') : value;
} }
return processedNode; return processedNode;
} }
/** /**
* Process edge data with enhanced styling and metadata * Process edge data with styling and metadata
* @param {Object} edge - Raw edge data * @param {Object} edge - Raw edge data
* @returns {Object} Processed edge data * @returns {Object} Processed edge data
*/ */
@ -478,7 +480,7 @@ class GraphManager {
} }
/** /**
* Get enhanced node shape based on type * Get node shape based on type
* @param {string} nodeType - Node type * @param {string} nodeType - Node type
* @returns {string} Shape name * @returns {string} Shape name
*/ */

8
static/js/main.js
View File

@ -243,7 +243,7 @@ class DNSReconApp {
} }
/** /**
* Enhanced start scan with better error handling * Start scan with error handling
*/ */
async startScan(clearGraph = true) { async startScan(clearGraph = true) {
console.log('=== STARTING SCAN ==='); console.log('=== STARTING SCAN ===');
@ -318,7 +318,7 @@ class DNSReconApp {
} }
} }
/** /**
* Enhanced scan stop with immediate UI feedback * Scan stop with immediate UI feedback
*/ */
async stopScan() { async stopScan() {
try { try {
@ -427,7 +427,7 @@ class DNSReconApp {
} }
/** /**
* Enhanced status update with better error handling * Status update with better error handling
*/ */
async updateStatus() { async updateStatus() {
try { try {
@ -668,7 +668,7 @@ class DNSReconApp {
} }
/** /**
* Enhanced UI state management with immediate button updates * UI state management with immediate button updates
*/ */
setUIState(state) { setUIState(state) {
console.log(`Setting UI state to: ${state}`); console.log(`Setting UI state to: ${state}`);