mstoeck3/dnsrecon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: mstoeck3:2185177a84ceb41d96d514906f32b07ea017dbff
Branches Tags
mstoeck3:main
mstoeck3:database_crtsh
mstoeck3:try-fix
..
compare: mstoeck3:41d556e2ce335889bec00446e4cccda2131f35b4
Branches Tags
mstoeck3:main
mstoeck3:database_crtsh
mstoeck3:try-fix

No commits in common. "2185177a84ceb41d96d514906f32b07ea017dbff" and "41d556e2ce335889bec00446e4cccda2131f35b4" have entirely different histories.
2185177a84 ... 41d556e2ce

12 changed files with 194 additions and 378 deletions
Show Stats Expand all files Collapse all files

15
app.py
View File

@ -1,6 +1,7 @@
"""
Flask application entry point for DNSRecon web interface.
Provides REST API endpoints and serves the web interface with user session support.
Enhanced with better session debugging and isolation.
"""
import json
@ -19,7 +20,7 @@ app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=2) # 2 hour session
def get_user_scanner():
"""
User scanner retrieval with better error handling and debugging.
Enhanced user scanner retrieval with better error handling and debugging.
"""
# Get current Flask session info for debugging
current_flask_session_id = session.get('dnsrecon_session_id')
@ -183,7 +184,7 @@ def stop_scan():
if not scanner.session_id:
scanner.session_id = user_session_id
# Use the stop mechanism
# Use the enhanced stop mechanism
success = scanner.stop_scan()
# Also set the Redis stop signal directly for extra reliability
@ -202,7 +203,7 @@ def stop_scan():
'message': 'Scan stop requested - termination initiated',
'user_session_id': user_session_id,
'scanner_status': scanner.status,
'stop_method': 'cross_process'
'stop_method': 'enhanced_cross_process'
})
except Exception as e:
@ -216,7 +217,7 @@ def stop_scan():
@app.route('/api/scan/status', methods=['GET'])
def get_scan_status():
"""Get current scan status with error handling."""
"""Get current scan status with enhanced error handling."""
try:
# Get user-specific scanner
user_session_id, scanner = get_user_scanner()
@ -278,7 +279,7 @@ def get_scan_status():
@app.route('/api/graph', methods=['GET'])
def get_graph_data():
"""Get current graph data with error handling."""
"""Get current graph data with enhanced error handling."""
try:
# Get user-specific scanner
user_session_id, scanner = get_user_scanner()
@ -523,7 +524,7 @@ def list_sessions():
@app.route('/api/health', methods=['GET'])
def health_check():
"""Health check endpoint."""
"""Health check endpoint with enhanced Phase 2 information."""
try:
# Get session stats
session_stats = session_manager.get_statistics()
@ -539,7 +540,7 @@ def health_check():
'concurrent_processing': True,
'real_time_updates': True,
'api_key_management': True,
'visualization': True,
'enhanced_visualization': True,
'retry_logic': True,
'user_sessions': True,
'session_isolation': True

17
core/__init__.py
View File

@ -1,25 +1,28 @@
"""
Core modules for DNSRecon passive reconnaissance tool.
Contains graph management, scanning orchestration, and forensic logging.
Phase 2: Enhanced with concurrent processing and real-time capabilities.
"""
from .graph_manager import GraphManager, NodeType
from .scanner import Scanner, ScanStatus
from .graph_manager import GraphManager, NodeType, RelationshipType
from .scanner import Scanner, ScanStatus # Remove 'scanner' global instance
from .logger import ForensicLogger, get_forensic_logger, new_session
from .session_manager import session_manager
from .session_config import SessionConfig, create_session_config
from .session_manager import session_manager # Add session manager
from .session_config import SessionConfig, create_session_config # Add session config
__all__ = [
'GraphManager',
'NodeType',
'RelationshipType',
'Scanner',
'ScanStatus',
# 'scanner', # Remove this - no more global scanner
'ForensicLogger',
'get_forensic_logger',
'new_session',
'session_manager',
'SessionConfig',
'create_session_config'
'session_manager', # Add this
'SessionConfig', # Add this
'create_session_config' # Add this
]
__version__ = "1.0.0-phase2"

222
core/graph_manager.py
View File

@ -22,6 +22,28 @@ class NodeType(Enum):
return self.value
class RelationshipType(Enum):
"""Enumeration of supported relationship types with confidence scores."""
SAN_CERTIFICATE = ("san", 0.9)
A_RECORD = ("a_record", 0.8)
AAAA_RECORD = ("aaaa_record", 0.8)
CNAME_RECORD = ("cname", 0.8)
MX_RECORD = ("mx_record", 0.7)
NS_RECORD = ("ns_record", 0.7)
PTR_RECORD = ("ptr_record", 0.8)
SOA_RECORD = ("soa_record", 0.7)
PASSIVE_DNS = ("passive_dns", 0.6)
ASN_MEMBERSHIP = ("asn", 0.7)
CORRELATED_TO = ("correlated_to", 0.9)
def __init__(self, relationship_name: str, default_confidence: float):
self.relationship_name = relationship_name
self.default_confidence = default_confidence
def __repr__(self):
return self.relationship_name
class GraphManager:
"""
Thread-safe graph manager for DNSRecon infrastructure mapping.
@ -130,8 +152,8 @@ class GraphManager:
})
return all_correlations
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None,
description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None,
description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
"""Add a node to the graph, update attributes, and process correlations."""
is_new_node = not self.graph.has_node(node_id)
if is_new_node:
@ -157,176 +179,54 @@ class GraphManager:
correlations = self._check_for_correlations(node_id, attributes)
for corr in correlations:
value = corr['value']
# STEP 1: Substring check against all existing nodes
if self._correlation_value_matches_existing_node(value):
# Skip creating correlation node - would be redundant
continue
# STEP 2: Filter out node pairs that already have direct edges
eligible_nodes = self._filter_nodes_without_direct_edges(set(corr['nodes']))
if len(eligible_nodes) < 2:
# Need at least 2 nodes to create a correlation
continue
# STEP 3: Check for existing correlation node with same connection pattern
correlation_nodes_with_pattern = self._find_correlation_nodes_with_same_pattern(eligible_nodes)
if correlation_nodes_with_pattern:
# STEP 4: Merge with existing correlation node
target_correlation_node = correlation_nodes_with_pattern[0]
self._merge_correlation_values(target_correlation_node, value, corr)
else:
# STEP 5: Create new correlation node for eligible nodes only
correlation_node_id = f"corr_{abs(hash(str(sorted(eligible_nodes))))}"
# FIXED: Check if the correlation value contains an existing node ID.
found_major_node_id = None
if isinstance(value, str):
for existing_node in self.graph.nodes():
if existing_node in value:
found_major_node_id = existing_node
break
if found_major_node_id:
# An existing major node is part of the value; link to it directly.
for c_node_id in set(corr['nodes']):
if self.graph.has_node(c_node_id) and c_node_id != found_major_node_id:
self.add_edge(c_node_id, found_major_node_id, RelationshipType.CORRELATED_TO)
continue # Skip creating a redundant correlation node
# Proceed to create a new correlation node if no major node was found.
correlation_node_id = f"{value}"
if not self.graph.has_node(correlation_node_id):
self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT,
metadata={'values': [value], 'sources': corr['sources'],
'correlated_nodes': list(eligible_nodes)})
# Create edges from eligible nodes to this correlation node
for c_node_id in eligible_nodes:
if self.graph.has_node(c_node_id):
attribute = corr['sources'][0]['path'].split('.')[-1]
relationship_type = f"c_{attribute}"
self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9)
metadata={'value': value, 'sources': corr['sources'],
'correlated_nodes': list(set(corr['nodes']))})
else: # Update existing correlation node
existing_meta = self.graph.nodes[correlation_node_id]['metadata']
existing_nodes = set(existing_meta.get('correlated_nodes', []))
existing_meta['correlated_nodes'] = list(existing_nodes.union(set(corr['nodes'])))
existing_sources = {(s['node_id'], s['path']) for s in existing_meta.get('sources', [])}
for s in corr['sources']:
existing_sources.add((s['node_id'], s['path']))
existing_meta['sources'] = [{'node_id': nid, 'path': p} for nid, p in existing_sources]
for c_node_id in set(corr['nodes']):
self.add_edge(c_node_id, correlation_node_id, RelationshipType.CORRELATED_TO)
self._update_correlation_index(node_id, attributes)
self.last_modified = datetime.now(timezone.utc).isoformat()
return is_new_node
def _filter_nodes_without_direct_edges(self, node_set: set) -> set:
"""
Filter out nodes that already have direct edges between them.
Returns set of nodes that should be included in correlation.
"""
nodes_list = list(node_set)
eligible_nodes = set(node_set) # Start with all nodes
# Check all pairs of nodes
for i in range(len(nodes_list)):
for j in range(i + 1, len(nodes_list)):
node_a = nodes_list[i]
node_b = nodes_list[j]
# Check if direct edge exists in either direction
if self._has_direct_edge_bidirectional(node_a, node_b):
# Remove both nodes from eligible set since they're already connected
eligible_nodes.discard(node_a)
eligible_nodes.discard(node_b)
return eligible_nodes
def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool:
"""
Check if there's a direct edge between two nodes in either direction.
Returns True if node_a→node_b OR node_b→node_a exists.
"""
return (self.graph.has_edge(node_a, node_b) or
self.graph.has_edge(node_b, node_a))
def _correlation_value_matches_existing_node(self, correlation_value: str) -> bool:
"""
Check if correlation value contains any existing node ID as substring.
Returns True if match found (correlation node should NOT be created).
"""
correlation_str = str(correlation_value).lower()
# Check against all existing nodes
for existing_node_id in self.graph.nodes():
if existing_node_id.lower() in correlation_str:
return True
return False
def _find_correlation_nodes_with_same_pattern(self, node_set: set) -> List[str]:
"""
Find existing correlation nodes that have the exact same pattern of connected nodes.
Returns list of correlation node IDs with matching patterns.
"""
correlation_nodes = self.get_nodes_by_type(NodeType.CORRELATION_OBJECT)
matching_nodes = []
for corr_node_id in correlation_nodes:
# Get all nodes connected to this correlation node
connected_nodes = set()
# Add all predecessors (nodes pointing TO the correlation node)
connected_nodes.update(self.graph.predecessors(corr_node_id))
# Add all successors (nodes pointed TO by the correlation node)
connected_nodes.update(self.graph.successors(corr_node_id))
# Check if the pattern matches exactly
if connected_nodes == node_set:
matching_nodes.append(corr_node_id)
return matching_nodes
def _merge_correlation_values(self, target_node_id: str, new_value: Any, corr_data: Dict) -> None:
"""
Merge a new correlation value into an existing correlation node.
Uses same logic as large entity merging.
"""
if not self.graph.has_node(target_node_id):
return
target_metadata = self.graph.nodes[target_node_id]['metadata']
# Get existing values (ensure it's a list)
existing_values = target_metadata.get('values', [])
if not isinstance(existing_values, list):
existing_values = [existing_values]
# Add new value if not already present
if new_value not in existing_values:
existing_values.append(new_value)
# Merge sources
existing_sources = target_metadata.get('sources', [])
new_sources = corr_data.get('sources', [])
# Create set of unique sources based on (node_id, path) tuples
source_set = set()
for source in existing_sources + new_sources:
source_tuple = (source['node_id'], source['path'])
source_set.add(source_tuple)
# Convert back to list of dictionaries
merged_sources = [{'node_id': nid, 'path': path} for nid, path in source_set]
# Update metadata
target_metadata.update({
'values': existing_values,
'sources': merged_sources,
'correlated_nodes': list(set(target_metadata.get('correlated_nodes', []) + corr_data.get('nodes', []))),
'merge_count': len(existing_values),
'last_merge_timestamp': datetime.now(timezone.utc).isoformat()
})
# Update description to reflect merged nature
value_count = len(existing_values)
node_count = len(target_metadata['correlated_nodes'])
self.graph.nodes[target_node_id]['description'] = (
f"Correlation container with {value_count} merged values "
f"across {node_count} nodes"
)
def add_edge(self, source_id: str, target_id: str, relationship_type: str,
confidence_score: float = 0.5, source_provider: str = "unknown",
def add_edge(self, source_id: str, target_id: str, relationship_type: RelationshipType,
confidence_score: Optional[float] = None, source_provider: str = "unknown",
raw_data: Optional[Dict[str, Any]] = None) -> bool:
"""Add or update an edge between two nodes, ensuring nodes exist."""
# LOGIC FIX: Ensure both source and target nodes exist before adding an edge.
if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
return False
new_confidence = confidence_score
if relationship_type.startswith("c_"):
edge_label = relationship_type
else:
edge_label = f"{source_provider}_{relationship_type}"
new_confidence = confidence_score or relationship_type.default_confidence
if self.graph.has_edge(source_id, target_id):
# If edge exists, update confidence if the new score is higher.
if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0):
@ -337,7 +237,7 @@ class GraphManager:
# Add a new edge with all attributes.
self.graph.add_edge(source_id, target_id,
relationship_type=edge_label,
relationship_type=relationship_type.relationship_name,
confidence_score=new_confidence,
source_provider=source_provider,
discovery_timestamp=datetime.now(timezone.utc).isoformat(),

24
core/scanner.py
View File

@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError,
from collections import defaultdict, deque
from datetime import datetime, timezone
from core.graph_manager import GraphManager, NodeType
from core.graph_manager import GraphManager, NodeType, RelationshipType
from core.logger import get_forensic_logger, new_session
from utils.helpers import _is_valid_ip, _is_valid_domain
from providers.base_provider import BaseProvider
@ -28,6 +28,7 @@ class ScanStatus:
class Scanner:
"""
Main scanning orchestrator for DNSRecon passive reconnaissance.
Enhanced with reliable cross-process termination capabilities.
"""
def __init__(self, session_config=None):
@ -506,7 +507,7 @@ class Scanner:
self.logger.log_relationship_discovery(
source_node=source,
target_node=rel_target,
relationship_type=rel_type,
relationship_type=rel_type.relationship_name,
confidence_score=confidence,
provider=provider_name,
raw_data=raw_data,
@ -518,18 +519,18 @@ class Scanner:
if _is_valid_ip(rel_target):
self.graph.add_node(rel_target, NodeType.IP)
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
print(f"Added IP relationship: {source} -> {rel_target} ({rel_type})")
print(f"Added IP relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
discovered_targets.add(rel_target)
elif rel_target.startswith('AS') and rel_target[2:].isdigit():
self.graph.add_node(rel_target, NodeType.ASN)
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type})")
print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
elif _is_valid_domain(rel_target):
self.graph.add_node(rel_target, NodeType.DOMAIN)
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
print(f"Added domain relationship: {source} -> {rel_target} ({rel_type})")
print(f"Added domain relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
discovered_targets.add(rel_target)
self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target])
@ -576,10 +577,10 @@ class Scanner:
return set(targets)
def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: str,
def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: RelationshipType,
target: str, raw_data: Dict[str, Any], attributes: Dict[str, Any]) -> None:
"""Collect and organize attributes for a node."""
self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type}")
self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type.relationship_name}")
if provider_name == 'dns':
record_type = raw_data.get('query_type', 'UNKNOWN')
@ -589,7 +590,7 @@ class Scanner:
attributes.setdefault('dns_records', []).append(dns_entry)
elif provider_name == 'crtsh':
if rel_type == "san_certificate":
if rel_type == RelationshipType.SAN_CERTIFICATE:
domain_certs = raw_data.get('domain_certificates', {})
if node_id in domain_certs:
cert_summary = domain_certs[node_id]
@ -603,7 +604,7 @@ class Scanner:
if key not in shodan_attributes or not shodan_attributes.get(key):
shodan_attributes[key] = value
if rel_type == "asn_membership":
if rel_type == RelationshipType.ASN_MEMBERSHIP:
attributes['asn'] = {
'id': target,
'description': raw_data.get('org', ''),
@ -611,7 +612,7 @@ class Scanner:
'country': raw_data.get('country', '')
}
record_type_name = rel_type
record_type_name = rel_type.relationship_name
if record_type_name not in attributes:
attributes[record_type_name] = []
@ -718,7 +719,8 @@ class Scanner:
'final_status': self.status,
'total_indicators_processed': self.indicators_processed,
'enabled_providers': list(provider_stats.keys()),
'session_id': self.session_id
'session_id': self.session_id,
'forensic_note': 'Enhanced scanner with reliable cross-process termination'
},
'graph_data': graph_data,
'forensic_audit': audit_trail,

3
core/session_manager.py
View File

@ -16,6 +16,7 @@ from core.scanner import Scanner
class SessionManager:
"""
Manages multiple scanner instances for concurrent user sessions using Redis.
Enhanced with reliable cross-process stop signal management and immediate state updates.
"""
def __init__(self, session_timeout_minutes: int = 60):
@ -249,7 +250,7 @@ class SessionManager:
def get_session(self, session_id: str) -> Optional[Scanner]:
"""
Get scanner instance for a session from Redis with session ID management.
Get scanner instance for a session from Redis with enhanced session ID management.
"""
if not session_id:
return None

9
providers/base_provider.py
View File

@ -9,6 +9,7 @@ from abc import ABC, abstractmethod
from typing import List, Dict, Any, Optional, Tuple
from core.logger import get_forensic_logger
from core.graph_manager import RelationshipType
class RateLimiter:
@ -146,7 +147,7 @@ class BaseProvider(ABC):
pass
@abstractmethod
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query the provider for information about a domain.
@ -159,7 +160,7 @@ class BaseProvider(ABC):
pass
@abstractmethod
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query the provider for information about an IP address.
@ -418,7 +419,7 @@ class BaseProvider(ABC):
return False
def log_relationship_discovery(self, source_node: str, target_node: str,
relationship_type: str,
relationship_type: RelationshipType,
confidence_score: float,
raw_data: Dict[str, Any],
discovery_method: str) -> None:
@ -438,7 +439,7 @@ class BaseProvider(ABC):
self.logger.log_relationship_discovery(
source_node=source_node,
target_node=target_node,
relationship_type=relationship_type,
relationship_type=relationship_type.relationship_name,
confidence_score=confidence_score,
provider=self.name,
raw_data=raw_data,

21
providers/crtsh_provider.py
View File

@ -9,10 +9,10 @@ import re
from typing import List, Dict, Any, Tuple, Set
from urllib.parse import quote
from datetime import datetime, timezone
import requests
from .base_provider import BaseProvider
from utils.helpers import _is_valid_domain
from core.graph_manager import RelationshipType
class CrtShProvider(BaseProvider):
@ -145,6 +145,7 @@ class CrtShProvider(BaseProvider):
'source': 'crt.sh'
}
# Add computed fields
try:
if metadata['not_before'] and metadata['not_after']:
not_before = self._parse_certificate_date(metadata['not_before'])
@ -165,9 +166,10 @@ class CrtShProvider(BaseProvider):
return metadata
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query crt.sh for certificates containing the domain.
Enhanced with more frequent stop signal checking for reliable termination.
"""
if not _is_valid_domain(domain):
return []
@ -182,7 +184,7 @@ class CrtShProvider(BaseProvider):
try:
# Query crt.sh for certificates
url = f"{self.base_url}?q={quote(domain)}&output=json"
response = self.make_request(url, target_indicator=domain, max_retries=3)
response = self.make_request(url, target_indicator=domain, max_retries=1) # Reduce retries for faster cancellation
if not response or response.status_code != 200:
return []
@ -206,7 +208,7 @@ class CrtShProvider(BaseProvider):
domain_certificates = {}
all_discovered_domains = set()
# Process certificates with cancellation checking
# Process certificates with enhanced cancellation checking
for i, cert_data in enumerate(certificates):
# Check for cancellation every 5 certificates instead of 10 for faster response
if i % 5 == 0 and self._stop_event and self._stop_event.is_set():
@ -281,7 +283,7 @@ class CrtShProvider(BaseProvider):
relationships.append((
domain,
discovered_domain,
'san_certificate',
RelationshipType.SAN_CERTIFICATE,
confidence,
relationship_raw_data
))
@ -290,7 +292,7 @@ class CrtShProvider(BaseProvider):
self.log_relationship_discovery(
source_node=domain,
target_node=discovered_domain,
relationship_type='san_certificate',
relationship_type=RelationshipType.SAN_CERTIFICATE,
confidence_score=confidence,
raw_data=relationship_raw_data,
discovery_method="certificate_transparency_analysis"
@ -298,9 +300,6 @@ class CrtShProvider(BaseProvider):
except json.JSONDecodeError as e:
self.logger.logger.error(f"Failed to parse JSON response from crt.sh: {e}")
except requests.exceptions.RequestException as e:
self.logger.logger.error(f"HTTP request to crt.sh failed: {e}")
return relationships
@ -395,7 +394,7 @@ class CrtShProvider(BaseProvider):
Returns:
Confidence score between 0.0 and 1.0
"""
base_confidence = 0.9
base_confidence = RelationshipType.SAN_CERTIFICATE.default_confidence
# Adjust confidence based on domain relationship context
relationship_context = self._determine_relationship_context(domain2, domain1)
@ -463,7 +462,7 @@ class CrtShProvider(BaseProvider):
else:
return 'related_domain'
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query crt.sh for certificates containing the IP address.
Note: crt.sh doesn't typically index by IP, so this returns empty results.

60
providers/dns_provider.py
View File

@ -5,6 +5,7 @@ import dns.reversename
from typing import List, Dict, Any, Tuple
from .base_provider import BaseProvider
from utils.helpers import _is_valid_ip, _is_valid_domain
from core.graph_manager import RelationshipType
class DNSProvider(BaseProvider):
@ -48,7 +49,7 @@ class DNSProvider(BaseProvider):
"""DNS is always available - no API key required."""
return True
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query DNS records for the domain to discover relationships.
@ -69,7 +70,7 @@ class DNSProvider(BaseProvider):
return relationships
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query reverse DNS for the IP address.
@ -105,16 +106,16 @@ class DNSProvider(BaseProvider):
relationships.append((
ip,
hostname,
'ptr_record',
0.8,
RelationshipType.PTR_RECORD,
RelationshipType.PTR_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type='ptr_record',
confidence_score=0.8,
relationship_type=RelationshipType.PTR_RECORD,
confidence_score=RelationshipType.PTR_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="reverse_dns_lookup"
)
@ -125,7 +126,7 @@ class DNSProvider(BaseProvider):
return relationships
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query a specific type of DNS record for the domain.
"""
@ -146,8 +147,7 @@ class DNSProvider(BaseProvider):
elif record_type == 'SOA':
target = str(record.mname).rstrip('.')
elif record_type in ['TXT']:
# TXT records are treated as metadata, not relationships.
continue
target = b' '.join(record.strings).decode('utf-8', 'ignore')
elif record_type == 'SRV':
target = str(record.target).rstrip('.')
elif record_type == 'CAA':
@ -155,6 +155,7 @@ class DNSProvider(BaseProvider):
else:
target = str(record)
if target:
raw_data = {
'query_type': record_type,
@ -162,25 +163,32 @@ class DNSProvider(BaseProvider):
'value': target,
'ttl': response.ttl
}
relationship_type = f"{record_type.lower()}_record"
confidence = 0.8 # Default confidence for DNS records
try:
relationship_type_enum_name = f"{record_type}_RECORD"
# Handle TXT records as metadata, not relationships
if record_type == 'TXT':
relationship_type_enum = RelationshipType.A_RECORD # Dummy value, won't be used
else:
relationship_type_enum = getattr(RelationshipType, relationship_type_enum_name)
relationships.append((
domain,
target,
relationship_type,
confidence,
raw_data
))
relationships.append((
domain,
target,
relationship_type_enum,
relationship_type_enum.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=target,
relationship_type=relationship_type,
confidence_score=confidence,
raw_data=raw_data,
discovery_method=f"dns_{record_type.lower()}_record"
)
self.log_relationship_discovery(
source_node=domain,
target_node=target,
relationship_type=relationship_type_enum,
confidence_score=relationship_type_enum.default_confidence,
raw_data=raw_data,
discovery_method=f"dns_{record_type.lower()}_record"
)
except AttributeError:
self.logger.logger.error(f"Unsupported record type '{record_type}' encountered for domain {domain}")
except Exception as e:
self.failed_requests += 1

33
providers/shodan_provider.py
View File

@ -7,6 +7,7 @@ import json
from typing import List, Dict, Any, Tuple
from .base_provider import BaseProvider
from utils.helpers import _is_valid_ip, _is_valid_domain
from core.graph_manager import RelationshipType
class ShodanProvider(BaseProvider):
@ -46,7 +47,7 @@ class ShodanProvider(BaseProvider):
"""Return a dictionary indicating if the provider can query domains and/or IPs."""
return {'domains': True, 'ips': True}
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query Shodan for information about a domain.
Uses Shodan's hostname search to find associated IPs.
@ -102,16 +103,16 @@ class ShodanProvider(BaseProvider):
relationships.append((
domain,
ip_address,
'a_record', # Domain resolves to IP
0.8,
RelationshipType.A_RECORD, # Domain resolves to IP
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ip_address,
relationship_type='a_record',
confidence_score=0.8,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="shodan_hostname_search"
)
@ -128,7 +129,7 @@ class ShodanProvider(BaseProvider):
relationships.append((
domain,
hostname,
'passive_dns', # Shared hosting relationship
RelationshipType.PASSIVE_DNS, # Shared hosting relationship
0.6, # Lower confidence for shared hosting
hostname_raw_data
))
@ -136,7 +137,7 @@ class ShodanProvider(BaseProvider):
self.log_relationship_discovery(
source_node=domain,
target_node=hostname,
relationship_type='passive_dns',
relationship_type=RelationshipType.PASSIVE_DNS,
confidence_score=0.6,
raw_data=hostname_raw_data,
discovery_method="shodan_shared_hosting"
@ -147,7 +148,7 @@ class ShodanProvider(BaseProvider):
return relationships
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query Shodan for information about an IP address.
@ -194,16 +195,16 @@ class ShodanProvider(BaseProvider):
relationships.append((
ip,
hostname,
'a_record', # IP resolves to hostname
0.8,
RelationshipType.A_RECORD, # IP resolves to hostname
RelationshipType.A_RECORD.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type='a_record',
confidence_score=0.8,
relationship_type=RelationshipType.A_RECORD,
confidence_score=RelationshipType.A_RECORD.default_confidence,
raw_data=raw_data,
discovery_method="shodan_host_lookup"
)
@ -229,16 +230,16 @@ class ShodanProvider(BaseProvider):
relationships.append((
ip,
asn_name,
'asn_membership',
0.7,
RelationshipType.ASN_MEMBERSHIP,
RelationshipType.ASN_MEMBERSHIP.default_confidence,
asn_raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=asn_name,
relationship_type='asn_membership',
confidence_score=0.7,
relationship_type=RelationshipType.ASN_MEMBERSHIP,
confidence_score=RelationshipType.ASN_MEMBERSHIP.default_confidence,
raw_data=asn_raw_data,
discovery_method="shodan_asn_lookup"
)

40
static/css/main.css
View File

@ -1000,46 +1000,6 @@ input[type="text"]:focus, select:focus {
font-style: italic;
}
.correlation-values-list {
margin-top: 1rem;
}
.correlation-value-details {
margin-bottom: 0.5rem;
border: 1px solid #333;
border-radius: 3px;
}
.correlation-value-details summary {
padding: 0.5rem;
background-color: #3a3a3a;
cursor: pointer;
outline: none;
color: #c7c7c7;
}
.correlation-value-details summary:hover {
background-color: #4a4a4a;
}
.correlation-value-details .detail-row {
margin-left: 1rem;
margin-right: 1rem;
padding: 0.5rem 0;
}
.correlation-value-details .detail-label {
color: #999;
font-weight: 500;
}
.correlation-value-details .detail-value {
color: #c7c7c7;
word-break: break-all;
font-family: 'Roboto Mono', monospace;
font-size: 0.9em;
}
@keyframes fadeIn {
from {opacity: 0; transform: scale(0.95);}
to {opacity: 1; transform: scale(1);}

41
static/js/graph.js
View File

@ -1,6 +1,6 @@
/**
* Graph visualization module for DNSRecon
* Handles network graph rendering using vis.js
* Handles network graph rendering using vis.js with enhanced Phase 2 features
*/
class GraphManager {
@ -130,7 +130,7 @@ class GraphManager {
}
/**
* Initialize the network graph
* Initialize the network graph with enhanced features
*/
initialize() {
if (this.isInitialized) {
@ -156,7 +156,7 @@ class GraphManager {
// Add graph controls
this.addGraphControls();
console.log('Graph initialized successfully');
console.log('Enhanced graph initialized successfully');
} catch (error) {
console.error('Failed to initialize graph:', error);
this.showError('Failed to initialize visualization');
@ -184,12 +184,12 @@ class GraphManager {
}
/**
* Setup network event handlers
* Setup enhanced network event handlers
*/
setupNetworkEvents() {
if (!this.network) return;
// Node click event with details
// Node click event with enhanced details
this.network.on('click', (params) => {
if (params.nodes.length > 0) {
const nodeId = params.nodes[0];
@ -207,7 +207,7 @@ class GraphManager {
}
});
// Hover events
// Enhanced hover events
this.network.on('hoverNode', (params) => {
const nodeId = params.node;
const node = this.nodes.get(nodeId);
@ -242,6 +242,7 @@ class GraphManager {
}
/**
* Update graph with new data and enhanced processing
* @param {Object} graphData - Graph data from backend
*/
updateGraph(graphData) {
@ -325,15 +326,15 @@ class GraphManager {
setTimeout(() => this.fitView(), 800);
}
console.log(`Graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
console.log(`Enhanced graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
} catch (error) {
console.error('Failed to update graph:', error);
console.error('Failed to update enhanced graph:', error);
this.showError('Failed to update visualization');
}
}
/**
* Process node data with styling and metadata
* Process node data with enhanced styling and metadata
* @param {Object} node - Raw node data
* @returns {Object} Processed node data
*/
@ -365,31 +366,15 @@ class GraphManager {
}
}
// Handle merged correlation objects (similar to large entities)
if (node.type === 'correlation_object') {
const metadata = node.metadata || {};
const values = metadata.values || [];
const mergeCount = metadata.merge_count || 1;
if (mergeCount > 1) {
// Display as merged correlation container
processedNode.label = `Correlations (${mergeCount})`;
processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`;
processedNode.borderWidth = 3; // Thicker border for merged nodes
} else {
// Single correlation value
const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown');
const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value;
processedNode.label = `Corr: ${displayValue}`;
processedNode.title = `Correlation: ${value}`;
}
processedNode.label = this.formatNodeLabel(node.metadata.value, node.type);
}
return processedNode;
}
/**
* Process edge data with styling and metadata
* Process edge data with enhanced styling and metadata
* @param {Object} edge - Raw edge data
* @returns {Object} Processed edge data
*/
@ -493,7 +478,7 @@ class GraphManager {
}
/**
* Get node shape based on type
* Get enhanced node shape based on type
* @param {string} nodeType - Node type
* @returns {string} Shape name
*/

87
static/js/main.js
View File

@ -243,7 +243,7 @@ class DNSReconApp {
}
/**
* Start scan with error handling
* Enhanced start scan with better error handling
*/
async startScan(clearGraph = true) {
console.log('=== STARTING SCAN ===');
@ -318,7 +318,7 @@ class DNSReconApp {
}
}
/**
* Scan stop with immediate UI feedback
* Enhanced scan stop with immediate UI feedback
*/
async stopScan() {
try {
@ -427,7 +427,7 @@ class DNSReconApp {
}
/**
* Status update with better error handling
* Enhanced status update with better error handling
*/
async updateStatus() {
try {
@ -668,7 +668,7 @@ class DNSReconApp {
}
/**
* UI state management with immediate button updates
* Enhanced UI state management with immediate button updates
*/
setUIState(state) {
console.log(`Setting UI state to: ${state}`);
@ -799,51 +799,10 @@ class DNSReconApp {
*/
generateNodeDetailsHtml(node) {
if (!node) return '<div class="detail-row"><span class="detail-value">Details not available.</span></div>';
let detailsHtml = '<div class="modal-details-grid">';
// Handle merged correlation objects similar to large entities
if (node.type === 'correlation_object') {
const metadata = node.metadata || {};
const values = metadata.values || [];
const mergeCount = metadata.merge_count || 1;
detailsHtml += '<div class="modal-section">';
detailsHtml += '<h4>Correlation Details</h4>';
if (mergeCount > 1) {
detailsHtml += `<p><strong>Merged Correlations:</strong> ${mergeCount} values</p>`;
detailsHtml += '<div class="correlation-values-list">';
values.forEach((value, index) => {
detailsHtml += `<details class="correlation-value-details">`;
detailsHtml += `<summary>Value ${index + 1}: ${typeof value === 'string' && value.length > 50 ? value.substring(0, 47) + '...' : value}</summary>`;
detailsHtml += `<div class="detail-row"><span class="detail-label">Full Value:</span><span class="detail-value">${value}</span></div>`;
detailsHtml += `</details>`;
});
detailsHtml += '</div>';
} else {
const singleValue = values.length > 0 ? values[0] : (metadata.value || 'Unknown');
detailsHtml += `<div class="detail-row"><span class="detail-label">Correlation Value:</span><span class="detail-value">${singleValue}</span></div>`;
}
// Show correlated nodes
const correlatedNodes = metadata.correlated_nodes || [];
if (correlatedNodes.length > 0) {
detailsHtml += `<div class="detail-row"><span class="detail-label">Correlated Nodes:</span><span class="detail-value">${correlatedNodes.length}</span></div>`;
detailsHtml += '<ul>';
correlatedNodes.forEach(nodeId => {
detailsHtml += `<li><a href="#" class="node-link" data-node-id="${nodeId}">${nodeId}</a></li>`;
});
detailsHtml += '</ul>';
}
detailsHtml += '</div>';
}
// Continue with standard node details for all node types
// Section for Incoming Edges (Source Nodes)
// Section for Incoming Edges (Source Nodes)
if (node.incoming_edges && node.incoming_edges.length > 0) {
detailsHtml += '<div class="modal-section">';
detailsHtml += '<h4>Source Nodes (Incoming)</h4>';
@ -853,7 +812,7 @@ class DNSReconApp {
});
detailsHtml += '</ul></div>';
}
// Section for Outgoing Edges (Destination Nodes)
if (node.outgoing_edges && node.outgoing_edges.length > 0) {
detailsHtml += '<div class="modal-section">';
@ -864,29 +823,25 @@ class DNSReconApp {
});
detailsHtml += '</ul></div>';
}
// Section for Attributes (skip for correlation objects - already handled above)
if (node.type !== 'correlation_object') {
detailsHtml += '<div class="modal-section">';
detailsHtml += '<h4>Attributes</h4>';
detailsHtml += this.formatObjectToHtml(node.attributes);
detailsHtml += '</div>';
}
// Section for Attributes
detailsHtml += '<div class="modal-section">';
detailsHtml += '<h4>Attributes</h4>';
detailsHtml += this.formatObjectToHtml(node.attributes);
detailsHtml += '</div>';
// Section for Description
detailsHtml += '<div class="modal-section">';
detailsHtml += '<h4>Description</h4>';
detailsHtml += `<p class="description-text">${node.description || 'No description available.'}</p>`;
detailsHtml += '</div>';
// Section for Metadata (skip detailed metadata for correlation objects - already handled above)
if (node.type !== 'correlation_object') {
detailsHtml += '<div class="modal-section">';
detailsHtml += '<h4>Metadata</h4>';
detailsHtml += this.formatObjectToHtml(node.metadata);
detailsHtml += '</div>';
}
// Section for Metadata
detailsHtml += '<div class="modal-section">';
detailsHtml += '<h4>Metadata</h4>';
detailsHtml += this.formatObjectToHtml(node.metadata);
detailsHtml += '</div>';
detailsHtml += '</div>';
return detailsHtml;
}