it
This commit is contained in:
parent
41d556e2ce
commit
b7a57f1552
15
app.py
15
app.py
@ -1,7 +1,6 @@
|
||||
"""
|
||||
Flask application entry point for DNSRecon web interface.
|
||||
Provides REST API endpoints and serves the web interface with user session support.
|
||||
Enhanced with better session debugging and isolation.
|
||||
"""
|
||||
|
||||
import json
|
||||
@ -20,7 +19,7 @@ app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=2) # 2 hour session
|
||||
|
||||
def get_user_scanner():
|
||||
"""
|
||||
Enhanced user scanner retrieval with better error handling and debugging.
|
||||
User scanner retrieval with better error handling and debugging.
|
||||
"""
|
||||
# Get current Flask session info for debugging
|
||||
current_flask_session_id = session.get('dnsrecon_session_id')
|
||||
@ -184,7 +183,7 @@ def stop_scan():
|
||||
if not scanner.session_id:
|
||||
scanner.session_id = user_session_id
|
||||
|
||||
# Use the enhanced stop mechanism
|
||||
# Use the stop mechanism
|
||||
success = scanner.stop_scan()
|
||||
|
||||
# Also set the Redis stop signal directly for extra reliability
|
||||
@ -203,7 +202,7 @@ def stop_scan():
|
||||
'message': 'Scan stop requested - termination initiated',
|
||||
'user_session_id': user_session_id,
|
||||
'scanner_status': scanner.status,
|
||||
'stop_method': 'enhanced_cross_process'
|
||||
'stop_method': 'cross_process'
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
@ -217,7 +216,7 @@ def stop_scan():
|
||||
|
||||
@app.route('/api/scan/status', methods=['GET'])
|
||||
def get_scan_status():
|
||||
"""Get current scan status with enhanced error handling."""
|
||||
"""Get current scan status with error handling."""
|
||||
try:
|
||||
# Get user-specific scanner
|
||||
user_session_id, scanner = get_user_scanner()
|
||||
@ -279,7 +278,7 @@ def get_scan_status():
|
||||
|
||||
@app.route('/api/graph', methods=['GET'])
|
||||
def get_graph_data():
|
||||
"""Get current graph data with enhanced error handling."""
|
||||
"""Get current graph data with error handling."""
|
||||
try:
|
||||
# Get user-specific scanner
|
||||
user_session_id, scanner = get_user_scanner()
|
||||
@ -524,7 +523,7 @@ def list_sessions():
|
||||
|
||||
@app.route('/api/health', methods=['GET'])
|
||||
def health_check():
|
||||
"""Health check endpoint with enhanced Phase 2 information."""
|
||||
"""Health check endpoint."""
|
||||
try:
|
||||
# Get session stats
|
||||
session_stats = session_manager.get_statistics()
|
||||
@ -540,7 +539,7 @@ def health_check():
|
||||
'concurrent_processing': True,
|
||||
'real_time_updates': True,
|
||||
'api_key_management': True,
|
||||
'enhanced_visualization': True,
|
||||
'visualization': True,
|
||||
'retry_logic': True,
|
||||
'user_sessions': True,
|
||||
'session_isolation': True
|
||||
|
@ -1,28 +1,25 @@
|
||||
"""
|
||||
Core modules for DNSRecon passive reconnaissance tool.
|
||||
Contains graph management, scanning orchestration, and forensic logging.
|
||||
Phase 2: Enhanced with concurrent processing and real-time capabilities.
|
||||
"""
|
||||
|
||||
from .graph_manager import GraphManager, NodeType, RelationshipType
|
||||
from .scanner import Scanner, ScanStatus # Remove 'scanner' global instance
|
||||
from .graph_manager import GraphManager, NodeType
|
||||
from .scanner import Scanner, ScanStatus
|
||||
from .logger import ForensicLogger, get_forensic_logger, new_session
|
||||
from .session_manager import session_manager # Add session manager
|
||||
from .session_config import SessionConfig, create_session_config # Add session config
|
||||
from .session_manager import session_manager
|
||||
from .session_config import SessionConfig, create_session_config
|
||||
|
||||
__all__ = [
|
||||
'GraphManager',
|
||||
'NodeType',
|
||||
'RelationshipType',
|
||||
'Scanner',
|
||||
'ScanStatus',
|
||||
# 'scanner', # Remove this - no more global scanner
|
||||
'ForensicLogger',
|
||||
'get_forensic_logger',
|
||||
'new_session',
|
||||
'session_manager', # Add this
|
||||
'SessionConfig', # Add this
|
||||
'create_session_config' # Add this
|
||||
'session_manager',
|
||||
'SessionConfig',
|
||||
'create_session_config'
|
||||
]
|
||||
|
||||
__version__ = "1.0.0-phase2"
|
@ -22,28 +22,6 @@ class NodeType(Enum):
|
||||
return self.value
|
||||
|
||||
|
||||
class RelationshipType(Enum):
|
||||
"""Enumeration of supported relationship types with confidence scores."""
|
||||
SAN_CERTIFICATE = ("san", 0.9)
|
||||
A_RECORD = ("a_record", 0.8)
|
||||
AAAA_RECORD = ("aaaa_record", 0.8)
|
||||
CNAME_RECORD = ("cname", 0.8)
|
||||
MX_RECORD = ("mx_record", 0.7)
|
||||
NS_RECORD = ("ns_record", 0.7)
|
||||
PTR_RECORD = ("ptr_record", 0.8)
|
||||
SOA_RECORD = ("soa_record", 0.7)
|
||||
PASSIVE_DNS = ("passive_dns", 0.6)
|
||||
ASN_MEMBERSHIP = ("asn", 0.7)
|
||||
CORRELATED_TO = ("correlated_to", 0.9)
|
||||
|
||||
def __init__(self, relationship_name: str, default_confidence: float):
|
||||
self.relationship_name = relationship_name
|
||||
self.default_confidence = default_confidence
|
||||
|
||||
def __repr__(self):
|
||||
return self.relationship_name
|
||||
|
||||
|
||||
class GraphManager:
|
||||
"""
|
||||
Thread-safe graph manager for DNSRecon infrastructure mapping.
|
||||
@ -152,7 +130,7 @@ class GraphManager:
|
||||
})
|
||||
return all_correlations
|
||||
|
||||
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None,
|
||||
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[Dict[str, Any]] = None,
|
||||
description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""Add a node to the graph, update attributes, and process correlations."""
|
||||
is_new_node = not self.graph.has_node(node_id)
|
||||
@ -180,19 +158,23 @@ class GraphManager:
|
||||
for corr in correlations:
|
||||
value = corr['value']
|
||||
|
||||
# FIXED: Check if the correlation value contains an existing node ID.
|
||||
found_major_node_id = None
|
||||
if isinstance(value, str):
|
||||
# Check if the value contains ANY existing major node ID from the entire graph
|
||||
for existing_node in self.graph.nodes():
|
||||
if existing_node in value:
|
||||
# Ensure the existing_node is a major type (domain/ip/asn) and is a substring of the correlation value
|
||||
if (self.graph.nodes[existing_node].get('type') in [NodeType.DOMAIN.value, NodeType.IP.value, NodeType.ASN.value] and
|
||||
existing_node in value):
|
||||
found_major_node_id = existing_node
|
||||
break
|
||||
break # Found a major node, no need to check further
|
||||
|
||||
if found_major_node_id:
|
||||
# An existing major node is part of the value; link to it directly.
|
||||
for c_node_id in set(corr['nodes']):
|
||||
if self.graph.has_node(c_node_id) and c_node_id != found_major_node_id:
|
||||
self.add_edge(c_node_id, found_major_node_id, RelationshipType.CORRELATED_TO)
|
||||
attribute = corr['sources'][0]['path'].split('.')[-1]
|
||||
relationship_type = f"c_{attribute}"
|
||||
self.add_edge(c_node_id, found_major_node_id, relationship_type, confidence_score=0.9)
|
||||
continue # Skip creating a redundant correlation node
|
||||
|
||||
# Proceed to create a new correlation node if no major node was found.
|
||||
@ -211,22 +193,29 @@ class GraphManager:
|
||||
existing_meta['sources'] = [{'node_id': nid, 'path': p} for nid, p in existing_sources]
|
||||
|
||||
for c_node_id in set(corr['nodes']):
|
||||
self.add_edge(c_node_id, correlation_node_id, RelationshipType.CORRELATED_TO)
|
||||
attribute = corr['sources'][0]['path'].split('.')[-1]
|
||||
relationship_type = f"c_{attribute}"
|
||||
self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9)
|
||||
|
||||
self._update_correlation_index(node_id, attributes)
|
||||
|
||||
self.last_modified = datetime.now(timezone.utc).isoformat()
|
||||
return is_new_node
|
||||
|
||||
def add_edge(self, source_id: str, target_id: str, relationship_type: RelationshipType,
|
||||
confidence_score: Optional[float] = None, source_provider: str = "unknown",
|
||||
def add_edge(self, source_id: str, target_id: str, relationship_type: str,
|
||||
confidence_score: float = 0.5, source_provider: str = "unknown",
|
||||
raw_data: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""Add or update an edge between two nodes, ensuring nodes exist."""
|
||||
# LOGIC FIX: Ensure both source and target nodes exist before adding an edge.
|
||||
if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
|
||||
return False
|
||||
|
||||
new_confidence = confidence_score or relationship_type.default_confidence
|
||||
new_confidence = confidence_score
|
||||
|
||||
if relationship_type.startswith("c_"):
|
||||
edge_label = relationship_type
|
||||
else:
|
||||
edge_label = f"{source_provider}_{relationship_type}"
|
||||
|
||||
if self.graph.has_edge(source_id, target_id):
|
||||
# If edge exists, update confidence if the new score is higher.
|
||||
if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0):
|
||||
@ -237,7 +226,7 @@ class GraphManager:
|
||||
|
||||
# Add a new edge with all attributes.
|
||||
self.graph.add_edge(source_id, target_id,
|
||||
relationship_type=relationship_type.relationship_name,
|
||||
relationship_type=edge_label,
|
||||
confidence_score=new_confidence,
|
||||
source_provider=source_provider,
|
||||
discovery_timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
|
@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError,
|
||||
from collections import defaultdict, deque
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from core.graph_manager import GraphManager, NodeType, RelationshipType
|
||||
from core.graph_manager import GraphManager, NodeType
|
||||
from core.logger import get_forensic_logger, new_session
|
||||
from utils.helpers import _is_valid_ip, _is_valid_domain
|
||||
from providers.base_provider import BaseProvider
|
||||
@ -28,7 +28,6 @@ class ScanStatus:
|
||||
class Scanner:
|
||||
"""
|
||||
Main scanning orchestrator for DNSRecon passive reconnaissance.
|
||||
Enhanced with reliable cross-process termination capabilities.
|
||||
"""
|
||||
|
||||
def __init__(self, session_config=None):
|
||||
@ -507,7 +506,7 @@ class Scanner:
|
||||
self.logger.log_relationship_discovery(
|
||||
source_node=source,
|
||||
target_node=rel_target,
|
||||
relationship_type=rel_type.relationship_name,
|
||||
relationship_type=rel_type,
|
||||
confidence_score=confidence,
|
||||
provider=provider_name,
|
||||
raw_data=raw_data,
|
||||
@ -519,18 +518,18 @@ class Scanner:
|
||||
if _is_valid_ip(rel_target):
|
||||
self.graph.add_node(rel_target, NodeType.IP)
|
||||
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
|
||||
print(f"Added IP relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
|
||||
print(f"Added IP relationship: {source} -> {rel_target} ({rel_type})")
|
||||
discovered_targets.add(rel_target)
|
||||
|
||||
elif rel_target.startswith('AS') and rel_target[2:].isdigit():
|
||||
self.graph.add_node(rel_target, NodeType.ASN)
|
||||
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
|
||||
print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
|
||||
print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type})")
|
||||
|
||||
elif _is_valid_domain(rel_target):
|
||||
self.graph.add_node(rel_target, NodeType.DOMAIN)
|
||||
if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
|
||||
print(f"Added domain relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
|
||||
print(f"Added domain relationship: {source} -> {rel_target} ({rel_type})")
|
||||
discovered_targets.add(rel_target)
|
||||
self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target])
|
||||
|
||||
@ -577,10 +576,10 @@ class Scanner:
|
||||
|
||||
return set(targets)
|
||||
|
||||
def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: RelationshipType,
|
||||
def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: str,
|
||||
target: str, raw_data: Dict[str, Any], attributes: Dict[str, Any]) -> None:
|
||||
"""Collect and organize attributes for a node."""
|
||||
self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type.relationship_name}")
|
||||
self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type}")
|
||||
|
||||
if provider_name == 'dns':
|
||||
record_type = raw_data.get('query_type', 'UNKNOWN')
|
||||
@ -590,7 +589,7 @@ class Scanner:
|
||||
attributes.setdefault('dns_records', []).append(dns_entry)
|
||||
|
||||
elif provider_name == 'crtsh':
|
||||
if rel_type == RelationshipType.SAN_CERTIFICATE:
|
||||
if rel_type == "san_certificate":
|
||||
domain_certs = raw_data.get('domain_certificates', {})
|
||||
if node_id in domain_certs:
|
||||
cert_summary = domain_certs[node_id]
|
||||
@ -604,7 +603,7 @@ class Scanner:
|
||||
if key not in shodan_attributes or not shodan_attributes.get(key):
|
||||
shodan_attributes[key] = value
|
||||
|
||||
if rel_type == RelationshipType.ASN_MEMBERSHIP:
|
||||
if rel_type == "asn_membership":
|
||||
attributes['asn'] = {
|
||||
'id': target,
|
||||
'description': raw_data.get('org', ''),
|
||||
@ -612,7 +611,7 @@ class Scanner:
|
||||
'country': raw_data.get('country', '')
|
||||
}
|
||||
|
||||
record_type_name = rel_type.relationship_name
|
||||
record_type_name = rel_type
|
||||
if record_type_name not in attributes:
|
||||
attributes[record_type_name] = []
|
||||
|
||||
@ -719,8 +718,7 @@ class Scanner:
|
||||
'final_status': self.status,
|
||||
'total_indicators_processed': self.indicators_processed,
|
||||
'enabled_providers': list(provider_stats.keys()),
|
||||
'session_id': self.session_id,
|
||||
'forensic_note': 'Enhanced scanner with reliable cross-process termination'
|
||||
'session_id': self.session_id
|
||||
},
|
||||
'graph_data': graph_data,
|
||||
'forensic_audit': audit_trail,
|
||||
|
@ -16,7 +16,6 @@ from core.scanner import Scanner
|
||||
class SessionManager:
|
||||
"""
|
||||
Manages multiple scanner instances for concurrent user sessions using Redis.
|
||||
Enhanced with reliable cross-process stop signal management and immediate state updates.
|
||||
"""
|
||||
|
||||
def __init__(self, session_timeout_minutes: int = 60):
|
||||
@ -250,7 +249,7 @@ class SessionManager:
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[Scanner]:
|
||||
"""
|
||||
Get scanner instance for a session from Redis with enhanced session ID management.
|
||||
Get scanner instance for a session from Redis with session ID management.
|
||||
"""
|
||||
if not session_id:
|
||||
return None
|
||||
|
@ -9,7 +9,6 @@ from abc import ABC, abstractmethod
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
|
||||
from core.logger import get_forensic_logger
|
||||
from core.graph_manager import RelationshipType
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
@ -147,7 +146,7 @@ class BaseProvider(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query the provider for information about a domain.
|
||||
|
||||
@ -160,7 +159,7 @@ class BaseProvider(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query the provider for information about an IP address.
|
||||
|
||||
@ -419,7 +418,7 @@ class BaseProvider(ABC):
|
||||
return False
|
||||
|
||||
def log_relationship_discovery(self, source_node: str, target_node: str,
|
||||
relationship_type: RelationshipType,
|
||||
relationship_type: str,
|
||||
confidence_score: float,
|
||||
raw_data: Dict[str, Any],
|
||||
discovery_method: str) -> None:
|
||||
@ -439,7 +438,7 @@ class BaseProvider(ABC):
|
||||
self.logger.log_relationship_discovery(
|
||||
source_node=source_node,
|
||||
target_node=target_node,
|
||||
relationship_type=relationship_type.relationship_name,
|
||||
relationship_type=relationship_type,
|
||||
confidence_score=confidence_score,
|
||||
provider=self.name,
|
||||
raw_data=raw_data,
|
||||
|
@ -9,10 +9,10 @@ import re
|
||||
from typing import List, Dict, Any, Tuple, Set
|
||||
from urllib.parse import quote
|
||||
from datetime import datetime, timezone
|
||||
import requests
|
||||
|
||||
from .base_provider import BaseProvider
|
||||
from utils.helpers import _is_valid_domain
|
||||
from core.graph_manager import RelationshipType
|
||||
|
||||
|
||||
class CrtShProvider(BaseProvider):
|
||||
@ -145,7 +145,6 @@ class CrtShProvider(BaseProvider):
|
||||
'source': 'crt.sh'
|
||||
}
|
||||
|
||||
# Add computed fields
|
||||
try:
|
||||
if metadata['not_before'] and metadata['not_after']:
|
||||
not_before = self._parse_certificate_date(metadata['not_before'])
|
||||
@ -166,10 +165,9 @@ class CrtShProvider(BaseProvider):
|
||||
|
||||
return metadata
|
||||
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query crt.sh for certificates containing the domain.
|
||||
Enhanced with more frequent stop signal checking for reliable termination.
|
||||
"""
|
||||
if not _is_valid_domain(domain):
|
||||
return []
|
||||
@ -184,7 +182,7 @@ class CrtShProvider(BaseProvider):
|
||||
try:
|
||||
# Query crt.sh for certificates
|
||||
url = f"{self.base_url}?q={quote(domain)}&output=json"
|
||||
response = self.make_request(url, target_indicator=domain, max_retries=1) # Reduce retries for faster cancellation
|
||||
response = self.make_request(url, target_indicator=domain, max_retries=3)
|
||||
|
||||
if not response or response.status_code != 200:
|
||||
return []
|
||||
@ -208,7 +206,7 @@ class CrtShProvider(BaseProvider):
|
||||
domain_certificates = {}
|
||||
all_discovered_domains = set()
|
||||
|
||||
# Process certificates with enhanced cancellation checking
|
||||
# Process certificates with cancellation checking
|
||||
for i, cert_data in enumerate(certificates):
|
||||
# Check for cancellation every 5 certificates instead of 10 for faster response
|
||||
if i % 5 == 0 and self._stop_event and self._stop_event.is_set():
|
||||
@ -283,7 +281,7 @@ class CrtShProvider(BaseProvider):
|
||||
relationships.append((
|
||||
domain,
|
||||
discovered_domain,
|
||||
RelationshipType.SAN_CERTIFICATE,
|
||||
'san_certificate',
|
||||
confidence,
|
||||
relationship_raw_data
|
||||
))
|
||||
@ -292,7 +290,7 @@ class CrtShProvider(BaseProvider):
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=discovered_domain,
|
||||
relationship_type=RelationshipType.SAN_CERTIFICATE,
|
||||
relationship_type='san_certificate',
|
||||
confidence_score=confidence,
|
||||
raw_data=relationship_raw_data,
|
||||
discovery_method="certificate_transparency_analysis"
|
||||
@ -300,6 +298,9 @@ class CrtShProvider(BaseProvider):
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
self.logger.logger.error(f"Failed to parse JSON response from crt.sh: {e}")
|
||||
except requests.exceptions.RequestException as e:
|
||||
self.logger.logger.error(f"HTTP request to crt.sh failed: {e}")
|
||||
|
||||
|
||||
return relationships
|
||||
|
||||
@ -394,7 +395,7 @@ class CrtShProvider(BaseProvider):
|
||||
Returns:
|
||||
Confidence score between 0.0 and 1.0
|
||||
"""
|
||||
base_confidence = RelationshipType.SAN_CERTIFICATE.default_confidence
|
||||
base_confidence = 0.9
|
||||
|
||||
# Adjust confidence based on domain relationship context
|
||||
relationship_context = self._determine_relationship_context(domain2, domain1)
|
||||
@ -462,7 +463,7 @@ class CrtShProvider(BaseProvider):
|
||||
else:
|
||||
return 'related_domain'
|
||||
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query crt.sh for certificates containing the IP address.
|
||||
Note: crt.sh doesn't typically index by IP, so this returns empty results.
|
||||
|
@ -5,7 +5,6 @@ import dns.reversename
|
||||
from typing import List, Dict, Any, Tuple
|
||||
from .base_provider import BaseProvider
|
||||
from utils.helpers import _is_valid_ip, _is_valid_domain
|
||||
from core.graph_manager import RelationshipType
|
||||
|
||||
|
||||
class DNSProvider(BaseProvider):
|
||||
@ -49,7 +48,7 @@ class DNSProvider(BaseProvider):
|
||||
"""DNS is always available - no API key required."""
|
||||
return True
|
||||
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query DNS records for the domain to discover relationships.
|
||||
|
||||
@ -70,7 +69,7 @@ class DNSProvider(BaseProvider):
|
||||
|
||||
return relationships
|
||||
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query reverse DNS for the IP address.
|
||||
|
||||
@ -106,16 +105,16 @@ class DNSProvider(BaseProvider):
|
||||
relationships.append((
|
||||
ip,
|
||||
hostname,
|
||||
RelationshipType.PTR_RECORD,
|
||||
RelationshipType.PTR_RECORD.default_confidence,
|
||||
'ptr_record',
|
||||
0.8,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type=RelationshipType.PTR_RECORD,
|
||||
confidence_score=RelationshipType.PTR_RECORD.default_confidence,
|
||||
relationship_type='ptr_record',
|
||||
confidence_score=0.8,
|
||||
raw_data=raw_data,
|
||||
discovery_method="reverse_dns_lookup"
|
||||
)
|
||||
@ -126,7 +125,7 @@ class DNSProvider(BaseProvider):
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query a specific type of DNS record for the domain.
|
||||
"""
|
||||
@ -147,7 +146,8 @@ class DNSProvider(BaseProvider):
|
||||
elif record_type == 'SOA':
|
||||
target = str(record.mname).rstrip('.')
|
||||
elif record_type in ['TXT']:
|
||||
target = b' '.join(record.strings).decode('utf-8', 'ignore')
|
||||
# TXT records are treated as metadata, not relationships.
|
||||
continue
|
||||
elif record_type == 'SRV':
|
||||
target = str(record.target).rstrip('.')
|
||||
elif record_type == 'CAA':
|
||||
@ -155,7 +155,6 @@ class DNSProvider(BaseProvider):
|
||||
else:
|
||||
target = str(record)
|
||||
|
||||
|
||||
if target:
|
||||
raw_data = {
|
||||
'query_type': record_type,
|
||||
@ -163,32 +162,25 @@ class DNSProvider(BaseProvider):
|
||||
'value': target,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
try:
|
||||
relationship_type_enum_name = f"{record_type}_RECORD"
|
||||
# Handle TXT records as metadata, not relationships
|
||||
if record_type == 'TXT':
|
||||
relationship_type_enum = RelationshipType.A_RECORD # Dummy value, won't be used
|
||||
else:
|
||||
relationship_type_enum = getattr(RelationshipType, relationship_type_enum_name)
|
||||
relationship_type = f"{record_type.lower()}_record"
|
||||
confidence = 0.8 # Default confidence for DNS records
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
target,
|
||||
relationship_type_enum,
|
||||
relationship_type_enum.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
relationships.append((
|
||||
domain,
|
||||
target,
|
||||
relationship_type,
|
||||
confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=target,
|
||||
relationship_type=relationship_type_enum,
|
||||
confidence_score=relationship_type_enum.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method=f"dns_{record_type.lower()}_record"
|
||||
)
|
||||
except AttributeError:
|
||||
self.logger.logger.error(f"Unsupported record type '{record_type}' encountered for domain {domain}")
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=target,
|
||||
relationship_type=relationship_type,
|
||||
confidence_score=confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method=f"dns_{record_type.lower()}_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.failed_requests += 1
|
||||
|
@ -7,7 +7,6 @@ import json
|
||||
from typing import List, Dict, Any, Tuple
|
||||
from .base_provider import BaseProvider
|
||||
from utils.helpers import _is_valid_ip, _is_valid_domain
|
||||
from core.graph_manager import RelationshipType
|
||||
|
||||
|
||||
class ShodanProvider(BaseProvider):
|
||||
@ -47,7 +46,7 @@ class ShodanProvider(BaseProvider):
|
||||
"""Return a dictionary indicating if the provider can query domains and/or IPs."""
|
||||
return {'domains': True, 'ips': True}
|
||||
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query Shodan for information about a domain.
|
||||
Uses Shodan's hostname search to find associated IPs.
|
||||
@ -103,16 +102,16 @@ class ShodanProvider(BaseProvider):
|
||||
relationships.append((
|
||||
domain,
|
||||
ip_address,
|
||||
RelationshipType.A_RECORD, # Domain resolves to IP
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
'a_record', # Domain resolves to IP
|
||||
0.8,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ip_address,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
relationship_type='a_record',
|
||||
confidence_score=0.8,
|
||||
raw_data=raw_data,
|
||||
discovery_method="shodan_hostname_search"
|
||||
)
|
||||
@ -129,7 +128,7 @@ class ShodanProvider(BaseProvider):
|
||||
relationships.append((
|
||||
domain,
|
||||
hostname,
|
||||
RelationshipType.PASSIVE_DNS, # Shared hosting relationship
|
||||
'passive_dns', # Shared hosting relationship
|
||||
0.6, # Lower confidence for shared hosting
|
||||
hostname_raw_data
|
||||
))
|
||||
@ -137,7 +136,7 @@ class ShodanProvider(BaseProvider):
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=hostname,
|
||||
relationship_type=RelationshipType.PASSIVE_DNS,
|
||||
relationship_type='passive_dns',
|
||||
confidence_score=0.6,
|
||||
raw_data=hostname_raw_data,
|
||||
discovery_method="shodan_shared_hosting"
|
||||
@ -148,7 +147,7 @@ class ShodanProvider(BaseProvider):
|
||||
|
||||
return relationships
|
||||
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query Shodan for information about an IP address.
|
||||
|
||||
@ -195,16 +194,16 @@ class ShodanProvider(BaseProvider):
|
||||
relationships.append((
|
||||
ip,
|
||||
hostname,
|
||||
RelationshipType.A_RECORD, # IP resolves to hostname
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
'a_record', # IP resolves to hostname
|
||||
0.8,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
relationship_type='a_record',
|
||||
confidence_score=0.8,
|
||||
raw_data=raw_data,
|
||||
discovery_method="shodan_host_lookup"
|
||||
)
|
||||
@ -230,16 +229,16 @@ class ShodanProvider(BaseProvider):
|
||||
relationships.append((
|
||||
ip,
|
||||
asn_name,
|
||||
RelationshipType.ASN_MEMBERSHIP,
|
||||
RelationshipType.ASN_MEMBERSHIP.default_confidence,
|
||||
'asn_membership',
|
||||
0.7,
|
||||
asn_raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=asn_name,
|
||||
relationship_type=RelationshipType.ASN_MEMBERSHIP,
|
||||
confidence_score=RelationshipType.ASN_MEMBERSHIP.default_confidence,
|
||||
relationship_type='asn_membership',
|
||||
confidence_score=0.7,
|
||||
raw_data=asn_raw_data,
|
||||
discovery_method="shodan_asn_lookup"
|
||||
)
|
||||
|
@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Graph visualization module for DNSRecon
|
||||
* Handles network graph rendering using vis.js with enhanced Phase 2 features
|
||||
* Handles network graph rendering using vis.js
|
||||
*/
|
||||
|
||||
class GraphManager {
|
||||
@ -130,7 +130,7 @@ class GraphManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the network graph with enhanced features
|
||||
* Initialize the network graph
|
||||
*/
|
||||
initialize() {
|
||||
if (this.isInitialized) {
|
||||
@ -156,7 +156,7 @@ class GraphManager {
|
||||
// Add graph controls
|
||||
this.addGraphControls();
|
||||
|
||||
console.log('Enhanced graph initialized successfully');
|
||||
console.log('Graph initialized successfully');
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize graph:', error);
|
||||
this.showError('Failed to initialize visualization');
|
||||
@ -184,12 +184,12 @@ class GraphManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup enhanced network event handlers
|
||||
* Setup network event handlers
|
||||
*/
|
||||
setupNetworkEvents() {
|
||||
if (!this.network) return;
|
||||
|
||||
// Node click event with enhanced details
|
||||
// Node click event with details
|
||||
this.network.on('click', (params) => {
|
||||
if (params.nodes.length > 0) {
|
||||
const nodeId = params.nodes[0];
|
||||
@ -207,7 +207,7 @@ class GraphManager {
|
||||
}
|
||||
});
|
||||
|
||||
// Enhanced hover events
|
||||
// Hover events
|
||||
this.network.on('hoverNode', (params) => {
|
||||
const nodeId = params.node;
|
||||
const node = this.nodes.get(nodeId);
|
||||
@ -242,7 +242,6 @@ class GraphManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Update graph with new data and enhanced processing
|
||||
* @param {Object} graphData - Graph data from backend
|
||||
*/
|
||||
updateGraph(graphData) {
|
||||
@ -326,15 +325,15 @@ class GraphManager {
|
||||
setTimeout(() => this.fitView(), 800);
|
||||
}
|
||||
|
||||
console.log(`Enhanced graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
|
||||
console.log(`Graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
|
||||
} catch (error) {
|
||||
console.error('Failed to update enhanced graph:', error);
|
||||
console.error('Failed to update graph:', error);
|
||||
this.showError('Failed to update visualization');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process node data with enhanced styling and metadata
|
||||
* Process node data with styling and metadata
|
||||
* @param {Object} node - Raw node data
|
||||
* @returns {Object} Processed node data
|
||||
*/
|
||||
@ -367,14 +366,17 @@ class GraphManager {
|
||||
}
|
||||
|
||||
if (node.type === 'correlation_object') {
|
||||
processedNode.label = this.formatNodeLabel(node.metadata.value, node.type);
|
||||
const value = node.metadata.value;
|
||||
const label = Array.isArray(value) ? `Correlated (${value.length})` : String(value);
|
||||
processedNode.label = this.formatNodeLabel(label, node.type);
|
||||
processedNode.title = Array.isArray(value) ? value.join(', ') : value;
|
||||
}
|
||||
|
||||
return processedNode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process edge data with enhanced styling and metadata
|
||||
* Process edge data with styling and metadata
|
||||
* @param {Object} edge - Raw edge data
|
||||
* @returns {Object} Processed edge data
|
||||
*/
|
||||
@ -478,7 +480,7 @@ class GraphManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get enhanced node shape based on type
|
||||
* Get node shape based on type
|
||||
* @param {string} nodeType - Node type
|
||||
* @returns {string} Shape name
|
||||
*/
|
||||
|
@ -243,7 +243,7 @@ class DNSReconApp {
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced start scan with better error handling
|
||||
* Start scan with error handling
|
||||
*/
|
||||
async startScan(clearGraph = true) {
|
||||
console.log('=== STARTING SCAN ===');
|
||||
@ -318,7 +318,7 @@ class DNSReconApp {
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Enhanced scan stop with immediate UI feedback
|
||||
* Scan stop with immediate UI feedback
|
||||
*/
|
||||
async stopScan() {
|
||||
try {
|
||||
@ -427,7 +427,7 @@ class DNSReconApp {
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced status update with better error handling
|
||||
* Status update with better error handling
|
||||
*/
|
||||
async updateStatus() {
|
||||
try {
|
||||
@ -668,7 +668,7 @@ class DNSReconApp {
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced UI state management with immediate button updates
|
||||
* UI state management with immediate button updates
|
||||
*/
|
||||
setUIState(state) {
|
||||
console.log(`Setting UI state to: ${state}`);
|
||||
@ -823,7 +823,7 @@ class DNSReconApp {
|
||||
});
|
||||
detailsHtml += '</ul></div>';
|
||||
}
|
||||
|
||||
|
||||
// Section for Attributes
|
||||
detailsHtml += '<div class="modal-section">';
|
||||
detailsHtml += '<h4>Attributes</h4>';
|
||||
|
Loading…
x
Reference in New Issue
Block a user