it

app.py

@@ -1,7 +1,6 @@
 """
 Flask application entry point for DNSRecon web interface.
 Provides REST API endpoints and serves the web interface with user session support.
-Enhanced with better session debugging and isolation.
 """
 
 import json
@@ -20,7 +19,7 @@ app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=2)  # 2 hour session
 
 def get_user_scanner():
     """
-    Enhanced user scanner retrieval with better error handling and debugging.
+    User scanner retrieval with better error handling and debugging.
     """
     # Get current Flask session info for debugging
     current_flask_session_id = session.get('dnsrecon_session_id')
@@ -184,7 +183,7 @@ def stop_scan():
         if not scanner.session_id:
             scanner.session_id = user_session_id
         
-        # Use the enhanced stop mechanism
+        # Use the stop mechanism
         success = scanner.stop_scan()
         
         # Also set the Redis stop signal directly for extra reliability
@@ -203,7 +202,7 @@ def stop_scan():
             'message': 'Scan stop requested - termination initiated',
             'user_session_id': user_session_id,
             'scanner_status': scanner.status,
-            'stop_method': 'enhanced_cross_process'
+            'stop_method': 'cross_process'
         })
         
     except Exception as e:
@@ -217,7 +216,7 @@ def stop_scan():
 
 @app.route('/api/scan/status', methods=['GET'])
 def get_scan_status():
-    """Get current scan status with enhanced error handling."""
+    """Get current scan status with error handling."""
     try:
         # Get user-specific scanner
         user_session_id, scanner = get_user_scanner()
@@ -279,7 +278,7 @@ def get_scan_status():
 
 @app.route('/api/graph', methods=['GET'])
 def get_graph_data():
-    """Get current graph data with enhanced error handling."""
+    """Get current graph data with error handling."""
     try:
         # Get user-specific scanner
         user_session_id, scanner = get_user_scanner()
@@ -524,7 +523,7 @@ def list_sessions():
 
 @app.route('/api/health', methods=['GET'])
 def health_check():
-    """Health check endpoint with enhanced Phase 2 information."""
+    """Health check endpoint."""
     try:
         # Get session stats
         session_stats = session_manager.get_statistics()
@@ -540,7 +539,7 @@ def health_check():
                 'concurrent_processing': True,
                 'real_time_updates': True,
                 'api_key_management': True,
-                'enhanced_visualization': True,
+                'visualization': True,
                 'retry_logic': True,
                 'user_sessions': True,
                 'session_isolation': True

core/__init__.py

@@ -1,28 +1,25 @@
 """
 Core modules for DNSRecon passive reconnaissance tool.
 Contains graph management, scanning orchestration, and forensic logging.
-Phase 2: Enhanced with concurrent processing and real-time capabilities.
 """
 
-from .graph_manager import GraphManager, NodeType, RelationshipType
-from .scanner import Scanner, ScanStatus  # Remove 'scanner' global instance
+from .graph_manager import GraphManager, NodeType
+from .scanner import Scanner, ScanStatus  
 from .logger import ForensicLogger, get_forensic_logger, new_session
-from .session_manager import session_manager  # Add session manager
-from .session_config import SessionConfig, create_session_config  # Add session config
+from .session_manager import session_manager 
+from .session_config import SessionConfig, create_session_config 
 
 __all__ = [
     'GraphManager',
     'NodeType', 
-    'RelationshipType',
     'Scanner',
     'ScanStatus',
-    # 'scanner',  # Remove this - no more global scanner
     'ForensicLogger',
     'get_forensic_logger',
     'new_session',
-    'session_manager',  # Add this
-    'SessionConfig',    # Add this
-    'create_session_config'  # Add this
+    'session_manager', 
+    'SessionConfig',
+    'create_session_config'
 ]
 
 __version__ = "1.0.0-phase2"

core/graph_manager.py

@@ -22,28 +22,6 @@ class NodeType(Enum):
         return self.value
 
 
-class RelationshipType(Enum):
-    """Enumeration of supported relationship types with confidence scores."""
-    SAN_CERTIFICATE = ("san", 0.9)
-    A_RECORD = ("a_record", 0.8)
-    AAAA_RECORD = ("aaaa_record", 0.8)
-    CNAME_RECORD = ("cname", 0.8)
-    MX_RECORD = ("mx_record", 0.7)
-    NS_RECORD = ("ns_record", 0.7)
-    PTR_RECORD = ("ptr_record", 0.8)
-    SOA_RECORD = ("soa_record", 0.7)
-    PASSIVE_DNS = ("passive_dns", 0.6)
-    ASN_MEMBERSHIP = ("asn", 0.7)
-    CORRELATED_TO = ("correlated_to", 0.9)
-
-    def __init__(self, relationship_name: str, default_confidence: float):
-        self.relationship_name = relationship_name
-        self.default_confidence = default_confidence
-
-    def __repr__(self):
-        return self.relationship_name
-
-
 class GraphManager:
     """
     Thread-safe graph manager for DNSRecon infrastructure mapping.
@@ -180,53 +158,175 @@ class GraphManager:
             for corr in correlations:
                 value = corr['value']
                 
-                # FIXED: Check if the correlation value contains an existing node ID.
-                found_major_node_id = None
-                if isinstance(value, str):
-                    for existing_node in self.graph.nodes():
-                        if existing_node in value:
-                            found_major_node_id = existing_node
-                            break
+                # STEP 1: Substring check against all existing nodes
+                if self._correlation_value_matches_existing_node(value):
+                    # Skip creating correlation node - would be redundant
+                    continue
                 
-                if found_major_node_id:
-                    # An existing major node is part of the value; link to it directly.
-                    for c_node_id in set(corr['nodes']):
-                        if self.graph.has_node(c_node_id) and c_node_id != found_major_node_id:
-                            self.add_edge(c_node_id, found_major_node_id, RelationshipType.CORRELATED_TO)
-                    continue # Skip creating a redundant correlation node
+                # STEP 2: Filter out node pairs that already have direct edges
+                eligible_nodes = self._filter_nodes_without_direct_edges(set(corr['nodes']))
                 
-                # Proceed to create a new correlation node if no major node was found.
-                correlation_node_id = f"{value}"
-                if not self.graph.has_node(correlation_node_id):
+                if len(eligible_nodes) < 2:
+                    # Need at least 2 nodes to create a correlation
+                    continue
+                    
+                # STEP 3: Check for existing correlation node with same connection pattern
+                correlation_nodes_with_pattern = self._find_correlation_nodes_with_same_pattern(eligible_nodes)
+                
+                if correlation_nodes_with_pattern:
+                    # STEP 4: Merge with existing correlation node
+                    target_correlation_node = correlation_nodes_with_pattern[0]
+                    self._merge_correlation_values(target_correlation_node, value, corr)
+                else:
+                    # STEP 5: Create new correlation node for eligible nodes only
+                    correlation_node_id = f"corr_{abs(hash(str(sorted(eligible_nodes))))}"
                     self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT,
-                                  metadata={'value': value, 'sources': corr['sources'],
-                                            'correlated_nodes': list(set(corr['nodes']))})
-                else: # Update existing correlation node
-                    existing_meta = self.graph.nodes[correlation_node_id]['metadata']
-                    existing_nodes = set(existing_meta.get('correlated_nodes', []))
-                    existing_meta['correlated_nodes'] = list(existing_nodes.union(set(corr['nodes'])))
-                    existing_sources = {(s['node_id'], s['path']) for s in existing_meta.get('sources', [])}
-                    for s in corr['sources']:
-                        existing_sources.add((s['node_id'], s['path']))
-                    existing_meta['sources'] = [{'node_id': nid, 'path': p} for nid, p in existing_sources]
+                                metadata={'values': [value], 'sources': corr['sources'],
+                                            'correlated_nodes': list(eligible_nodes)})
                     
-                for c_node_id in set(corr['nodes']):
-                    self.add_edge(c_node_id, correlation_node_id, RelationshipType.CORRELATED_TO)
+                    # Create edges from eligible nodes to this correlation node
+                    for c_node_id in eligible_nodes:
+                        if self.graph.has_node(c_node_id):
+                            attribute = corr['sources'][0]['path'].split('.')[-1]
+                            relationship_type = f"c_{attribute}"
+                            self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9)
 
             self._update_correlation_index(node_id, attributes)
 
         self.last_modified = datetime.now(timezone.utc).isoformat()
         return is_new_node
 
-    def add_edge(self, source_id: str, target_id: str, relationship_type: RelationshipType,
-                 confidence_score: Optional[float] = None, source_provider: str = "unknown",
+    def _filter_nodes_without_direct_edges(self, node_set: set) -> set:
+        """
+        Filter out nodes that already have direct edges between them.
+        Returns set of nodes that should be included in correlation.
+        """
+        nodes_list = list(node_set)
+        eligible_nodes = set(node_set)  # Start with all nodes
+        
+        # Check all pairs of nodes
+        for i in range(len(nodes_list)):
+            for j in range(i + 1, len(nodes_list)):
+                node_a = nodes_list[i]
+                node_b = nodes_list[j]
+                
+                # Check if direct edge exists in either direction
+                if self._has_direct_edge_bidirectional(node_a, node_b):
+                    # Remove both nodes from eligible set since they're already connected
+                    eligible_nodes.discard(node_a)
+                    eligible_nodes.discard(node_b)
+        
+        return eligible_nodes
+
+    def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool:
+        """
+        Check if there's a direct edge between two nodes in either direction.
+        Returns True if node_a→node_b OR node_b→node_a exists.
+        """
+        return (self.graph.has_edge(node_a, node_b) or 
+                self.graph.has_edge(node_b, node_a))
+
+    def _correlation_value_matches_existing_node(self, correlation_value: str) -> bool:
+        """
+        Check if correlation value contains any existing node ID as substring.
+        Returns True if match found (correlation node should NOT be created).
+        """
+        correlation_str = str(correlation_value).lower()
+        
+        # Check against all existing nodes
+        for existing_node_id in self.graph.nodes():
+            if existing_node_id.lower() in correlation_str:
+                return True
+        
+        return False
+
+    def _find_correlation_nodes_with_same_pattern(self, node_set: set) -> List[str]:
+        """
+        Find existing correlation nodes that have the exact same pattern of connected nodes.
+        Returns list of correlation node IDs with matching patterns.
+        """
+        correlation_nodes = self.get_nodes_by_type(NodeType.CORRELATION_OBJECT)
+        matching_nodes = []
+        
+        for corr_node_id in correlation_nodes:
+            # Get all nodes connected to this correlation node
+            connected_nodes = set()
+            
+            # Add all predecessors (nodes pointing TO the correlation node)
+            connected_nodes.update(self.graph.predecessors(corr_node_id))
+            
+            # Add all successors (nodes pointed TO by the correlation node)  
+            connected_nodes.update(self.graph.successors(corr_node_id))
+            
+            # Check if the pattern matches exactly
+            if connected_nodes == node_set:
+                matching_nodes.append(corr_node_id)
+        
+        return matching_nodes
+
+    def _merge_correlation_values(self, target_node_id: str, new_value: Any, corr_data: Dict) -> None:
+        """
+        Merge a new correlation value into an existing correlation node.
+        Uses same logic as large entity merging.
+        """
+        if not self.graph.has_node(target_node_id):
+            return
+            
+        target_metadata = self.graph.nodes[target_node_id]['metadata']
+        
+        # Get existing values (ensure it's a list)
+        existing_values = target_metadata.get('values', [])
+        if not isinstance(existing_values, list):
+            existing_values = [existing_values]
+        
+        # Add new value if not already present
+        if new_value not in existing_values:
+            existing_values.append(new_value)
+            
+        # Merge sources
+        existing_sources = target_metadata.get('sources', [])
+        new_sources = corr_data.get('sources', [])
+        
+        # Create set of unique sources based on (node_id, path) tuples
+        source_set = set()
+        for source in existing_sources + new_sources:
+            source_tuple = (source['node_id'], source['path'])
+            source_set.add(source_tuple)
+        
+        # Convert back to list of dictionaries
+        merged_sources = [{'node_id': nid, 'path': path} for nid, path in source_set]
+        
+        # Update metadata
+        target_metadata.update({
+            'values': existing_values,
+            'sources': merged_sources,
+            'correlated_nodes': list(set(target_metadata.get('correlated_nodes', []) + corr_data.get('nodes', []))),
+            'merge_count': len(existing_values),
+            'last_merge_timestamp': datetime.now(timezone.utc).isoformat()
+        })
+        
+        # Update description to reflect merged nature
+        value_count = len(existing_values)
+        node_count = len(target_metadata['correlated_nodes'])
+        self.graph.nodes[target_node_id]['description'] = (
+            f"Correlation container with {value_count} merged values "
+            f"across {node_count} nodes"
+        )
+        
+    def add_edge(self, source_id: str, target_id: str, relationship_type: str,
+                 confidence_score: float = 0.5, source_provider: str = "unknown",
                  raw_data: Optional[Dict[str, Any]] = None) -> bool:
         """Add or update an edge between two nodes, ensuring nodes exist."""
-        # LOGIC FIX: Ensure both source and target nodes exist before adding an edge.
         if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
             return False
 
-        new_confidence = confidence_score or relationship_type.default_confidence
+        new_confidence = confidence_score
+        
+        if relationship_type.startswith("c_"):
+            edge_label = relationship_type
+        else:
+            edge_label = f"{source_provider}_{relationship_type}"
+        
         if self.graph.has_edge(source_id, target_id):
             # If edge exists, update confidence if the new score is higher.
             if new_confidence > self.graph.edges[source_id, target_id].get('confidence_score', 0):
@@ -237,7 +337,7 @@ class GraphManager:
 
         # Add a new edge with all attributes.
         self.graph.add_edge(source_id, target_id,
-                            relationship_type=relationship_type.relationship_name,
+                            relationship_type=edge_label,
                             confidence_score=new_confidence,
                             source_provider=source_provider,
                             discovery_timestamp=datetime.now(timezone.utc).isoformat(),

core/scanner.py

@@ -10,7 +10,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError,
 from collections import defaultdict, deque
 from datetime import datetime, timezone
 
-from core.graph_manager import GraphManager, NodeType, RelationshipType
+from core.graph_manager import GraphManager, NodeType
 from core.logger import get_forensic_logger, new_session
 from utils.helpers import _is_valid_ip, _is_valid_domain
 from providers.base_provider import BaseProvider
@@ -28,7 +28,6 @@ class ScanStatus:
 class Scanner:
     """
     Main scanning orchestrator for DNSRecon passive reconnaissance.
-    Enhanced with reliable cross-process termination capabilities.
     """
 
     def __init__(self, session_config=None):
@@ -507,7 +506,7 @@ class Scanner:
             self.logger.log_relationship_discovery(
                 source_node=source,
                 target_node=rel_target,
-                relationship_type=rel_type.relationship_name,
+                relationship_type=rel_type,
                 confidence_score=confidence,
                 provider=provider_name,
                 raw_data=raw_data,
@@ -519,18 +518,18 @@ class Scanner:
             if _is_valid_ip(rel_target):
                 self.graph.add_node(rel_target, NodeType.IP)
                 if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
-                    print(f"Added IP relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
+                    print(f"Added IP relationship: {source} -> {rel_target} ({rel_type})")
                 discovered_targets.add(rel_target)
 
             elif rel_target.startswith('AS') and rel_target[2:].isdigit():
                 self.graph.add_node(rel_target, NodeType.ASN)
                 if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
-                    print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
+                    print(f"Added ASN relationship: {source} -> {rel_target} ({rel_type})")
 
             elif _is_valid_domain(rel_target):
                 self.graph.add_node(rel_target, NodeType.DOMAIN)
                 if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data):
-                    print(f"Added domain relationship: {source} -> {rel_target} ({rel_type.relationship_name})")
+                    print(f"Added domain relationship: {source} -> {rel_target} ({rel_type})")
                 discovered_targets.add(rel_target)
                 self._collect_node_attributes(rel_target, provider_name, rel_type, source, raw_data, node_attributes[rel_target])
             
@@ -577,10 +576,10 @@ class Scanner:
         
         return set(targets)
 
-    def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: RelationshipType,
+    def _collect_node_attributes(self, node_id: str, provider_name: str, rel_type: str,
                                     target: str, raw_data: Dict[str, Any], attributes: Dict[str, Any]) -> None:
         """Collect and organize attributes for a node."""
-        self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type.relationship_name}")
+        self.logger.logger.debug(f"Collecting attributes for {node_id} from {provider_name}: {rel_type}")
 
         if provider_name == 'dns':
             record_type = raw_data.get('query_type', 'UNKNOWN')
@@ -590,7 +589,7 @@ class Scanner:
                 attributes.setdefault('dns_records', []).append(dns_entry)
 
         elif provider_name == 'crtsh':
-            if rel_type == RelationshipType.SAN_CERTIFICATE:
+            if rel_type == "san_certificate":
                 domain_certs = raw_data.get('domain_certificates', {})
                 if node_id in domain_certs:
                     cert_summary = domain_certs[node_id]
@@ -604,7 +603,7 @@ class Scanner:
                 if key not in shodan_attributes or not shodan_attributes.get(key):
                     shodan_attributes[key] = value
 
-        if rel_type == RelationshipType.ASN_MEMBERSHIP:
+        if rel_type == "asn_membership":
             attributes['asn'] = {
                 'id': target,
                 'description': raw_data.get('org', ''),
@@ -612,7 +611,7 @@ class Scanner:
                 'country': raw_data.get('country', '')
             }
         
-        record_type_name = rel_type.relationship_name
+        record_type_name = rel_type
         if record_type_name not in attributes:
             attributes[record_type_name] = []
         
@@ -719,8 +718,7 @@ class Scanner:
                 'final_status': self.status,
                 'total_indicators_processed': self.indicators_processed,
                 'enabled_providers': list(provider_stats.keys()),
-                'session_id': self.session_id,
-                'forensic_note': 'Enhanced scanner with reliable cross-process termination'
+                'session_id': self.session_id
             },
             'graph_data': graph_data,
             'forensic_audit': audit_trail,

core/session_manager.py

@@ -16,7 +16,6 @@ from core.scanner import Scanner
 class SessionManager:
     """
     Manages multiple scanner instances for concurrent user sessions using Redis.
-    Enhanced with reliable cross-process stop signal management and immediate state updates.
     """
     
     def __init__(self, session_timeout_minutes: int = 60):
@@ -250,7 +249,7 @@ class SessionManager:
 
     def get_session(self, session_id: str) -> Optional[Scanner]:
         """
-        Get scanner instance for a session from Redis with enhanced session ID management.
+        Get scanner instance for a session from Redis with session ID management.
         """
         if not session_id:
             return None

providers/base_provider.py

@@ -9,7 +9,6 @@ from abc import ABC, abstractmethod
 from typing import List, Dict, Any, Optional, Tuple
 
 from core.logger import get_forensic_logger
-from core.graph_manager import RelationshipType
 
 
 class RateLimiter:
@@ -147,7 +146,7 @@ class BaseProvider(ABC):
         pass
 
     @abstractmethod
-    def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query the provider for information about a domain.
 
@@ -160,7 +159,7 @@ class BaseProvider(ABC):
         pass
 
     @abstractmethod
-    def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query the provider for information about an IP address.
 
@@ -419,7 +418,7 @@ class BaseProvider(ABC):
         return False
 
     def log_relationship_discovery(self, source_node: str, target_node: str,
-                                 relationship_type: RelationshipType,
+                                 relationship_type: str,
                                  confidence_score: float,
                                  raw_data: Dict[str, Any],
                                  discovery_method: str) -> None:
@@ -439,7 +438,7 @@ class BaseProvider(ABC):
         self.logger.log_relationship_discovery(
             source_node=source_node,
             target_node=target_node,
-            relationship_type=relationship_type.relationship_name,
+            relationship_type=relationship_type,
             confidence_score=confidence_score,
             provider=self.name,
             raw_data=raw_data,

providers/crtsh_provider.py

@@ -9,10 +9,10 @@ import re
 from typing import List, Dict, Any, Tuple, Set
 from urllib.parse import quote
 from datetime import datetime, timezone
+import requests
 
 from .base_provider import BaseProvider
 from utils.helpers import _is_valid_domain
-from core.graph_manager import RelationshipType
 
 
 class CrtShProvider(BaseProvider):
@@ -145,7 +145,6 @@ class CrtShProvider(BaseProvider):
             'source': 'crt.sh'
         }
         
-        # Add computed fields
         try:
             if metadata['not_before'] and metadata['not_after']:
                 not_before = self._parse_certificate_date(metadata['not_before'])
@@ -166,10 +165,9 @@ class CrtShProvider(BaseProvider):
         
         return metadata
 
-    def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query crt.sh for certificates containing the domain.
-        Enhanced with more frequent stop signal checking for reliable termination.
         """
         if not _is_valid_domain(domain):
             return []
@@ -184,7 +182,7 @@ class CrtShProvider(BaseProvider):
         try:
             # Query crt.sh for certificates
             url = f"{self.base_url}?q={quote(domain)}&output=json"
-            response = self.make_request(url, target_indicator=domain, max_retries=1)  # Reduce retries for faster cancellation
+            response = self.make_request(url, target_indicator=domain, max_retries=3)
             
             if not response or response.status_code != 200:
                 return []
@@ -208,7 +206,7 @@ class CrtShProvider(BaseProvider):
             domain_certificates = {}
             all_discovered_domains = set()
             
-            # Process certificates with enhanced cancellation checking
+            # Process certificates with cancellation checking
             for i, cert_data in enumerate(certificates):
                 # Check for cancellation every 5 certificates instead of 10 for faster response
                 if i % 5 == 0 and self._stop_event and self._stop_event.is_set():
@@ -283,7 +281,7 @@ class CrtShProvider(BaseProvider):
                 relationships.append((
                     domain,
                     discovered_domain,
-                    RelationshipType.SAN_CERTIFICATE,
+                    'san_certificate',
                     confidence,
                     relationship_raw_data
                 ))
@@ -292,7 +290,7 @@ class CrtShProvider(BaseProvider):
                 self.log_relationship_discovery(
                     source_node=domain,
                     target_node=discovered_domain,
-                    relationship_type=RelationshipType.SAN_CERTIFICATE,
+                    relationship_type='san_certificate',
                     confidence_score=confidence,
                     raw_data=relationship_raw_data,
                     discovery_method="certificate_transparency_analysis"
@@ -300,6 +298,9 @@ class CrtShProvider(BaseProvider):
 
         except json.JSONDecodeError as e:
             self.logger.logger.error(f"Failed to parse JSON response from crt.sh: {e}")
+        except requests.exceptions.RequestException as e:
+            self.logger.logger.error(f"HTTP request to crt.sh failed: {e}")
+
         
         return relationships
 
@@ -394,7 +395,7 @@ class CrtShProvider(BaseProvider):
         Returns:
             Confidence score between 0.0 and 1.0
         """
-        base_confidence = RelationshipType.SAN_CERTIFICATE.default_confidence
+        base_confidence = 0.9
         
         # Adjust confidence based on domain relationship context
         relationship_context = self._determine_relationship_context(domain2, domain1)
@@ -462,7 +463,7 @@ class CrtShProvider(BaseProvider):
         else:
             return 'related_domain'
     
-    def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query crt.sh for certificates containing the IP address.
         Note: crt.sh doesn't typically index by IP, so this returns empty results.

providers/dns_provider.py

@@ -5,7 +5,6 @@ import dns.reversename
 from typing import List, Dict, Any, Tuple
 from .base_provider import BaseProvider
 from utils.helpers import _is_valid_ip, _is_valid_domain
-from core.graph_manager import RelationshipType
 
 
 class DNSProvider(BaseProvider):
@@ -49,7 +48,7 @@ class DNSProvider(BaseProvider):
         """DNS is always available - no API key required."""
         return True
 
-    def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query DNS records for the domain to discover relationships.
 
@@ -70,7 +69,7 @@ class DNSProvider(BaseProvider):
 
         return relationships
 
-    def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query reverse DNS for the IP address.
 
@@ -106,16 +105,16 @@ class DNSProvider(BaseProvider):
                     relationships.append((
                         ip,
                         hostname,
-                        RelationshipType.PTR_RECORD,
-                        RelationshipType.PTR_RECORD.default_confidence,
+                        'ptr_record',
+                        0.8,
                         raw_data
                     ))
 
                     self.log_relationship_discovery(
                         source_node=ip,
                         target_node=hostname,
-                        relationship_type=RelationshipType.PTR_RECORD,
-                        confidence_score=RelationshipType.PTR_RECORD.default_confidence,
+                        relationship_type='ptr_record',
+                        confidence_score=0.8,
                         raw_data=raw_data,
                         discovery_method="reverse_dns_lookup"
                     )
@@ -126,7 +125,7 @@ class DNSProvider(BaseProvider):
 
         return relationships
 
-    def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query a specific type of DNS record for the domain.
         """
@@ -147,7 +146,8 @@ class DNSProvider(BaseProvider):
                 elif record_type == 'SOA':
                     target = str(record.mname).rstrip('.')
                 elif record_type in ['TXT']:
-                    target = b' '.join(record.strings).decode('utf-8', 'ignore')
+                    # TXT records are treated as metadata, not relationships.
+                    continue
                 elif record_type == 'SRV':
                     target = str(record.target).rstrip('.')
                 elif record_type == 'CAA':
@@ -155,7 +155,6 @@ class DNSProvider(BaseProvider):
                 else:
                     target = str(record)
 
-
                 if target:
                     raw_data = {
                         'query_type': record_type,
@@ -163,32 +162,25 @@ class DNSProvider(BaseProvider):
                         'value': target,
                         'ttl': response.ttl
                     }
-                    try:
-                        relationship_type_enum_name = f"{record_type}_RECORD"
-                        # Handle TXT records as metadata, not relationships
-                        if record_type == 'TXT':
-                            relationship_type_enum = RelationshipType.A_RECORD # Dummy value, won't be used
-                        else:
-                            relationship_type_enum = getattr(RelationshipType, relationship_type_enum_name)
+                    relationship_type = f"{record_type.lower()}_record"
+                    confidence = 0.8  # Default confidence for DNS records
 
                     relationships.append((
                         domain,
                         target,
-                            relationship_type_enum,
-                            relationship_type_enum.default_confidence,
+                        relationship_type,
+                        confidence,
                         raw_data
                     ))
 
                     self.log_relationship_discovery(
                         source_node=domain,
                         target_node=target,
-                            relationship_type=relationship_type_enum,
-                            confidence_score=relationship_type_enum.default_confidence,
+                        relationship_type=relationship_type,
+                        confidence_score=confidence,
                         raw_data=raw_data,
                         discovery_method=f"dns_{record_type.lower()}_record"
                     )
-                    except AttributeError:
-                        self.logger.logger.error(f"Unsupported record type '{record_type}' encountered for domain {domain}")
 
         except Exception as e:
             self.failed_requests += 1

providers/shodan_provider.py

@@ -7,7 +7,6 @@ import json
 from typing import List, Dict, Any, Tuple
 from .base_provider import BaseProvider
 from utils.helpers import _is_valid_ip, _is_valid_domain
-from core.graph_manager import RelationshipType
 
 
 class ShodanProvider(BaseProvider):
@@ -47,7 +46,7 @@ class ShodanProvider(BaseProvider):
         """Return a dictionary indicating if the provider can query domains and/or IPs."""
         return {'domains': True, 'ips': True}
 
-    def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query Shodan for information about a domain.
         Uses Shodan's hostname search to find associated IPs.
@@ -103,16 +102,16 @@ class ShodanProvider(BaseProvider):
                     relationships.append((
                         domain,
                         ip_address,
-                        RelationshipType.A_RECORD,  # Domain resolves to IP
-                        RelationshipType.A_RECORD.default_confidence,
+                        'a_record',  # Domain resolves to IP
+                        0.8,
                         raw_data
                     ))
 
                     self.log_relationship_discovery(
                         source_node=domain,
                         target_node=ip_address,
-                        relationship_type=RelationshipType.A_RECORD,
-                        confidence_score=RelationshipType.A_RECORD.default_confidence,
+                        relationship_type='a_record',
+                        confidence_score=0.8,
                         raw_data=raw_data,
                         discovery_method="shodan_hostname_search"
                     )
@@ -129,7 +128,7 @@ class ShodanProvider(BaseProvider):
                             relationships.append((
                                 domain,
                                 hostname,
-                                RelationshipType.PASSIVE_DNS,  # Shared hosting relationship
+                                'passive_dns',  # Shared hosting relationship
                                 0.6,  # Lower confidence for shared hosting
                                 hostname_raw_data
                             ))
@@ -137,7 +136,7 @@ class ShodanProvider(BaseProvider):
                             self.log_relationship_discovery(
                                 source_node=domain,
                                 target_node=hostname,
-                                relationship_type=RelationshipType.PASSIVE_DNS,
+                                relationship_type='passive_dns',
                                 confidence_score=0.6,
                                 raw_data=hostname_raw_data,
                                 discovery_method="shodan_shared_hosting"
@@ -148,7 +147,7 @@ class ShodanProvider(BaseProvider):
 
         return relationships
 
-    def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
+    def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
         """
         Query Shodan for information about an IP address.
 
@@ -195,16 +194,16 @@ class ShodanProvider(BaseProvider):
                     relationships.append((
                         ip,
                         hostname,
-                        RelationshipType.A_RECORD,  # IP resolves to hostname
-                        RelationshipType.A_RECORD.default_confidence,
+                        'a_record',  # IP resolves to hostname
+                        0.8,
                         raw_data
                     ))
 
                     self.log_relationship_discovery(
                         source_node=ip,
                         target_node=hostname,
-                        relationship_type=RelationshipType.A_RECORD,
-                        confidence_score=RelationshipType.A_RECORD.default_confidence,
+                        relationship_type='a_record',
+                        confidence_score=0.8,
                         raw_data=raw_data,
                         discovery_method="shodan_host_lookup"
                     )
@@ -230,16 +229,16 @@ class ShodanProvider(BaseProvider):
                 relationships.append((
                     ip,
                     asn_name,
-                    RelationshipType.ASN_MEMBERSHIP,
-                    RelationshipType.ASN_MEMBERSHIP.default_confidence,
+                    'asn_membership',
+                    0.7,
                     asn_raw_data
                 ))
 
                 self.log_relationship_discovery(
                     source_node=ip,
                     target_node=asn_name,
-                    relationship_type=RelationshipType.ASN_MEMBERSHIP,
-                    confidence_score=RelationshipType.ASN_MEMBERSHIP.default_confidence,
+                    relationship_type='asn_membership',
+                    confidence_score=0.7,
                     raw_data=asn_raw_data,
                     discovery_method="shodan_asn_lookup"
                 )

static/css/main.css

@@ -1000,6 +1000,46 @@ input[type="text"]:focus, select:focus {
     font-style: italic;
 }
 
+.correlation-values-list {
+    margin-top: 1rem;
+}
+
+.correlation-value-details {
+    margin-bottom: 0.5rem;
+    border: 1px solid #333;
+    border-radius: 3px;
+}
+
+.correlation-value-details summary {
+    padding: 0.5rem;
+    background-color: #3a3a3a;
+    cursor: pointer;
+    outline: none;
+    color: #c7c7c7;
+}
+
+.correlation-value-details summary:hover {
+    background-color: #4a4a4a;
+}
+
+.correlation-value-details .detail-row {
+    margin-left: 1rem;
+    margin-right: 1rem;
+    padding: 0.5rem 0;
+}
+
+.correlation-value-details .detail-label {
+    color: #999;
+    font-weight: 500;
+}
+
+.correlation-value-details .detail-value {
+    color: #c7c7c7;
+    word-break: break-all;
+    font-family: 'Roboto Mono', monospace;
+    font-size: 0.9em;
+}
+
 @keyframes fadeIn {
     from {opacity: 0; transform: scale(0.95);}
     to {opacity: 1; transform: scale(1);}

static/js/graph.js

@@ -1,6 +1,6 @@
 /**
  * Graph visualization module for DNSRecon
- * Handles network graph rendering using vis.js with enhanced Phase 2 features
+ * Handles network graph rendering using vis.js
  */
 
 class GraphManager {
@@ -130,7 +130,7 @@ class GraphManager {
     }
 
     /**
-     * Initialize the network graph with enhanced features
+     * Initialize the network graph
      */
     initialize() {
         if (this.isInitialized) {
@@ -156,7 +156,7 @@ class GraphManager {
             // Add graph controls
             this.addGraphControls();
 
-            console.log('Enhanced graph initialized successfully');
+            console.log('Graph initialized successfully');
         } catch (error) {
             console.error('Failed to initialize graph:', error);
             this.showError('Failed to initialize visualization');
@@ -184,12 +184,12 @@ class GraphManager {
     }
 
     /**
-     * Setup enhanced network event handlers
+     * Setup network event handlers
      */
     setupNetworkEvents() {
         if (!this.network) return;
 
-        // Node click event with enhanced details
+        // Node click event with details
         this.network.on('click', (params) => {
             if (params.nodes.length > 0) {
                 const nodeId = params.nodes[0];
@@ -207,7 +207,7 @@ class GraphManager {
             }
         });
 
-        // Enhanced hover events
+        // Hover events
         this.network.on('hoverNode', (params) => {
             const nodeId = params.node;
             const node = this.nodes.get(nodeId);
@@ -242,7 +242,6 @@ class GraphManager {
     }
 
     /**
-     * Update graph with new data and enhanced processing
      * @param {Object} graphData - Graph data from backend
      */
     updateGraph(graphData) {
@@ -326,15 +325,15 @@ class GraphManager {
                 setTimeout(() => this.fitView(), 800);
             }
 
-            console.log(`Enhanced graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
+            console.log(`Graph updated: ${processedNodes.length} nodes, ${processedEdges.length} edges (${newNodes.length} new nodes, ${newEdges.length} new edges)`);
         } catch (error) {
-            console.error('Failed to update enhanced graph:', error);
+            console.error('Failed to update graph:', error);
             this.showError('Failed to update visualization');
         }
     }
 
     /**
-     * Process node data with enhanced styling and metadata
+     * Process node data with styling and metadata
      * @param {Object} node - Raw node data
      * @returns {Object} Processed node data
      */
@@ -366,15 +365,31 @@ class GraphManager {
             }
         }
         
+        // Handle merged correlation objects (similar to large entities)
         if (node.type === 'correlation_object') {
-            processedNode.label = this.formatNodeLabel(node.metadata.value, node.type);
+            const metadata = node.metadata || {};
+            const values = metadata.values || [];
+            const mergeCount = metadata.merge_count || 1;
+            
+            if (mergeCount > 1) {
+                // Display as merged correlation container
+                processedNode.label = `Correlations (${mergeCount})`;
+                processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`;
+                processedNode.borderWidth = 3; // Thicker border for merged nodes
+            } else {
+                // Single correlation value
+                const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown');
+                const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value;
+                processedNode.label = `Corr: ${displayValue}`;
+                processedNode.title = `Correlation: ${value}`;
+            }
         }
 
         return processedNode;
     }
 
     /**
-     * Process edge data with enhanced styling and metadata
+     * Process edge data with styling and metadata
      * @param {Object} edge - Raw edge data
      * @returns {Object} Processed edge data
      */
@@ -478,7 +493,7 @@ class GraphManager {
     }
     
     /**
-     * Get enhanced node shape based on type
+     * Get node shape based on type
      * @param {string} nodeType - Node type
      * @returns {string} Shape name
      */

static/js/main.js

@@ -243,7 +243,7 @@ class DNSReconApp {
     }
     
     /**
-     * Enhanced start scan with better error handling
+     * Start scan with error handling
      */
     async startScan(clearGraph = true) {
         console.log('=== STARTING SCAN ===');
@@ -318,7 +318,7 @@ class DNSReconApp {
         }
     }
     /**
-     * Enhanced scan stop with immediate UI feedback
+     * Scan stop with immediate UI feedback
      */
     async stopScan() {
         try {
@@ -427,7 +427,7 @@ class DNSReconApp {
     }
     
     /**
-     * Enhanced status update with better error handling
+     * Status update with better error handling
      */
     async updateStatus() {
         try {
@@ -668,7 +668,7 @@ class DNSReconApp {
     }
     
     /**
-     * Enhanced UI state management with immediate button updates
+     * UI state management with immediate button updates
      */
     setUIState(state) {
         console.log(`Setting UI state to: ${state}`);
@@ -802,6 +802,47 @@ class DNSReconApp {
 
         let detailsHtml = '<div class="modal-details-grid">';
 
+        // Handle merged correlation objects similar to large entities
+        if (node.type === 'correlation_object') {
+            const metadata = node.metadata || {};
+            const values = metadata.values || [];
+            const mergeCount = metadata.merge_count || 1;
+            
+            detailsHtml += '<div class="modal-section">';
+            detailsHtml += '<h4>Correlation Details</h4>';
+            
+            if (mergeCount > 1) {
+                detailsHtml += `<p><strong>Merged Correlations:</strong> ${mergeCount} values</p>`;
+                detailsHtml += '<div class="correlation-values-list">';
+                
+                values.forEach((value, index) => {
+                    detailsHtml += `<details class="correlation-value-details">`;
+                    detailsHtml += `<summary>Value ${index + 1}: ${typeof value === 'string' && value.length > 50 ? value.substring(0, 47) + '...' : value}</summary>`;
+                    detailsHtml += `<div class="detail-row"><span class="detail-label">Full Value:</span><span class="detail-value">${value}</span></div>`;
+                    detailsHtml += `</details>`;
+                });
+                
+                detailsHtml += '</div>';
+            } else {
+                const singleValue = values.length > 0 ? values[0] : (metadata.value || 'Unknown');
+                detailsHtml += `<div class="detail-row"><span class="detail-label">Correlation Value:</span><span class="detail-value">${singleValue}</span></div>`;
+            }
+            
+            // Show correlated nodes
+            const correlatedNodes = metadata.correlated_nodes || [];
+            if (correlatedNodes.length > 0) {
+                detailsHtml += `<div class="detail-row"><span class="detail-label">Correlated Nodes:</span><span class="detail-value">${correlatedNodes.length}</span></div>`;
+                detailsHtml += '<ul>';
+                correlatedNodes.forEach(nodeId => {
+                    detailsHtml += `<li><a href="#" class="node-link" data-node-id="${nodeId}">${nodeId}</a></li>`;
+                });
+                detailsHtml += '</ul>';
+            }
+            
+            detailsHtml += '</div>';
+        }
+
+        // Continue with standard node details for all node types
         // Section for Incoming Edges (Source Nodes)
         if (node.incoming_edges && node.incoming_edges.length > 0) {
             detailsHtml += '<div class="modal-section">';
@@ -824,11 +865,13 @@ class DNSReconApp {
             detailsHtml += '</ul></div>';
         }
 
-        // Section for Attributes
+        // Section for Attributes (skip for correlation objects - already handled above)
+        if (node.type !== 'correlation_object') {
             detailsHtml += '<div class="modal-section">';
             detailsHtml += '<h4>Attributes</h4>';
             detailsHtml += this.formatObjectToHtml(node.attributes);
             detailsHtml += '</div>';
+        }
 
         // Section for Description
         detailsHtml += '<div class="modal-section">';
@@ -836,11 +879,13 @@ class DNSReconApp {
         detailsHtml += `<p class="description-text">${node.description || 'No description available.'}</p>`;
         detailsHtml += '</div>';
 
-        // Section for Metadata
+        // Section for Metadata (skip detailed metadata for correlation objects - already handled above)
+        if (node.type !== 'correlation_object') {
             detailsHtml += '<div class="modal-section">';
             detailsHtml += '<h4>Metadata</h4>';
             detailsHtml += this.formatObjectToHtml(node.metadata);
             detailsHtml += '</div>';
+        }
 
         detailsHtml += '</div>';
         return detailsHtml;