fix large entity

README.md

@@ -148,7 +148,6 @@ export FLASK_ENV='production'
 export FLASK_DEBUG=False
 
 # API keys (optional, but recommended for full functionality)
-export VIRUSTOTAL_API_KEY="your_virustotal_key"
 export SHODAN_API_KEY="your_shodan_key"
 ```
 
@@ -224,7 +223,6 @@ Restart=always
 Environment="SECRET_KEY=your-super-secret-and-random-key"
 Environment="FLASK_ENV=production"
 Environment="FLASK_DEBUG=False"
-Environment="VIRUSTOTAL_API_KEY=your_virustotal_key"
 Environment="SHODAN_API_KEY=your_shodan_key"
 
 [Install]

app.py

@@ -384,7 +384,7 @@ def get_providers():
                 'statistics': stats,
                 'enabled': config.is_provider_enabled(provider_name),
                 'rate_limit': config.get_rate_limit(provider_name),
-                'requires_api_key': provider_name in ['shodan', 'virustotal']
+                'requires_api_key': provider_name in ['shodan']
             }
         
         return jsonify({
@@ -423,7 +423,7 @@ def set_api_keys():
         updated_providers = []
         
         for provider, api_key in data.items():
-            if provider in ['shodan', 'virustotal'] and api_key.strip():
+            if provider in ['shodan'] and api_key.strip():
                 success = session_config.set_api_key(provider, api_key.strip())
                 if success:
                     updated_providers.append(provider)

config.py

@@ -13,8 +13,7 @@ class Config:
     def __init__(self):
         """Initialize configuration with default values."""
         self.api_keys: Dict[str, Optional[str]] = {
-            'shodan': None,
+            'shodan': None
-            'virustotal': None
         }
         
         # Default settings
@@ -26,7 +25,6 @@ class Config:
         # Rate limiting settings (requests per minute)
         self.rate_limits = {
             'crtsh': 60,          # Free service, be respectful
-            'virustotal': 4,      # Free tier limit
             'shodan': 60,         # API dependent
             'dns': 100            # Local DNS queries
         }
@@ -35,7 +33,6 @@ class Config:
         self.enabled_providers = {
             'crtsh': True,        # Always enabled (free)
             'dns': True,          # Always enabled (free)
-            'virustotal': False,  # Requires API key
             'shodan': False       # Requires API key
         }
         
@@ -53,7 +50,7 @@ class Config:
         Set API key for a provider.
         
         Args:
-            provider: Provider name (shodan, virustotal)
+            provider: Provider name (shodan, etc)
             api_key: API key string
             
         Returns:
@@ -103,9 +100,6 @@ class Config:
     
     def load_from_env(self):
         """Load configuration from environment variables."""
-        if os.getenv('VIRUSTOTAL_API_KEY'):
-            self.set_api_key('virustotal', os.getenv('VIRUSTOTAL_API_KEY'))
-        
         if os.getenv('SHODAN_API_KEY'):
             self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))
         

core/scanner.py

@@ -14,7 +14,6 @@ from utils.helpers import _is_valid_ip, _is_valid_domain
 from providers.crtsh_provider import CrtShProvider
 from providers.dns_provider import DNSProvider
 from providers.shodan_provider import ShodanProvider
-from providers.virustotal_provider import VirusTotalProvider
 
 
 class ScanStatus:
@@ -66,8 +65,7 @@ class Scanner:
             self.provider_eligibility = {
                 'dns': {'domains': True, 'ips': True},
                 'crtsh': {'domains': True, 'ips': False},
-                'shodan': {'domains': True, 'ips': True},
+                'shodan': {'domains': True, 'ips': True}
-                'virustotal': {'domains': True, 'ips': True}
             }
 
             # Initialize providers with session config
@@ -169,8 +167,7 @@ class Scanner:
         provider_classes = {
             'dns': DNSProvider,
             'crtsh': CrtShProvider,
-            'shodan': ShodanProvider,
+            'shodan': ShodanProvider
-            'virustotal': VirusTotalProvider
         }
 
         for provider_name, provider_class in provider_classes.items():
@@ -757,18 +754,24 @@ class Scanner:
 
     def _create_large_entity(self, source: str, provider_name: str, results: List, current_depth: int) -> None:
         """Create a large entity node for forensic tracking."""
-        entity_id = f"large_entity_{provider_name}_{hash(source) & 0x7FFFFFFF}"
+        entity_id = f"Large Entity: {provider_name}"
         
         # Extract targets from results
-        targets = [rel[1] for rel in results if len(rel) > 1]
+        targets = []
-        
-        # Determine node type
         node_type = 'unknown'
-        if targets:
+        
-            if _is_valid_domain(targets[0]):
+        for rel in results:
+            if len(rel) > 1:
+                target = rel[1]
+                targets.append(target)
+                
+                # Determine node type and add node to graph
+                if _is_valid_domain(target):
                     node_type = 'domain'
-            elif _is_valid_ip(targets[0]):
+                    self.graph.add_node(target, NodeType.DOMAIN)
+                elif _is_valid_ip(target):
                     node_type = 'ip'
+                    self.graph.add_node(target, NodeType.IP)
         
         # Create large entity metadata
         metadata = {

core/session_config.py

@@ -17,8 +17,7 @@ class SessionConfig:
         """Initialize session config with global defaults."""
         # Copy all attributes from global config
         self.api_keys: Dict[str, Optional[str]] = {
-            'shodan': None,
+            'shodan': None
-            'virustotal': None
         }
         
         # Default settings (copied from global config)
@@ -30,7 +29,6 @@ class SessionConfig:
         # Rate limiting settings (per session)
         self.rate_limits = {
             'crtsh': 60,
-            'virustotal': 4,
             'shodan': 60,
             'dns': 100
         }
@@ -39,7 +37,6 @@ class SessionConfig:
         self.enabled_providers = {
             'crtsh': True,
             'dns': True,
-            'virustotal': False,
             'shodan': False
         }
         
@@ -57,7 +54,7 @@ class SessionConfig:
         Set API key for a provider in this session.
         
         Args:
-            provider: Provider name (shodan, virustotal)
+            provider: Provider name (shodan, etc)
             api_key: API key string
             
         Returns:
@@ -107,9 +104,6 @@ class SessionConfig:
     
     def load_from_env(self):
         """Load configuration from environment variables (only if not already set)."""
-        if os.getenv('VIRUSTOTAL_API_KEY') and not self.api_keys['virustotal']:
-            self.set_api_key('virustotal', os.getenv('VIRUSTOTAL_API_KEY'))
-        
         if os.getenv('SHODAN_API_KEY') and not self.api_keys['shodan']:
             self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))
         

providers/__init__.py

@@ -7,15 +7,13 @@ from .base_provider import BaseProvider, RateLimiter
 from .crtsh_provider import CrtShProvider
 from .dns_provider import DNSProvider
 from .shodan_provider import ShodanProvider
-from .virustotal_provider import VirusTotalProvider
 
 __all__ = [
     'BaseProvider',
     'RateLimiter', 
     'CrtShProvider',
     'DNSProvider',
-    'ShodanProvider',
+    'ShodanProvider'
-    'VirusTotalProvider'
 ]
 
 __version__ = "1.0.0-phase2"

providers/virustotal_provider.py

@@ -1,333 +0,0 @@
-"""
-VirusTotal provider for DNSRecon.
-Discovers domain relationships through passive DNS and URL analysis.
-"""
-
-import json
-from typing import List, Dict, Any, Tuple
-from .base_provider import BaseProvider
-from utils.helpers import _is_valid_ip, _is_valid_domain
-from core.graph_manager import RelationshipType
-
-
-class VirusTotalProvider(BaseProvider):
-    """
-    Provider for querying VirusTotal API for passive DNS and domain reputation data.
-    Now uses session-specific API keys and rate limits.
-    """
-    
-    def __init__(self, session_config=None):
-        """Initialize VirusTotal provider with session-specific configuration."""
-        super().__init__(
-            name="virustotal",
-            rate_limit=4,  # Free tier: 4 requests per minute
-            timeout=30,
-            session_config=session_config
-        )
-        self.base_url = "https://www.virustotal.com/vtapi/v2"
-        self.api_key = self.config.get_api_key('virustotal')
-    
-    def is_available(self) -> bool:
-        """Check if VirusTotal provider is available (has valid API key in this session)."""
-        return self.api_key is not None and len(self.api_key.strip()) > 0
-    
-    def get_name(self) -> str:
-        """Return the provider name."""
-        return "virustotal"
-    
-    def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
-        """
-        Query VirusTotal for domain information including passive DNS.
-        
-        Args:
-            domain: Domain to investigate
-            
-        Returns:
-            List of relationships discovered from VirusTotal data
-        """
-        if not _is_valid_domain(domain) or not self.is_available():
-            return []
-        
-        relationships = []
-        
-        # Query domain report
-        domain_relationships = self._query_domain_report(domain)
-        relationships.extend(domain_relationships)
-        
-        # Query passive DNS for the domain
-        passive_dns_relationships = self._query_passive_dns_domain(domain)
-        relationships.extend(passive_dns_relationships)
-        
-        return relationships
-    
-    def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
-        """
-        Query VirusTotal for IP address information including passive DNS.
-        
-        Args:
-            ip: IP address to investigate
-            
-        Returns:
-            List of relationships discovered from VirusTotal IP data
-        """
-        if not _is_valid_ip(ip) or not self.is_available():
-            return []
-        
-        relationships = []
-        
-        # Query IP report
-        ip_relationships = self._query_ip_report(ip)
-        relationships.extend(ip_relationships)
-        
-        # Query passive DNS for the IP
-        passive_dns_relationships = self._query_passive_dns_ip(ip)
-        relationships.extend(passive_dns_relationships)
-        
-        return relationships
-    
-    def _query_domain_report(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
-        """Query VirusTotal domain report."""
-        relationships = []
-        
-        try:
-            url = f"{self.base_url}/domain/report"
-            params = {
-                'apikey': self.api_key,
-                'domain': domain,
-                'allinfo': 1  # Get comprehensive information
-            }
-            
-            response = self.make_request(url, method="GET", params=params, target_indicator=domain)
-            
-            if not response or response.status_code != 200:
-                return []
-            
-            data = response.json()
-            
-            if data.get('response_code') != 1:
-                return []
-            
-            # Extract resolved IPs
-            resolutions = data.get('resolutions', [])
-            for resolution in resolutions:
-                ip_address = resolution.get('ip_address')
-                last_resolved = resolution.get('last_resolved')
-                
-                if ip_address and _is_valid_ip(ip_address):
-                    raw_data = {
-                        'domain': domain,
-                        'ip_address': ip_address,
-                        'last_resolved': last_resolved,
-                        'source': 'virustotal_domain_report'
-                    }
-                    
-                    relationships.append((
-                        domain,
-                        ip_address,
-                        RelationshipType.PASSIVE_DNS,
-                        RelationshipType.PASSIVE_DNS.default_confidence,
-                        raw_data
-                    ))
-                    
-                    self.log_relationship_discovery(
-                        source_node=domain,
-                        target_node=ip_address,
-                        relationship_type=RelationshipType.PASSIVE_DNS,
-                        confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
-                        raw_data=raw_data,
-                        discovery_method="virustotal_domain_resolution"
-                    )
-            
-            # Extract subdomains
-            subdomains = data.get('subdomains', [])
-            for subdomain in subdomains:
-                if subdomain != domain and _is_valid_domain(subdomain):
-                    raw_data = {
-                        'parent_domain': domain,
-                        'subdomain': subdomain,
-                        'source': 'virustotal_subdomain_discovery'
-                    }
-                    
-                    relationships.append((
-                        domain,
-                        subdomain,
-                        RelationshipType.PASSIVE_DNS,
-                        0.7,  # Medium-high confidence for subdomains
-                        raw_data
-                    ))
-                    
-                    self.log_relationship_discovery(
-                        source_node=domain,
-                        target_node=subdomain,
-                        relationship_type=RelationshipType.PASSIVE_DNS,
-                        confidence_score=0.7,
-                        raw_data=raw_data,
-                        discovery_method="virustotal_subdomain_discovery"
-                    )
-        
-        except json.JSONDecodeError as e:
-            self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
-        except Exception as e:
-            self.logger.logger.error(f"Error querying VirusTotal domain report for {domain}: {e}")
-        
-        return relationships
-    
-    def _query_ip_report(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
-        """Query VirusTotal IP report."""
-        relationships = []
-        
-        try:
-            url = f"{self.base_url}/ip-address/report"
-            params = {
-                'apikey': self.api_key,
-                'ip': ip
-            }
-            
-            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
-            
-            if not response or response.status_code != 200:
-                return []
-            
-            data = response.json()
-            
-            if data.get('response_code') != 1:
-                return []
-            
-            # Extract resolved domains
-            resolutions = data.get('resolutions', [])
-            for resolution in resolutions:
-                hostname = resolution.get('hostname')
-                last_resolved = resolution.get('last_resolved')
-                
-                if hostname and _is_valid_domain(hostname):
-                    raw_data = {
-                        'ip_address': ip,
-                        'hostname': hostname,
-                        'last_resolved': last_resolved,
-                        'source': 'virustotal_ip_report'
-                    }
-                    
-                    relationships.append((
-                        ip,
-                        hostname,
-                        RelationshipType.PASSIVE_DNS,
-                        RelationshipType.PASSIVE_DNS.default_confidence,
-                        raw_data
-                    ))
-                    
-                    self.log_relationship_discovery(
-                        source_node=ip,
-                        target_node=hostname,
-                        relationship_type=RelationshipType.PASSIVE_DNS,
-                        confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
-                        raw_data=raw_data,
-                        discovery_method="virustotal_ip_resolution"
-                    )
-        
-        except json.JSONDecodeError as e:
-            self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
-        except Exception as e:
-            self.logger.logger.error(f"Error querying VirusTotal IP report for {ip}: {e}")
-        
-        return relationships
-    
-    def _query_passive_dns_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
-        """Query VirusTotal passive DNS for domain."""
-        # Note: VirusTotal's passive DNS API might require a premium subscription
-        # This is a placeholder for the endpoint structure
-        return []
-    
-    def _query_passive_dns_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
-        """Query VirusTotal passive DNS for IP."""
-        # Note: VirusTotal's passive DNS API might require a premium subscription
-        # This is a placeholder for the endpoint structure
-        return []
-    
-    def get_domain_reputation(self, domain: str) -> Dict[str, Any]:
-        """
-        Get domain reputation information from VirusTotal.
-        
-        Args:
-            domain: Domain to check reputation for
-            
-        Returns:
-            Dictionary containing reputation data
-        """
-        if not _is_valid_domain(domain) or not self.is_available():
-            return {}
-        
-        try:
-            url = f"{self.base_url}/domain/report"
-            params = {
-                'apikey': self.api_key,
-                'domain': domain
-            }
-            
-            response = self.make_request(url, method="GET", params=params, target_indicator=domain)
-            
-            if response and response.status_code == 200:
-                data = response.json()
-                
-                if data.get('response_code') == 1:
-                    return {
-                        'positives': data.get('positives', 0),
-                        'total': data.get('total', 0),
-                        'scan_date': data.get('scan_date', ''),
-                        'permalink': data.get('permalink', ''),
-                        'reputation_score': self._calculate_reputation_score(data)
-                    }
-        
-        except Exception as e:
-            self.logger.logger.error(f"Error getting VirusTotal reputation for domain {domain}: {e}")
-        
-        return {}
-    
-    def get_ip_reputation(self, ip: str) -> Dict[str, Any]:
-        """
-        Get IP reputation information from VirusTotal.
-        
-        Args:
-            ip: IP address to check reputation for
-            
-        Returns:
-            Dictionary containing reputation data
-        """
-        if not _is_valid_ip(ip) or not self.is_available():
-            return {}
-        
-        try:
-            url = f"{self.base_url}/ip-address/report"
-            params = {
-                'apikey': self.api_key,
-                'ip': ip
-            }
-            
-            response = self.make_request(url, method="GET", params=params, target_indicator=ip)
-            
-            if response and response.status_code == 200:
-                data = response.json()
-                
-                if data.get('response_code') == 1:
-                    return {
-                        'positives': data.get('positives', 0),
-                        'total': data.get('total', 0),
-                        'scan_date': data.get('scan_date', ''),
-                        'permalink': data.get('permalink', ''),
-                        'reputation_score': self._calculate_reputation_score(data)
-                    }
-        
-        except Exception as e:
-            self.logger.logger.error(f"Error getting VirusTotal reputation for IP {ip}: {e}")
-        
-        return {}
-    
-    def _calculate_reputation_score(self, data: Dict[str, Any]) -> float:
-        """Calculate a normalized reputation score (0.0 to 1.0)."""
-        positives = data.get('positives', 0)
-        total = data.get('total', 1)  # Avoid division by zero
-        
-        if total == 0:
-            return 1.0  # No data means neutral
-        
-        # Score is inverse of detection ratio (lower detection = higher reputation)
-        return max(0.0, 1.0 - (positives / total))

static/js/graph.js

@@ -270,8 +270,23 @@ class GraphManager {
                 this.initialize();
             }
 
-            // Process nodes with enhanced attributes
+            // Find all aggregated node IDs first
-            const processedNodes = graphData.nodes.map(node => this.processNode(node));
+            const aggregatedNodeIds = new Set();
+            graphData.nodes.forEach(node => {
+                if (node.type === 'large_entity' && node.metadata && Array.isArray(node.metadata.nodes)) {
+                    node.metadata.nodes.forEach(nodeId => aggregatedNodeIds.add(nodeId));
+                }
+            });
+
+            // Process nodes, hiding the ones that are aggregated
+            const processedNodes = graphData.nodes.map(node => {
+                const processed = this.processNode(node);
+                if (aggregatedNodeIds.has(node.id)) {
+                    processed.hidden = true; // Mark node as hidden
+                }
+                return processed;
+            });
+            
             const processedEdges = graphData.edges.map(edge => this.processEdge(edge));
 
             // Update datasets with animation
@@ -441,7 +456,8 @@ class GraphManager {
             'domain': 12,
             'ip': 14,
             'asn': 16,
-            'correlation_object': 8
+            'correlation_object': 8,
+            'large_entity': 12
         };
         return sizes[nodeType] || 12;
     }
@@ -456,7 +472,8 @@ class GraphManager {
             'domain': 'dot',
             'ip': 'square',
             'asn': 'triangle',
-            'correlation_object': 'hexagon'
+            'correlation_object': 'hexagon',
+            'large_entity': 'database'
         };
         return shapes[nodeType] || 'dot';
     }

static/js/main.js

@@ -80,7 +80,6 @@ class DNSReconApp {
             // API Key Modal elements
             apiKeyModal: document.getElementById('api-key-modal'),
             apiKeyModalClose: document.getElementById('api-key-modal-close'),
-            virustotalApiKey: document.getElementById('virustotal-api-key'),
             shodanApiKey: document.getElementById('shodan-api-key'),
             saveApiKeys: document.getElementById('save-api-keys'),
             resetApiKeys: document.getElementById('reset-api-keys'),
@@ -851,13 +850,11 @@ class DNSReconApp {
                 detailsHtml += createDetailRow('Related Domains (SAN)', metadata.related_domains_san);
                 detailsHtml += createDetailRow('Passive DNS', metadata.passive_dns);
                 detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
-                detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
                 break;
             case 'ip':
                 detailsHtml += createDetailRow('Hostnames', metadata.hostnames);
                 detailsHtml += createDetailRow('Passive DNS', metadata.passive_dns);
                 detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
-                detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
                 break;
             case 'correlation_object':
                 detailsHtml += createDetailRow('Correlated Value', metadata.value);
@@ -974,11 +971,9 @@ class DNSReconApp {
      */
     async saveApiKeys() {
         const shodanKey = this.elements.shodanApiKey.value.trim();
-        const virustotalKey = this.elements.virustotalApiKey.value.trim();
 
         const keys = {};
         if (shodanKey) keys.shodan = shodanKey;
-        if (virustotalKey) keys.virustotal = virustotalKey;
 
         if (Object.keys(keys).length === 0) {
             this.showWarning('No API keys were entered.');
@@ -1004,7 +999,6 @@ class DNSReconApp {
      */
     resetApiKeys() {
         this.elements.shodanApiKey.value = '';
-        this.elements.virustotalApiKey.value = '';
     }
     
     /**

templates/index.html

@@ -217,11 +217,6 @@
                     <p class="modal-description">
                         Enter your API keys for enhanced data providers. Keys are stored in memory for the current session only and are never saved to disk.
                     </p>
-                    <div class="apikey-section">
-                        <label for="virustotal-api-key">VirusTotal API Key</label>
-                        <input type="password" id="virustotal-api-key" placeholder="Enter VirusTotal API Key">
-                        <p class="apikey-help">Enables passive DNS and domain reputation lookups.</p>
-                    </div>
                     <div class="apikey-section">
                         <label for="shodan-api-key">Shodan API Key</label>
                         <input type="password" id="shodan-api-key" placeholder="Enter Shodan API Key">