diff --git a/README.md b/README.md
index decc1ae..65338fa 100644
--- a/README.md
+++ b/README.md
@@ -148,7 +148,6 @@ export FLASK_ENV='production'
export FLASK_DEBUG=False
# API keys (optional, but recommended for full functionality)
-export VIRUSTOTAL_API_KEY="your_virustotal_key"
export SHODAN_API_KEY="your_shodan_key"
```
@@ -224,7 +223,6 @@ Restart=always
Environment="SECRET_KEY=your-super-secret-and-random-key"
Environment="FLASK_ENV=production"
Environment="FLASK_DEBUG=False"
-Environment="VIRUSTOTAL_API_KEY=your_virustotal_key"
Environment="SHODAN_API_KEY=your_shodan_key"
[Install]
diff --git a/app.py b/app.py
index 2885167..8bc0162 100644
--- a/app.py
+++ b/app.py
@@ -384,7 +384,7 @@ def get_providers():
'statistics': stats,
'enabled': config.is_provider_enabled(provider_name),
'rate_limit': config.get_rate_limit(provider_name),
- 'requires_api_key': provider_name in ['shodan', 'virustotal']
+ 'requires_api_key': provider_name in ['shodan']
}
return jsonify({
@@ -423,7 +423,7 @@ def set_api_keys():
updated_providers = []
for provider, api_key in data.items():
- if provider in ['shodan', 'virustotal'] and api_key.strip():
+ if provider in ['shodan'] and api_key.strip():
success = session_config.set_api_key(provider, api_key.strip())
if success:
updated_providers.append(provider)
diff --git a/config.py b/config.py
index df4a06c..4c1aca4 100644
--- a/config.py
+++ b/config.py
@@ -13,8 +13,7 @@ class Config:
def __init__(self):
"""Initialize configuration with default values."""
self.api_keys: Dict[str, Optional[str]] = {
- 'shodan': None,
- 'virustotal': None
+ 'shodan': None
}
# Default settings
@@ -26,7 +25,6 @@ class Config:
# Rate limiting settings (requests per minute)
self.rate_limits = {
'crtsh': 60, # Free service, be respectful
- 'virustotal': 4, # Free tier limit
'shodan': 60, # API dependent
'dns': 100 # Local DNS queries
}
@@ -35,7 +33,6 @@ class Config:
self.enabled_providers = {
'crtsh': True, # Always enabled (free)
'dns': True, # Always enabled (free)
- 'virustotal': False, # Requires API key
'shodan': False # Requires API key
}
@@ -53,7 +50,7 @@ class Config:
Set API key for a provider.
Args:
- provider: Provider name (shodan, virustotal)
+ provider: Provider name (shodan, etc)
api_key: API key string
Returns:
@@ -103,9 +100,6 @@ class Config:
def load_from_env(self):
"""Load configuration from environment variables."""
- if os.getenv('VIRUSTOTAL_API_KEY'):
- self.set_api_key('virustotal', os.getenv('VIRUSTOTAL_API_KEY'))
-
if os.getenv('SHODAN_API_KEY'):
self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))
diff --git a/core/scanner.py b/core/scanner.py
index 47f8f8c..4048e96 100644
--- a/core/scanner.py
+++ b/core/scanner.py
@@ -14,7 +14,6 @@ from utils.helpers import _is_valid_ip, _is_valid_domain
from providers.crtsh_provider import CrtShProvider
from providers.dns_provider import DNSProvider
from providers.shodan_provider import ShodanProvider
-from providers.virustotal_provider import VirusTotalProvider
class ScanStatus:
@@ -66,8 +65,7 @@ class Scanner:
self.provider_eligibility = {
'dns': {'domains': True, 'ips': True},
'crtsh': {'domains': True, 'ips': False},
- 'shodan': {'domains': True, 'ips': True},
- 'virustotal': {'domains': True, 'ips': True}
+ 'shodan': {'domains': True, 'ips': True}
}
# Initialize providers with session config
@@ -169,8 +167,7 @@ class Scanner:
provider_classes = {
'dns': DNSProvider,
'crtsh': CrtShProvider,
- 'shodan': ShodanProvider,
- 'virustotal': VirusTotalProvider
+ 'shodan': ShodanProvider
}
for provider_name, provider_class in provider_classes.items():
@@ -757,18 +754,24 @@ class Scanner:
def _create_large_entity(self, source: str, provider_name: str, results: List, current_depth: int) -> None:
"""Create a large entity node for forensic tracking."""
- entity_id = f"large_entity_{provider_name}_{hash(source) & 0x7FFFFFFF}"
+ entity_id = f"Large Entity: {provider_name}"
# Extract targets from results
- targets = [rel[1] for rel in results if len(rel) > 1]
-
- # Determine node type
+ targets = []
node_type = 'unknown'
- if targets:
- if _is_valid_domain(targets[0]):
- node_type = 'domain'
- elif _is_valid_ip(targets[0]):
- node_type = 'ip'
+
+ for rel in results:
+ if len(rel) > 1:
+ target = rel[1]
+ targets.append(target)
+
+ # Determine node type and add node to graph
+ if _is_valid_domain(target):
+ node_type = 'domain'
+ self.graph.add_node(target, NodeType.DOMAIN)
+ elif _is_valid_ip(target):
+ node_type = 'ip'
+ self.graph.add_node(target, NodeType.IP)
# Create large entity metadata
metadata = {
diff --git a/core/session_config.py b/core/session_config.py
index dbf698e..3545b14 100644
--- a/core/session_config.py
+++ b/core/session_config.py
@@ -17,8 +17,7 @@ class SessionConfig:
"""Initialize session config with global defaults."""
# Copy all attributes from global config
self.api_keys: Dict[str, Optional[str]] = {
- 'shodan': None,
- 'virustotal': None
+ 'shodan': None
}
# Default settings (copied from global config)
@@ -30,7 +29,6 @@ class SessionConfig:
# Rate limiting settings (per session)
self.rate_limits = {
'crtsh': 60,
- 'virustotal': 4,
'shodan': 60,
'dns': 100
}
@@ -39,7 +37,6 @@ class SessionConfig:
self.enabled_providers = {
'crtsh': True,
'dns': True,
- 'virustotal': False,
'shodan': False
}
@@ -57,7 +54,7 @@ class SessionConfig:
Set API key for a provider in this session.
Args:
- provider: Provider name (shodan, virustotal)
+ provider: Provider name (shodan, etc)
api_key: API key string
Returns:
@@ -107,9 +104,6 @@ class SessionConfig:
def load_from_env(self):
"""Load configuration from environment variables (only if not already set)."""
- if os.getenv('VIRUSTOTAL_API_KEY') and not self.api_keys['virustotal']:
- self.set_api_key('virustotal', os.getenv('VIRUSTOTAL_API_KEY'))
-
if os.getenv('SHODAN_API_KEY') and not self.api_keys['shodan']:
self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))
diff --git a/dump.rdb b/dump.rdb
new file mode 100644
index 0000000..2fd4d59
Binary files /dev/null and b/dump.rdb differ
diff --git a/providers/__init__.py b/providers/__init__.py
index a071381..b56306c 100644
--- a/providers/__init__.py
+++ b/providers/__init__.py
@@ -7,15 +7,13 @@ from .base_provider import BaseProvider, RateLimiter
from .crtsh_provider import CrtShProvider
from .dns_provider import DNSProvider
from .shodan_provider import ShodanProvider
-from .virustotal_provider import VirusTotalProvider
__all__ = [
'BaseProvider',
'RateLimiter',
'CrtShProvider',
'DNSProvider',
- 'ShodanProvider',
- 'VirusTotalProvider'
+ 'ShodanProvider'
]
__version__ = "1.0.0-phase2"
\ No newline at end of file
diff --git a/providers/virustotal_provider.py b/providers/virustotal_provider.py
deleted file mode 100644
index 0d75f0d..0000000
--- a/providers/virustotal_provider.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-VirusTotal provider for DNSRecon.
-Discovers domain relationships through passive DNS and URL analysis.
-"""
-
-import json
-from typing import List, Dict, Any, Tuple
-from .base_provider import BaseProvider
-from utils.helpers import _is_valid_ip, _is_valid_domain
-from core.graph_manager import RelationshipType
-
-
-class VirusTotalProvider(BaseProvider):
- """
- Provider for querying VirusTotal API for passive DNS and domain reputation data.
- Now uses session-specific API keys and rate limits.
- """
-
- def __init__(self, session_config=None):
- """Initialize VirusTotal provider with session-specific configuration."""
- super().__init__(
- name="virustotal",
- rate_limit=4, # Free tier: 4 requests per minute
- timeout=30,
- session_config=session_config
- )
- self.base_url = "https://www.virustotal.com/vtapi/v2"
- self.api_key = self.config.get_api_key('virustotal')
-
- def is_available(self) -> bool:
- """Check if VirusTotal provider is available (has valid API key in this session)."""
- return self.api_key is not None and len(self.api_key.strip()) > 0
-
- def get_name(self) -> str:
- """Return the provider name."""
- return "virustotal"
-
- def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
- """
- Query VirusTotal for domain information including passive DNS.
-
- Args:
- domain: Domain to investigate
-
- Returns:
- List of relationships discovered from VirusTotal data
- """
- if not _is_valid_domain(domain) or not self.is_available():
- return []
-
- relationships = []
-
- # Query domain report
- domain_relationships = self._query_domain_report(domain)
- relationships.extend(domain_relationships)
-
- # Query passive DNS for the domain
- passive_dns_relationships = self._query_passive_dns_domain(domain)
- relationships.extend(passive_dns_relationships)
-
- return relationships
-
- def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
- """
- Query VirusTotal for IP address information including passive DNS.
-
- Args:
- ip: IP address to investigate
-
- Returns:
- List of relationships discovered from VirusTotal IP data
- """
- if not _is_valid_ip(ip) or not self.is_available():
- return []
-
- relationships = []
-
- # Query IP report
- ip_relationships = self._query_ip_report(ip)
- relationships.extend(ip_relationships)
-
- # Query passive DNS for the IP
- passive_dns_relationships = self._query_passive_dns_ip(ip)
- relationships.extend(passive_dns_relationships)
-
- return relationships
-
- def _query_domain_report(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
- """Query VirusTotal domain report."""
- relationships = []
-
- try:
- url = f"{self.base_url}/domain/report"
- params = {
- 'apikey': self.api_key,
- 'domain': domain,
- 'allinfo': 1 # Get comprehensive information
- }
-
- response = self.make_request(url, method="GET", params=params, target_indicator=domain)
-
- if not response or response.status_code != 200:
- return []
-
- data = response.json()
-
- if data.get('response_code') != 1:
- return []
-
- # Extract resolved IPs
- resolutions = data.get('resolutions', [])
- for resolution in resolutions:
- ip_address = resolution.get('ip_address')
- last_resolved = resolution.get('last_resolved')
-
- if ip_address and _is_valid_ip(ip_address):
- raw_data = {
- 'domain': domain,
- 'ip_address': ip_address,
- 'last_resolved': last_resolved,
- 'source': 'virustotal_domain_report'
- }
-
- relationships.append((
- domain,
- ip_address,
- RelationshipType.PASSIVE_DNS,
- RelationshipType.PASSIVE_DNS.default_confidence,
- raw_data
- ))
-
- self.log_relationship_discovery(
- source_node=domain,
- target_node=ip_address,
- relationship_type=RelationshipType.PASSIVE_DNS,
- confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
- raw_data=raw_data,
- discovery_method="virustotal_domain_resolution"
- )
-
- # Extract subdomains
- subdomains = data.get('subdomains', [])
- for subdomain in subdomains:
- if subdomain != domain and _is_valid_domain(subdomain):
- raw_data = {
- 'parent_domain': domain,
- 'subdomain': subdomain,
- 'source': 'virustotal_subdomain_discovery'
- }
-
- relationships.append((
- domain,
- subdomain,
- RelationshipType.PASSIVE_DNS,
- 0.7, # Medium-high confidence for subdomains
- raw_data
- ))
-
- self.log_relationship_discovery(
- source_node=domain,
- target_node=subdomain,
- relationship_type=RelationshipType.PASSIVE_DNS,
- confidence_score=0.7,
- raw_data=raw_data,
- discovery_method="virustotal_subdomain_discovery"
- )
-
- except json.JSONDecodeError as e:
- self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
- except Exception as e:
- self.logger.logger.error(f"Error querying VirusTotal domain report for {domain}: {e}")
-
- return relationships
-
- def _query_ip_report(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
- """Query VirusTotal IP report."""
- relationships = []
-
- try:
- url = f"{self.base_url}/ip-address/report"
- params = {
- 'apikey': self.api_key,
- 'ip': ip
- }
-
- response = self.make_request(url, method="GET", params=params, target_indicator=ip)
-
- if not response or response.status_code != 200:
- return []
-
- data = response.json()
-
- if data.get('response_code') != 1:
- return []
-
- # Extract resolved domains
- resolutions = data.get('resolutions', [])
- for resolution in resolutions:
- hostname = resolution.get('hostname')
- last_resolved = resolution.get('last_resolved')
-
- if hostname and _is_valid_domain(hostname):
- raw_data = {
- 'ip_address': ip,
- 'hostname': hostname,
- 'last_resolved': last_resolved,
- 'source': 'virustotal_ip_report'
- }
-
- relationships.append((
- ip,
- hostname,
- RelationshipType.PASSIVE_DNS,
- RelationshipType.PASSIVE_DNS.default_confidence,
- raw_data
- ))
-
- self.log_relationship_discovery(
- source_node=ip,
- target_node=hostname,
- relationship_type=RelationshipType.PASSIVE_DNS,
- confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
- raw_data=raw_data,
- discovery_method="virustotal_ip_resolution"
- )
-
- except json.JSONDecodeError as e:
- self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
- except Exception as e:
- self.logger.logger.error(f"Error querying VirusTotal IP report for {ip}: {e}")
-
- return relationships
-
- def _query_passive_dns_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
- """Query VirusTotal passive DNS for domain."""
- # Note: VirusTotal's passive DNS API might require a premium subscription
- # This is a placeholder for the endpoint structure
- return []
-
- def _query_passive_dns_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
- """Query VirusTotal passive DNS for IP."""
- # Note: VirusTotal's passive DNS API might require a premium subscription
- # This is a placeholder for the endpoint structure
- return []
-
- def get_domain_reputation(self, domain: str) -> Dict[str, Any]:
- """
- Get domain reputation information from VirusTotal.
-
- Args:
- domain: Domain to check reputation for
-
- Returns:
- Dictionary containing reputation data
- """
- if not _is_valid_domain(domain) or not self.is_available():
- return {}
-
- try:
- url = f"{self.base_url}/domain/report"
- params = {
- 'apikey': self.api_key,
- 'domain': domain
- }
-
- response = self.make_request(url, method="GET", params=params, target_indicator=domain)
-
- if response and response.status_code == 200:
- data = response.json()
-
- if data.get('response_code') == 1:
- return {
- 'positives': data.get('positives', 0),
- 'total': data.get('total', 0),
- 'scan_date': data.get('scan_date', ''),
- 'permalink': data.get('permalink', ''),
- 'reputation_score': self._calculate_reputation_score(data)
- }
-
- except Exception as e:
- self.logger.logger.error(f"Error getting VirusTotal reputation for domain {domain}: {e}")
-
- return {}
-
- def get_ip_reputation(self, ip: str) -> Dict[str, Any]:
- """
- Get IP reputation information from VirusTotal.
-
- Args:
- ip: IP address to check reputation for
-
- Returns:
- Dictionary containing reputation data
- """
- if not _is_valid_ip(ip) or not self.is_available():
- return {}
-
- try:
- url = f"{self.base_url}/ip-address/report"
- params = {
- 'apikey': self.api_key,
- 'ip': ip
- }
-
- response = self.make_request(url, method="GET", params=params, target_indicator=ip)
-
- if response and response.status_code == 200:
- data = response.json()
-
- if data.get('response_code') == 1:
- return {
- 'positives': data.get('positives', 0),
- 'total': data.get('total', 0),
- 'scan_date': data.get('scan_date', ''),
- 'permalink': data.get('permalink', ''),
- 'reputation_score': self._calculate_reputation_score(data)
- }
-
- except Exception as e:
- self.logger.logger.error(f"Error getting VirusTotal reputation for IP {ip}: {e}")
-
- return {}
-
- def _calculate_reputation_score(self, data: Dict[str, Any]) -> float:
- """Calculate a normalized reputation score (0.0 to 1.0)."""
- positives = data.get('positives', 0)
- total = data.get('total', 1) # Avoid division by zero
-
- if total == 0:
- return 1.0 # No data means neutral
-
- # Score is inverse of detection ratio (lower detection = higher reputation)
- return max(0.0, 1.0 - (positives / total))
\ No newline at end of file
diff --git a/static/js/graph.js b/static/js/graph.js
index f9fdeb2..7d4bd64 100644
--- a/static/js/graph.js
+++ b/static/js/graph.js
@@ -270,8 +270,23 @@ class GraphManager {
this.initialize();
}
- // Process nodes with enhanced attributes
- const processedNodes = graphData.nodes.map(node => this.processNode(node));
+ // Find all aggregated node IDs first
+ const aggregatedNodeIds = new Set();
+ graphData.nodes.forEach(node => {
+ if (node.type === 'large_entity' && node.metadata && Array.isArray(node.metadata.nodes)) {
+ node.metadata.nodes.forEach(nodeId => aggregatedNodeIds.add(nodeId));
+ }
+ });
+
+ // Process nodes, hiding the ones that are aggregated
+ const processedNodes = graphData.nodes.map(node => {
+ const processed = this.processNode(node);
+ if (aggregatedNodeIds.has(node.id)) {
+ processed.hidden = true; // Mark node as hidden
+ }
+ return processed;
+ });
+
const processedEdges = graphData.edges.map(edge => this.processEdge(edge));
// Update datasets with animation
@@ -441,7 +456,8 @@ class GraphManager {
'domain': 12,
'ip': 14,
'asn': 16,
- 'correlation_object': 8
+ 'correlation_object': 8,
+ 'large_entity': 12
};
return sizes[nodeType] || 12;
}
@@ -456,7 +472,8 @@ class GraphManager {
'domain': 'dot',
'ip': 'square',
'asn': 'triangle',
- 'correlation_object': 'hexagon'
+ 'correlation_object': 'hexagon',
+ 'large_entity': 'database'
};
return shapes[nodeType] || 'dot';
}
diff --git a/static/js/main.js b/static/js/main.js
index 815aa15..96b109b 100644
--- a/static/js/main.js
+++ b/static/js/main.js
@@ -80,7 +80,6 @@ class DNSReconApp {
// API Key Modal elements
apiKeyModal: document.getElementById('api-key-modal'),
apiKeyModalClose: document.getElementById('api-key-modal-close'),
- virustotalApiKey: document.getElementById('virustotal-api-key'),
shodanApiKey: document.getElementById('shodan-api-key'),
saveApiKeys: document.getElementById('save-api-keys'),
resetApiKeys: document.getElementById('reset-api-keys'),
@@ -851,13 +850,11 @@ class DNSReconApp {
detailsHtml += createDetailRow('Related Domains (SAN)', metadata.related_domains_san);
detailsHtml += createDetailRow('Passive DNS', metadata.passive_dns);
detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
- detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
break;
case 'ip':
detailsHtml += createDetailRow('Hostnames', metadata.hostnames);
detailsHtml += createDetailRow('Passive DNS', metadata.passive_dns);
detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
- detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
break;
case 'correlation_object':
detailsHtml += createDetailRow('Correlated Value', metadata.value);
@@ -974,11 +971,9 @@ class DNSReconApp {
*/
async saveApiKeys() {
const shodanKey = this.elements.shodanApiKey.value.trim();
- const virustotalKey = this.elements.virustotalApiKey.value.trim();
const keys = {};
if (shodanKey) keys.shodan = shodanKey;
- if (virustotalKey) keys.virustotal = virustotalKey;
if (Object.keys(keys).length === 0) {
this.showWarning('No API keys were entered.');
@@ -1004,7 +999,6 @@ class DNSReconApp {
*/
resetApiKeys() {
this.elements.shodanApiKey.value = '';
- this.elements.virustotalApiKey.value = '';
}
/**
diff --git a/templates/index.html b/templates/index.html
index 7b61632..1e9bdda 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -217,11 +217,6 @@
Enter your API keys for enhanced data providers. Keys are stored in memory for the current session only and are never saved to disk.
-
-
-
-
Enables passive DNS and domain reputation lookups.
-