mstoeck3/dnsrecon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix large entity

Browse Source
This commit is contained in:
overcuriousity 2025-09-13 15:38:05 +02:00
parent 53baf2e291
commit 612f414d2a
11 changed files with 45 additions and 385 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -148,7 +148,6 @@ export FLASK_ENV='production'
export FLASK_DEBUG=False
# API keys (optional, but recommended for full functionality)
export VIRUSTOTAL_API_KEY="your_virustotal_key"
export SHODAN_API_KEY="your_shodan_key"
```
@ -224,7 +223,6 @@ Restart=always
Environment="SECRET_KEY=your-super-secret-and-random-key"
Environment="FLASK_ENV=production"
Environment="FLASK_DEBUG=False"
Environment="VIRUSTOTAL_API_KEY=your_virustotal_key"
Environment="SHODAN_API_KEY=your_shodan_key"
[Install]

4
app.py
View File

@ -384,7 +384,7 @@ def get_providers():
'statistics': stats,
'enabled': config.is_provider_enabled(provider_name),
'rate_limit': config.get_rate_limit(provider_name),
'requires_api_key': provider_name in ['shodan', 'virustotal']
'requires_api_key': provider_name in ['shodan']
}
return jsonify({
@ -423,7 +423,7 @@ def set_api_keys():
updated_providers = []
for provider, api_key in data.items():
if provider in ['shodan', 'virustotal'] and api_key.strip():
if provider in ['shodan'] and api_key.strip():
success = session_config.set_api_key(provider, api_key.strip())
if success:
updated_providers.append(provider)

10
config.py
View File

@ -13,8 +13,7 @@ class Config:
def __init__(self):
"""Initialize configuration with default values."""
self.api_keys: Dict[str, Optional[str]] = {
'shodan': None,
'virustotal': None
'shodan': None
}
# Default settings
@ -26,7 +25,6 @@ class Config:
# Rate limiting settings (requests per minute)
self.rate_limits = {
'crtsh': 60, # Free service, be respectful
'virustotal': 4, # Free tier limit
'shodan': 60, # API dependent
'dns': 100 # Local DNS queries
}
@ -35,7 +33,6 @@ class Config:
self.enabled_providers = {
'crtsh': True, # Always enabled (free)
'dns': True, # Always enabled (free)
'virustotal': False, # Requires API key
'shodan': False # Requires API key
}
@ -53,7 +50,7 @@ class Config:
Set API key for a provider.
Args:
provider: Provider name (shodan, virustotal)
provider: Provider name (shodan, etc)
api_key: API key string
Returns:
@ -103,9 +100,6 @@ class Config:
def load_from_env(self):
"""Load configuration from environment variables."""
if os.getenv('VIRUSTOTAL_API_KEY'):
self.set_api_key('virustotal', os.getenv('VIRUSTOTAL_API_KEY'))
if os.getenv('SHODAN_API_KEY'):
self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))

31
core/scanner.py
View File

@ -14,7 +14,6 @@ from utils.helpers import _is_valid_ip, _is_valid_domain
from providers.crtsh_provider import CrtShProvider
from providers.dns_provider import DNSProvider
from providers.shodan_provider import ShodanProvider
from providers.virustotal_provider import VirusTotalProvider
class ScanStatus:
@ -66,8 +65,7 @@ class Scanner:
self.provider_eligibility = {
'dns': {'domains': True, 'ips': True},
'crtsh': {'domains': True, 'ips': False},
'shodan': {'domains': True, 'ips': True},
'virustotal': {'domains': True, 'ips': True}
'shodan': {'domains': True, 'ips': True}
}
# Initialize providers with session config
@ -169,8 +167,7 @@ class Scanner:
provider_classes = {
'dns': DNSProvider,
'crtsh': CrtShProvider,
'shodan': ShodanProvider,
'virustotal': VirusTotalProvider
'shodan': ShodanProvider
}
for provider_name, provider_class in provider_classes.items():
@ -757,18 +754,24 @@ class Scanner:
def _create_large_entity(self, source: str, provider_name: str, results: List, current_depth: int) -> None:
"""Create a large entity node for forensic tracking."""
entity_id = f"large_entity_{provider_name}_{hash(source) & 0x7FFFFFFF}"
entity_id = f"Large Entity: {provider_name}"
# Extract targets from results
targets = [rel[1] for rel in results if len(rel) > 1]
# Determine node type
targets = []
node_type = 'unknown'
if targets:
if _is_valid_domain(targets[0]):
node_type = 'domain'
elif _is_valid_ip(targets[0]):
node_type = 'ip'
for rel in results:
if len(rel) > 1:
target = rel[1]
targets.append(target)
# Determine node type and add node to graph
if _is_valid_domain(target):
node_type = 'domain'
self.graph.add_node(target, NodeType.DOMAIN)
elif _is_valid_ip(target):
node_type = 'ip'
self.graph.add_node(target, NodeType.IP)
# Create large entity metadata
metadata = {

10
core/session_config.py
View File

@ -17,8 +17,7 @@ class SessionConfig:
"""Initialize session config with global defaults."""
# Copy all attributes from global config
self.api_keys: Dict[str, Optional[str]] = {
'shodan': None,
'virustotal': None
'shodan': None
}
# Default settings (copied from global config)
@ -30,7 +29,6 @@ class SessionConfig:
# Rate limiting settings (per session)
self.rate_limits = {
'crtsh': 60,
'virustotal': 4,
'shodan': 60,
'dns': 100
}
@ -39,7 +37,6 @@ class SessionConfig:
self.enabled_providers = {
'crtsh': True,
'dns': True,
'virustotal': False,
'shodan': False
}
@ -57,7 +54,7 @@ class SessionConfig:
Set API key for a provider in this session.
Args:
provider: Provider name (shodan, virustotal)
provider: Provider name (shodan, etc)
api_key: API key string
Returns:
@ -107,9 +104,6 @@ class SessionConfig:
def load_from_env(self):
"""Load configuration from environment variables (only if not already set)."""
if os.getenv('VIRUSTOTAL_API_KEY') and not self.api_keys['virustotal']:
self.set_api_key('virustotal', os.getenv('VIRUSTOTAL_API_KEY'))
if os.getenv('SHODAN_API_KEY') and not self.api_keys['shodan']:
self.set_api_key('shodan', os.getenv('SHODAN_API_KEY'))

BIN
dump.rdb Normal file
View File

Binary file not shown.

4
providers/__init__.py
View File

@ -7,15 +7,13 @@ from .base_provider import BaseProvider, RateLimiter
from .crtsh_provider import CrtShProvider
from .dns_provider import DNSProvider
from .shodan_provider import ShodanProvider
from .virustotal_provider import VirusTotalProvider
__all__ = [
'BaseProvider',
'RateLimiter',
'CrtShProvider',
'DNSProvider',
'ShodanProvider',
'VirusTotalProvider'
'ShodanProvider'
]
__version__ = "1.0.0-phase2"

333
providers/virustotal_provider.py
View File

@ -1,333 +0,0 @@
"""
VirusTotal provider for DNSRecon.
Discovers domain relationships through passive DNS and URL analysis.
"""
import json
from typing import List, Dict, Any, Tuple
from .base_provider import BaseProvider
from utils.helpers import _is_valid_ip, _is_valid_domain
from core.graph_manager import RelationshipType
class VirusTotalProvider(BaseProvider):
"""
Provider for querying VirusTotal API for passive DNS and domain reputation data.
Now uses session-specific API keys and rate limits.
"""
def __init__(self, session_config=None):
"""Initialize VirusTotal provider with session-specific configuration."""
super().__init__(
name="virustotal",
rate_limit=4, # Free tier: 4 requests per minute
timeout=30,
session_config=session_config
)
self.base_url = "https://www.virustotal.com/vtapi/v2"
self.api_key = self.config.get_api_key('virustotal')
def is_available(self) -> bool:
"""Check if VirusTotal provider is available (has valid API key in this session)."""
return self.api_key is not None and len(self.api_key.strip()) > 0
def get_name(self) -> str:
"""Return the provider name."""
return "virustotal"
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query VirusTotal for domain information including passive DNS.
Args:
domain: Domain to investigate
Returns:
List of relationships discovered from VirusTotal data
"""
if not _is_valid_domain(domain) or not self.is_available():
return []
relationships = []
# Query domain report
domain_relationships = self._query_domain_report(domain)
relationships.extend(domain_relationships)
# Query passive DNS for the domain
passive_dns_relationships = self._query_passive_dns_domain(domain)
relationships.extend(passive_dns_relationships)
return relationships
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""
Query VirusTotal for IP address information including passive DNS.
Args:
ip: IP address to investigate
Returns:
List of relationships discovered from VirusTotal IP data
"""
if not _is_valid_ip(ip) or not self.is_available():
return []
relationships = []
# Query IP report
ip_relationships = self._query_ip_report(ip)
relationships.extend(ip_relationships)
# Query passive DNS for the IP
passive_dns_relationships = self._query_passive_dns_ip(ip)
relationships.extend(passive_dns_relationships)
return relationships
def _query_domain_report(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query VirusTotal domain report."""
relationships = []
try:
url = f"{self.base_url}/domain/report"
params = {
'apikey': self.api_key,
'domain': domain,
'allinfo': 1 # Get comprehensive information
}
response = self.make_request(url, method="GET", params=params, target_indicator=domain)
if not response or response.status_code != 200:
return []
data = response.json()
if data.get('response_code') != 1:
return []
# Extract resolved IPs
resolutions = data.get('resolutions', [])
for resolution in resolutions:
ip_address = resolution.get('ip_address')
last_resolved = resolution.get('last_resolved')
if ip_address and _is_valid_ip(ip_address):
raw_data = {
'domain': domain,
'ip_address': ip_address,
'last_resolved': last_resolved,
'source': 'virustotal_domain_report'
}
relationships.append((
domain,
ip_address,
RelationshipType.PASSIVE_DNS,
RelationshipType.PASSIVE_DNS.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=ip_address,
relationship_type=RelationshipType.PASSIVE_DNS,
confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
raw_data=raw_data,
discovery_method="virustotal_domain_resolution"
)
# Extract subdomains
subdomains = data.get('subdomains', [])
for subdomain in subdomains:
if subdomain != domain and _is_valid_domain(subdomain):
raw_data = {
'parent_domain': domain,
'subdomain': subdomain,
'source': 'virustotal_subdomain_discovery'
}
relationships.append((
domain,
subdomain,
RelationshipType.PASSIVE_DNS,
0.7, # Medium-high confidence for subdomains
raw_data
))
self.log_relationship_discovery(
source_node=domain,
target_node=subdomain,
relationship_type=RelationshipType.PASSIVE_DNS,
confidence_score=0.7,
raw_data=raw_data,
discovery_method="virustotal_subdomain_discovery"
)
except json.JSONDecodeError as e:
self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
except Exception as e:
self.logger.logger.error(f"Error querying VirusTotal domain report for {domain}: {e}")
return relationships
def _query_ip_report(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query VirusTotal IP report."""
relationships = []
try:
url = f"{self.base_url}/ip-address/report"
params = {
'apikey': self.api_key,
'ip': ip
}
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
if not response or response.status_code != 200:
return []
data = response.json()
if data.get('response_code') != 1:
return []
# Extract resolved domains
resolutions = data.get('resolutions', [])
for resolution in resolutions:
hostname = resolution.get('hostname')
last_resolved = resolution.get('last_resolved')
if hostname and _is_valid_domain(hostname):
raw_data = {
'ip_address': ip,
'hostname': hostname,
'last_resolved': last_resolved,
'source': 'virustotal_ip_report'
}
relationships.append((
ip,
hostname,
RelationshipType.PASSIVE_DNS,
RelationshipType.PASSIVE_DNS.default_confidence,
raw_data
))
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type=RelationshipType.PASSIVE_DNS,
confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
raw_data=raw_data,
discovery_method="virustotal_ip_resolution"
)
except json.JSONDecodeError as e:
self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
except Exception as e:
self.logger.logger.error(f"Error querying VirusTotal IP report for {ip}: {e}")
return relationships
def _query_passive_dns_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query VirusTotal passive DNS for domain."""
# Note: VirusTotal's passive DNS API might require a premium subscription
# This is a placeholder for the endpoint structure
return []
def _query_passive_dns_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
"""Query VirusTotal passive DNS for IP."""
# Note: VirusTotal's passive DNS API might require a premium subscription
# This is a placeholder for the endpoint structure
return []
def get_domain_reputation(self, domain: str) -> Dict[str, Any]:
"""
Get domain reputation information from VirusTotal.
Args:
domain: Domain to check reputation for
Returns:
Dictionary containing reputation data
"""
if not _is_valid_domain(domain) or not self.is_available():
return {}
try:
url = f"{self.base_url}/domain/report"
params = {
'apikey': self.api_key,
'domain': domain
}
response = self.make_request(url, method="GET", params=params, target_indicator=domain)
if response and response.status_code == 200:
data = response.json()
if data.get('response_code') == 1:
return {
'positives': data.get('positives', 0),
'total': data.get('total', 0),
'scan_date': data.get('scan_date', ''),
'permalink': data.get('permalink', ''),
'reputation_score': self._calculate_reputation_score(data)
}
except Exception as e:
self.logger.logger.error(f"Error getting VirusTotal reputation for domain {domain}: {e}")
return {}
def get_ip_reputation(self, ip: str) -> Dict[str, Any]:
"""
Get IP reputation information from VirusTotal.
Args:
ip: IP address to check reputation for
Returns:
Dictionary containing reputation data
"""
if not _is_valid_ip(ip) or not self.is_available():
return {}
try:
url = f"{self.base_url}/ip-address/report"
params = {
'apikey': self.api_key,
'ip': ip
}
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
if response and response.status_code == 200:
data = response.json()
if data.get('response_code') == 1:
return {
'positives': data.get('positives', 0),
'total': data.get('total', 0),
'scan_date': data.get('scan_date', ''),
'permalink': data.get('permalink', ''),
'reputation_score': self._calculate_reputation_score(data)
}
except Exception as e:
self.logger.logger.error(f"Error getting VirusTotal reputation for IP {ip}: {e}")
return {}
def _calculate_reputation_score(self, data: Dict[str, Any]) -> float:
"""Calculate a normalized reputation score (0.0 to 1.0)."""
positives = data.get('positives', 0)
total = data.get('total', 1) # Avoid division by zero
if total == 0:
return 1.0 # No data means neutral
# Score is inverse of detection ratio (lower detection = higher reputation)
return max(0.0, 1.0 - (positives / total))

25
static/js/graph.js
View File

@ -270,8 +270,23 @@ class GraphManager {
this.initialize();
}
// Process nodes with enhanced attributes
const processedNodes = graphData.nodes.map(node => this.processNode(node));
// Find all aggregated node IDs first
const aggregatedNodeIds = new Set();
graphData.nodes.forEach(node => {
if (node.type === 'large_entity' && node.metadata && Array.isArray(node.metadata.nodes)) {
node.metadata.nodes.forEach(nodeId => aggregatedNodeIds.add(nodeId));
}
});
// Process nodes, hiding the ones that are aggregated
const processedNodes = graphData.nodes.map(node => {
const processed = this.processNode(node);
if (aggregatedNodeIds.has(node.id)) {
processed.hidden = true; // Mark node as hidden
}
return processed;
});
const processedEdges = graphData.edges.map(edge => this.processEdge(edge));
// Update datasets with animation
@ -441,7 +456,8 @@ class GraphManager {
'domain': 12,
'ip': 14,
'asn': 16,
'correlation_object': 8
'correlation_object': 8,
'large_entity': 12
};
return sizes[nodeType] || 12;
}
@ -456,7 +472,8 @@ class GraphManager {
'domain': 'dot',
'ip': 'square',
'asn': 'triangle',
'correlation_object': 'hexagon'
'correlation_object': 'hexagon',
'large_entity': 'database'
};
return shapes[nodeType] || 'dot';
}

6
static/js/main.js
View File

@ -80,7 +80,6 @@ class DNSReconApp {
// API Key Modal elements
apiKeyModal: document.getElementById('api-key-modal'),
apiKeyModalClose: document.getElementById('api-key-modal-close'),
virustotalApiKey: document.getElementById('virustotal-api-key'),
shodanApiKey: document.getElementById('shodan-api-key'),
saveApiKeys: document.getElementById('save-api-keys'),
resetApiKeys: document.getElementById('reset-api-keys'),
@ -851,13 +850,11 @@ class DNSReconApp {
detailsHtml += createDetailRow('Related Domains (SAN)', metadata.related_domains_san);
detailsHtml += createDetailRow('Passive DNS', metadata.passive_dns);
detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
break;
case 'ip':
detailsHtml += createDetailRow('Hostnames', metadata.hostnames);
detailsHtml += createDetailRow('Passive DNS', metadata.passive_dns);
detailsHtml += createDetailRow('Shodan Data', metadata.shodan);
detailsHtml += createDetailRow('VirusTotal Data', metadata.virustotal);
break;
case 'correlation_object':
detailsHtml += createDetailRow('Correlated Value', metadata.value);
@ -974,11 +971,9 @@ class DNSReconApp {
*/
async saveApiKeys() {
const shodanKey = this.elements.shodanApiKey.value.trim();
const virustotalKey = this.elements.virustotalApiKey.value.trim();
const keys = {};
if (shodanKey) keys.shodan = shodanKey;
if (virustotalKey) keys.virustotal = virustotalKey;
if (Object.keys(keys).length === 0) {
this.showWarning('No API keys were entered.');
@ -1004,7 +999,6 @@ class DNSReconApp {
*/
resetApiKeys() {
this.elements.shodanApiKey.value = '';
this.elements.virustotalApiKey.value = '';
}
/**

5
templates/index.html
View File

@ -217,11 +217,6 @@
<p class="modal-description">
Enter your API keys for enhanced data providers. Keys are stored in memory for the current session only and are never saved to disk.
</p>
<div class="apikey-section">
<label for="virustotal-api-key">VirusTotal API Key</label>
<input type="password" id="virustotal-api-key" placeholder="Enter VirusTotal API Key">
<p class="apikey-help">Enables passive DNS and domain reputation lookups.</p>
</div>
<div class="apikey-section">
<label for="shodan-api-key">Shodan API Key</label>
<input type="password" id="shodan-api-key" placeholder="Enter Shodan API Key">