progress
This commit is contained in:
overcuriousity
@@ -5,11 +5,17 @@ Contains implementations for various reconnaissance data sources.
|
||||
|
||||
from .base_provider import BaseProvider, RateLimiter
|
||||
from .crtsh_provider import CrtShProvider
|
||||
from .dns_provider import DNSProvider
|
||||
from .shodan_provider import ShodanProvider
|
||||
from .virustotal_provider import VirusTotalProvider
|
||||
|
||||
__all__ = [
|
||||
'BaseProvider',
|
||||
'RateLimiter',
|
||||
'CrtShProvider'
|
||||
'CrtShProvider',
|
||||
'DNSProvider',
|
||||
'ShodanProvider',
|
||||
'VirusTotalProvider'
|
||||
]
|
||||
|
||||
__version__ = "1.0.0-phase1"
|
||||
__version__ = "1.0.0-phase2"
|
||||
@@ -113,86 +113,134 @@ class BaseProvider(ABC):
|
||||
pass
|
||||
|
||||
def make_request(self, url: str, method: str = "GET",
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
target_indicator: str = "") -> Optional[requests.Response]:
|
||||
"""
|
||||
Make a rate-limited HTTP request with forensic logging.
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
target_indicator: str = "",
|
||||
max_retries: int = 3) -> Optional[requests.Response]:
|
||||
"""
|
||||
Make a rate-limited HTTP request with forensic logging and retry logic.
|
||||
|
||||
Args:
|
||||
url: Request URL
|
||||
method: HTTP method
|
||||
params: Query parameters
|
||||
headers: Additional headers
|
||||
target_indicator: The indicator being investigated
|
||||
max_retries: Maximum number of retry attempts
|
||||
|
||||
Returns:
|
||||
Response object or None if request failed
|
||||
"""
|
||||
for attempt in range(max_retries + 1):
|
||||
# Apply rate limiting
|
||||
self.rate_limiter.wait_if_needed()
|
||||
|
||||
start_time = time.time()
|
||||
response = None
|
||||
error = None
|
||||
|
||||
try:
|
||||
self.total_requests += 1
|
||||
|
||||
# Prepare request
|
||||
request_headers = self.session.headers.copy()
|
||||
if headers:
|
||||
request_headers.update(headers)
|
||||
|
||||
print(f"Making {method} request to: {url} (attempt {attempt + 1})")
|
||||
|
||||
# Make request
|
||||
if method.upper() == "GET":
|
||||
response = self.session.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=request_headers,
|
||||
timeout=self.timeout
|
||||
)
|
||||
elif method.upper() == "POST":
|
||||
response = self.session.post(
|
||||
url,
|
||||
json=params,
|
||||
headers=request_headers,
|
||||
timeout=self.timeout
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported HTTP method: {method}")
|
||||
|
||||
print(f"Response status: {response.status_code}")
|
||||
response.raise_for_status()
|
||||
self.successful_requests += 1
|
||||
|
||||
# Success - log and return
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
self.logger.log_api_request(
|
||||
provider=self.name,
|
||||
url=url,
|
||||
method=method.upper(),
|
||||
status_code=response.status_code,
|
||||
response_size=len(response.content),
|
||||
duration_ms=duration_ms,
|
||||
error=None,
|
||||
target_indicator=target_indicator
|
||||
)
|
||||
return response
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
error = str(e)
|
||||
self.failed_requests += 1
|
||||
print(f"Request failed (attempt {attempt + 1}): {error}")
|
||||
|
||||
# Check if we should retry
|
||||
if attempt < max_retries and self._should_retry(e):
|
||||
backoff_time = (2 ** attempt) * 1 # Exponential backoff: 1s, 2s, 4s
|
||||
print(f"Retrying in {backoff_time} seconds...")
|
||||
time.sleep(backoff_time)
|
||||
continue
|
||||
else:
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
error = f"Unexpected error: {str(e)}"
|
||||
self.failed_requests += 1
|
||||
print(f"Unexpected error: {error}")
|
||||
break
|
||||
|
||||
# All attempts failed - log and return None
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
self.logger.log_api_request(
|
||||
provider=self.name,
|
||||
url=url,
|
||||
method=method.upper(),
|
||||
status_code=response.status_code if response else None,
|
||||
response_size=len(response.content) if response else None,
|
||||
duration_ms=duration_ms,
|
||||
error=error,
|
||||
target_indicator=target_indicator
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _should_retry(self, exception: requests.exceptions.RequestException) -> bool:
|
||||
"""
|
||||
Determine if a request should be retried based on the exception.
|
||||
|
||||
Args:
|
||||
url: Request URL
|
||||
method: HTTP method
|
||||
params: Query parameters
|
||||
headers: Additional headers
|
||||
target_indicator: The indicator being investigated
|
||||
|
||||
exception: The request exception that occurred
|
||||
|
||||
Returns:
|
||||
Response object or None if request failed
|
||||
True if the request should be retried
|
||||
"""
|
||||
# Apply rate limiting
|
||||
self.rate_limiter.wait_if_needed()
|
||||
|
||||
start_time = time.time()
|
||||
response = None
|
||||
error = None
|
||||
|
||||
try:
|
||||
self.total_requests += 1
|
||||
|
||||
# Prepare request
|
||||
request_headers = self.session.headers.copy()
|
||||
if headers:
|
||||
request_headers.update(headers)
|
||||
|
||||
print(f"Making {method} request to: {url}")
|
||||
|
||||
# Make request
|
||||
if method.upper() == "GET":
|
||||
response = self.session.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=request_headers,
|
||||
timeout=self.timeout
|
||||
)
|
||||
elif method.upper() == "POST":
|
||||
response = self.session.post(
|
||||
url,
|
||||
json=params,
|
||||
headers=request_headers,
|
||||
timeout=self.timeout
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported HTTP method: {method}")
|
||||
|
||||
print(f"Response status: {response.status_code}")
|
||||
response.raise_for_status()
|
||||
self.successful_requests += 1
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
error = str(e)
|
||||
self.failed_requests += 1
|
||||
print(f"Request failed: {error}")
|
||||
|
||||
except Exception as e:
|
||||
error = f"Unexpected error: {str(e)}"
|
||||
self.failed_requests += 1
|
||||
print(f"Unexpected error: {error}")
|
||||
|
||||
# Calculate duration and log request
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
|
||||
self.logger.log_api_request(
|
||||
provider=self.name,
|
||||
url=url,
|
||||
method=method.upper(),
|
||||
status_code=response.status_code if response else None,
|
||||
response_size=len(response.content) if response else None,
|
||||
duration_ms=duration_ms,
|
||||
error=error,
|
||||
target_indicator=target_indicator
|
||||
)
|
||||
|
||||
return response if error is None else None
|
||||
# Retry on connection errors, timeouts, and 5xx server errors
|
||||
if isinstance(exception, (requests.exceptions.ConnectionError,
|
||||
requests.exceptions.Timeout)):
|
||||
return True
|
||||
|
||||
if isinstance(exception, requests.exceptions.HTTPError):
|
||||
if hasattr(exception, 'response') and exception.response:
|
||||
# Retry on server errors (5xx) but not client errors (4xx)
|
||||
return exception.response.status_code >= 500
|
||||
|
||||
return False
|
||||
|
||||
def log_relationship_discovery(self, source_node: str, target_node: str,
|
||||
relationship_type: RelationshipType,
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
"""
|
||||
DNS resolution provider for DNSRecon.
|
||||
Discovers domain relationships through DNS record analysis.
|
||||
"""
|
||||
|
||||
import socket
|
||||
import dns.resolver
|
||||
import dns.reversename
|
||||
from typing import List, Dict, Any, Tuple, Optional
|
||||
from .base_provider import BaseProvider
|
||||
from core.graph_manager import RelationshipType, NodeType
|
||||
|
||||
|
||||
class DNSProvider(BaseProvider):
|
||||
"""
|
||||
Provider for standard DNS resolution and reverse DNS lookups.
|
||||
Discovers domain-to-IP and IP-to-domain relationships through DNS records.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize DNS provider with appropriate rate limiting."""
|
||||
super().__init__(
|
||||
name="dns",
|
||||
rate_limit=100, # DNS queries can be faster
|
||||
timeout=10
|
||||
)
|
||||
|
||||
# Configure DNS resolver
|
||||
self.resolver = dns.resolver.Resolver()
|
||||
self.resolver.timeout = 5
|
||||
self.resolver.lifetime = 10
|
||||
|
||||
def get_name(self) -> str:
|
||||
"""Return the provider name."""
|
||||
return "dns"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""DNS is always available - no API key required."""
|
||||
return True
|
||||
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query DNS records for the domain to discover relationships.
|
||||
|
||||
Args:
|
||||
domain: Domain to investigate
|
||||
|
||||
Returns:
|
||||
List of relationships discovered from DNS analysis
|
||||
"""
|
||||
if not self._is_valid_domain(domain):
|
||||
return []
|
||||
|
||||
relationships = []
|
||||
|
||||
# Query A records
|
||||
relationships.extend(self._query_a_records(domain))
|
||||
|
||||
# Query AAAA records (IPv6)
|
||||
relationships.extend(self._query_aaaa_records(domain))
|
||||
|
||||
# Query CNAME records
|
||||
relationships.extend(self._query_cname_records(domain))
|
||||
|
||||
# Query MX records
|
||||
relationships.extend(self._query_mx_records(domain))
|
||||
|
||||
# Query NS records
|
||||
relationships.extend(self._query_ns_records(domain))
|
||||
|
||||
return relationships
|
||||
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query reverse DNS for the IP address.
|
||||
|
||||
Args:
|
||||
ip: IP address to investigate
|
||||
|
||||
Returns:
|
||||
List of relationships discovered from reverse DNS
|
||||
"""
|
||||
if not self._is_valid_ip(ip):
|
||||
return []
|
||||
|
||||
relationships = []
|
||||
|
||||
try:
|
||||
# Perform reverse DNS lookup
|
||||
reverse_name = dns.reversename.from_address(ip)
|
||||
response = self.resolver.resolve(reverse_name, 'PTR')
|
||||
|
||||
for ptr_record in response:
|
||||
hostname = str(ptr_record).rstrip('.')
|
||||
|
||||
if self._is_valid_domain(hostname):
|
||||
raw_data = {
|
||||
'query_type': 'PTR',
|
||||
'ip_address': ip,
|
||||
'hostname': hostname,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
ip,
|
||||
hostname,
|
||||
RelationshipType.A_RECORD, # Reverse relationship
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="reverse_dns_lookup"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_a_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query A records for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
response = self.resolver.resolve(domain, 'A')
|
||||
|
||||
for a_record in response:
|
||||
ip_address = str(a_record)
|
||||
|
||||
raw_data = {
|
||||
'query_type': 'A',
|
||||
'domain': domain,
|
||||
'ip_address': ip_address,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
ip_address,
|
||||
RelationshipType.A_RECORD,
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ip_address,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_a_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.debug(f"A record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_aaaa_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query AAAA records (IPv6) for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
response = self.resolver.resolve(domain, 'AAAA')
|
||||
|
||||
for aaaa_record in response:
|
||||
ip_address = str(aaaa_record)
|
||||
|
||||
raw_data = {
|
||||
'query_type': 'AAAA',
|
||||
'domain': domain,
|
||||
'ip_address': ip_address,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
ip_address,
|
||||
RelationshipType.A_RECORD, # Using same type for IPv6
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ip_address,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_aaaa_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.debug(f"AAAA record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_cname_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query CNAME records for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
response = self.resolver.resolve(domain, 'CNAME')
|
||||
|
||||
for cname_record in response:
|
||||
target_domain = str(cname_record).rstrip('.')
|
||||
|
||||
if self._is_valid_domain(target_domain):
|
||||
raw_data = {
|
||||
'query_type': 'CNAME',
|
||||
'source_domain': domain,
|
||||
'target_domain': target_domain,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
target_domain,
|
||||
RelationshipType.CNAME_RECORD,
|
||||
RelationshipType.CNAME_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=target_domain,
|
||||
relationship_type=RelationshipType.CNAME_RECORD,
|
||||
confidence_score=RelationshipType.CNAME_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_cname_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.debug(f"CNAME record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_mx_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query MX records for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
response = self.resolver.resolve(domain, 'MX')
|
||||
|
||||
for mx_record in response:
|
||||
mx_host = str(mx_record.exchange).rstrip('.')
|
||||
|
||||
if self._is_valid_domain(mx_host):
|
||||
raw_data = {
|
||||
'query_type': 'MX',
|
||||
'domain': domain,
|
||||
'mx_host': mx_host,
|
||||
'priority': mx_record.preference,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
mx_host,
|
||||
RelationshipType.MX_RECORD,
|
||||
RelationshipType.MX_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=mx_host,
|
||||
relationship_type=RelationshipType.MX_RECORD,
|
||||
confidence_score=RelationshipType.MX_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_mx_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.debug(f"MX record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_ns_records(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query NS records for the domain."""
|
||||
relationships = []
|
||||
|
||||
#if not DNS_AVAILABLE:
|
||||
# return relationships
|
||||
|
||||
try:
|
||||
response = self.resolver.resolve(domain, 'NS')
|
||||
|
||||
for ns_record in response:
|
||||
ns_host = str(ns_record).rstrip('.')
|
||||
|
||||
if self._is_valid_domain(ns_host):
|
||||
raw_data = {
|
||||
'query_type': 'NS',
|
||||
'domain': domain,
|
||||
'ns_host': ns_host,
|
||||
'ttl': response.ttl
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
ns_host,
|
||||
RelationshipType.NS_RECORD,
|
||||
RelationshipType.NS_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ns_host,
|
||||
relationship_type=RelationshipType.NS_RECORD,
|
||||
confidence_score=RelationshipType.NS_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="dns_ns_record"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.debug(f"NS record query failed for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
299
providers/shodan_provider.py
Normal file
299
providers/shodan_provider.py
Normal file
@@ -0,0 +1,299 @@
|
||||
"""
|
||||
Shodan provider for DNSRecon.
|
||||
Discovers IP relationships and infrastructure context through Shodan API.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import List, Dict, Any, Tuple, Optional
|
||||
from urllib.parse import quote
|
||||
from .base_provider import BaseProvider
|
||||
from core.graph_manager import RelationshipType
|
||||
from config import config
|
||||
|
||||
|
||||
class ShodanProvider(BaseProvider):
|
||||
"""
|
||||
Provider for querying Shodan API for IP address and hostname information.
|
||||
Requires valid API key and respects Shodan's rate limits.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Shodan provider with appropriate rate limiting."""
|
||||
super().__init__(
|
||||
name="shodan",
|
||||
rate_limit=60, # Shodan API has various rate limits depending on plan
|
||||
timeout=30
|
||||
)
|
||||
self.base_url = "https://api.shodan.io"
|
||||
self.api_key = config.get_api_key('shodan')
|
||||
|
||||
def get_name(self) -> str:
|
||||
"""Return the provider name."""
|
||||
return "shodan"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""
|
||||
Check if Shodan provider is available (has valid API key).
|
||||
"""
|
||||
return self.api_key is not None and len(self.api_key.strip()) > 0
|
||||
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query Shodan for information about a domain.
|
||||
Uses Shodan's hostname search to find associated IPs.
|
||||
|
||||
Args:
|
||||
domain: Domain to investigate
|
||||
|
||||
Returns:
|
||||
List of relationships discovered from Shodan data
|
||||
"""
|
||||
if not self._is_valid_domain(domain) or not self.is_available():
|
||||
return []
|
||||
|
||||
relationships = []
|
||||
|
||||
try:
|
||||
# Search for hostname in Shodan
|
||||
search_query = f"hostname:{domain}"
|
||||
url = f"{self.base_url}/shodan/host/search"
|
||||
params = {
|
||||
'key': self.api_key,
|
||||
'query': search_query,
|
||||
'minify': True # Get minimal data to reduce bandwidth
|
||||
}
|
||||
|
||||
response = self.make_request(url, method="GET", params=params, target_indicator=domain)
|
||||
|
||||
if not response or response.status_code != 200:
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'matches' not in data:
|
||||
return []
|
||||
|
||||
# Process search results
|
||||
for match in data['matches']:
|
||||
ip_address = match.get('ip_str')
|
||||
hostnames = match.get('hostnames', [])
|
||||
|
||||
if ip_address and domain in hostnames:
|
||||
raw_data = {
|
||||
'ip_address': ip_address,
|
||||
'hostnames': hostnames,
|
||||
'country': match.get('location', {}).get('country_name', ''),
|
||||
'city': match.get('location', {}).get('city', ''),
|
||||
'isp': match.get('isp', ''),
|
||||
'org': match.get('org', ''),
|
||||
'ports': match.get('ports', []),
|
||||
'last_update': match.get('last_update', '')
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
ip_address,
|
||||
RelationshipType.A_RECORD, # Domain resolves to IP
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ip_address,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="shodan_hostname_search"
|
||||
)
|
||||
|
||||
# Also create relationships to other hostnames on the same IP
|
||||
for hostname in hostnames:
|
||||
if hostname != domain and self._is_valid_domain(hostname):
|
||||
hostname_raw_data = {
|
||||
'shared_ip': ip_address,
|
||||
'all_hostnames': hostnames,
|
||||
'discovery_context': 'shared_hosting'
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
hostname,
|
||||
RelationshipType.PASSIVE_DNS, # Shared hosting relationship
|
||||
0.6, # Lower confidence for shared hosting
|
||||
hostname_raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=hostname,
|
||||
relationship_type=RelationshipType.PASSIVE_DNS,
|
||||
confidence_score=0.6,
|
||||
raw_data=hostname_raw_data,
|
||||
discovery_method="shodan_shared_hosting"
|
||||
)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
|
||||
except Exception as e:
|
||||
self.logger.logger.error(f"Error querying Shodan for domain {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query Shodan for information about an IP address.
|
||||
|
||||
Args:
|
||||
ip: IP address to investigate
|
||||
|
||||
Returns:
|
||||
List of relationships discovered from Shodan IP data
|
||||
"""
|
||||
if not self._is_valid_ip(ip) or not self.is_available():
|
||||
return []
|
||||
|
||||
relationships = []
|
||||
|
||||
try:
|
||||
# Query Shodan host information
|
||||
url = f"{self.base_url}/shodan/host/{ip}"
|
||||
params = {'key': self.api_key}
|
||||
|
||||
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
|
||||
|
||||
if not response or response.status_code != 200:
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Extract hostname relationships
|
||||
hostnames = data.get('hostnames', [])
|
||||
for hostname in hostnames:
|
||||
if self._is_valid_domain(hostname):
|
||||
raw_data = {
|
||||
'ip_address': ip,
|
||||
'hostname': hostname,
|
||||
'country': data.get('country_name', ''),
|
||||
'city': data.get('city', ''),
|
||||
'isp': data.get('isp', ''),
|
||||
'org': data.get('org', ''),
|
||||
'asn': data.get('asn', ''),
|
||||
'ports': data.get('ports', []),
|
||||
'last_update': data.get('last_update', ''),
|
||||
'os': data.get('os', '')
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
ip,
|
||||
hostname,
|
||||
RelationshipType.A_RECORD, # IP resolves to hostname
|
||||
RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type=RelationshipType.A_RECORD,
|
||||
confidence_score=RelationshipType.A_RECORD.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="shodan_host_lookup"
|
||||
)
|
||||
|
||||
# Extract ASN relationship if available
|
||||
asn = data.get('asn')
|
||||
if asn:
|
||||
asn_name = f"AS{asn}"
|
||||
|
||||
asn_raw_data = {
|
||||
'ip_address': ip,
|
||||
'asn': asn,
|
||||
'isp': data.get('isp', ''),
|
||||
'org': data.get('org', '')
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
ip,
|
||||
asn_name,
|
||||
RelationshipType.ASN_MEMBERSHIP,
|
||||
RelationshipType.ASN_MEMBERSHIP.default_confidence,
|
||||
asn_raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=asn_name,
|
||||
relationship_type=RelationshipType.ASN_MEMBERSHIP,
|
||||
confidence_score=RelationshipType.ASN_MEMBERSHIP.default_confidence,
|
||||
raw_data=asn_raw_data,
|
||||
discovery_method="shodan_asn_lookup"
|
||||
)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
self.logger.logger.error(f"Failed to parse JSON response from Shodan: {e}")
|
||||
except Exception as e:
|
||||
self.logger.logger.error(f"Error querying Shodan for IP {ip}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def search_by_organization(self, org_name: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search Shodan for hosts belonging to a specific organization.
|
||||
|
||||
Args:
|
||||
org_name: Organization name to search for
|
||||
|
||||
Returns:
|
||||
List of host information dictionaries
|
||||
"""
|
||||
if not self.is_available():
|
||||
return []
|
||||
|
||||
try:
|
||||
search_query = f"org:\"{org_name}\""
|
||||
url = f"{self.base_url}/shodan/host/search"
|
||||
params = {
|
||||
'key': self.api_key,
|
||||
'query': search_query,
|
||||
'minify': True
|
||||
}
|
||||
|
||||
response = self.make_request(url, method="GET", params=params, target_indicator=org_name)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
data = response.json()
|
||||
return data.get('matches', [])
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.error(f"Error searching Shodan by organization {org_name}: {e}")
|
||||
|
||||
return []
|
||||
|
||||
def get_host_services(self, ip: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get service information for a specific IP address.
|
||||
|
||||
Args:
|
||||
ip: IP address to query
|
||||
|
||||
Returns:
|
||||
List of service information dictionaries
|
||||
"""
|
||||
if not self._is_valid_ip(ip) or not self.is_available():
|
||||
return []
|
||||
|
||||
try:
|
||||
url = f"{self.base_url}/shodan/host/{ip}"
|
||||
params = {'key': self.api_key}
|
||||
|
||||
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
data = response.json()
|
||||
return data.get('data', []) # Service banners
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.error(f"Error getting Shodan services for IP {ip}: {e}")
|
||||
|
||||
return []
|
||||
334
providers/virustotal_provider.py
Normal file
334
providers/virustotal_provider.py
Normal file
@@ -0,0 +1,334 @@
|
||||
"""
|
||||
VirusTotal provider for DNSRecon.
|
||||
Discovers domain relationships through passive DNS and URL analysis.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import List, Dict, Any, Tuple, Optional
|
||||
from .base_provider import BaseProvider
|
||||
from core.graph_manager import RelationshipType
|
||||
from config import config
|
||||
|
||||
|
||||
class VirusTotalProvider(BaseProvider):
|
||||
"""
|
||||
Provider for querying VirusTotal API for passive DNS and domain reputation data.
|
||||
Requires valid API key and strictly respects free tier rate limits.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize VirusTotal provider with strict rate limiting for free tier."""
|
||||
super().__init__(
|
||||
name="virustotal",
|
||||
rate_limit=4, # Free tier: 4 requests per minute
|
||||
timeout=30
|
||||
)
|
||||
self.base_url = "https://www.virustotal.com/vtapi/v2"
|
||||
self.api_key = config.get_api_key('virustotal')
|
||||
|
||||
def get_name(self) -> str:
|
||||
"""Return the provider name."""
|
||||
return "virustotal"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""
|
||||
Check if VirusTotal provider is available (has valid API key).
|
||||
"""
|
||||
return self.api_key is not None and len(self.api_key.strip()) > 0
|
||||
|
||||
def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query VirusTotal for domain information including passive DNS.
|
||||
|
||||
Args:
|
||||
domain: Domain to investigate
|
||||
|
||||
Returns:
|
||||
List of relationships discovered from VirusTotal data
|
||||
"""
|
||||
if not self._is_valid_domain(domain) or not self.is_available():
|
||||
return []
|
||||
|
||||
relationships = []
|
||||
|
||||
# Query domain report
|
||||
domain_relationships = self._query_domain_report(domain)
|
||||
relationships.extend(domain_relationships)
|
||||
|
||||
# Query passive DNS for the domain
|
||||
passive_dns_relationships = self._query_passive_dns_domain(domain)
|
||||
relationships.extend(passive_dns_relationships)
|
||||
|
||||
return relationships
|
||||
|
||||
def query_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""
|
||||
Query VirusTotal for IP address information including passive DNS.
|
||||
|
||||
Args:
|
||||
ip: IP address to investigate
|
||||
|
||||
Returns:
|
||||
List of relationships discovered from VirusTotal IP data
|
||||
"""
|
||||
if not self._is_valid_ip(ip) or not self.is_available():
|
||||
return []
|
||||
|
||||
relationships = []
|
||||
|
||||
# Query IP report
|
||||
ip_relationships = self._query_ip_report(ip)
|
||||
relationships.extend(ip_relationships)
|
||||
|
||||
# Query passive DNS for the IP
|
||||
passive_dns_relationships = self._query_passive_dns_ip(ip)
|
||||
relationships.extend(passive_dns_relationships)
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_domain_report(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query VirusTotal domain report."""
|
||||
relationships = []
|
||||
|
||||
try:
|
||||
url = f"{self.base_url}/domain/report"
|
||||
params = {
|
||||
'apikey': self.api_key,
|
||||
'domain': domain,
|
||||
'allinfo': 1 # Get comprehensive information
|
||||
}
|
||||
|
||||
response = self.make_request(url, method="GET", params=params, target_indicator=domain)
|
||||
|
||||
if not response or response.status_code != 200:
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
|
||||
if data.get('response_code') != 1:
|
||||
return []
|
||||
|
||||
# Extract resolved IPs
|
||||
resolutions = data.get('resolutions', [])
|
||||
for resolution in resolutions:
|
||||
ip_address = resolution.get('ip_address')
|
||||
last_resolved = resolution.get('last_resolved')
|
||||
|
||||
if ip_address and self._is_valid_ip(ip_address):
|
||||
raw_data = {
|
||||
'domain': domain,
|
||||
'ip_address': ip_address,
|
||||
'last_resolved': last_resolved,
|
||||
'source': 'virustotal_domain_report'
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
ip_address,
|
||||
RelationshipType.PASSIVE_DNS,
|
||||
RelationshipType.PASSIVE_DNS.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=ip_address,
|
||||
relationship_type=RelationshipType.PASSIVE_DNS,
|
||||
confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="virustotal_domain_resolution"
|
||||
)
|
||||
|
||||
# Extract subdomains
|
||||
subdomains = data.get('subdomains', [])
|
||||
for subdomain in subdomains:
|
||||
if subdomain != domain and self._is_valid_domain(subdomain):
|
||||
raw_data = {
|
||||
'parent_domain': domain,
|
||||
'subdomain': subdomain,
|
||||
'source': 'virustotal_subdomain_discovery'
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
domain,
|
||||
subdomain,
|
||||
RelationshipType.PASSIVE_DNS,
|
||||
0.7, # Medium-high confidence for subdomains
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=subdomain,
|
||||
relationship_type=RelationshipType.PASSIVE_DNS,
|
||||
confidence_score=0.7,
|
||||
raw_data=raw_data,
|
||||
discovery_method="virustotal_subdomain_discovery"
|
||||
)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
|
||||
except Exception as e:
|
||||
self.logger.logger.error(f"Error querying VirusTotal domain report for {domain}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_ip_report(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query VirusTotal IP report."""
|
||||
relationships = []
|
||||
|
||||
try:
|
||||
url = f"{self.base_url}/ip-address/report"
|
||||
params = {
|
||||
'apikey': self.api_key,
|
||||
'ip': ip
|
||||
}
|
||||
|
||||
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
|
||||
|
||||
if not response or response.status_code != 200:
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
|
||||
if data.get('response_code') != 1:
|
||||
return []
|
||||
|
||||
# Extract resolved domains
|
||||
resolutions = data.get('resolutions', [])
|
||||
for resolution in resolutions:
|
||||
hostname = resolution.get('hostname')
|
||||
last_resolved = resolution.get('last_resolved')
|
||||
|
||||
if hostname and self._is_valid_domain(hostname):
|
||||
raw_data = {
|
||||
'ip_address': ip,
|
||||
'hostname': hostname,
|
||||
'last_resolved': last_resolved,
|
||||
'source': 'virustotal_ip_report'
|
||||
}
|
||||
|
||||
relationships.append((
|
||||
ip,
|
||||
hostname,
|
||||
RelationshipType.PASSIVE_DNS,
|
||||
RelationshipType.PASSIVE_DNS.default_confidence,
|
||||
raw_data
|
||||
))
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type=RelationshipType.PASSIVE_DNS,
|
||||
confidence_score=RelationshipType.PASSIVE_DNS.default_confidence,
|
||||
raw_data=raw_data,
|
||||
discovery_method="virustotal_ip_resolution"
|
||||
)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
self.logger.logger.error(f"Failed to parse JSON response from VirusTotal: {e}")
|
||||
except Exception as e:
|
||||
self.logger.logger.error(f"Error querying VirusTotal IP report for {ip}: {e}")
|
||||
|
||||
return relationships
|
||||
|
||||
def _query_passive_dns_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query VirusTotal passive DNS for domain."""
|
||||
# Note: VirusTotal's passive DNS API might require a premium subscription
|
||||
# This is a placeholder for the endpoint structure
|
||||
return []
|
||||
|
||||
def _query_passive_dns_ip(self, ip: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
|
||||
"""Query VirusTotal passive DNS for IP."""
|
||||
# Note: VirusTotal's passive DNS API might require a premium subscription
|
||||
# This is a placeholder for the endpoint structure
|
||||
return []
|
||||
|
||||
def get_domain_reputation(self, domain: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get domain reputation information from VirusTotal.
|
||||
|
||||
Args:
|
||||
domain: Domain to check reputation for
|
||||
|
||||
Returns:
|
||||
Dictionary containing reputation data
|
||||
"""
|
||||
if not self._is_valid_domain(domain) or not self.is_available():
|
||||
return {}
|
||||
|
||||
try:
|
||||
url = f"{self.base_url}/domain/report"
|
||||
params = {
|
||||
'apikey': self.api_key,
|
||||
'domain': domain
|
||||
}
|
||||
|
||||
response = self.make_request(url, method="GET", params=params, target_indicator=domain)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
data = response.json()
|
||||
|
||||
if data.get('response_code') == 1:
|
||||
return {
|
||||
'positives': data.get('positives', 0),
|
||||
'total': data.get('total', 0),
|
||||
'scan_date': data.get('scan_date', ''),
|
||||
'permalink': data.get('permalink', ''),
|
||||
'reputation_score': self._calculate_reputation_score(data)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.error(f"Error getting VirusTotal reputation for domain {domain}: {e}")
|
||||
|
||||
return {}
|
||||
|
||||
def get_ip_reputation(self, ip: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get IP reputation information from VirusTotal.
|
||||
|
||||
Args:
|
||||
ip: IP address to check reputation for
|
||||
|
||||
Returns:
|
||||
Dictionary containing reputation data
|
||||
"""
|
||||
if not self._is_valid_ip(ip) or not self.is_available():
|
||||
return {}
|
||||
|
||||
try:
|
||||
url = f"{self.base_url}/ip-address/report"
|
||||
params = {
|
||||
'apikey': self.api_key,
|
||||
'ip': ip
|
||||
}
|
||||
|
||||
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
|
||||
|
||||
if response and response.status_code == 200:
|
||||
data = response.json()
|
||||
|
||||
if data.get('response_code') == 1:
|
||||
return {
|
||||
'positives': data.get('positives', 0),
|
||||
'total': data.get('total', 0),
|
||||
'scan_date': data.get('scan_date', ''),
|
||||
'permalink': data.get('permalink', ''),
|
||||
'reputation_score': self._calculate_reputation_score(data)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.logger.error(f"Error getting VirusTotal reputation for IP {ip}: {e}")
|
||||
|
||||
return {}
|
||||
|
||||
def _calculate_reputation_score(self, data: Dict[str, Any]) -> float:
|
||||
"""Calculate a normalized reputation score (0.0 to 1.0)."""
|
||||
positives = data.get('positives', 0)
|
||||
total = data.get('total', 1) # Avoid division by zero
|
||||
|
||||
if total == 0:
|
||||
return 1.0 # No data means neutral
|
||||
|
||||
# Score is inverse of detection ratio (lower detection = higher reputation)
|
||||
return max(0.0, 1.0 - (positives / total))
|
||||
Reference in New Issue
Block a user