331 lines
12 KiB
Python
331 lines
12 KiB
Python
# dnsrecon/providers/shodan_provider.py
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Dict, Any
|
|
from datetime import datetime, timezone
|
|
import requests
|
|
|
|
from .base_provider import BaseProvider
|
|
from core.provider_result import ProviderResult
|
|
from utils.helpers import _is_valid_ip, _is_valid_domain
|
|
|
|
|
|
class ShodanProvider(BaseProvider):
|
|
"""
|
|
Provider for querying Shodan API for IP address information.
|
|
Now returns standardized ProviderResult objects with caching support.
|
|
"""
|
|
|
|
def __init__(self, name=None, session_config=None):
|
|
"""Initialize Shodan provider with session-specific configuration."""
|
|
super().__init__(
|
|
name="shodan",
|
|
rate_limit=60,
|
|
timeout=30,
|
|
session_config=session_config
|
|
)
|
|
self.base_url = "https://api.shodan.io"
|
|
self.api_key = self.config.get_api_key('shodan')
|
|
|
|
# Initialize cache directory
|
|
self.cache_dir = Path('cache') / 'shodan'
|
|
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
def is_available(self) -> bool:
|
|
"""Check if Shodan provider is available (has valid API key in this session)."""
|
|
return self.api_key is not None and len(self.api_key.strip()) > 0
|
|
|
|
def get_name(self) -> str:
|
|
"""Return the provider name."""
|
|
return "shodan"
|
|
|
|
def get_display_name(self) -> str:
|
|
"""Return the provider display name for the UI."""
|
|
return "Shodan"
|
|
|
|
def requires_api_key(self) -> bool:
|
|
"""Return True if the provider requires an API key."""
|
|
return True
|
|
|
|
def get_eligibility(self) -> Dict[str, bool]:
|
|
"""Return a dictionary indicating if the provider can query domains and/or IPs."""
|
|
return {'domains': False, 'ips': True}
|
|
|
|
def _get_cache_file_path(self, ip: str) -> Path:
|
|
"""Generate cache file path for an IP address."""
|
|
safe_ip = ip.replace('.', '_').replace(':', '_')
|
|
return self.cache_dir / f"{safe_ip}.json"
|
|
|
|
def _get_cache_status(self, cache_file_path: Path) -> str:
|
|
"""
|
|
Check cache status for an IP.
|
|
Returns: 'not_found', 'fresh', or 'stale'
|
|
"""
|
|
if not cache_file_path.exists():
|
|
return "not_found"
|
|
|
|
try:
|
|
with open(cache_file_path, 'r') as f:
|
|
cache_data = json.load(f)
|
|
|
|
last_query_str = cache_data.get("last_upstream_query")
|
|
if not last_query_str:
|
|
return "stale"
|
|
|
|
last_query = datetime.fromisoformat(last_query_str.replace('Z', '+00:00'))
|
|
hours_since_query = (datetime.now(timezone.utc) - last_query).total_seconds() / 3600
|
|
|
|
cache_timeout = self.config.cache_timeout_hours
|
|
if hours_since_query < cache_timeout:
|
|
return "fresh"
|
|
else:
|
|
return "stale"
|
|
|
|
except (json.JSONDecodeError, ValueError, KeyError):
|
|
return "stale"
|
|
|
|
def query_domain(self, domain: str) -> ProviderResult:
|
|
"""
|
|
Domain queries are no longer supported for the Shodan provider.
|
|
|
|
Args:
|
|
domain: Domain to investigate
|
|
|
|
Returns:
|
|
Empty ProviderResult
|
|
"""
|
|
return ProviderResult()
|
|
|
|
def query_ip(self, ip: str) -> ProviderResult:
|
|
"""
|
|
Query Shodan for information about an IP address, with caching of processed data.
|
|
|
|
Args:
|
|
ip: IP address to investigate
|
|
|
|
Returns:
|
|
ProviderResult containing discovered relationships and attributes
|
|
"""
|
|
if not _is_valid_ip(ip) or not self.is_available():
|
|
return ProviderResult()
|
|
|
|
cache_file = self._get_cache_file_path(ip)
|
|
cache_status = self._get_cache_status(cache_file)
|
|
|
|
result = ProviderResult()
|
|
|
|
try:
|
|
if cache_status == "fresh":
|
|
result = self._load_from_cache(cache_file)
|
|
self.logger.logger.info(f"Using cached Shodan data for {ip}")
|
|
else: # "stale" or "not_found"
|
|
url = f"{self.base_url}/shodan/host/{ip}"
|
|
params = {'key': self.api_key}
|
|
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
|
|
|
|
if response and response.status_code == 200:
|
|
data = response.json()
|
|
# Process the data into ProviderResult BEFORE caching
|
|
result = self._process_shodan_data(ip, data)
|
|
self._save_to_cache(cache_file, result, data) # Save both result and raw data
|
|
elif cache_status == "stale":
|
|
# If API fails on a stale cache, use the old data
|
|
result = self._load_from_cache(cache_file)
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
self.logger.logger.error(f"Shodan API query failed for {ip}: {e}")
|
|
if cache_status == "stale":
|
|
result = self._load_from_cache(cache_file)
|
|
|
|
return result
|
|
|
|
def _load_from_cache(self, cache_file_path: Path) -> ProviderResult:
|
|
"""Load processed Shodan data from a cache file."""
|
|
try:
|
|
with open(cache_file_path, 'r') as f:
|
|
cache_content = json.load(f)
|
|
|
|
result = ProviderResult()
|
|
|
|
# Reconstruct relationships
|
|
for rel_data in cache_content.get("relationships", []):
|
|
result.add_relationship(
|
|
source_node=rel_data["source_node"],
|
|
target_node=rel_data["target_node"],
|
|
relationship_type=rel_data["relationship_type"],
|
|
provider=rel_data["provider"],
|
|
confidence=rel_data["confidence"],
|
|
raw_data=rel_data.get("raw_data", {})
|
|
)
|
|
|
|
# Reconstruct attributes
|
|
for attr_data in cache_content.get("attributes", []):
|
|
result.add_attribute(
|
|
target_node=attr_data["target_node"],
|
|
name=attr_data["name"],
|
|
value=attr_data["value"],
|
|
attr_type=attr_data["type"],
|
|
provider=attr_data["provider"],
|
|
confidence=attr_data["confidence"],
|
|
metadata=attr_data.get("metadata", {})
|
|
)
|
|
|
|
return result
|
|
|
|
except (json.JSONDecodeError, FileNotFoundError, KeyError):
|
|
return ProviderResult()
|
|
|
|
def _save_to_cache(self, cache_file_path: Path, result: ProviderResult, raw_data: Dict[str, Any]) -> None:
|
|
"""Save processed Shodan data to a cache file."""
|
|
try:
|
|
cache_data = {
|
|
"last_upstream_query": datetime.now(timezone.utc).isoformat(),
|
|
"raw_data": raw_data, # Preserve original for forensic purposes
|
|
"relationships": [
|
|
{
|
|
"source_node": rel.source_node,
|
|
"target_node": rel.target_node,
|
|
"relationship_type": rel.relationship_type,
|
|
"confidence": rel.confidence,
|
|
"provider": rel.provider,
|
|
"raw_data": rel.raw_data
|
|
} for rel in result.relationships
|
|
],
|
|
"attributes": [
|
|
{
|
|
"target_node": attr.target_node,
|
|
"name": attr.name,
|
|
"value": attr.value,
|
|
"type": attr.type,
|
|
"provider": attr.provider,
|
|
"confidence": attr.confidence,
|
|
"metadata": attr.metadata
|
|
} for attr in result.attributes
|
|
]
|
|
}
|
|
with open(cache_file_path, 'w') as f:
|
|
json.dump(cache_data, f, separators=(',', ':'), default=str)
|
|
except Exception as e:
|
|
self.logger.logger.warning(f"Failed to save Shodan cache for {cache_file_path.name}: {e}")
|
|
|
|
def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult:
|
|
"""
|
|
Process Shodan data to extract relationships and attributes.
|
|
|
|
Args:
|
|
ip: IP address queried
|
|
data: Raw Shodan response data
|
|
|
|
Returns:
|
|
ProviderResult with relationships and attributes
|
|
"""
|
|
result = ProviderResult()
|
|
|
|
# Extract hostname relationships
|
|
hostnames = data.get('hostnames', [])
|
|
for hostname in hostnames:
|
|
if _is_valid_domain(hostname):
|
|
result.add_relationship(
|
|
source_node=ip,
|
|
target_node=hostname,
|
|
relationship_type='a_record',
|
|
provider=self.name,
|
|
confidence=0.8,
|
|
raw_data=data
|
|
)
|
|
|
|
self.log_relationship_discovery(
|
|
source_node=ip,
|
|
target_node=hostname,
|
|
relationship_type='a_record',
|
|
confidence_score=0.8,
|
|
raw_data=data,
|
|
discovery_method="shodan_host_lookup"
|
|
)
|
|
|
|
# Extract ASN relationship
|
|
asn = data.get('asn')
|
|
if asn:
|
|
asn_name = f"AS{asn[2:]}" if isinstance(asn, str) and asn.startswith('AS') else f"AS{asn}"
|
|
result.add_relationship(
|
|
source_node=ip,
|
|
target_node=asn_name,
|
|
relationship_type='asn_membership',
|
|
provider=self.name,
|
|
confidence=0.7,
|
|
raw_data=data
|
|
)
|
|
|
|
self.log_relationship_discovery(
|
|
source_node=ip,
|
|
target_node=asn_name,
|
|
relationship_type='asn_membership',
|
|
confidence_score=0.7,
|
|
raw_data=data,
|
|
discovery_method="shodan_asn_lookup"
|
|
)
|
|
|
|
# Add comprehensive Shodan host information as attributes
|
|
if 'ports' in data:
|
|
result.add_attribute(
|
|
target_node=ip,
|
|
name='ports',
|
|
value=data['ports'],
|
|
attr_type='network_info',
|
|
provider=self.name,
|
|
confidence=0.9
|
|
)
|
|
|
|
if 'os' in data and data['os']:
|
|
result.add_attribute(
|
|
target_node=ip,
|
|
name='operating_system',
|
|
value=data['os'],
|
|
attr_type='system_info',
|
|
provider=self.name,
|
|
confidence=0.8
|
|
)
|
|
|
|
if 'org' in data:
|
|
result.add_attribute(
|
|
target_node=ip,
|
|
name='organization',
|
|
value=data['org'],
|
|
attr_type='network_info',
|
|
provider=self.name,
|
|
confidence=0.8
|
|
)
|
|
|
|
if 'country_name' in data:
|
|
result.add_attribute(
|
|
target_node=ip,
|
|
name='country',
|
|
value=data['country_name'],
|
|
attr_type='location_info',
|
|
provider=self.name,
|
|
confidence=0.9
|
|
)
|
|
|
|
if 'city' in data:
|
|
result.add_attribute(
|
|
target_node=ip,
|
|
name='city',
|
|
value=data['city'],
|
|
attr_type='location_info',
|
|
provider=self.name,
|
|
confidence=0.8
|
|
)
|
|
|
|
# Store complete Shodan data as a comprehensive attribute
|
|
result.add_attribute(
|
|
target_node=ip,
|
|
name='shodan_host_info',
|
|
value=data, # Complete Shodan response for full forensic detail
|
|
attr_type='comprehensive_data',
|
|
provider=self.name,
|
|
confidence=0.9,
|
|
metadata={'data_source': 'shodan_api', 'query_type': 'host_lookup'}
|
|
)
|
|
|
|
return result |