mstoeck3/dnscope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implement new data api

Browse Source
This commit is contained in:
overcuriousity
2025-09-16 20:21:08 +02:00
parent 15421dd4a5
commit 97aa18f788
11 changed files with 1206 additions and 1120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
providers/base_provider.py
View File

@@ -4,16 +4,17 @@ import time
import requests
import threading
from abc import ABC, abstractmethod
from typing import List, Dict, Any, Optional, Tuple
from typing import Dict, Any, Optional
from core.logger import get_forensic_logger
from core.rate_limiter import GlobalRateLimiter
from core.provider_result import ProviderResult
class BaseProvider(ABC):
"""
Abstract base class for all DNSRecon data providers.
Now supports session-specific configuration.
Now supports session-specific configuration and returns standardized ProviderResult objects.
"""
def __init__(self, name: str, rate_limit: int = 60, timeout: int = 30, session_config=None):
@@ -101,7 +102,7 @@ class BaseProvider(ABC):
pass
@abstractmethod
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_domain(self, domain: str) -> ProviderResult:
"""
Query the provider for information about a domain.
@@ -109,12 +110,12 @@ class BaseProvider(ABC):
domain: Domain to investigate
Returns:
List of tuples: (source_node, target_node, relationship_type, confidence, raw_data)
ProviderResult containing standardized attributes and relationships
"""
pass
@abstractmethod
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_ip(self, ip: str) -> ProviderResult:
"""
Query the provider for information about an IP address.
@@ -122,7 +123,7 @@ class BaseProvider(ABC):
ip: IP address to investigate
Returns:
List of tuples: (source_node, target_node, relationship_type, confidence, raw_data)
ProviderResult containing standardized attributes and relationships
"""
pass

845
providers/crtsh_provider.py
View File

File diff suppressed because it is too large Load Diff

139
providers/dns_provider.py
View File

@@ -1,15 +1,16 @@
# dnsrecon/providers/dns_provider.py
from dns import resolver, reversename
from typing import List, Dict, Any, Tuple
from typing import Dict
from .base_provider import BaseProvider
from core.provider_result import ProviderResult
from utils.helpers import _is_valid_ip, _is_valid_domain
class DNSProvider(BaseProvider):
"""
Provider for standard DNS resolution and reverse DNS lookups.
Now uses session-specific configuration.
Now returns standardized ProviderResult objects.
"""
def __init__(self, name=None, session_config=None):
@@ -25,7 +26,6 @@ class DNSProvider(BaseProvider):
self.resolver = resolver.Resolver()
self.resolver.timeout = 5
self.resolver.lifetime = 10
#self.resolver.nameservers = ['127.0.0.1']
def get_name(self) -> str:
"""Return the provider name."""
@@ -47,31 +47,35 @@ class DNSProvider(BaseProvider):
"""DNS is always available - no API key required."""
return True
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_domain(self, domain: str) -> ProviderResult:
"""
Query DNS records for the domain to discover relationships.
...
Query DNS records for the domain to discover relationships and attributes.
Args:
domain: Domain to investigate
Returns:
ProviderResult containing discovered relationships and attributes
"""
if not _is_valid_domain(domain):
return []
return ProviderResult()
relationships = []
result = ProviderResult()
# Query all record types
for record_type in ['A', 'AAAA', 'CNAME', 'MX', 'NS', 'SOA', 'TXT', 'SRV', 'CAA']:
try:
relationships.extend(self._query_record(domain, record_type))
self._query_record(domain, record_type, result)
except resolver.NoAnswer:
# This is not an error, just a confirmation that the record doesn't exist.
self.logger.logger.debug(f"No {record_type} record found for {domain}")
except Exception as e:
self.failed_requests += 1
self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
# Optionally, you might want to re-raise other, more serious exceptions.
return relationships
return result
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_ip(self, ip: str) -> ProviderResult:
"""
Query reverse DNS for the IP address.
@@ -79,12 +83,12 @@ class DNSProvider(BaseProvider):
ip: IP address to investigate
Returns:
List of relationships discovered from reverse DNS
ProviderResult containing discovered relationships and attributes
"""
if not _is_valid_ip(ip):
return []
return ProviderResult()
relationships = []
result = ProviderResult()
try:
# Perform reverse DNS lookup
@@ -97,27 +101,44 @@ class DNSProvider(BaseProvider):
hostname = str(ptr_record).rstrip('.')
if _is_valid_domain(hostname):
raw_data = {
'query_type': 'PTR',
'ip_address': ip,
'hostname': hostname,
'ttl': response.ttl
}
# Add the relationship
result.add_relationship(
source_node=ip,
target_node=hostname,
relationship_type='ptr_record',
provider=self.name,
confidence=0.8,
raw_data={
'query_type': 'PTR',
'ip_address': ip,
'hostname': hostname,
'ttl': response.ttl
}
)
relationships.append((
ip,
hostname,
'ptr_record',
0.8,
raw_data
))
# Add PTR record as attribute to the IP
result.add_attribute(
target_node=ip,
name='ptr_record',
value=hostname,
attr_type='dns_record',
provider=self.name,
confidence=0.8,
metadata={'ttl': response.ttl}
)
# Log the relationship discovery
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
relationship_type='ptr_record',
confidence_score=0.8,
raw_data=raw_data,
raw_data={
'query_type': 'PTR',
'ip_address': ip,
'hostname': hostname,
'ttl': response.ttl
},
discovery_method="reverse_dns_lookup"
)
@@ -130,18 +151,24 @@ class DNSProvider(BaseProvider):
# Re-raise the exception so the scanner can handle the failure
raise e
return relationships
return result
def _query_record(self, domain: str, record_type: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None:
"""
Query a specific type of DNS record for the domain.
Query a specific type of DNS record for the domain and add results to ProviderResult.
Args:
domain: Domain to query
record_type: DNS record type (A, AAAA, CNAME, etc.)
result: ProviderResult to populate
"""
relationships = []
try:
self.total_requests += 1
response = self.resolver.resolve(domain, record_type)
self.successful_requests += 1
dns_records = []
for record in response:
target = ""
if record_type in ['A', 'AAAA']:
@@ -153,12 +180,16 @@ class DNSProvider(BaseProvider):
elif record_type == 'SOA':
target = str(record.mname).rstrip('.')
elif record_type in ['TXT']:
# TXT records are treated as metadata, not relationships.
# TXT records are treated as attributes, not relationships
txt_value = str(record).strip('"')
dns_records.append(f"TXT: {txt_value}")
continue
elif record_type == 'SRV':
target = str(record.target).rstrip('.')
elif record_type == 'CAA':
target = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
dns_records.append(f"CAA: {caa_value}")
continue
else:
target = str(record)
@@ -170,16 +201,22 @@ class DNSProvider(BaseProvider):
'ttl': response.ttl
}
relationship_type = f"{record_type.lower()}_record"
confidence = 0.8 # Default confidence for DNS records
confidence = 0.8 # Standard confidence for DNS records
relationships.append((
domain,
target,
relationship_type,
confidence,
raw_data
))
# Add relationship
result.add_relationship(
source_node=domain,
target_node=target,
relationship_type=relationship_type,
provider=self.name,
confidence=confidence,
raw_data=raw_data
)
# Add DNS record as attribute to the source domain
dns_records.append(f"{record_type}: {target}")
# Log relationship discovery
self.log_relationship_discovery(
source_node=domain,
target_node=target,
@@ -189,10 +226,20 @@ class DNSProvider(BaseProvider):
discovery_method=f"dns_{record_type.lower()}_record"
)
# Add DNS records as a consolidated attribute
if dns_records:
result.add_attribute(
target_node=domain,
name='dns_records',
value=dns_records,
attr_type='dns_record_list',
provider=self.name,
confidence=0.8,
metadata={'record_types': [record_type]}
)
except Exception as e:
self.failed_requests += 1
self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
# Re-raise the exception so the scanner can handle it
raise e
return relationships
raise e

224
providers/shodan_provider.py
View File

@@ -1,20 +1,20 @@
# dnsrecon/providers/shodan_provider.py
import json
import os
from pathlib import Path
from typing import List, Dict, Any, Tuple
from typing import Dict, Any
from datetime import datetime, timezone
import requests
from .base_provider import BaseProvider
from core.provider_result import ProviderResult
from utils.helpers import _is_valid_ip, _is_valid_domain
class ShodanProvider(BaseProvider):
"""
Provider for querying Shodan API for IP address information.
Now uses session-specific API keys, is limited to IP-only queries, and includes caching.
Now returns standardized ProviderResult objects with caching support.
"""
def __init__(self, name=None, session_config=None):
@@ -85,88 +85,156 @@ class ShodanProvider(BaseProvider):
except (json.JSONDecodeError, ValueError, KeyError):
return "stale"
def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_domain(self, domain: str) -> ProviderResult:
"""
Domain queries are no longer supported for the Shodan provider.
Args:
domain: Domain to investigate
Returns:
Empty ProviderResult
"""
return []
return ProviderResult()
def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def query_ip(self, ip: str) -> ProviderResult:
"""
Query Shodan for information about an IP address, with caching of processed relationships.
Query Shodan for information about an IP address, with caching of processed data.
Args:
ip: IP address to investigate
Returns:
ProviderResult containing discovered relationships and attributes
"""
if not _is_valid_ip(ip) or not self.is_available():
return []
return ProviderResult()
cache_file = self._get_cache_file_path(ip)
cache_status = self._get_cache_status(cache_file)
relationships = []
result = ProviderResult()
try:
if cache_status == "fresh":
relationships = self._load_from_cache(cache_file)
self.logger.logger.info(f"Using cached Shodan relationships for {ip}")
else: # "stale" or "not_found"
result = self._load_from_cache(cache_file)
self.logger.logger.info(f"Using cached Shodan data for {ip}")
else: # "stale" or "not_found"
url = f"{self.base_url}/shodan/host/{ip}"
params = {'key': self.api_key}
response = self.make_request(url, method="GET", params=params, target_indicator=ip)
if response and response.status_code == 200:
data = response.json()
# Process the data into relationships BEFORE caching
relationships = self._process_shodan_data(ip, data)
self._save_to_cache(cache_file, relationships) # Save the processed relationships
# Process the data into ProviderResult BEFORE caching
result = self._process_shodan_data(ip, data)
self._save_to_cache(cache_file, result, data) # Save both result and raw data
elif cache_status == "stale":
# If API fails on a stale cache, use the old data
relationships = self._load_from_cache(cache_file)
result = self._load_from_cache(cache_file)
except requests.exceptions.RequestException as e:
self.logger.logger.error(f"Shodan API query failed for {ip}: {e}")
if cache_status == "stale":
relationships = self._load_from_cache(cache_file)
result = self._load_from_cache(cache_file)
return relationships
return result
def _load_from_cache(self, cache_file_path: Path) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
"""Load processed Shodan relationships from a cache file."""
def _load_from_cache(self, cache_file_path: Path) -> ProviderResult:
"""Load processed Shodan data from a cache file."""
try:
with open(cache_file_path, 'r') as f:
cache_content = json.load(f)
# The entire file content is the list of relationships
return cache_content.get("relationships", [])
result = ProviderResult()
# Reconstruct relationships
for rel_data in cache_content.get("relationships", []):
result.add_relationship(
source_node=rel_data["source_node"],
target_node=rel_data["target_node"],
relationship_type=rel_data["relationship_type"],
provider=rel_data["provider"],
confidence=rel_data["confidence"],
raw_data=rel_data.get("raw_data", {})
)
# Reconstruct attributes
for attr_data in cache_content.get("attributes", []):
result.add_attribute(
target_node=attr_data["target_node"],
name=attr_data["name"],
value=attr_data["value"],
attr_type=attr_data["type"],
provider=attr_data["provider"],
confidence=attr_data["confidence"],
metadata=attr_data.get("metadata", {})
)
return result
except (json.JSONDecodeError, FileNotFoundError, KeyError):
return []
return ProviderResult()
def _save_to_cache(self, cache_file_path: Path, relationships: List[Tuple[str, str, str, float, Dict[str, Any]]]) -> None:
"""Save processed Shodan relationships to a cache file."""
def _save_to_cache(self, cache_file_path: Path, result: ProviderResult, raw_data: Dict[str, Any]) -> None:
"""Save processed Shodan data to a cache file."""
try:
cache_data = {
"last_upstream_query": datetime.now(timezone.utc).isoformat(),
"relationships": relationships
"raw_data": raw_data, # Preserve original for forensic purposes
"relationships": [
{
"source_node": rel.source_node,
"target_node": rel.target_node,
"relationship_type": rel.relationship_type,
"confidence": rel.confidence,
"provider": rel.provider,
"raw_data": rel.raw_data
} for rel in result.relationships
],
"attributes": [
{
"target_node": attr.target_node,
"name": attr.name,
"value": attr.value,
"type": attr.type,
"provider": attr.provider,
"confidence": attr.confidence,
"metadata": attr.metadata
} for attr in result.attributes
]
}
with open(cache_file_path, 'w') as f:
json.dump(cache_data, f, separators=(',', ':'))
json.dump(cache_data, f, separators=(',', ':'), default=str)
except Exception as e:
self.logger.logger.warning(f"Failed to save Shodan cache for {cache_file_path.name}: {e}")
def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult:
"""
Process Shodan data to extract relationships.
Process Shodan data to extract relationships and attributes.
Args:
ip: IP address queried
data: Raw Shodan response data
Returns:
ProviderResult with relationships and attributes
"""
relationships = []
result = ProviderResult()
# Extract hostname relationships
hostnames = data.get('hostnames', [])
for hostname in hostnames:
if _is_valid_domain(hostname):
relationships.append((
ip,
hostname,
'a_record',
0.8,
data
))
result.add_relationship(
source_node=ip,
target_node=hostname,
relationship_type='a_record',
provider=self.name,
confidence=0.8,
raw_data=data
)
self.log_relationship_discovery(
source_node=ip,
target_node=hostname,
@@ -180,13 +248,15 @@ class ShodanProvider(BaseProvider):
asn = data.get('asn')
if asn:
asn_name = f"AS{asn[2:]}" if isinstance(asn, str) and asn.startswith('AS') else f"AS{asn}"
relationships.append((
ip,
asn_name,
'asn_membership',
0.7,
data
))
result.add_relationship(
source_node=ip,
target_node=asn_name,
relationship_type='asn_membership',
provider=self.name,
confidence=0.7,
raw_data=data
)
self.log_relationship_discovery(
source_node=ip,
target_node=asn_name,
@@ -195,5 +265,67 @@ class ShodanProvider(BaseProvider):
raw_data=data,
discovery_method="shodan_asn_lookup"
)
return relationships
# Add comprehensive Shodan host information as attributes
if 'ports' in data:
result.add_attribute(
target_node=ip,
name='ports',
value=data['ports'],
attr_type='network_info',
provider=self.name,
confidence=0.9
)
if 'os' in data and data['os']:
result.add_attribute(
target_node=ip,
name='operating_system',
value=data['os'],
attr_type='system_info',
provider=self.name,
confidence=0.8
)
if 'org' in data:
result.add_attribute(
target_node=ip,
name='organization',
value=data['org'],
attr_type='network_info',
provider=self.name,
confidence=0.8
)
if 'country_name' in data:
result.add_attribute(
target_node=ip,
name='country',
value=data['country_name'],
attr_type='location_info',
provider=self.name,
confidence=0.9
)
if 'city' in data:
result.add_attribute(
target_node=ip,
name='city',
value=data['city'],
attr_type='location_info',
provider=self.name,
confidence=0.8
)
# Store complete Shodan data as a comprehensive attribute
result.add_attribute(
target_node=ip,
name='shodan_host_info',
value=data, # Complete Shodan response for full forensic detail
attr_type='comprehensive_data',
provider=self.name,
confidence=0.9,
metadata={'data_source': 'shodan_api', 'query_type': 'host_lookup'}
)
return result