many improvements

providers/base_provider.py

@@ -1,11 +1,10 @@
-"""
-Abstract base provider class for DNSRecon data sources.
-Defines the interface and common functionality for all providers.
-"""
+# dnsrecon/providers/base_provider.py
 
 import time
 import requests
 import threading
+import os
+import json
 from abc import ABC, abstractmethod
 from typing import List, Dict, Any, Optional, Tuple
 from datetime import datetime
@@ -61,12 +60,18 @@ class BaseProvider(ABC):
         self._local = threading.local()
         self.logger = get_forensic_logger()
 
+        # Caching configuration
+        self.cache_dir = '.cache'
+        self.cache_expiry = 12 * 3600  # 12 hours in seconds
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
+
         # Statistics
         self.total_requests = 0
         self.successful_requests = 0
         self.failed_requests = 0
         self.total_relationships_found = 0
-    
+
     @property
     def session(self):
         if not hasattr(self._local, 'session'):
@@ -131,6 +136,23 @@ class BaseProvider(ABC):
             Returns:
                 Response object or None if request failed
             """
+            # Create a unique cache key
+            cache_key = f"{self.name}_{hash(f'{method}:{url}:{json.dumps(params, sort_keys=True)}')}.json"
+            cache_path = os.path.join(self.cache_dir, cache_key)
+
+            # Check cache
+            if os.path.exists(cache_path):
+                cache_age = time.time() - os.path.getmtime(cache_path)
+                if cache_age < self.cache_expiry:
+                    print(f"Returning cached response for: {url}")
+                    with open(cache_path, 'r') as f:
+                        cached_data = json.load(f)
+                        response = requests.Response()
+                        response.status_code = cached_data['status_code']
+                        response._content = cached_data['content'].encode('utf-8')
+                        response.headers = cached_data['headers']
+                        return response
+
             for attempt in range(max_retries + 1):
                 # Apply rate limiting
                 self.rate_limiter.wait_if_needed()
@@ -171,7 +193,7 @@ class BaseProvider(ABC):
                     response.raise_for_status()
                     self.successful_requests += 1
                     
-                    # Success - log and return
+                    # Success - log, cache, and return
                     duration_ms = (time.time() - start_time) * 1000
                     self.logger.log_api_request(
                         provider=self.name,
@@ -183,6 +205,13 @@ class BaseProvider(ABC):
                         error=None,
                         target_indicator=target_indicator
                     )
+                    # Cache the successful response to disk
+                    with open(cache_path, 'w') as f:
+                        json.dump({
+                            'status_code': response.status_code,
+                            'content': response.text,
+                            'headers': dict(response.headers)
+                        }, f)
                     return response
 
                 except requests.exceptions.RequestException as e:

providers/dns_provider.py

@@ -1,7 +1,4 @@
-"""
-DNS resolution provider for DNSRecon.
-Discovers domain relationships through DNS record analysis.
-"""
+# dnsrecon/providers/dns_provider.py
 
 import socket
 import dns.resolver
@@ -87,8 +84,10 @@ class DNSProvider(BaseProvider):
         
         try:
             # Perform reverse DNS lookup
+            self.total_requests += 1
             reverse_name = dns.reversename.from_address(ip)
             response = self.resolver.resolve(reverse_name, 'PTR')
+            self.successful_requests += 1
             
             for ptr_record in response:
                 hostname = str(ptr_record).rstrip('.')
@@ -119,6 +118,7 @@ class DNSProvider(BaseProvider):
                     )
         
         except Exception as e:
+            self.failed_requests += 1
             self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}")
         
         return relationships
@@ -131,7 +131,9 @@ class DNSProvider(BaseProvider):
         #    return relationships
         
         try:
+            self.total_requests += 1
             response = self.resolver.resolve(domain, 'A')
+            self.successful_requests += 1
             
             for a_record in response:
                 ip_address = str(a_record)
@@ -161,6 +163,7 @@ class DNSProvider(BaseProvider):
                 )
         
         except Exception as e:
+            self.failed_requests += 1
             self.logger.logger.debug(f"A record query failed for {domain}: {e}")
         
         return relationships
@@ -173,7 +176,9 @@ class DNSProvider(BaseProvider):
         #    return relationships
         
         try:
+            self.total_requests += 1
             response = self.resolver.resolve(domain, 'AAAA')
+            self.successful_requests += 1
             
             for aaaa_record in response:
                 ip_address = str(aaaa_record)
@@ -203,6 +208,7 @@ class DNSProvider(BaseProvider):
                 )
         
         except Exception as e:
+            self.failed_requests += 1
             self.logger.logger.debug(f"AAAA record query failed for {domain}: {e}")
         
         return relationships
@@ -215,7 +221,9 @@ class DNSProvider(BaseProvider):
         #    return relationships
         
         try:
+            self.total_requests += 1
             response = self.resolver.resolve(domain, 'CNAME')
+            self.successful_requests += 1
             
             for cname_record in response:
                 target_domain = str(cname_record).rstrip('.')
@@ -246,6 +254,7 @@ class DNSProvider(BaseProvider):
                     )
         
         except Exception as e:
+            self.failed_requests += 1
             self.logger.logger.debug(f"CNAME record query failed for {domain}: {e}")
         
         return relationships
@@ -258,7 +267,9 @@ class DNSProvider(BaseProvider):
         #    return relationships
         
         try:
+            self.total_requests += 1
             response = self.resolver.resolve(domain, 'MX')
+            self.successful_requests += 1
             
             for mx_record in response:
                 mx_host = str(mx_record.exchange).rstrip('.')
@@ -290,6 +301,7 @@ class DNSProvider(BaseProvider):
                     )
         
         except Exception as e:
+            self.failed_requests += 1
             self.logger.logger.debug(f"MX record query failed for {domain}: {e}")
         
         return relationships
@@ -302,7 +314,9 @@ class DNSProvider(BaseProvider):
         #    return relationships
         
         try:
+            self.total_requests += 1
             response = self.resolver.resolve(domain, 'NS')
+            self.successful_requests += 1
             
             for ns_record in response:
                 ns_host = str(ns_record).rstrip('.')
@@ -333,6 +347,7 @@ class DNSProvider(BaseProvider):
                     )
         
         except Exception as e:
+            self.failed_requests += 1
             self.logger.logger.debug(f"NS record query failed for {domain}: {e}")
         
         return relationships