progress

src/certificate_checker.py

@@ -5,6 +5,7 @@ import requests
 import json
 import time
 import logging
+import socket
 from datetime import datetime
 from typing import List, Optional, Set
 from .data_structures import Certificate
@@ -22,8 +23,51 @@ class CertificateChecker:
         self.config = config
         self.last_request = 0
         self.query_count = 0
+        self.connection_failures = 0
+        self.max_connection_failures = 3  # Stop trying after 3 consecutive failures
         
         logger.info("🔐 Certificate checker initialized")
+        
+        # Test connectivity to crt.sh on initialization
+        self._test_connectivity()
+    
+    def _test_connectivity(self):
+        """Test if we can reach crt.sh."""
+        try:
+            logger.info("🔗 Testing connectivity to crt.sh...")
+            
+            # First test DNS resolution
+            try:
+                socket.gethostbyname('crt.sh')
+                logger.debug("✅ DNS resolution for crt.sh successful")
+            except socket.gaierror as e:
+                logger.warning(f"⚠️ DNS resolution failed for crt.sh: {e}")
+                return False
+            
+            # Test HTTP connection with a simple request
+            response = requests.get(
+                self.CRT_SH_URL,
+                params={'q': 'example.com', 'output': 'json'},
+                timeout=10,
+                headers={'User-Agent': 'DNS-Recon-Tool/1.0'}
+            )
+            
+            if response.status_code in [200, 404]:  # 404 is also acceptable (no results)
+                logger.info("✅ crt.sh connectivity test successful")
+                return True
+            else:
+                logger.warning(f"⚠️ crt.sh returned status {response.status_code}")
+                return False
+                
+        except requests.exceptions.ConnectionError as e:
+            logger.warning(f"⚠️ Cannot reach crt.sh: {e}")
+            return False
+        except requests.exceptions.Timeout:
+            logger.warning("⚠️ crt.sh connectivity test timed out")
+            return False
+        except Exception as e:
+            logger.warning(f"⚠️ Unexpected error testing crt.sh connectivity: {e}")
+            return False
     
     def _rate_limit(self):
         """Apply rate limiting for crt.sh."""
@@ -33,7 +77,7 @@ class CertificateChecker:
         
         if time_since_last < min_interval:
             sleep_time = min_interval - time_since_last
-            logger.debug(f"⏸️  crt.sh rate limiting: sleeping for {sleep_time:.2f}s")
+            logger.debug(f"⏸️ crt.sh rate limiting: sleeping for {sleep_time:.2f}s")
             time.sleep(sleep_time)
         
         self.last_request = time.time()
@@ -41,7 +85,12 @@ class CertificateChecker:
     
     def get_certificates(self, domain: str) -> List[Certificate]:
         """Get certificates for a domain from crt.sh."""
-        logger.debug(f"🔐 Getting certificates for domain: {domain}")
+        logger.debug(f"🔍 Getting certificates for domain: {domain}")
+        
+        # Skip if we've had too many connection failures
+        if self.connection_failures >= self.max_connection_failures:
+            logger.warning(f"⚠️ Skipping certificate lookup for {domain} due to repeated connection failures")
+            return []
         
         certificates = []
         
@@ -49,9 +98,10 @@ class CertificateChecker:
         domain_certs = self._query_crt_sh(domain)
         certificates.extend(domain_certs)
         
-        # Also query for wildcard certificates
-        wildcard_certs = self._query_crt_sh(f"%.{domain}")
-        certificates.extend(wildcard_certs)
+        # Also query for wildcard certificates (if the main query succeeded)
+        if domain_certs or self.connection_failures < self.max_connection_failures:
+            wildcard_certs = self._query_crt_sh(f"%.{domain}")
+            certificates.extend(wildcard_certs)
         
         # Remove duplicates based on certificate ID
         unique_certs = {cert.id: cert for cert in certificates}
@@ -65,13 +115,15 @@ class CertificateChecker:
         return final_certs
     
     def _query_crt_sh(self, query: str) -> List[Certificate]:
-        """Query crt.sh API with retry logic."""
+        """Query crt.sh API with retry logic and better error handling."""
         certificates = []
         self._rate_limit()
         
         logger.debug(f"📡 Querying crt.sh for: {query}")
         
-        max_retries = 3
+        max_retries = 2  # Reduced retries for faster failure
+        backoff_delays = [1, 3]  # Shorter delays
+        
         for attempt in range(max_retries):
             try:
                 params = {
@@ -111,50 +163,68 @@ class CertificateChecker:
                                     certificates.append(certificate)
                                     logger.debug(f"✅ Parsed certificate ID {certificate.id} for {query}")
                                 else:
-                                    logger.debug(f"⚠️  Skipped certificate with invalid dates: {cert_data.get('id')}")
+                                    logger.debug(f"⚠️ Skipped certificate with invalid dates: {cert_data.get('id')}")
                                     
                             except (ValueError, TypeError, KeyError) as e:
-                                logger.debug(f"⚠️  Error parsing certificate data: {e}")
+                                logger.debug(f"⚠️ Error parsing certificate data: {e}")
                                 continue  # Skip malformed certificate data
                         
+                        # Success! Reset connection failure counter
+                        self.connection_failures = 0
                         logger.info(f"✅ Successfully processed {len(certificates)} certificates from crt.sh for {query}")
-                        return certificates # Success, exit retry loop
+                        return certificates
                     
                     except json.JSONDecodeError as e:
                         logger.warning(f"❌ Invalid JSON response from crt.sh for {query}: {e}")
                         if attempt < max_retries - 1:
-                            time.sleep(2 ** attempt)  # Exponential backoff
+                            time.sleep(backoff_delays[attempt])
                             continue
                         return certificates
                 
+                elif response.status_code == 404:
+                    # 404 is normal - no certificates found
+                    logger.debug(f"ℹ️ No certificates found for {query} (404)")
+                    self.connection_failures = 0  # Reset counter for successful connection
+                    return certificates
+                
                 elif response.status_code == 429:
-                    logger.warning(f"⚠️  crt.sh rate limit exceeded for {query}")
+                    logger.warning(f"⚠️ crt.sh rate limit exceeded for {query}")
                     if attempt < max_retries - 1:
                         time.sleep(5)  # Wait longer for rate limits
                         continue
                     return certificates
                     
                 else:
-                    logger.warning(f"⚠️  crt.sh HTTP error for {query}: {response.status_code}")
+                    logger.warning(f"⚠️ crt.sh HTTP error for {query}: {response.status_code}")
                     if attempt < max_retries - 1:
-                        time.sleep(2)
+                        time.sleep(backoff_delays[attempt])
                         continue
                     return certificates
                             
-            except requests.exceptions.Timeout:
-                logger.warning(f"⏱️  crt.sh query timeout for {query} (attempt {attempt+1}/{max_retries})")
+            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
+                error_type = "Connection Error" if isinstance(e, requests.exceptions.ConnectionError) else "Timeout"
+                logger.warning(f"🌐 crt.sh {error_type} for {query} (attempt {attempt+1}/{max_retries}): {e}")
+                
+                # Track connection failures
+                if isinstance(e, requests.exceptions.ConnectionError):
+                    self.connection_failures += 1
+                    if self.connection_failures >= self.max_connection_failures:
+                        logger.error(f"❌ Too many connection failures to crt.sh. Disabling certificate lookups.")
+                        return certificates
+                
                 if attempt < max_retries - 1:
-                    time.sleep(2)
+                    time.sleep(backoff_delays[attempt])
                     continue
+                    
             except requests.exceptions.RequestException as e:
                 logger.warning(f"🌐 crt.sh network error for {query} (attempt {attempt+1}/{max_retries}): {e}")
                 if attempt < max_retries - 1:
-                    time.sleep(2)
+                    time.sleep(backoff_delays[attempt])
                     continue
             except Exception as e:
                 logger.error(f"❌ Unexpected error querying crt.sh for {query}: {e}")
                 if attempt < max_retries - 1:
-                    time.sleep(2)
+                    time.sleep(backoff_delays[attempt])
                     continue
         
         # If we get here, all retries failed
@@ -187,7 +257,7 @@ class CertificateChecker:
         except ValueError:
             pass
         
-        logger.debug(f"⚠️  Could not parse date: {date_str}")
+        logger.debug(f"⚠️ Could not parse date: {date_str}")
         return None
     
     def extract_subdomains_from_certificates(self, certificates: List[Certificate]) -> Set[str]:

src/reconnaissance.py

@@ -34,23 +34,31 @@ class ReconnaissanceEngine:
             self.shodan_client = ShodanClient(config.shodan_key, config)
             logger.info("✅ Shodan client initialized")
         else:
-            logger.info("⚠️  Shodan API key not provided, skipping Shodan integration")
+            logger.info("⚠️ Shodan API key not provided, skipping Shodan integration")
         
         self.virustotal_client = None
         if config.virustotal_key:
             self.virustotal_client = VirusTotalClient(config.virustotal_key, config)
             logger.info("✅ VirusTotal client initialized")
         else:
-            logger.info("⚠️  VirusTotal API key not provided, skipping VirusTotal integration")
+            logger.info("⚠️ VirusTotal API key not provided, skipping VirusTotal integration")
         
         # Progress tracking
         self.progress_callback = None
         self._lock = threading.Lock()
+        
+        # Shared data object for live updates
+        self.shared_data = None
     
     def set_progress_callback(self, callback):
         """Set callback for progress updates."""
         self.progress_callback = callback
     
+    def set_shared_data(self, shared_data: ReconData):
+        """Set shared data object for live updates during web interface usage."""
+        self.shared_data = shared_data
+        logger.info("📊 Using shared data object for live updates")
+    
     def _update_progress(self, message: str, percentage: int = None):
         """Update progress if callback is set."""
         logger.info(f"Progress: {message} ({percentage}%)" if percentage else f"Progress: {message}")
@@ -59,7 +67,14 @@ class ReconnaissanceEngine:
     
     def run_reconnaissance(self, target: str) -> ReconData:
         """Run full reconnaissance on target."""
-        self.data = ReconData()
+        # Use shared data object if available, otherwise create new one
+        if self.shared_data is not None:
+            self.data = self.shared_data
+            logger.info("📊 Using shared data object for reconnaissance")
+        else:
+            self.data = ReconData()
+            logger.info("📊 Created new data object for reconnaissance")
+            
         self.data.start_time = datetime.now()
         
         logger.info(f"🚀 Starting reconnaissance for target: {target}")
@@ -100,7 +115,7 @@ class ReconnaissanceEngine:
         finally:
             self.data.end_time = datetime.now()
             duration = self.data.end_time - self.data.start_time
-            logger.info(f"⏱️  Total reconnaissance time: {duration}")
+            logger.info(f"⏱️ Total reconnaissance time: {duration}")
         
         return self.data
     
@@ -108,7 +123,7 @@ class ReconnaissanceEngine:
         """Expand hostname to all possible TLDs."""
         logger.info(f"🌐 Fetching TLD list for hostname expansion")
         tlds = self.tld_fetcher.get_tlds()
-        logger.info(f"📝 Testing against {len(tlds)} TLDs")
+        logger.info(f"🔍 Testing against {len(tlds)} TLDs")
         
         targets = set()
         tested_count = 0
@@ -182,7 +197,7 @@ class ReconnaissanceEngine:
                 self.data.add_ip_address(record.value)
         
         # Get certificates
-        logger.debug(f"🔐 Checking certificates for {hostname}")
+        logger.debug(f"🔍 Checking certificates for {hostname}")
         certificates = self.cert_checker.get_certificates(hostname)
         if certificates:
             self.data.certificates[hostname] = certificates
@@ -234,7 +249,7 @@ class ReconnaissanceEngine:
         
         # Shodan lookups
         if self.shodan_client:
-            logger.info(f"🕵️  Starting Shodan lookups for {len(self.data.ip_addresses)} IPs")
+            logger.info(f"🕵️ Starting Shodan lookups for {len(self.data.ip_addresses)} IPs")
             shodan_success_count = 0
             
             for ip in self.data.ip_addresses:
@@ -248,16 +263,16 @@ class ReconnaissanceEngine:
                     else:
                         logger.debug(f"❌ No Shodan data for {ip}")
                 except Exception as e:
-                    logger.warning(f"⚠️  Error querying Shodan for {ip}: {e}")
+                    logger.warning(f"⚠️ Error querying Shodan for {ip}: {e}")
             
             logger.info(f"✅ Shodan lookups complete: {shodan_success_count}/{len(self.data.ip_addresses)} successful")
         else:
-            logger.info("⚠️  Skipping Shodan lookups (no API key)")
+            logger.info("⚠️ Skipping Shodan lookups (no API key)")
         
         # VirusTotal lookups
         if self.virustotal_client:
             total_resources = len(self.data.ip_addresses) + len(self.data.hostnames)
-            logger.info(f"🛡️  Starting VirusTotal lookups for {total_resources} resources")
+            logger.info(f"🛡️ Starting VirusTotal lookups for {total_resources} resources")
             vt_success_count = 0
             
             # Check IPs
@@ -268,11 +283,11 @@ class ReconnaissanceEngine:
                     if result:
                         self.data.add_virustotal_result(ip, result)
                         vt_success_count += 1
-                        logger.info(f"🛡️  VirusTotal result for {ip}: {result.positives}/{result.total} detections")
+                        logger.info(f"🛡️ VirusTotal result for {ip}: {result.positives}/{result.total} detections")
                     else:
                         logger.debug(f"❌ No VirusTotal data for {ip}")
                 except Exception as e:
-                    logger.warning(f"⚠️  Error querying VirusTotal for IP {ip}: {e}")
+                    logger.warning(f"⚠️ Error querying VirusTotal for IP {ip}: {e}")
             
             # Check domains
             for hostname in self.data.hostnames:
@@ -282,15 +297,15 @@ class ReconnaissanceEngine:
                     if result:
                         self.data.add_virustotal_result(hostname, result)
                         vt_success_count += 1
-                        logger.info(f"🛡️  VirusTotal result for {hostname}: {result.positives}/{result.total} detections")
+                        logger.info(f"🛡️ VirusTotal result for {hostname}: {result.positives}/{result.total} detections")
                     else:
                         logger.debug(f"❌ No VirusTotal data for {hostname}")
                 except Exception as e:
-                    logger.warning(f"⚠️  Error querying VirusTotal for domain {hostname}: {e}")
+                    logger.warning(f"⚠️ Error querying VirusTotal for domain {hostname}: {e}")
             
             logger.info(f"✅ VirusTotal lookups complete: {vt_success_count}/{total_resources} successful")
         else:
-            logger.info("⚠️  Skipping VirusTotal lookups (no API key)")
+            logger.info("⚠️ Skipping VirusTotal lookups (no API key)")
         
         # Final external lookup summary
         ext_stats = {

src/web_app.py

@@ -8,6 +8,7 @@ import logging
 from .config import Config
 from .reconnaissance import ReconnaissanceEngine
 from .report_generator import ReportGenerator
+from .data_structures import ReconData
 
 # Set up logging for this module
 logger = logging.getLogger(__name__)
@@ -46,20 +47,23 @@ def create_app(config: Config):
             )
             
             if not target:
-                logger.warning("⚠️  Scan request missing target")
+                logger.warning("⚠️ Scan request missing target")
                 return jsonify({'error': 'Target is required'}), 400
             
             # Generate scan ID
             scan_id = f"{target}_{int(time.time())}"
             logger.info(f"🚀 Starting new scan: {scan_id} for target: {target}")
             
-            # Initialize scan data
+            # Create shared ReconData object for live updates
+            shared_data = ReconData()
+            
+            # Initialize scan data with the shared data object
             with scan_lock:
                 active_scans[scan_id] = {
                     'status': 'starting',
                     'progress': 0,
                     'message': 'Initializing...',
-                    'data': None,
+                    'data': shared_data,  # Share the data object from the start!
                     'error': None,
                     'live_stats': {
                         'hostnames': 0,
@@ -75,7 +79,7 @@ def create_app(config: Config):
             # Start reconnaissance in background thread
             thread = threading.Thread(
                 target=run_reconnaissance_background,
-                args=(scan_id, target, scan_config)
+                args=(scan_id, target, scan_config, shared_data)
             )
             thread.daemon = True
             thread.start()
@@ -118,7 +122,7 @@ def create_app(config: Config):
             report_gen = ReportGenerator(scan_data['data'])
             
             return jsonify({
-                'json_report': scan_data['data'].to_json(),  # This should now work properly
+                'json_report': scan_data['data'].to_json(),
                 'text_report': report_gen.generate_text_report()
             })
         except Exception as e:
@@ -134,37 +138,41 @@ def create_app(config: Config):
             
             scan_data = active_scans[scan_id]
             
-            if not scan_data['data']:
+            # Now we always have a data object, even if it's empty initially
+            data_obj = scan_data['data']
+            
+            if not data_obj:
                 return jsonify({
                     'hostnames': [],
                     'ip_addresses': [],
-                    'stats': scan_data['live_stats']
+                    'stats': scan_data['live_stats'],
+                    'latest_discoveries': []
                 })
             
-            # Return current discoveries
+            # Return current discoveries from the shared data object
             return jsonify({
-                'hostnames': sorted(list(scan_data['data'].hostnames)),
-                'ip_addresses': sorted(list(scan_data['data'].ip_addresses)),
-                'stats': scan_data['data'].get_stats(),
+                'hostnames': sorted(list(data_obj.hostnames)),
+                'ip_addresses': sorted(list(data_obj.ip_addresses)),
+                'stats': data_obj.get_stats(),
                 'latest_discoveries': scan_data.get('latest_discoveries', [])
             })
     
     return app
 
-def run_reconnaissance_background(scan_id: str, target: str, config: Config):
-    """Run reconnaissance in background thread."""
+def run_reconnaissance_background(scan_id: str, target: str, config: Config, shared_data: ReconData):
+    """Run reconnaissance in background thread with shared data object."""
     
     def update_progress(message: str, percentage: int = None):
-        """Update scan progress."""
+        """Update scan progress and live statistics."""
         with scan_lock:
             if scan_id in active_scans:
                 active_scans[scan_id]['message'] = message
                 if percentage is not None:
                     active_scans[scan_id]['progress'] = percentage
                 
-                # Update live stats if we have data
-                if active_scans[scan_id]['data']:
-                    active_scans[scan_id]['live_stats'] = active_scans[scan_id]['data'].get_stats()
+                # Update live stats from the shared data object
+                if shared_data:
+                    active_scans[scan_id]['live_stats'] = shared_data.get_stats()
                     
                     # Add to latest discoveries (keep last 10)
                     if 'latest_discoveries' not in active_scans[scan_id]:
@@ -188,27 +196,30 @@ def run_reconnaissance_background(scan_id: str, target: str, config: Config):
         engine = ReconnaissanceEngine(config)
         engine.set_progress_callback(update_progress)
         
+        # IMPORTANT: Pass the shared data object to the engine
+        engine.set_shared_data(shared_data)
+        
         # Update status
         with scan_lock:
             active_scans[scan_id]['status'] = 'running'
         
         logger.info(f"🚀 Starting reconnaissance for: {target}")
         
-        # Run reconnaissance
-        data = engine.run_reconnaissance(target)
+        # Run reconnaissance - this will populate the shared_data object incrementally
+        final_data = engine.run_reconnaissance(target)
         
         logger.info(f"✅ Reconnaissance completed for scan: {scan_id}")
         
-        # Update with results
+        # Update with final results (the shared_data should already be populated)
         with scan_lock:
             active_scans[scan_id]['status'] = 'completed'
             active_scans[scan_id]['progress'] = 100
             active_scans[scan_id]['message'] = 'Reconnaissance completed'
-            active_scans[scan_id]['data'] = data
-            active_scans[scan_id]['live_stats'] = data.get_stats()
+            active_scans[scan_id]['data'] = final_data  # This should be the same as shared_data
+            active_scans[scan_id]['live_stats'] = final_data.get_stats()
         
         # Log final statistics
-        final_stats = data.get_stats()
+        final_stats = final_data.get_stats()
         logger.info(f"📊 Final stats for {scan_id}: {final_stats}")
     
     except Exception as e: