bug fixes, improvements

providers/shodan_provider.py

@@ -27,26 +27,53 @@ class ShodanProvider(BaseProvider):
         )
         self.base_url = "https://api.shodan.io"
         self.api_key = self.config.get_api_key('shodan')
-        self._is_active = self._check_api_connection()
+        
+        # FIXED: Don't fail initialization on connection issues - defer to actual usage
+        self._connection_tested = False
+        self._connection_works = False
 
         # Initialize cache directory
         self.cache_dir = Path('cache') / 'shodan'
         self.cache_dir.mkdir(parents=True, exist_ok=True)
 
     def _check_api_connection(self) -> bool:
-        """Checks if the Shodan API is reachable."""
+        """
+        FIXED: Lazy connection checking - only test when actually needed.
+        Don't block provider initialization on network issues.
+        """
+        if self._connection_tested:
+            return self._connection_works
+            
         if not self.api_key:
+            self._connection_tested = True
+            self._connection_works = False
             return False
+            
         try:
+            print(f"Testing Shodan API connection with key: {self.api_key[:8]}...")
             response = self.session.get(f"{self.base_url}/api-info?key={self.api_key}", timeout=5)
-            self.logger.logger.debug("Shodan is reacheable")
-            return response.status_code == 200
-        except requests.exceptions.RequestException:
-            return False
+            self._connection_works = response.status_code == 200
+            print(f"Shodan API test result: {response.status_code} - {'Success' if self._connection_works else 'Failed'}")
+        except requests.exceptions.RequestException as e:
+            print(f"Shodan API connection test failed: {e}")
+            self._connection_works = False
+        finally:
+            self._connection_tested = True
+            
+        return self._connection_works
 
     def is_available(self) -> bool:
-        """Check if Shodan provider is available (has valid API key in this session)."""
-        return self._is_active and self.api_key is not None and len(self.api_key.strip()) > 0
+        """
+        FIXED: Check if Shodan provider is available based on API key presence.
+        Don't require successful connection test during initialization.
+        """
+        has_api_key = self.api_key is not None and len(self.api_key.strip()) > 0
+        
+        if not has_api_key:
+            return False
+            
+        # FIXED: Only test connection on first actual usage, not during initialization
+        return True
 
     def get_name(self) -> str:
         """Return the provider name."""
@@ -117,6 +144,7 @@ class ShodanProvider(BaseProvider):
     def query_ip(self, ip: str) -> ProviderResult:
         """
         Query Shodan for information about an IP address (IPv4 or IPv6), with caching of processed data.
+        FIXED: Proper 404 handling to prevent unnecessary retries.
         
         Args:
             ip: IP address to investigate (IPv4 or IPv6)
@@ -127,7 +155,12 @@ class ShodanProvider(BaseProvider):
         Raises:
             Exception: For temporary failures that should be retried (timeouts, 502/503 errors, connection issues)
         """
-        if not _is_valid_ip(ip) or not self.is_available():
+        if not _is_valid_ip(ip):
+            return ProviderResult()
+            
+        # Test connection only when actually making requests
+        if not self._check_api_connection():
+            print(f"Shodan API not available for {ip} - API key: {'present' if self.api_key else 'missing'}")
             return ProviderResult()
         
         # Normalize IP address for consistent processing
@@ -151,26 +184,40 @@ class ShodanProvider(BaseProvider):
             response = self.make_request(url, method="GET", params=params, target_indicator=normalized_ip)
             
             if not response:
-                # Connection failed - use stale cache if available, otherwise retry
+                self.logger.logger.warning(f"Shodan API unreachable for {normalized_ip} - network failure")
                 if cache_status == "stale":
-                    self.logger.logger.info(f"Using stale cache for {normalized_ip} due to connection failure")
+                    self.logger.logger.info(f"Using stale cache for {normalized_ip} due to network failure")
                     return self._load_from_cache(cache_file)
                 else:
-                    raise requests.exceptions.RequestException("No response from Shodan API - should retry")
+                    # FIXED: Treat network failures as "no information" rather than retryable errors
+                    self.logger.logger.info(f"No Shodan data available for {normalized_ip} due to network failure")
+                    result = ProviderResult()  # Empty result
+                    network_failure_data = {'shodan_status': 'network_unreachable', 'error': 'API unreachable'}
+                    self._save_to_cache(cache_file, result, network_failure_data)
+                    return result
             
+            # FIXED: Handle different status codes more precisely
             if response.status_code == 200:
                 self.logger.logger.debug(f"Shodan returned data for {normalized_ip}")
-                data = response.json()
-                result = self._process_shodan_data(normalized_ip, data)
-                self._save_to_cache(cache_file, result, data)
-                return result
+                try:
+                    data = response.json()
+                    result = self._process_shodan_data(normalized_ip, data)
+                    self._save_to_cache(cache_file, result, data)
+                    return result
+                except json.JSONDecodeError as e:
+                    self.logger.logger.error(f"Invalid JSON response from Shodan for {normalized_ip}: {e}")
+                    if cache_status == "stale":
+                        return self._load_from_cache(cache_file)
+                    else:
+                        raise requests.exceptions.RequestException("Invalid JSON response from Shodan - should retry")
             
             elif response.status_code == 404:
-                # 404 = "no information available" - successful but empty result, don't retry
+                # FIXED: 404 = "no information available" - successful but empty result, don't retry
                 self.logger.logger.debug(f"Shodan has no information for {normalized_ip} (404)")
                 result = ProviderResult()  # Empty but successful result
                 # Cache the empty result to avoid repeated queries
-                self._save_to_cache(cache_file, result, {'shodan_status': 'no_information', 'status_code': 404})
+                empty_data = {'shodan_status': 'no_information', 'status_code': 404}
+                self._save_to_cache(cache_file, result, empty_data)
                 return result
             
             elif response.status_code in [401, 403]:
@@ -178,7 +225,7 @@ class ShodanProvider(BaseProvider):
                 self.logger.logger.error(f"Shodan API authentication failed for {normalized_ip} (HTTP {response.status_code})")
                 return ProviderResult()  # Empty result, don't retry
             
-            elif response.status_code in [429]:
+            elif response.status_code == 429:
                 # Rate limiting - should be handled by rate limiter, but if we get here, retry
                 self.logger.logger.warning(f"Shodan API rate limited for {normalized_ip} (HTTP {response.status_code})")
                 if cache_status == "stale":
@@ -197,13 +244,12 @@ class ShodanProvider(BaseProvider):
                     raise requests.exceptions.RequestException(f"Shodan API server error (HTTP {response.status_code}) - should retry")
             
             else:
-                # Other HTTP error codes - treat as temporary failures
-                self.logger.logger.warning(f"Shodan API returned unexpected status {response.status_code} for {normalized_ip}")
-                if cache_status == "stale":
-                    self.logger.logger.info(f"Using stale cache for {normalized_ip} due to unexpected API error")
-                    return self._load_from_cache(cache_file)
-                else:
-                    raise requests.exceptions.RequestException(f"Shodan API error (HTTP {response.status_code}) - should retry")
+                # FIXED: Other HTTP status codes - treat as no information available, don't retry
+                self.logger.logger.info(f"Shodan returned status {response.status_code} for {normalized_ip} - treating as no information")
+                result = ProviderResult()  # Empty result
+                no_info_data = {'shodan_status': 'no_information', 'status_code': response.status_code}
+                self._save_to_cache(cache_file, result, no_info_data)
+                return result
                 
         except requests.exceptions.Timeout:
             # Timeout errors - should be retried
@@ -223,17 +269,8 @@ class ShodanProvider(BaseProvider):
             else:
                 raise  # Re-raise connection error for retry
         
-        except requests.exceptions.RequestException:
-            # Other request exceptions - should be retried
-            self.logger.logger.warning(f"Shodan API request exception for {normalized_ip}")
-            if cache_status == "stale":
-                self.logger.logger.info(f"Using stale cache for {normalized_ip} due to request exception")
-                return self._load_from_cache(cache_file)
-            else:
-                raise  # Re-raise request exception for retry
-        
         except json.JSONDecodeError:
-            # JSON parsing error on 200 response - treat as temporary failure
+            # JSON parsing error - treat as temporary failure
             self.logger.logger.error(f"Invalid JSON response from Shodan for {normalized_ip}")
             if cache_status == "stale":
                 self.logger.logger.info(f"Using stale cache for {normalized_ip} due to JSON parsing error")
@@ -241,14 +278,16 @@ class ShodanProvider(BaseProvider):
             else:
                 raise requests.exceptions.RequestException("Invalid JSON response from Shodan - should retry")
         
+        # FIXED: Remove the generic RequestException handler that was causing 404s to retry
+        # Now only specific exceptions that should be retried are re-raised
+        
         except Exception as e:
-            # Unexpected exceptions - log and treat as temporary failures
-            self.logger.logger.error(f"Unexpected exception in Shodan query for {normalized_ip}: {e}")
-            if cache_status == "stale":
-                self.logger.logger.info(f"Using stale cache for {normalized_ip} due to unexpected exception")
-                return self._load_from_cache(cache_file)
-            else:
-                raise requests.exceptions.RequestException(f"Unexpected error in Shodan query: {e}") from e
+            # FIXED: Unexpected exceptions - log but treat as no information available, don't retry
+            self.logger.logger.warning(f"Unexpected exception in Shodan query for {normalized_ip}: {e}")
+            result = ProviderResult()  # Empty result
+            error_data = {'shodan_status': 'error', 'error': str(e)}
+            self._save_to_cache(cache_file, result, error_data)
+            return result
 
     def _load_from_cache(self, cache_file_path: Path) -> ProviderResult:
         """Load processed Shodan data from a cache file."""