modularize, shodan qs

app.py

@@ -374,17 +374,7 @@ def get_providers():
         # Get user-specific scanner
         user_session_id, scanner = get_user_scanner()
         
-        provider_stats = scanner.get_provider_statistics()
+        provider_info = scanner.get_provider_info()
-        
-        # Add configuration information
-        provider_info = {}
-        for provider_name, stats in provider_stats.items():
-            provider_info[provider_name] = {
-                'statistics': stats,
-                'enabled': config.is_provider_enabled(provider_name),
-                'rate_limit': config.get_rate_limit(provider_name),
-                'requires_api_key': provider_name in ['shodan']
-            }
         
         return jsonify({
             'success': True,
@@ -409,7 +399,7 @@ def set_api_keys():
     try:
         data = request.get_json()
         
-        if not data:
+        if data is None:
             return jsonify({
                 'success': False,
                 'error': 'No API keys provided'
@@ -421,16 +411,23 @@ def set_api_keys():
         
         updated_providers = []
         
-        for provider, api_key in data.items():
+        # Iterate over the API keys provided in the request data
-            if provider in ['shodan'] and api_key.strip():
+        for provider_name, api_key in data.items():
-                success = session_config.set_api_key(provider, api_key.strip())
+            # This allows us to both set and clear keys. The config
+            # handles enabling/disabling based on if the key is empty.
+            api_key_value = str(api_key or '').strip()
+            success = session_config.set_api_key(provider_name.lower(), api_key_value)
+            
             if success:
-                    updated_providers.append(provider)
+                updated_providers.append(provider_name)
         
         if updated_providers:
-            # Reinitialize scanner providers for this session only
+            # Reinitialize scanner providers to apply the new keys
             scanner._initialize_providers()
             
+            # Persist the updated scanner object back to the user's session
+            session_manager.update_session_scanner(user_session_id, scanner)
+            
             return jsonify({
                 'success': True,
                 'message': f'API keys updated for session {user_session_id}: {", ".join(updated_providers)}',
@@ -440,7 +437,7 @@ def set_api_keys():
         else:
             return jsonify({
                 'success': False,
-                'error': 'No valid API keys were provided'
+                'error': 'No valid API keys were provided or provider names were incorrect.'
             }), 400
     
     except Exception as e:
@@ -451,14 +448,6 @@ def set_api_keys():
             'error': f'Internal server error: {str(e)}'
         }), 500
 
-    except Exception as e:
-        print(f"ERROR: Exception in set_api_keys endpoint: {e}")
-        traceback.print_exc()
-        return jsonify({
-            'success': False,
-            'error': f'Internal server error: {str(e)}'
-        }), 500
-
 
 @app.route('/api/session/info', methods=['GET'])
 def get_session_info():

core/scanner.py

@@ -3,6 +3,8 @@
 import threading
 import traceback
 import time
+import os
+import importlib
 from typing import List, Set, Dict, Any, Tuple
 from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError, Future
 from collections import defaultdict, deque
@@ -11,9 +13,7 @@ from datetime import datetime, timezone
 from core.graph_manager import GraphManager, NodeType, RelationshipType
 from core.logger import get_forensic_logger, new_session
 from utils.helpers import _is_valid_ip, _is_valid_domain
-from providers.crtsh_provider import CrtShProvider
+from providers.base_provider import BaseProvider
-from providers.dns_provider import DNSProvider
-from providers.shodan_provider import ShodanProvider
 
 
 class ScanStatus:
@@ -61,13 +61,6 @@ class Scanner:
             self.max_workers = self.config.max_concurrent_requests
             self.executor = None
 
-            # Provider eligibility mapping
-            self.provider_eligibility = {
-                'dns': {'domains': True, 'ips': True},
-                'crtsh': {'domains': True, 'ips': False},
-                'shodan': {'domains': True, 'ips': True}
-            }
-
             # Initialize providers with session config
             print("Calling _initialize_providers with session config...")
             self._initialize_providers()
@@ -163,25 +156,27 @@ class Scanner:
         self.providers = []
         print("Initializing providers with session config...")
 
-        # Provider classes mapping
+        provider_dir = os.path.join(os.path.dirname(__file__), '..', 'providers')
-        provider_classes = {
+        for filename in os.listdir(provider_dir):
-            'dns': DNSProvider,
+            if filename.endswith('_provider.py') and not filename.startswith('base'):
-            'crtsh': CrtShProvider,
+                module_name = f"providers.{filename[:-3]}"
-            'shodan': ShodanProvider
-        }
-
-        for provider_name, provider_class in provider_classes.items():
-            if self.config.is_provider_enabled(provider_name):
                 try:
+                    module = importlib.import_module(module_name)
+                    for attribute_name in dir(module):
+                        attribute = getattr(module, attribute_name)
+                        if isinstance(attribute, type) and issubclass(attribute, BaseProvider) and attribute is not BaseProvider:
+                            provider_class = attribute
+                            provider_name = provider_class(session_config=self.config).get_name()
+                            if self.config.is_provider_enabled(provider_name):
                                 provider = provider_class(session_config=self.config)
                                 if provider.is_available():
                                     provider.set_stop_event(self.stop_event)
                                     self.providers.append(provider)
-                        print(f"✓ {provider_name.title()} provider initialized successfully for session")
+                                    print(f"✓ {provider.get_display_name()} provider initialized successfully for session")
                                 else:
-                        print(f"✗ {provider_name.title()} provider is not available")
+                                    print(f"✗ {provider.get_display_name()} provider is not available")
                 except Exception as e:
-                    print(f"✗ Failed to initialize {provider_name.title()} provider: {e}")
+                    print(f"✗ Failed to initialize provider from {filename}: {e}")
                     traceback.print_exc()
 
         print(f"Initialized {len(self.providers)} providers for session")
@@ -417,13 +412,11 @@ class Scanner:
         target_key = 'ips' if is_ip else 'domains'
         
         for provider in self.providers:
-            provider_name = provider.get_name()
+            if provider.get_eligibility().get(target_key):
-            if provider_name in self.provider_eligibility:
+                if not self._already_queried_provider(target, provider.get_name()):
-                if self.provider_eligibility[provider_name][target_key]:
-                    if not self._already_queried_provider(target, provider_name):
                     eligible.append(provider)
                 else:
-                        print(f"Skipping {provider_name} for {target} - already queried")
+                    print(f"Skipping {provider.get_name()} for {target} - already queried")
         
         return eligible
 
@@ -741,3 +734,35 @@ class Scanner:
         for provider in self.providers:
             stats[provider.get_name()] = provider.get_statistics()
         return stats
+
+    def get_provider_info(self) -> Dict[str, Dict[str, Any]]:
+        """Get information about all available providers."""
+        info = {}
+        provider_dir = os.path.join(os.path.dirname(__file__), '..', 'providers')
+        for filename in os.listdir(provider_dir):
+            if filename.endswith('_provider.py') and not filename.startswith('base'):
+                module_name = f"providers.{filename[:-3]}"
+                try:
+                    module = importlib.import_module(module_name)
+                    for attribute_name in dir(module):
+                        attribute = getattr(module, attribute_name)
+                        if isinstance(attribute, type) and issubclass(attribute, BaseProvider) and attribute is not BaseProvider:
+                            provider_class = attribute
+                            # Instantiate to get metadata, even if not fully configured
+                            temp_provider = provider_class(session_config=self.config)
+                            provider_name = temp_provider.get_name()
+                            
+                            # Find the actual provider instance if it exists, to get live stats
+                            live_provider = next((p for p in self.providers if p.get_name() == provider_name), None)
+                            
+                            info[provider_name] = {
+                                'display_name': temp_provider.get_display_name(),
+                                'requires_api_key': temp_provider.requires_api_key(),
+                                'statistics': live_provider.get_statistics() if live_provider else temp_provider.get_statistics(),
+                                'enabled': self.config.is_provider_enabled(provider_name),
+                                'rate_limit': self.config.get_rate_limit(provider_name),
+                            }
+                except Exception as e:
+                    print(f"✗ Failed to get info for provider from {filename}: {e}")
+                    traceback.print_exc()
+        return info

providers/__init__.py

@@ -16,4 +16,4 @@ __all__ = [
     'ShodanProvider'
 ]
 
-__version__ = "1.0.0-phase2"
+__version__ = "0.0.0-rc"

providers/base_provider.py

@@ -126,6 +126,21 @@ class BaseProvider(ABC):
         """Return the provider name."""
         pass
 
+    @abstractmethod
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        pass
+
+    @abstractmethod
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        pass
+
+    @abstractmethod
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        pass
+
     @abstractmethod
     def is_available(self) -> bool:
         """Check if the provider is available and properly configured."""

providers/crtsh_provider.py

@@ -36,6 +36,18 @@ class CrtShProvider(BaseProvider):
         """Return the provider name."""
         return "crtsh"
     
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        return "crt.sh"
+
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        return False
+
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        return {'domains': True, 'ips': False}
+
     def is_available(self) -> bool:
         """
         Check if the provider is configured to be used.

providers/dns_provider.py

@@ -33,6 +33,18 @@ class DNSProvider(BaseProvider):
         """Return the provider name."""
         return "dns"
 
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        return "DNS"
+
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        return False
+
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        return {'domains': True, 'ips': True}
+
     def is_available(self) -> bool:
         """DNS is always available - no API key required."""
         return True

providers/shodan_provider.py

@@ -35,6 +35,17 @@ class ShodanProvider(BaseProvider):
         """Return the provider name."""
         return "shodan"
 
+    def get_display_name(self) -> str:
+        """Return the provider display name for the UI."""
+        return "shodan"
+
+    def requires_api_key(self) -> bool:
+        """Return True if the provider requires an API key."""
+        return True
+
+    def get_eligibility(self) -> Dict[str, bool]:
+        """Return a dictionary indicating if the provider can query domains and/or IPs."""
+        return {'domains': True, 'ips': True}
 
     def query_domain(self, domain: str) -> List[Tuple[str, str, RelationshipType, float, Dict[str, Any]]]:
         """
@@ -201,11 +212,17 @@ class ShodanProvider(BaseProvider):
             # Extract ASN relationship if available
             asn = data.get('asn')
             if asn:
+                # Ensure the ASN starts with "AS"
+                if isinstance(asn, str) and asn.startswith('AS'):
+                    asn_name = asn
+                    asn_number = asn[2:]
+                else:
                     asn_name = f"AS{asn}"
+                    asn_number = str(asn)
 
                 asn_raw_data = {
                     'ip_address': ip,
-                    'asn': asn,
+                    'asn': asn_number,
                     'isp': data.get('isp', ''),
                     'org': data.get('org', '')
                 }

static/js/graph.js

@@ -13,7 +13,6 @@ class GraphManager {
         this.currentLayout = 'physics';
         this.nodeInfoPopup = null;
 
-        // Enhanced graph options for Phase 2
         this.options = {
             nodes: {
                 shape: 'dot',
@@ -214,20 +213,7 @@ class GraphManager {
             }
         });
 
-        this.network.on('blurNode', (params) => {
+        // TODO Context menu (right-click)
-            this.hideNodeInfoPopup();
-            this.clearHoverHighlights();
-        });
-
-        // Double-click to focus on node
-        this.network.on('doubleClick', (params) => {
-            if (params.nodes.length > 0) {
-                const nodeId = params.nodes[0];
-                this.focusOnNode(nodeId);
-            }
-        });
-
-        // Context menu (right-click)
         this.network.on('oncontext', (params) => {
             params.event.preventDefault();
             if (params.nodes.length > 0) {

static/js/main.js

@@ -12,10 +12,8 @@ class DNSReconApp {
         this.pollInterval = null;
         this.currentSessionId = null;
         
-        // UI Elements
         this.elements = {};
         
-        // Application state
         this.isScanning = false;
         this.lastGraphUpdate = null;
         
@@ -80,7 +78,7 @@ class DNSReconApp {
             // API Key Modal elements
             apiKeyModal: document.getElementById('api-key-modal'),
             apiKeyModalClose: document.getElementById('api-key-modal-close'),
-            shodanApiKey: document.getElementById('shodan-api-key'),
+            apiKeyInputs: document.getElementById('api-key-inputs'),
             saveApiKeys: document.getElementById('save-api-keys'),
             resetApiKeys: document.getElementById('reset-api-keys'),
 
@@ -732,6 +730,7 @@ class DNSReconApp {
             
             if (response.success) {
                 this.updateProviderDisplay(response.providers);
+                this.buildApiKeyModal(response.providers);
                 console.log('Providers loaded successfully');
             }
             
@@ -766,7 +765,7 @@ class DNSReconApp {
             
             providerItem.innerHTML = `
                 <div class="provider-header">
-                    <div class="provider-name">${name.toUpperCase()}</div>
+                    <div class="provider-name">${info.display_name}</div>
                     <div class="provider-status ${statusClass}">${statusText}</div>
                 </div>
                 <div class="provider-stats">
@@ -970,10 +969,15 @@ class DNSReconApp {
      * Save API Keys
      */
     async saveApiKeys() {
-        const shodanKey = this.elements.shodanApiKey.value.trim();
+        const inputs = this.elements.apiKeyInputs.querySelectorAll('input');
-
         const keys = {};
-        if (shodanKey) keys.shodan = shodanKey;
+        inputs.forEach(input => {
+            const provider = input.dataset.provider;
+            const value = input.value.trim();
+            if (provider && value) {
+                keys[provider] = value;
+            }
+        });
 
         if (Object.keys(keys).length === 0) {
             this.showWarning('No API keys were entered.');
@@ -998,7 +1002,10 @@ class DNSReconApp {
      * Reset API Key fields
      */
     resetApiKeys() {
-        this.elements.shodanApiKey.value = '';
+        const inputs = this.elements.apiKeyInputs.querySelectorAll('input');
+        inputs.forEach(input => {
+            input.value = '';
+        });
     }
     
     /**
@@ -1295,6 +1302,74 @@ class DNSReconApp {
         };
         return colors[type] || colors.info;
     }
+
+    /**
+     * Build the API key modal dynamically
+     * @param {Object} providers - Provider information
+     */
+    buildApiKeyModal(providers) {
+        if (!this.elements.apiKeyInputs) return;
+        this.elements.apiKeyInputs.innerHTML = ''; // Clear existing inputs
+        let hasApiKeyProviders = false;
+
+        for (const [name, info] of Object.entries(providers)) {
+            if (info.requires_api_key) {
+                hasApiKeyProviders = true;
+                const inputGroup = document.createElement('div');
+                inputGroup.className = 'apikey-section';
+
+                if (info.enabled) {
+                    // If the API key is set and the provider is enabled
+                    inputGroup.innerHTML = `
+                        <label for="${name}-api-key">${info.display_name} API Key</label>
+                        <div class="api-key-set-message">
+                            <span class="api-key-set-text">API Key is set</span>
+                            <button class="clear-api-key-btn" data-provider="${name}">Clear</button>
+                        </div>
+                        <p class="apikey-help">Provides infrastructure context and service information.</p>
+                    `;
+                } else {
+                    // If the API key is not set
+                    inputGroup.innerHTML = `
+                        <label for="${name}-api-key">${info.display_name} API Key</label>
+                        <input type="password" id="${name}-api-key" data-provider="${name}" placeholder="Enter ${info.display_name} API Key">
+                        <p class="apikey-help">Provides infrastructure context and service information.</p>
+                    `;
+                }
+                this.elements.apiKeyInputs.appendChild(inputGroup);
+            }
+        }
+        
+        // Add event listeners for the new clear buttons
+        this.elements.apiKeyInputs.querySelectorAll('.clear-api-key-btn').forEach(button => {
+            button.addEventListener('click', (e) => {
+                const provider = e.target.dataset.provider;
+                this.clearApiKey(provider);
+            });
+        });
+
+        if (!hasApiKeyProviders) {
+            this.elements.apiKeyInputs.innerHTML = '<p>No providers require API keys.</p>';
+        }
+    }
+
+    /**
+     * Clear an API key for a specific provider
+     * @param {string} provider The name of the provider to clear the API key for
+     */
+    async clearApiKey(provider) {
+        try {
+            const response = await this.apiCall('/api/config/api-keys', 'POST', { [provider]: '' });
+            if (response.success) {
+                this.showSuccess(`API key for ${provider} has been cleared.`);
+                this.loadProviders(); // This will rebuild the modal with the updated state
+            } else {
+                throw new Error(response.error || 'Failed to clear API key');
+            }
+        } catch (error) {
+            this.showError(`Error clearing API key: ${error.message}`);
+        }
+    }
 }
 
 // Add CSS animations for message toasts

templates/index.html

@@ -186,7 +186,7 @@
 
         <footer class="footer">
             <div class="footer-content">
-                <span>DNSRecon v1.0 - Phase 1 Implementation</span>
+                <span>v0.0.0rc</span>
                 <span class="footer-separator">|</span>
                 <span>Passive Infrastructure Reconnaissance</span>
                 <span class="footer-separator">|</span>
@@ -217,10 +217,7 @@
                     <p class="modal-description">
                         Enter your API keys for enhanced data providers. Keys are stored in memory for the current session only and are never saved to disk.
                     </p>
-                    <div class="apikey-section">
+                    <div id="api-key-inputs">
-                        <label for="shodan-api-key">Shodan API Key</label>
-                        <input type="password" id="shodan-api-key" placeholder="Enter Shodan API Key">
-                        <p class="apikey-help">Provides infrastructure context and service information.</p>
                         </div>
                     <div class="button-group" style="flex-direction: row; justify-content: flex-end;">
                         <button id="reset-api-keys" class="btn btn-secondary">